qq27800734/taoyao
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[*] 视频采集编码

Browse Source
This commit is contained in:
acgist
2024-05-06 15:21:13 +08:00
parent 8f5cb55479
commit e318866834
15 changed files with 453 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt
View File

@@ -40,6 +40,7 @@ add_library(
media/Capturer.cpp
media/AudioCapturer.cpp
media/VideoCapturer.cpp
media/VideoEncoder.cpp
media/MediaManager.cpp
)

120
taoyao-client-openharmony/taoyao/media/src/main/cpp/bind.cpp
View File

@@ -1,5 +1,7 @@
/**
* 方法绑定
* NAPI(NODE-API)
*
* https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/reference/native-lib/napi.md
*/
#include <map>
@@ -71,7 +73,7 @@ static void init() {
/**
* 卸载系统
*/
static napi_value shutdown(napi_env env, napi_callback_info info) {
static napi_value shutdown(napi_env env, napi_callback_info info) {
if(!initTaoyao) {
OH_LOG_INFO(LOG_APP, "已经卸载libtaoyao");
return 0;
@@ -80,7 +82,13 @@ static napi_value shutdown(napi_env env, napi_callback_info info) {
OH_LOG_INFO(LOG_APP, "卸载libtaoyao");
OH_LOG_INFO(LOG_APP, "释放mediasoupclient");
mediasoupclient::Cleanup();
// this->roomMap
// 删除房间
for(auto iterator = roomMap.begin(); iterator != roomMap.end(); ++iterator) {
delete iterator->second;
iterator->second = nullptr;
}
roomMap.clear();
// 关闭媒体
if (mediaManager != nullptr) {
delete mediaManager;
mediaManager = nullptr;
@@ -90,6 +98,9 @@ static napi_value shutdown(napi_env env, napi_callback_info info) {
return 0;
}
/**
* 发送消息
*/
static void send(const std::string& signal, const std::string& body) {
napi_value ret;
napi_value callback = nullptr;
@@ -101,6 +112,9 @@ static void send(const std::string& signal, const std::string& body) {
// napi_get_undefined(acgist::env, &ret);
}
/**
* 发送请求
*/
static std::string request(const std::string& signal, const std::string& body) {
napi_value ret;
napi_value callback = nullptr;
@@ -115,34 +129,27 @@ static std::string request(const std::string& signal, const std::string& body) {
return chars;
}
static napi_value mediaConsume(napi_env env, napi_callback_info info) { return 0; }
static napi_value mediaConsumerClose(napi_env env, napi_callback_info info) { return 0; }
static napi_value mediaConsumerPause(napi_env env, napi_callback_info info) { return 0; }
static napi_value mediaConsumerResume(napi_env env, napi_callback_info info) { return 0; }
static napi_value mediaProducerClose(napi_env env, napi_callback_info info) { return 0; }
static napi_value mediaProducerPause(napi_env env, napi_callback_info info) { return 0; }
static napi_value mediaProducerResume(napi_env env, napi_callback_info info) { return 0; }
static napi_value roomClientList(napi_env env, napi_callback_info info) { return 0; }
static napi_value roomClose(napi_env env, napi_callback_info info) { return 0; }
/**
* 房间关闭
*/
static napi_value roomClose(napi_env env, napi_callback_info info) { return 0; }
/**
* 其他终端进入房间
* 进入房间
* 其他终端进入房间，自己进入房间逻辑参考房间邀请。
*/
static napi_value roomEnter(napi_env env, napi_callback_info info) {
return 0;
}
static napi_value roomExpel(napi_env env, napi_callback_info info) { return 0; }
/**
* 踢出房间
* 踢出房间以后终端离开房间
*/
static napi_value roomExpel(napi_env env, napi_callback_info info) { return 0; }
/**
* 房间邀请
* 邀请终端进入房间
*/
static napi_value roomInvite(napi_env env, napi_callback_info info) {
@@ -174,8 +181,56 @@ static napi_value roomInvite(napi_env env, napi_callback_info info) {
return ret;
}
static napi_value roomLeave(napi_env env, napi_callback_info info) { return 0; }
/**
* 离开房间
* 其他终端离开房间
*/
static napi_value roomLeave(napi_env env, napi_callback_info info) { return 0; }
/**
* 终端列表
* 房间所有终端列表首次进入方便加载终端列表信息
*/
static napi_value roomClientList(napi_env env, napi_callback_info info) { return 0; }
/**
* 媒体消费（被动通知）
*/
static napi_value mediaConsume(napi_env env, napi_callback_info info) { return 0; }
/**
* 消费者关闭（被动通知）
*/
static napi_value mediaConsumerClose(napi_env env, napi_callback_info info) { return 0; }
/**
* 消费者暂停（被动通知）
*/
static napi_value mediaConsumerPause(napi_env env, napi_callback_info info) { return 0; }
/**
* 消费者恢复（被动通知）
*/
static napi_value mediaConsumerResume(napi_env env, napi_callback_info info) { return 0; }
/**
* 生产者关闭（被动通知）
*/
static napi_value mediaProducerClose(napi_env env, napi_callback_info info) { return 0; }
/**
* 生产者暂停（被动通知）
*/
static napi_value mediaProducerPause(napi_env env, napi_callback_info info) { return 0; }
/**
* 生产者恢复（被动通知）
*/
static napi_value mediaProducerResume(napi_env env, napi_callback_info info) { return 0; }
/**
* 注册发送回调
*/
static napi_value registerSend(napi_env env, napi_callback_info info) {
size_t argc = 1;
napi_value args[1] = { nullptr };
@@ -184,7 +239,10 @@ static napi_value registerSend(napi_env env, napi_callback_info info) {
return 0;
}
static napi_value registerRequest(napi_env env, napi_callback_info info) {
/**
* 注册请求回调
*/
static napi_value registerRequest(napi_env env, napi_callback_info info) {
size_t argc = 1;
napi_value args[1] = { nullptr };
napi_get_cb_info(env, info, &argc, args, nullptr, nullptr);
@@ -200,6 +258,13 @@ EXTERN_C_START
static napi_value Init(napi_env env, napi_value exports) {
acgist::env = env;
napi_property_descriptor desc[] = {
{ "shutdown", nullptr, acgist::shutdown, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "roomClose", nullptr, acgist::roomClose, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "roomEnter", nullptr, acgist::roomEnter, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "roomExpel", nullptr, acgist::roomExpel, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "roomInvite", nullptr, acgist::roomInvite, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "roomLeave", nullptr, acgist::roomLeave, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "roomClientList", nullptr, acgist::roomClientList, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "mediaConsume", nullptr, acgist::mediaConsume, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "mediaConsumerClose", nullptr, acgist::mediaConsumerClose, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "mediaConsumerPause", nullptr, acgist::mediaConsumerPause, nullptr, nullptr, nullptr, napi_default, nullptr },
@@ -207,13 +272,6 @@ static napi_value Init(napi_env env, napi_value exports) {
{ "mediaProducerClose", nullptr, acgist::mediaProducerClose, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "mediaProducerPause", nullptr, acgist::mediaProducerPause, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "mediaProducerResume", nullptr, acgist::mediaProducerResume, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "roomClientList", nullptr, acgist::roomClientList, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "roomClose", nullptr, acgist::roomClose, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "roomEnter", nullptr, acgist::roomEnter, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "roomExpel", nullptr, acgist::roomExpel, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "roomInvite", nullptr, acgist::roomInvite, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "roomLeave", nullptr, acgist::roomLeave, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "shutdown", nullptr, acgist::shutdown, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "registerSend", nullptr, acgist::registerSend, nullptr, nullptr, nullptr, napi_default, nullptr },
{ "registerRequest", nullptr, acgist::registerRequest, nullptr, nullptr, nullptr, napi_default, nullptr },
};

109
taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp
View File

@@ -3,16 +3,15 @@
*
* @author acgist
*
* https://docs.openharmony.cn/pages/v4.0/zh-cn/application-dev/media/audio-encoding.md
* https://docs.openharmony.cn/pages/v4.0/zh-cn/application-dev/media/video-encoding.md
* https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/media/avcodec/audio-encoding.md
* https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/media/avcodec/video-encoding.md
* https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/media/media/avscreen-capture.md
* https://docs.openharmony.cn/pages/v4.0/zh-cn/application-dev/media/obtain-supported-codecs.md
* https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/reference/native-lib/opengles.md
* https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/reference/native-lib/opensles.md
* https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/media/avcodec/obtain-supported-codecs.md
* https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/media/camera/native-camera-recording.md
* https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/media/audio/using-ohaudio-for-recording.md
* https://docs.openharmony.cn/pages/v4.0/zh-cn/application-dev/reference/native-lib/third_party_opengl/opengles.md
* https://docs.openharmony.cn/pages/v4.0/zh-cn/application-dev/reference/native-lib/third_party_opensles/opensles.md
*/
#ifndef taoyao_Capturer_HPP
#define taoyao_Capturer_HPP
@@ -26,6 +25,8 @@
#include <ohaudio/native_audiocapturer.h>
#include <ohaudio/native_audiostreambuilder.h>
#include <multimedia/player_framework/native_avcodec_videoencoder.h>
namespace acgist {
/**
@@ -37,6 +38,7 @@ template <typename Sink>
class Capturer {
public:
// rtc::scoped_refptr
std::map<std::string, Sink*> map;
public:
@@ -44,6 +46,10 @@ public:
virtual ~Capturer();
public:
// 添加管道
virtual bool add(const std::string& id, Sink* sink);
// 删除管道
virtual bool remove(const std::string& id);
// 开始采集
virtual bool start() = 0;
// 结束采集
@@ -51,6 +57,38 @@ public:
};
template <typename Sink>
acgist::Capturer<Sink>::Capturer() {}
template <typename Sink>
acgist::Capturer<Sink>::~Capturer() {
for (auto iterator = this->map.begin(); iterator != this->map.end(); ++iterator) {
// TODO：释放
// delete iterator->second;
// iterator->second = nullptr;
}
this->map.clear();
}
template <typename Sink>
bool acgist::Capturer<Sink>::add(const std::string& id, Sink* sink) {
this->map.insert(std::make_pair(id, sink));
return true;
}
template <typename Sink>
bool acgist::Capturer<Sink>::remove(const std::string& id) {
auto iterator = this->map.find(id);
if (iterator == this->map.end()) {
return false;
}
// TODO：释放
// delete iterator->second;
// iterator->second = nullptr;
this->map.erase(iterator);
return true;
}
/**
* 音频采集器
*/
@@ -67,30 +105,69 @@ public:
virtual ~AudioCapturer();
public:
virtual bool start();
virtual bool stop();
virtual bool start() override;
virtual bool stop() override;
};
/**
* 视频编码
*/
class VideoEncoder {
public:
// 视频编码器
OH_AVCodec* avCodec;
// 视频窗口
OHNativeWindow* nativeWindow;
public:
VideoEncoder();
virtual ~VideoEncoder();
public:
// 重新开始
void restart();
// 动态配置
void reset(OH_AVFormat *format);
// 动态配置
void resetIntConfig(const char* key, int32_t value);
// 动态配置
void resetLongConfig(const char* key, int64_t value);
// 动态配置
void resetDoubleConfig(const char* key, double value);
// 开始编码
bool start();
// 结束编码
bool stop();
};
/**
* 视频采集器
*/
class VideoCapturer: public Capturer<rtc::VideoSinkInterface<webrtc::RecordableEncodedFrame>> {
class VideoCapturer: public Capturer<rtc::VideoSinkInterface<webrtc::VideoFrame>> {
public:
Camera_Device* camera_Device;
Camera_Manager* camera_Manager;
Camera_VideoOutput* camera_VideoOutput;
Camera_VideoProfile* camera_VideoProfile;
Camera_OutputCapability* camera_OutputCapability;
// 摄像头设备数量
uint32_t size = 0;
// 摄像头索引
uint32_t cameraIndex = 0;
// 摄像头设备列表
Camera_Device* cameraDevice = nullptr;
// 摄像头管理器
Camera_Manager* cameraManager = nullptr;
// 摄像头视频输出
Camera_VideoOutput* cameraVideoOutput = nullptr;
// 摄像头输出能力
Camera_OutputCapability* cameraOutputCapability = nullptr;
public:
VideoCapturer();
virtual ~VideoCapturer();
public:
virtual bool start();
virtual bool stop();
virtual bool start() override;
virtual bool stop() override;
};
}

1
taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Client.hpp
View File

@@ -3,7 +3,6 @@
*
* @author acgist
*/
#ifndef taoyao_Client_HPP
#define taoyao_Client_HPP

3
taoyao-client-openharmony/taoyao/media/src/main/cpp/include/MediaManager.hpp
View File

@@ -4,9 +4,8 @@
*
* @author acgist
*
* https://docs.openharmony.cn/pages/v4.0/zh-cn/application-dev/media/obtain-supported-codecs.md
* https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/media/avcodec/obtain-supported-codecs.md
*/
#ifndef taoyao_MediaManager_HPP
#define taoyao_MediaManager_HPP

1
taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Palyer.hpp
View File

@@ -5,7 +5,6 @@
* https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/media/audio/audio-playback-overview.md
* https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/media/media/using-ndk-avplayer-for-playerback.md
*/
#ifndef TAOYAO_PALYER_H
#define TAOYAO_PALYER_H

1
taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Room.hpp
View File

@@ -3,7 +3,6 @@
*
* @author acgist
*/
#ifndef taoyao_Room_HPP
#define taoyao_Room_HPP

33
taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp
View File

@@ -19,23 +19,22 @@ static int32_t OnStreamEvent(OH_AudioCapturer* capturer, void* userData, OH_Audi
static int32_t OnInterruptEvent(OH_AudioCapturer* capturer, void* userData, OH_AudioInterrupt_ForceType type, OH_AudioInterrupt_Hint hint);
acgist::AudioCapturer::AudioCapturer() {
OH_AudioStream_Result ret = OH_AudioStreamBuilder_Create(&builder, AUDIOSTREAM_TYPE_RENDERER);
OH_AudioStream_Result ret = OH_AudioStreamBuilder_Create(&this->builder, AUDIOSTREAM_TYPE_RENDERER);
OH_LOG_INFO(LOG_APP, "构造音频采集：%o", ret);
}
acgist::AudioCapturer::~AudioCapturer() {
OH_AudioStream_Result ret = OH_AudioStreamBuilder_Destroy(builder);
// TODO: 是否需要delete
builder = nullptr;
OH_AudioStream_Result ret = OH_AudioStreamBuilder_Destroy(this->builder);
this->builder = nullptr;
OH_LOG_INFO(LOG_APP, "释放音频采集：%o", ret);
}
bool acgist::AudioCapturer::start() {
// 配置采集参数
OH_AudioStreamBuilder_SetSamplingRate(builder, samplingRate);
OH_AudioStreamBuilder_SetChannelCount(builder, channelCount);
OH_AudioStreamBuilder_SetLatencyMode(builder, latencyMode);
OH_AudioStreamBuilder_SetSampleFormat(builder, sampleFormat);
OH_AudioStreamBuilder_SetSamplingRate(this->builder, samplingRate);
OH_AudioStreamBuilder_SetChannelCount(this->builder, channelCount);
OH_AudioStreamBuilder_SetLatencyMode(this->builder, latencyMode);
OH_AudioStreamBuilder_SetSampleFormat(this->builder, sampleFormat);
OH_LOG_DEBUG(LOG_APP, "配置音频格式：%d %d %o %o", samplingRate, channelCount, latencyMode, sampleFormat);
// 设置回调函数
OH_AudioCapturer_Callbacks callbacks;
@@ -43,24 +42,24 @@ bool acgist::AudioCapturer::start() {
callbacks.OH_AudioCapturer_OnReadData = OnReadData;
callbacks.OH_AudioCapturer_OnStreamEvent = OnStreamEvent;
callbacks.OH_AudioCapturer_OnInterruptEvent = OnInterruptEvent;
OH_AudioStream_Result ret = OH_AudioStreamBuilder_SetCapturerCallback(builder, callbacks, this);
OH_AudioStream_Result ret = OH_AudioStreamBuilder_SetCapturerCallback(this->builder, callbacks, this);
OH_LOG_DEBUG(LOG_APP, "设置回调函数：%o", ret);
// 构造音频采集器
ret = OH_AudioStreamBuilder_GenerateCapturer(builder, &audioCapturer);
ret = OH_AudioStreamBuilder_GenerateCapturer(this->builder, &this->audioCapturer);
OH_LOG_DEBUG(LOG_APP, "构造音频采集器：%o", ret);
// 开始录制
ret = OH_AudioCapturer_Start(audioCapturer);
ret = OH_AudioCapturer_Start(this->audioCapturer);
OH_LOG_DEBUG(LOG_APP, "开始录制：%o", ret);
return ret == OH_AudioStream_Result::AUDIOSTREAM_SUCCESS;
}
bool acgist::AudioCapturer::stop() {
// 停止录制
OH_AudioStream_Result ret = OH_AudioCapturer_Stop(audioCapturer);
OH_AudioStream_Result ret = OH_AudioCapturer_Stop(this->audioCapturer);
OH_LOG_DEBUG(LOG_APP, "停止录制：%o", ret);
// 释放音频采集器
ret = OH_AudioCapturer_Release(audioCapturer);
audioCapturer = nullptr;
ret = OH_AudioCapturer_Release(this->audioCapturer);
this->audioCapturer = nullptr;
OH_LOG_DEBUG(LOG_APP, "释放音频采集器：%o", ret);
return ret == OH_AudioStream_Result::AUDIOSTREAM_SUCCESS;
}
@@ -73,11 +72,7 @@ static int32_t OnError(OH_AudioCapturer* capturer, void* userData, OH_AudioStrea
static int32_t OnReadData(OH_AudioCapturer* capturer, void* userData, void* buffer, int32_t length) {
acgist::AudioCapturer* audioCapturer = (acgist::AudioCapturer*) userData;
int64_t timeMillis = rtc::TimeMillis();
for (
auto iterator = audioCapturer->map.begin();
iterator != audioCapturer->map.end();
++iterator
) {
for (auto iterator = audioCapturer->map.begin(); iterator != audioCapturer->map.end(); ++iterator) {
iterator->second->OnData(buffer, 16, samplingRate, channelCount, sizeof(buffer) / 2, timeMillis);
}
return 0;

1
taoyao-client-openharmony/taoyao/media/src/main/cpp/media/Capturer.cpp
View File

@@ -0,0 +1 @@
#include "../include/Capturer.hpp"

4
taoyao-client-openharmony/taoyao/media/src/main/cpp/media/MediaManager.cpp
View File

@@ -65,6 +65,10 @@ bool acgist::MediaManager::startAudioCapture() {
return true;
}
bool acgist::MediaManager::startVideoCapture() {
return true;
}
rtc::scoped_refptr<webrtc::AudioTrackInterface> acgist::MediaManager::getAudioTrack() {
cricket::AudioOptions options;
options.highpass_filter = true;

66
taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoCapturer.cpp
View File

@@ -1,8 +1,74 @@
/**
* https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/reference/apis-camera-kit/camera_8h.md
* https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/reference/apis-camera-kit/video__output_8h.md
* https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/reference/apis-camera-kit/camera__manager_8h.md
* https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/reference/apis-camera-kit/capture__session_8h.md
*/
#include "../include/Capturer.hpp"
#include "hilog/log.h"
#include "ohcamera/camera_input.h"
#include "ohcamera/camera_manager.h"
#include "ohcamera/capture_session.h"
static void onError(Camera_VideoOutput* videoOutput, Camera_ErrorCode errorCode);
static void onFrameEnd(Camera_VideoOutput* videoOutput, int32_t frameCount);
static void onFrameStart(Camera_VideoOutput* videoOutput);
acgist::VideoCapturer::VideoCapturer() {
Camera_ErrorCode ret = OH_Camera_GetCameraManager(&this->cameraManager);
OH_LOG_INFO(LOG_APP, "构造摄像头管理器：%o", ret);
ret = OH_CameraManager_GetSupportedCameras(this->cameraManager, &this->cameraDevice, &this->size);
OH_LOG_INFO(LOG_APP, "摄像头设备列表：%o %d", ret, size);
ret = OH_CameraManager_GetSupportedCameraOutputCapability(this->cameraManager, &this->cameraDevice[this->cameraIndex], &this->cameraOutputCapability);
OH_LOG_INFO(LOG_APP, "摄像头输出功能：%o %d %d", ret, this->cameraIndex, this->cameraOutputCapability->videoProfilesSize);
// 二次处理：createImageReceiver/getReceivingSurfaceId
// ret = OH_CameraManager_CreateVideoOutput(this->cameraManager, this->cameraOutputCapability->videoProfiles[0],
// this->nativeWindow, &this->cameraVideoOutput);
}
acgist::VideoCapturer::~VideoCapturer() {
Camera_ErrorCode ret = OH_VideoOutput_Release(this->cameraVideoOutput);
this->cameraVideoOutput = nullptr;
OH_LOG_INFO(LOG_APP, "释放摄像头视频输出：%o", ret);
ret = OH_CameraManager_DeleteSupportedCameraOutputCapability(this->cameraManager, this->cameraOutputCapability);
this->cameraOutputCapability = nullptr;
OH_LOG_INFO(LOG_APP, "释放摄像头输出能力：%o", ret);
ret = OH_CameraManager_DeleteSupportedCameras(this->cameraManager, this->cameraDevice, this->size);
this->cameraDevice = nullptr;
OH_LOG_INFO(LOG_APP, "释放摄像头设备列表：%o", ret);
ret = OH_Camera_DeleteCameraManager(this->cameraManager);
this->cameraManager = nullptr;
OH_LOG_INFO(LOG_APP, "释放摄像头管理器：%o", ret);
}
bool acgist::VideoCapturer::start() {
Camera_ErrorCode ret = OH_VideoOutput_Start(this->cameraVideoOutput);
OH_LOG_INFO(LOG_APP, "开始视频捕获：%o", ret);
VideoOutput_Callbacks callbacks;
callbacks.onError = onError;
callbacks.onFrameEnd = onFrameEnd;
callbacks.onFrameStart = onFrameStart;
ret = OH_VideoOutput_RegisterCallback(this->cameraVideoOutput, &callbacks);
OH_LOG_INFO(LOG_APP, "视频捕获回调：%o", ret);
return true;
}
bool acgist::VideoCapturer::stop() {
Camera_ErrorCode ret = OH_VideoOutput_Stop(this->cameraVideoOutput);
OH_LOG_INFO(LOG_APP, "结束视频捕获：%o", ret);
return true;
}
static void onError(Camera_VideoOutput* videoOutput, Camera_ErrorCode errorCode) {
OH_LOG_WARN(LOG_APP, "视频捕获数据帧失败：%d", errorCode);
}
static void onFrameEnd(Camera_VideoOutput* videoOutput, int32_t frameCount) {
OH_LOG_DEBUG(LOG_APP, "结束视频捕获数据帧");
}
static void onFrameStart(Camera_VideoOutput* videoOutput) {
OH_LOG_DEBUG(LOG_APP, "开始视频捕获数据帧");
}

164
taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp Normal file
View File

@@ -0,0 +1,164 @@
#include "../include/Capturer.hpp"
#include "hilog/log.h"
#include "rtc_base/time_utils.h"
#include "api/video/nv12_buffer.h"
#include "api/video/i420_buffer.h"
#include <multimedia/player_framework/native_avbuffer.h>
#include <multimedia/player_framework/native_avformat.h>
#include <multimedia/player_framework/native_avcodec_base.h>
#include <multimedia/player_framework/native_avcapability.h>
static uint32_t width = 720;
static uint32_t height = 480;
static uint64_t bitrate = 3'000'000L;
static double frameRate = 30.0;
static void OnError(OH_AVCodec* codec, int32_t errorCode, void* userData);
static void OnStreamChanged(OH_AVCodec* codec, OH_AVFormat* format, void* userData);
static void OnNeedInputBuffer(OH_AVCodec* codec, uint32_t index, OH_AVBuffer* buffer, void* userData);
static void OnNewOutputBuffer(OH_AVCodec* codec, uint32_t index, OH_AVBuffer* buffer, void* userData);
acgist::VideoEncoder::VideoEncoder() {
OH_AVCapability* capability = OH_AVCodec_GetCapability(OH_AVCODEC_MIMETYPE_VIDEO_AVC, true);
const char* codecName = OH_AVCapability_GetName(capability);
this->avCodec = OH_VideoEncoder_CreateByName(codecName);
OH_LOG_INFO(LOG_APP, "视频编码格式：%s", codecName);
// 注册回调
OH_AVCodecCallback callback = { &OnError, &OnStreamChanged, &OnNeedInputBuffer, &OnNewOutputBuffer };
OH_AVErrCode ret = OH_VideoEncoder_RegisterCallback(this->avCodec, callback, this);
OH_LOG_INFO(LOG_APP, "注册编码回调：%o", ret);
// 配置编码参数：https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/media/avcodec/video-encoding.md
OH_AVFormat* format = OH_AVFormat_Create();
// 配置视频帧宽度（必须）
OH_AVFormat_SetIntValue(format, OH_MD_KEY_WIDTH, width);
// 配置视频帧高度（必须）
OH_AVFormat_SetIntValue(format, OH_MD_KEY_HEIGHT, height);
// 配置视频颜色格式（必须）
OH_AVFormat_SetIntValue(format, OH_MD_KEY_PIXEL_FORMAT, AV_PIXEL_FORMAT_YUVI420);
// OH_AVFormat_SetIntValue(format, OH_MD_KEY_PIXEL_FORMAT, AV_PIXEL_FORMAT_NV12);
// 配置视频帧速率
OH_AVFormat_SetDoubleValue(format, OH_MD_KEY_FRAME_RATE, frameRate);
// 配置视频YUV值范围标志
OH_AVFormat_SetIntValue(format, OH_MD_KEY_RANGE_FLAG, false);
// 配置视频原色
OH_AVFormat_SetIntValue(format, OH_MD_KEY_COLOR_PRIMARIES, static_cast<int32_t>(OH_ColorPrimary::COLOR_PRIMARY_BT709));
// 配置传输特性
OH_AVFormat_SetIntValue(format, OH_MD_KEY_TRANSFER_CHARACTERISTICS, static_cast<int32_t>(OH_TransferCharacteristic::TRANSFER_CHARACTERISTIC_BT709));
// 配置最大矩阵系数
OH_AVFormat_SetIntValue(format, OH_MD_KEY_MATRIX_COEFFICIENTS, static_cast<int32_t>(OH_MatrixCoefficient::MATRIX_COEFFICIENT_IDENTITY));
// 配置关键帧的间隔（单位为：毫秒）
OH_AVFormat_SetIntValue(format, OH_MD_KEY_I_FRAME_INTERVAL, 5000);
// 配置编码Profile
OH_AVFormat_SetIntValue(format, OH_MD_KEY_PROFILE, static_cast<int32_t>(OH_AVCProfile::AVC_PROFILE_BASELINE));
// 配置编码比特率模式
OH_AVFormat_SetIntValue(format, OH_MD_KEY_VIDEO_ENCODE_BITRATE_MODE, static_cast<int32_t>(OH_VideoEncodeBitrateMode::CBR));
// 配置比特率
OH_AVFormat_SetLongValue(format, OH_MD_KEY_BITRATE, bitrate);
// 配置所需的编码质量：只有在恒定质量模式下配置的编码器才支持此配置
OH_AVFormat_SetIntValue(format, OH_MD_KEY_QUALITY, 0);
ret = OH_VideoEncoder_Configure(this->avCodec, format);
OH_AVFormat_Destroy(format);
OH_LOG_INFO(LOG_APP, "配置编码参数：%o %d %d %f %ld", ret, width, height, frameRate, bitrate);
ret = OH_VideoEncoder_Prepare(this->avCodec);
OH_LOG_INFO(LOG_APP, "视频编码准备就绪：%o", ret);
ret = OH_VideoEncoder_GetSurface(this->avCodec, &this->nativeWindow);
OH_LOG_INFO(LOG_APP, "配置surface：%o", ret);
}
acgist::VideoEncoder::~VideoEncoder() {
OH_AVErrCode ret = OH_VideoEncoder_Destroy(this->avCodec);
this->avCodec = nullptr;
OH_LOG_INFO(LOG_APP, "释放视频编码器：%o", ret);
// TODO: delete nativeWindow
}
void acgist::VideoEncoder::restart() {
OH_AVErrCode ret = OH_VideoEncoder_Flush(this->avCodec);
OH_LOG_INFO(LOG_APP, "清空编码队列：%o", ret);
ret = OH_VideoEncoder_Start(this->avCodec);
OH_LOG_INFO(LOG_APP, "开始视频编码：%o", ret);
}
void acgist::VideoEncoder::reset(OH_AVFormat* format) {
OH_AVErrCode ret = OH_VideoEncoder_Reset(this->avCodec);
OH_LOG_INFO(LOG_APP, "重置视频编码：%o", ret);
ret = OH_VideoEncoder_Configure(this->avCodec, format);
OH_LOG_INFO(LOG_APP, "配置视频编码：%o", ret);
}
void acgist::VideoEncoder::resetIntConfig(const char* key, int32_t value) {
OH_AVFormat* format = OH_AVFormat_Create();
OH_AVFormat_SetIntValue(format, key, value);
OH_AVErrCode ret = OH_VideoEncoder_SetParameter(this->avCodec, format);
OH_LOG_INFO(LOG_APP, "动态配置视频编码：%o %s %d", ret, key, value);
OH_AVFormat_Destroy(format);
}
void acgist::VideoEncoder::resetLongConfig(const char* key, int64_t value) {
OH_AVFormat* format = OH_AVFormat_Create();
OH_AVFormat_SetLongValue(format, key, value);
OH_AVErrCode ret = OH_VideoEncoder_SetParameter(this->avCodec, format);
OH_LOG_INFO(LOG_APP, "动态配置视频编码：%o %s %ld", ret, key, value);
OH_AVFormat_Destroy(format);
}
void acgist::VideoEncoder::resetDoubleConfig(const char* key, double value) {
OH_AVFormat* format = OH_AVFormat_Create();
OH_AVFormat_SetDoubleValue(format, key, value);
OH_AVErrCode ret = OH_VideoEncoder_SetParameter(this->avCodec, format);
OH_LOG_INFO(LOG_APP, "动态配置视频编码：%o %s %f", ret, key, value);
OH_AVFormat_Destroy(format);
}
bool acgist::VideoEncoder::start() {
OH_AVErrCode ret = OH_VideoEncoder_Start(this->avCodec);
OH_LOG_INFO(LOG_APP, "开始视频编码：%o", ret);
return true;
}
bool acgist::VideoEncoder::stop() {
OH_AVErrCode ret = OH_VideoEncoder_NotifyEndOfStream(this->avCodec);
OH_LOG_INFO(LOG_APP, "通知视频编码结束：%o", ret);
ret = OH_VideoEncoder_Stop(this->avCodec);
OH_LOG_INFO(LOG_APP, "结束视频编码：%o", ret);
return true;
}
static void OnError(OH_AVCodec* codec, int32_t errorCode, void* userData) {
OH_LOG_WARN(LOG_APP, "视频编码发送错误：%d", errorCode);
}
static void OnStreamChanged(OH_AVCodec* codec, OH_AVFormat* format, void* userData) {
OH_LOG_DEBUG(LOG_APP, "视频编码配置变化");
}
static void OnNeedInputBuffer(OH_AVCodec* codec, uint32_t index, OH_AVBuffer* buffer, void* userData) {
// 忽略
}
static void OnNewOutputBuffer(OH_AVCodec* codec, uint32_t index, OH_AVBuffer* buffer, void* userData) {
acgist::VideoCapturer* videoCapturer = (acgist::VideoCapturer*) userData;
// TODO: 全局是否性能更好
OH_AVCodecBufferAttr info;
OH_AVErrCode ret = OH_AVBuffer_GetBufferAttr(buffer, &info);
char* data = reinterpret_cast<char*>(OH_AVBuffer_GetAddr(buffer));
// TODO: 解析
rtc::scoped_refptr<webrtc::I420Buffer> videoFrameBuffer = webrtc::I420Buffer::Copy(width, height, (uint8_t*)data, 0, (uint8_t*)data, 0, (uint8_t*)data, 0);
// webrtc::NV12Buffer::Create(width, height);
webrtc::VideoFrame::Builder builder;
webrtc::VideoFrame videoFrame = builder
.set_timestamp_ms(rtc::TimeMillis())
.set_video_frame_buffer(videoFrameBuffer)
.set_rotation(webrtc::VideoRotation::kVideoRotation_0)
.build();
for (auto iterator = videoCapturer->map.begin(); iterator != videoCapturer->map.end(); ++iterator) {
iterator->second->OnFrame(videoFrame);
}
// TODO: 释放webrtc
videoFrameBuffer->Release();
ret = OH_VideoEncoder_FreeOutputBuffer(codec, index);
}

30
taoyao-client-openharmony/taoyao/media/src/main/cpp/types/libtaoyao/index.d.ts vendored
View File

@@ -1,16 +1,16 @@
export const mediaConsume : () => number;
export const mediaConsumerClose : () => number;
export const mediaConsumerPause : () => number;
export const mediaConsumerResume: () => number;
export const mediaProducerClose : () => number;
export const mediaProducerPause : () => number;
export const mediaProducerResume: () => number;
export const roomClientList : () => number;
export const roomClose : () => number;
export const roomEnter : () => number;
export const roomExpel : () => number;
export const shutdown : (json: string) => number;
export const roomClose : (json: string) => number;
export const roomEnter : (json: string) => number;
export const roomExpel : (json: string) => number;
export const roomInvite: (json: string) => number;
export const roomLeave : () => number;
export const shutdown : () => number;
export const registerSend : (callback: (signal: string, body: string) => void) => number;
export const registerRequest: (callback: (signal: string, body: string) => string) => number;
export const roomLeave : (json: string) => number;
export const roomClientList : (json: string) => number;
export const mediaConsume : (json: string) => number;
export const mediaConsumerClose : (json: string) => number;
export const mediaConsumerPause : (json: string) => number;
export const mediaConsumerResume: (json: string) => number;
export const mediaProducerClose : (json: string) => number;
export const mediaProducerPause : (json: string) => number;
export const mediaProducerResume: (json: string) => number;
export const registerSend : (callback: (signal: string, body: string) => void) => number;
export const registerRequest: (callback: (signal: string, body: string) => Promise<string>) => number;

8
taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/TaoyaoSignal.ets
View File

@@ -166,6 +166,7 @@ class TaoyaoSignal {
"body" : body
};
try {
hilog.debug(0x0000, "TaoyaoSignal", "发送消息：%o", message);
this.socket?.send(JSON.stringify(message));
} catch (error) {
hilog.error(0x0000, "TaoyaoSignal", "发送消息异常：%o", message, error);
@@ -206,6 +207,7 @@ class TaoyaoSignal {
});
// 发送消息
try {
hilog.debug(0x0000, "TaoyaoSignal", "发送请求：%o", message);
this.socket?.send(JSON.stringify(message));
} catch (error) {
hilog.error(0x0000, "TaoyaoSignal", "发送消息异常：%o", message, error);
@@ -222,7 +224,7 @@ class TaoyaoSignal {
onMessage(message: string) {
const json : Record<string, Object> = JSON.parse(message);
const header: Record<string, Object> = json.header as Record<string, Object>;
const body : Record<string, Object> = json.body as Record<string, Object>;
const body : Record<string, Object> = json.body as Record<string, Object>;
const id : number = header.id as number;
const signal: string = header.signal as string;
if (this.callbackMapping.has(id)) {
@@ -251,8 +253,8 @@ class TaoyaoSignal {
this.send(signal, JSON.parse(body));
}
nativeRequest(signal: string, body: string): string {
const response = this.request(signal, JSON.parse(body));
async nativeRequest(signal: string, body: string): Promise<string> {
const response = await this.request(signal, JSON.parse(body));
return JSON.stringify(response);
}

6
taoyao-client-openharmony/taoyao/media/src/main/module.json5
View File

@@ -36,42 +36,36 @@
"requestPermissions": [
{
"name": "ohos.permission.CAMERA",
"reason": "视频",
"usedScene": {
"when": "always"
}
},
{
"name": "ohos.permission.INTERNET",
"reason": "网络",
"usedScene": {
"when": "always"
}
},
{
"name": "ohos.permission.MICROPHONE",
"reason": "音频",
"usedScene": {
"when": "always"
}
},
{
"name": "ohos.permission.READ_MEDIA",
"reason": "录制文件",
"usedScene": {
"when": "always"
}
},
{
"name": "ohos.permission.WRITE_MEDIA",
"reason": "录制文件",
"usedScene": {
"when": "always"
}
},
{
"name": "ohos.permission.MEDIA_LOCATION",
"reason": "地理位置",
"usedScene": {
"when": "always"
}