diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt b/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt index 6d03422..67cd3cd 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/CMakeLists.txt @@ -57,10 +57,8 @@ target_include_directories( target_link_libraries( ${PROJECT_NAME} PUBLIC - # NAPI - libace_napi.z.so - # LOG - libhilog_ndk.z.so + # libuv + libuv.so # EGL libEGL.so # OpenGL ES @@ -71,6 +69,10 @@ target_link_libraries( libohaudio.so # 相机 libohcamera.so + # NAPI + libace_napi.z.so + # LOG + libhilog_ndk.z.so # 图片 libnative_image.so # NativeBuffer diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/bind.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/bind.cpp index a2ce835..1727731 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/bind.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/bind.cpp @@ -6,11 +6,19 @@ * @author acgist * * https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/reference/native-lib/napi.md + * https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/napi/use-napi-thread-safety.md + * https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/performance/native-threads-call-js.md + * https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/performance/develop-Native-modules-using-NAPI-safely-and-efficiently.md */ #include #include #include +#include +#include +#include + +#include #include @@ -42,20 +50,25 @@ static std::recursive_mutex taoyaoMutex; size_t length; \ char chars[2048] = { 0 }; \ napi_get_value_string_utf8(env, args[0], chars, sizeof(chars), &length); \ - OH_LOG_INFO(LOG_APP, "解析JSON:%s", chars); \ + if(length <= 0) { \ + OH_LOG_WARN(LOG_APP, "TAOYAO ERROR JSON: %{public}s", chars); \ + napi_create_int32(env, -1, &ret); \ + return ret; \ + } \ + OH_LOG_DEBUG(LOG_APP, "TAOYAO JSON: %{public}s", chars); \ nlohmann::json json = nlohmann::json::parse(chars, chars + length); \ nlohmann::json body = json["body"]; #endif // 房间检查 #ifndef TAOYAO_ROOM_CHECK -#define TAOYAO_ROOM_CHECK(action) \ - std::string roomId = body["roomId"]; \ - auto room = acgist::roomMap.find(roomId); \ - if(room == acgist::roomMap.end()) { \ - OH_LOG_WARN(LOG_APP, "房间无效:%s %s", #action, roomId.data()); \ - napi_create_int32(env, -1, &ret); \ - return ret; \ +#define TAOYAO_ROOM_CHECK(action) \ + std::string roomId = body["roomId"]; \ + auto room = acgist::roomMap.find(roomId); \ + if(room == acgist::roomMap.end()) { \ + OH_LOG_WARN(LOG_APP, "TAOYAO ERROR ROOM ID: %{public}s %{public}s", #action, roomId.data()); \ + napi_create_int32(env, -1, &ret); \ + return ret; \ } #endif @@ -64,6 +77,7 @@ namespace acgist { uint32_t width = 720; uint32_t height = 480; uint64_t bitrate = 3'000'000L; +uint32_t iFrameInterval = 5'000; double frameRate = 30.0; int32_t samplingRate = 48'000; int32_t channelCount = 2; @@ -71,18 +85,30 @@ int32_t bitsPerSample = 16; std::string clientId = ""; std::string name = ""; +// 索引:667-999 +static uint32_t index = 667; +// 最小索引 +static uint32_t minIndex = 667; +// 最大索引 +static uint32_t maxIndex = 999; +// 终端索引 +static uint32_t clientIndex = 99999; // ETS环境 static napi_env env = nullptr; // 是否加载 static bool initTaoyao = false; -// PUSH方法引用 +// push方法引用 static napi_ref pushRef = nullptr; -// REQUEST方法引用 +// request方法引用 static napi_ref requestRef = nullptr; +// request线程安全方法 +static napi_threadsafe_function requestFunction = nullptr; // 媒体功能 static acgist::MediaManager* mediaManager = nullptr; // 房间管理 static std::map roomMap; +// 异步回调 +static std::map*> promiseMap; /** * 支持的编解码 @@ -91,58 +117,135 @@ static void printSupportCodec() { // TODO: 验证是否需要释放 OH_AVCapability* format = nullptr; format = OH_AVCodec_GetCapability(OH_AVCODEC_MIMETYPE_AUDIO_OPUS, true); - OH_LOG_INFO(LOG_APP, "是否支持OPUS硬件解码:%o", OH_AVCapability_IsHardware(format)); + OH_LOG_INFO(LOG_APP, "是否支持OPUS硬件解码:%{public}o", OH_AVCapability_IsHardware(format)); format = OH_AVCodec_GetCapability(OH_AVCODEC_MIMETYPE_AUDIO_OPUS, false); - OH_LOG_INFO(LOG_APP, "是否支持OPUS硬件解码:%o", OH_AVCapability_IsHardware(format)); + OH_LOG_INFO(LOG_APP, "是否支持OPUS硬件解码:%{public}o", OH_AVCapability_IsHardware(format)); format = OH_AVCodec_GetCapability(OH_AVCODEC_MIMETYPE_AUDIO_G711MU, true); - OH_LOG_INFO(LOG_APP, "是否支持PCMU硬件编码:%o", OH_AVCapability_IsHardware(format)); + OH_LOG_INFO(LOG_APP, "是否支持PCMU硬件编码:%{public}o", OH_AVCapability_IsHardware(format)); format = OH_AVCodec_GetCapability(OH_AVCODEC_MIMETYPE_AUDIO_G711MU, false); - OH_LOG_INFO(LOG_APP, "是否支持PCMU硬件解码:%o", OH_AVCapability_IsHardware(format)); + OH_LOG_INFO(LOG_APP, "是否支持PCMU硬件解码:%{public}o", OH_AVCapability_IsHardware(format)); format = OH_AVCodec_GetCapability(OH_AVCODEC_MIMETYPE_VIDEO_AVC, true); - OH_LOG_INFO(LOG_APP, "是否支持H264硬件编码:%o", OH_AVCapability_IsHardware(format)); + OH_LOG_INFO(LOG_APP, "是否支持H264硬件编码:%{public}o", OH_AVCapability_IsHardware(format)); format = OH_AVCodec_GetCapability(OH_AVCODEC_MIMETYPE_VIDEO_AVC, false); - OH_LOG_INFO(LOG_APP, "是否支持H264硬件解码:%o", OH_AVCapability_IsHardware(format)); + OH_LOG_INFO(LOG_APP, "是否支持H264硬件解码:%{public}o", OH_AVCapability_IsHardware(format)); format = OH_AVCodec_GetCapability(OH_AVCODEC_MIMETYPE_VIDEO_HEVC, true); - OH_LOG_INFO(LOG_APP, "是否支持H265硬件编码:%o", OH_AVCapability_IsHardware(format)); + OH_LOG_INFO(LOG_APP, "是否支持H265硬件编码:%{public}o", OH_AVCapability_IsHardware(format)); format = OH_AVCodec_GetCapability(OH_AVCODEC_MIMETYPE_VIDEO_HEVC, false); - OH_LOG_INFO(LOG_APP, "是否支持H265硬件解码:%o", OH_AVCapability_IsHardware(format)); + OH_LOG_INFO(LOG_APP, "是否支持H265硬件解码:%{public}o", OH_AVCapability_IsHardware(format)); +} + +struct Message { + uint64_t id; + std::string signal; + std::string body; +}; + +static void pushCallback(uv_work_t* work) { + // 不能调用ETS函数 +} + +static void afterPushCallback(uv_work_t* work, int status) { + Message* message = (Message*) work->data; + napi_handle_scope scope = nullptr; + napi_open_handle_scope(acgist::env, &scope); + if(scope == nullptr) { + delete message; + delete work; + return; + }; + // 开始执行ETS函数 + napi_value ret; + napi_value callback = nullptr; + napi_get_reference_value(acgist::env, acgist::pushRef, &callback); + napi_value data[3]; + napi_create_string_utf8(acgist::env, message->signal.data(), NAPI_AUTO_LENGTH, &data[0]); + napi_create_string_utf8(acgist::env, message->body.data(), NAPI_AUTO_LENGTH, &data[1]); + napi_create_int64(acgist::env, message->id, &data[2]); + napi_call_function(acgist::env, nullptr, callback, 3, data, &ret); + napi_get_undefined(acgist::env, &ret); + // 释放资源 + napi_close_handle_scope(acgist::env, scope); + delete message; + delete work; } /** * 发送消息 */ void push(const std::string& signal, const std::string& body, uint64_t id) { - // TODO: 验证是否需要释放 + uv_loop_s* loop = nullptr; + napi_get_uv_event_loop(acgist::env, &loop); + uv_work_t* work = new uv_work_t{}; + work->data = new Message{ id, signal, body }; + uv_queue_work(loop, work, pushCallback, afterPushCallback); +} + +static void requestCallback(uv_work_t* work) { + // 不能调用ETS函数 +} + +static void afterRequestCallback(uv_work_t* work, int status) { + Message* message = (Message*) work->data; + napi_handle_scope scope = nullptr; + napi_open_handle_scope(acgist::env, &scope); + if(scope == nullptr) { + delete message; + delete work; + return; + }; + // 开始执行ETS函数 napi_value ret; napi_value callback = nullptr; - napi_get_reference_value(env, acgist::pushRef, &callback); + napi_get_reference_value(env, acgist::requestRef, &callback); napi_value data[3]; - napi_create_string_utf8(acgist::env, signal.data(), NAPI_AUTO_LENGTH, &data[0]); - napi_create_string_utf8(acgist::env, body.data(), NAPI_AUTO_LENGTH, &data[1]); - napi_create_int64(acgist::env, id, &data[2]); + napi_create_string_utf8(acgist::env, message->signal.data(), NAPI_AUTO_LENGTH, &data[0]); + napi_create_string_utf8(acgist::env, message->body.data(), NAPI_AUTO_LENGTH, &data[1]); + napi_create_int64(acgist::env, message->id, &data[2]); napi_call_function(acgist::env, nullptr, callback, 3, data, &ret); napi_get_undefined(acgist::env, &ret); + // 释放资源 + napi_close_handle_scope(acgist::env, scope); + delete message; + delete work; } /** * 发送请求 */ std::string request(const std::string& signal, const std::string& body, uint64_t id) { - napi_value ret; - napi_value callback = nullptr; - napi_get_reference_value(env, acgist::requestRef, &callback); - napi_value data[3]; - napi_create_string_utf8(acgist::env, signal.data(), NAPI_AUTO_LENGTH, &data[0]); - napi_create_string_utf8(acgist::env, body.data(), NAPI_AUTO_LENGTH, &data[1]); - napi_create_int64(acgist::env, id, &data[2]); - napi_call_function(acgist::env, nullptr, callback, 3, data, &ret); - char chars[TAOYAO_JSON_SIZE]; - size_t length; - napi_get_value_string_utf8(env, ret, chars, sizeof(chars), &length); - // TODO: promise - // napi_create_promise - // napi_resolve_deferred - return chars; + uv_loop_s* loop = nullptr; + napi_get_uv_event_loop(acgist::env, &loop); + uv_work_t* work = new uv_work_t{}; + if(id <= 0L) { + if (++acgist::index > acgist::maxIndex) { + acgist::index = acgist::minIndex; + } + auto now = std::chrono::system_clock::now(); + std::time_t time = std::chrono::system_clock::to_time_t(now); + std::tm* tm = std::localtime(&time); + id = + 100000000000000L * tm->tm_mday + + 1000000000000 * tm->tm_hour + + 10000000000 * tm->tm_min + + 100000000 * tm->tm_sec + + 1000 * acgist::clientIndex + + acgist::index; + } + std::promise* promise = new std::promise{}; + acgist::promiseMap.insert({ id, promise }); + work->data = new Message{ id, signal, body }; + uv_queue_work(loop, work, requestCallback, afterRequestCallback); + std::future future = promise->get_future(); + if(future.wait_for(std::chrono::seconds(5)) == std::future_status::timeout) { + OH_LOG_WARN(LOG_APP, "请求超时:%{public}s %{public}s", signal.data(), body.data()); + acgist::promiseMap.erase(id); + delete promise; + return "{}"; + } else { + acgist::promiseMap.erase(id); + delete promise; + return future.get(); + } } /** @@ -162,11 +265,12 @@ static napi_value init(napi_env env, napi_callback_info info) { napi_create_reference(env, args[1], 1, &acgist::pushRef); napi_create_reference(env, args[2], 1, &acgist::requestRef); printSupportCodec(); - acgist::clientId = json["clientId"]; - acgist::name = json["name"]; + acgist::clientId = json["clientId"]; + acgist::name = json["name"]; + acgist::clientIndex = json["clientIndex"]; OH_LOG_INFO(LOG_APP, "加载libtaoyao"); std::string version = mediasoupclient::Version(); - OH_LOG_INFO(LOG_APP, "加载MediasoupClient:%s", version.data()); + OH_LOG_INFO(LOG_APP, "加载MediasoupClient:%{public}s", version.data()); mediasoupclient::Initialize(); OH_LOG_INFO(LOG_APP, "加载媒体功能"); mediaManager = new MediaManager(); @@ -212,6 +316,23 @@ static napi_value shutdown(napi_env env, napi_callback_info info) { return ret; } +/** + * Promise回调 + */ +static napi_value callback(napi_env env, napi_callback_info info) { + TAOYAO_JSON_BODY(1); + nlohmann::json header = json["header"]; + uint64_t id = header["id"]; + auto promise = acgist::promiseMap.find(id); + if(promise == acgist::promiseMap.end()) { + napi_create_int32(env, -1, &ret); + } else { + napi_create_int32(env, 0, &ret); + promise->second->set_value(chars); + } + return ret; +} + /** * 房间关闭 */ @@ -448,6 +569,7 @@ static napi_value Init(napi_env env, napi_value exports) { acgist::env = env; napi_property_descriptor desc[] = { { "init", nullptr, acgist::init, nullptr, nullptr, nullptr, napi_default, nullptr }, + { "callback", nullptr, acgist::callback, nullptr, nullptr, nullptr, napi_default, nullptr }, { "shutdown", nullptr, acgist::shutdown, nullptr, nullptr, nullptr, napi_default, nullptr }, { "roomClose", nullptr, acgist::roomClose, nullptr, nullptr, nullptr, napi_default, nullptr }, { "roomEnter", nullptr, acgist::roomEnter, nullptr, nullptr, nullptr, napi_default, nullptr }, diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp index 732cd40..6b4c966 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Capturer.hpp @@ -20,10 +20,8 @@ #define TAOYAO_CAPTURER_HPP // OpenGL ES || VULKAN -#define __TAOYAO_VULKAN__ true -#ifndef __TAOYAO_VULKAN__ +#define __TAOYAO_VULKAN__ false #define __TAOYAO_OPENGL__ true -#endif // 本地音频采集 #define __TAOYAO_AUDIO_LOCAL__ false diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Signal.hpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Signal.hpp index 2c8e305..0205f53 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Signal.hpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/Signal.hpp @@ -21,6 +21,8 @@ extern uint32_t width; extern uint32_t height; // 视频码率 extern uint64_t bitrate; +// 关键帧的频率 +extern uint32_t iFrameInterval; // 视频帧率 extern double frameRate; // 采样率 diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/WebRTC.hpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/WebRTC.hpp index 5678467..897d2dc 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/WebRTC.hpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/include/WebRTC.hpp @@ -98,7 +98,7 @@ public: // 视频编码器 OH_AVCodec* avCodec = nullptr; // 缓冲数据索引 - uint32_t index = 0; + uint32_t index = 0; // 缓冲数据 OH_AVBuffer* buffer = nullptr; // 编码回调 @@ -108,12 +108,14 @@ public: TaoyaoVideoEncoder(); virtual ~TaoyaoVideoEncoder() override; -public: +protected: // 初始配置 void initFormatConfig(OH_AVFormat* format); + +public: // 重新开始 void restart(); - // 动态配置 + // 动态配置:format自己释放 void reset(OH_AVFormat* format); // 动态配置 void resetIntConfig(const char* key, int32_t value); @@ -140,6 +142,10 @@ public: */ class TaoyaoVideoDecoder : public webrtc::VideoDecoder { +public: + // 是否运行 + bool running = false; + public: TaoyaoVideoDecoder(); virtual ~TaoyaoVideoDecoder() override; diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp index bb9c479..5b4c9d1 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/AudioCapturer.cpp @@ -1,7 +1,3 @@ -/** - * 音频采集不用实现(系统已经实现) - * 这里只是用来学习使用 - */ #include "../include/Capturer.hpp" #include diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoCapturer.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoCapturer.cpp index 40943ed..6f5496d 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoCapturer.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoCapturer.cpp @@ -11,6 +11,8 @@ #include "../include/Capturer.hpp" +#include + #include "hilog/log.h" #include "rtc_base/time_utils.h" @@ -21,6 +23,8 @@ #include #include +static std::recursive_mutex videoMutex; + // 采集回调 static void onError(Camera_VideoOutput* videoOutput, Camera_ErrorCode errorCode); static void onFrameEnd(Camera_VideoOutput* videoOutput, int32_t frameCount); @@ -33,14 +37,19 @@ static void onFrame(void* context); static bool CheckEglExtension(const char* extensions, const char* extension); acgist::VideoCapturer::VideoCapturer() { + #if __TAOYAO_VULKAN__ + initVulkan(); + #endif + #if __TAOYAO_OPENGL__ initOpenGLES(); + #endif Camera_ErrorCode ret = OH_Camera_GetCameraManager(&this->cameraManager); - OH_LOG_INFO(LOG_APP, "获取摄像头管理器:%o", ret); + OH_LOG_INFO(LOG_APP, "配置摄像头管理器:%o", ret); ret = OH_CameraManager_GetSupportedCameras(this->cameraManager, &this->cameraDevice, &this->cameraSize); OH_LOG_INFO(LOG_APP, "获取摄像头设备列表:%o %d", ret, this->cameraSize); ret = OH_CameraManager_GetSupportedCameraOutputCapability(this->cameraManager, &this->cameraDevice[this->cameraIndex], &this->cameraOutputCapability); OH_LOG_INFO(LOG_APP, "获取摄像头输出功能:%o %d %d", ret, this->cameraIndex, this->cameraOutputCapability->videoProfilesSize); - // 注册相机状态回调 + // 注册相机状态回调:可以取消注册 // OH_CameraManager_RegisterCallback(this->cameraManager, CameraManager_Callbacks* callback); // char surfaceId[OH_XCOMPONENT_ID_LEN_MAX + 1] = {}; // uint64_t idSize = OH_XCOMPONENT_ID_LEN_MAX + 1; @@ -55,7 +64,12 @@ acgist::VideoCapturer::VideoCapturer() { } acgist::VideoCapturer::~VideoCapturer() { + #if __TAOYAO_VULKAN__ + releaseVulkan(); + #endif + #if __TAOYAO_OPENGL__ releaseOpenGLES(); + #endif Camera_ErrorCode ret = OH_CaptureSession_Release(this->cameraCaptureSession); this->cameraCaptureSession = nullptr; OH_LOG_INFO(LOG_APP, "释放摄像头视频会话:%o", ret); @@ -74,6 +88,7 @@ acgist::VideoCapturer::~VideoCapturer() { } bool acgist::VideoCapturer::start() { + std::lock_guard videoLock(videoMutex); if (this->running) { return true; } @@ -90,6 +105,7 @@ bool acgist::VideoCapturer::start() { callbacks.onError = onError; callbacks.onFrameEnd = onFrameEnd; callbacks.onFrameStart = onFrameStart; + // :可以取消注册 ret = OH_VideoOutput_RegisterCallback(this->cameraVideoOutput, &callbacks); OH_LOG_INFO(LOG_APP, "视频捕获回调:%o", ret); OH_NativeImage_AttachContext(this->nativeImage, this->textureId); @@ -97,6 +113,7 @@ bool acgist::VideoCapturer::start() { } bool acgist::VideoCapturer::stop() { + std::lock_guard videoLock(videoMutex); if (!this->running) { return true; } @@ -247,7 +264,7 @@ void acgist::VideoCapturer::releaseOpenGLES() { } static void onError(Camera_VideoOutput* videoOutput, Camera_ErrorCode errorCode) { - OH_LOG_WARN(LOG_APP, "视频捕获数据帧失败:%d", errorCode); + OH_LOG_ERROR(LOG_APP, "视频捕获数据帧失败:%d", errorCode); } static void onFrameEnd(Camera_VideoOutput* videoOutput, int32_t frameCount) { diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp index d91a0d3..55ab313 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/media/VideoEncoder.cpp @@ -1,6 +1,8 @@ #include "../include/WebRTC.hpp" -#include "hilog/log.h" +#include + +#include #include #include @@ -12,6 +14,8 @@ #include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_decoder.h" +static std::recursive_mutex videoMutex; + // 编码回调 static void OnError(OH_AVCodec* codec, int32_t errorCode, void* userData); static void OnStreamChanged(OH_AVCodec* codec, OH_AVFormat* format, void* userData); @@ -22,18 +26,18 @@ acgist::TaoyaoVideoEncoder::TaoyaoVideoEncoder() { OH_AVCapability* capability = OH_AVCodec_GetCapability(OH_AVCODEC_MIMETYPE_VIDEO_AVC, true); const char* codecName = OH_AVCapability_GetName(capability); this->avCodec = OH_VideoEncoder_CreateByName(codecName); - OH_LOG_INFO(LOG_APP, "视频编码格式:%s", codecName); - // 注册回调 + OH_LOG_INFO(LOG_APP, "配置视频编码格式:%s", codecName); + // 注册视频编码回调 OH_AVCodecCallback callback = { &OnError, &OnStreamChanged, &OnNeedInputBuffer, &OnNewOutputBuffer }; OH_AVErrCode ret = OH_VideoEncoder_RegisterCallback(this->avCodec, callback, this); - OH_LOG_INFO(LOG_APP, "注册编码回调:%o", ret); - // 配置编码参数 + OH_LOG_INFO(LOG_APP, "注册视频编码回调:%o", ret); + // 配置视频编码参数 OH_AVFormat* format = OH_AVFormat_Create(); this->initFormatConfig(format); ret = OH_VideoEncoder_Configure(this->avCodec, format); OH_AVFormat_Destroy(format); - OH_LOG_INFO(LOG_APP, "配置编码参数:%o %d %d %f %lld", ret, acgist::width, acgist::height, acgist::frameRate, acgist::bitrate); - // 准备就绪 + OH_LOG_INFO(LOG_APP, "配置视频编码参数:%o %d %d %f %d %lld", ret, acgist::width, acgist::height, acgist::frameRate, acgist::iFrameInterval, acgist::bitrate); + // 视频编码准备就绪 ret = OH_VideoEncoder_Prepare(this->avCodec); OH_LOG_INFO(LOG_APP, "视频编码准备就绪:%o", ret); } @@ -48,42 +52,31 @@ acgist::TaoyaoVideoEncoder::~TaoyaoVideoEncoder() { void acgist::TaoyaoVideoEncoder::initFormatConfig(OH_AVFormat* format) { // https://docs.openharmony.cn/pages/v4.1/zh-cn/application-dev/media/avcodec/video-encoding.md - // 配置视频宽度 - OH_AVFormat_SetIntValue(format, OH_MD_KEY_WIDTH, acgist::width); - // 配置视频高度 - OH_AVFormat_SetIntValue(format, OH_MD_KEY_HEIGHT, acgist::height); - // 配置视频比特率 - OH_AVFormat_SetLongValue(format, OH_MD_KEY_BITRATE, acgist::bitrate); - // 配置视频帧率 - OH_AVFormat_SetDoubleValue(format, OH_MD_KEY_FRAME_RATE, acgist::frameRate); - // 配置视频颜色格式 - OH_AVFormat_SetIntValue(format, OH_MD_KEY_PIXEL_FORMAT, AV_PIXEL_FORMAT_YUVI420); - // 配置视频YUV值范围标志 - OH_AVFormat_SetIntValue(format, OH_MD_KEY_RANGE_FLAG, false); - // 配置视频原色 - OH_AVFormat_SetIntValue(format, OH_MD_KEY_COLOR_PRIMARIES, static_cast(OH_ColorPrimary::COLOR_PRIMARY_BT709)); - // 配置传输特性 - OH_AVFormat_SetIntValue(format, OH_MD_KEY_TRANSFER_CHARACTERISTICS, static_cast(OH_TransferCharacteristic::TRANSFER_CHARACTERISTIC_BT709)); - // 配置最大矩阵系数 - OH_AVFormat_SetIntValue(format, OH_MD_KEY_MATRIX_COEFFICIENTS, static_cast(OH_MatrixCoefficient::MATRIX_COEFFICIENT_IDENTITY)); - // 配置关键帧的间隔(单位为:毫秒) - OH_AVFormat_SetIntValue(format, OH_MD_KEY_I_FRAME_INTERVAL, 5000); - // 配置编码Profile - OH_AVFormat_SetIntValue(format, OH_MD_KEY_PROFILE, static_cast(OH_AVCProfile::AVC_PROFILE_BASELINE)); - // 配置编码比特率模式 - OH_AVFormat_SetIntValue(format, OH_MD_KEY_VIDEO_ENCODE_BITRATE_MODE, static_cast(OH_VideoEncodeBitrateMode::CBR)); - // 配置所需的编码质量:只有在恒定质量模式下配置的编码器才支持此配置 - OH_AVFormat_SetIntValue(format, OH_MD_KEY_QUALITY, 0); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_WIDTH, acgist::width); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_HEIGHT, acgist::height); + OH_AVFormat_SetLongValue(format, OH_MD_KEY_BITRATE, acgist::bitrate); + OH_AVFormat_SetDoubleValue(format, OH_MD_KEY_FRAME_RATE, acgist::frameRate); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_I_FRAME_INTERVAL, acgist::iFrameInterval); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_QUALITY, 0); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_RANGE_FLAG, false); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_PIXEL_FORMAT, AV_PIXEL_FORMAT_YUVI420); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_PROFILE, static_cast(OH_AVCProfile::AVC_PROFILE_BASELINE)); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_COLOR_PRIMARIES, static_cast(OH_ColorPrimary::COLOR_PRIMARY_BT709)); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_MATRIX_COEFFICIENTS, static_cast(OH_MatrixCoefficient::MATRIX_COEFFICIENT_IDENTITY)); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_TRANSFER_CHARACTERISTICS, static_cast(OH_TransferCharacteristic::TRANSFER_CHARACTERISTIC_BT709)); + OH_AVFormat_SetIntValue(format, OH_MD_KEY_VIDEO_ENCODE_BITRATE_MODE, static_cast(OH_VideoEncodeBitrateMode::CBR)); } void acgist::TaoyaoVideoEncoder::restart() { + std::lock_guard videoLock(videoMutex); OH_AVErrCode ret = OH_VideoEncoder_Flush(this->avCodec); - OH_LOG_INFO(LOG_APP, "清空编码队列:%o", ret); + OH_LOG_INFO(LOG_APP, "清空视频编码队列:%o", ret); ret = OH_VideoEncoder_Start(this->avCodec); OH_LOG_INFO(LOG_APP, "开始视频编码:%o", ret); } void acgist::TaoyaoVideoEncoder::reset(OH_AVFormat* format) { + std::lock_guard videoLock(videoMutex); OH_AVErrCode ret = OH_VideoEncoder_Reset(this->avCodec); OH_LOG_INFO(LOG_APP, "重置视频编码:%o", ret); ret = OH_VideoEncoder_Configure(this->avCodec, format); @@ -91,6 +84,7 @@ void acgist::TaoyaoVideoEncoder::reset(OH_AVFormat* format) { } void acgist::TaoyaoVideoEncoder::resetIntConfig(const char* key, int32_t value) { + std::lock_guard videoLock(videoMutex); OH_AVFormat* format = OH_AVFormat_Create(); OH_AVFormat_SetIntValue(format, key, value); OH_AVErrCode ret = OH_VideoEncoder_SetParameter(this->avCodec, format); @@ -99,6 +93,7 @@ void acgist::TaoyaoVideoEncoder::resetIntConfig(const char* key, int32_t value) } void acgist::TaoyaoVideoEncoder::resetLongConfig(const char* key, int64_t value) { + std::lock_guard videoLock(videoMutex); OH_AVFormat* format = OH_AVFormat_Create(); OH_AVFormat_SetLongValue(format, key, value); OH_AVErrCode ret = OH_VideoEncoder_SetParameter(this->avCodec, format); @@ -107,6 +102,7 @@ void acgist::TaoyaoVideoEncoder::resetLongConfig(const char* key, int64_t value) } void acgist::TaoyaoVideoEncoder::resetDoubleConfig(const char* key, double value) { + std::lock_guard videoLock(videoMutex); OH_AVFormat* format = OH_AVFormat_Create(); OH_AVFormat_SetDoubleValue(format, key, value); OH_AVErrCode ret = OH_VideoEncoder_SetParameter(this->avCodec, format); @@ -115,6 +111,7 @@ void acgist::TaoyaoVideoEncoder::resetDoubleConfig(const char* key, double value } bool acgist::TaoyaoVideoEncoder::start() { + std::lock_guard videoLock(videoMutex); if(this->running) { return true; } @@ -125,20 +122,23 @@ bool acgist::TaoyaoVideoEncoder::start() { } bool acgist::TaoyaoVideoEncoder::stop() { + std::lock_guard videoLock(videoMutex); if(!this->running) { return true; } this->running = false; + // Buffer模式 + OH_AVErrCode ret = OH_VideoEncoder_Stop(this->avCodec); + OH_LOG_INFO(LOG_APP, "结束视频编码:%o", ret); // Surface模式 // OH_AVErrCode ret = OH_VideoEncoder_NotifyEndOfStream(this->avCodec); // OH_LOG_INFO(LOG_APP, "通知视频编码结束:%o", ret); - OH_AVErrCode ret = OH_VideoEncoder_Stop(this->avCodec); - OH_LOG_INFO(LOG_APP, "结束视频编码:%o", ret); return ret == OH_AVErrCode::AV_ERR_OK; } int32_t acgist::TaoyaoVideoEncoder::Release() { - // TODO: 释放资源 + // TODO: 释放 + delete this; return 0; } @@ -158,20 +158,19 @@ webrtc::VideoEncoder::EncoderInfo acgist::TaoyaoVideoEncoder::GetEncoderInfo() c } int32_t acgist::TaoyaoVideoEncoder::Encode(const webrtc::VideoFrame& videoFrame, const std::vector* frame_types) { -// frameSize = videoFrame.width * height * 3 / 2 - // 配置buffer info信息 OH_AVCodecBufferAttr info; - info.size = 0; // TODO + info.size = videoFrame.width() * videoFrame.height() * 3 / 2; info.offset = 0; info.pts = 0; - info.flags = 0; // TODO frame_types + info.flags = 0; + // TODO: videoFrame.video_frame_buffer-> OH_AVErrCode ret = OH_AVBuffer_SetBufferAttr(this->buffer, &info); ret = OH_VideoEncoder_PushInputBuffer(this->avCodec, index); return 0; } static void OnError(OH_AVCodec* codec, int32_t errorCode, void* userData) { - OH_LOG_WARN(LOG_APP, "视频编码发送错误:%d", errorCode); + OH_LOG_ERROR(LOG_APP, "视频编码发送错误:%d", errorCode); } static void OnStreamChanged(OH_AVCodec* codec, OH_AVFormat* format, void* userData) { @@ -210,6 +209,11 @@ static void OnNewOutputBuffer(OH_AVCodec* codec, uint32_t index, OH_AVBuffer* bu OH_AVCodecBufferAttr info; OH_AVErrCode ret = OH_AVBuffer_GetBufferAttr(buffer, &info); char* data = reinterpret_cast(OH_AVBuffer_GetAddr(buffer)); + // webrtc::VideoFrameType flags = + // info.flags == AVCODEC_BUFFER_FLAGS_SYNC_FRAME ? webrtc::VideoFrameType::kVideoFrameKey : + // info.flags == AVCODEC_BUFFER_FLAGS_INCOMPLETE_FRAME ? webrtc::VideoFrameType::kVideoFrameDelta : + // webrtc::VideoFrameType::kEmptyFrame; + // frame_types->push_back(std::move(flags)); // videoEncoder->encodedImageCallback // TODO: 继续处理 ret = OH_VideoEncoder_FreeOutputBuffer(codec, index); diff --git a/taoyao-client-openharmony/taoyao/media/src/main/cpp/types/libtaoyao/index.d.ts b/taoyao-client-openharmony/taoyao/media/src/main/cpp/types/libtaoyao/index.d.ts index aa14ee8..cf8d562 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/cpp/types/libtaoyao/index.d.ts +++ b/taoyao-client-openharmony/taoyao/media/src/main/cpp/types/libtaoyao/index.d.ts @@ -1,9 +1,10 @@ export const init: ( json : string, push : (signal: string, body: string, id: number) => void, - request: (signal: string, body: string, id: number) => Promise + request: (signal: string, body: string, id: number) => void ) => number; export const shutdown : (json: string) => number; +export const callback : (json: string) => number; export const roomClose : (json: string) => number; export const roomEnter : (json: string) => number; export const roomExpel : (json: string) => number; diff --git a/taoyao-client-openharmony/taoyao/media/src/main/ets/pages/Index.ets b/taoyao-client-openharmony/taoyao/media/src/main/ets/pages/Index.ets index 0466732..086f9ce 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/ets/pages/Index.ets +++ b/taoyao-client-openharmony/taoyao/media/src/main/ets/pages/Index.ets @@ -1,4 +1,4 @@ -import { signal } from "../taoyao/TaoyaoSignal"; +import { taoyaoSignal } from "../taoyao/TaoyaoSignal"; import { setting } from "../taoyao/Setting"; @@ -13,7 +13,7 @@ struct Index { .fontSize(20) .fontWeight(FontWeight.Bold) .onClick(() => { - signal.connect(); + taoyaoSignal.connect(); }); } .width("100%") @@ -23,7 +23,7 @@ struct Index { .fontSize(20) .fontWeight(FontWeight.Bold) .onClick(() => { - signal.close(); + taoyaoSignal.close(); }); } .width("100%") @@ -33,7 +33,7 @@ struct Index { .fontSize(20) .fontWeight(FontWeight.Bold) .onClick(() => { - signal.init(); + taoyaoSignal.init(); }); } .width("100%") @@ -43,7 +43,7 @@ struct Index { .fontSize(20) .fontWeight(FontWeight.Bold) .onClick(() => { - signal.shutdown(); + taoyaoSignal.shutdown(); }); } .width("100%") diff --git a/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/Setting.ets b/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/Setting.ets index d57fa31..1c36de8 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/Setting.ets +++ b/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/Setting.ets @@ -5,34 +5,37 @@ */ import fs from '@ohos.file.fs'; -import resourceManager from '@ohos.resourceManager'; -class Signal { +class Config { // 终端名称 - name : string = "鸿蒙"; + name : string = "鸿蒙"; // 终端ID - clientId : string = "harmony"; + clientId : string = "harmony"; // 终端类型 - clientType: string = "MOBILE"; + clientType : string = "MOBILE"; // 信令账号 - username : string = "taoyao"; + username : string = "taoyao"; // 信令密码 - password : string = "taoyao"; + password : string = "taoyao"; + // 当前终端索引 + clientIndex: number = 99999; }; class Setting { + // 信令配置 + config : Config = new Config(); + // CA证书 + caPath : string = "/cacert.pem"; + // 信令版本 + version: string = "1.0.0"; // 信令地址 // signalAddress: string = "wss://192.168.8.57:8888/websocket.signal"; signalAddress: string = "wss://192.168.1.100:8888/websocket.signal"; - // 信令版本 - version: string = "1.0.0"; - // 信令配置 - signal : Signal = new Signal(); - // CA证书 - caPath : string = "/cacert.pem"; + // 启动自动加载系统 + initOnLoad : boolean = true; }; diff --git a/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/TaoyaoSignal.ets b/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/TaoyaoSignal.ets index f0b1b04..f06af0f 100644 --- a/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/TaoyaoSignal.ets +++ b/taoyao-client-openharmony/taoyao/media/src/main/ets/taoyao/TaoyaoSignal.ets @@ -24,21 +24,28 @@ class TaoyaoSignal { heartbeatTimer: number = 0; // 同步请求 callbackMapping = new Map(); - // 当前消息索引 - index : number = 0; + // 当前消息索引:0-666 + index : number = 0; + // 最小消息索引 + minIndex: number = 0; // 最大消息索引 - maxIndex : number = 999; - // 当前终端索引 - clientIndex: number = 99999; + maxIndex: number = 666; + /** + * 加载系统 + * 建议注册成功以后加载 + */ init() { hilog.info(0x0000, "TaoyaoSignal", "加载系统"); - taoyaoModule.init(JSON.stringify(setting.signal), this.nativePush, this.nativeRequest); + taoyaoModule.init(JSON.stringify(setting.config), this.nativePush, this.nativeRequest); } + /** + * 卸载系统 + */ shutdown() { hilog.info(0x0000, "TaoyaoSignal", "卸载系统"); - taoyaoModule.shutdown(JSON.stringify(setting.signal)); + taoyaoModule.shutdown(JSON.stringify(setting.config)); } /** @@ -125,19 +132,22 @@ class TaoyaoSignal { */ async register() { const response: Record = await this.request("client::register", { - "name" : setting.signal.name, - "clientId" : setting.signal.clientId, - "clientType": setting.signal.clientType, - "username" : setting.signal.username, - "password" : setting.signal.password, + "name" : setting.config.name, + "clientId" : setting.config.clientId, + "clientType": setting.config.clientType, + "username" : setting.config.username, + "password" : setting.config.password, "battery" : 100, "charging" : true }); const body = response.body as Record; const index = body.index as number; - this.clientIndex = index; + setting.config.clientIndex = index; hilog.info(0x0000, "TaoyaoSignal", "信令注册成功:%{public}d", index); this.heartbeat(); + if(setting.initOnLoad) { + taoyaoSignal.init(); + } } /** @@ -160,15 +170,15 @@ class TaoyaoSignal { */ buildId(): number { if (++this.index > this.maxIndex) { - this.index = 0; + this.index = this.minIndex; } const date = new Date(); return ( - 100000000000000 * date.getDate() + - 1000000000000 * date.getHours() + - 10000000000 * date.getMinutes() + - 100000000 * date.getSeconds() + - 1000 * this.clientIndex + + 100000000000000 * date.getDate() + + 1000000000000 * date.getHours() + + 10000000000 * date.getMinutes() + + 100000000 * date.getSeconds() + + 1000 * setting.config.clientIndex + this.index ); } @@ -323,7 +333,7 @@ class TaoyaoSignal { * @param id ID */ nativePush(signal: string, body: string, id: number = 0) { - this.push(signal, JSON.parse(body), id); + taoyaoSignal.push(signal, JSON.parse(body), id); } /** @@ -332,18 +342,21 @@ class TaoyaoSignal { * @param signal 信令 * @param body 主体 * @param id ID - * - * @returns 响应 */ - async nativeRequest(signal: string, body: string, id: number = 0): Promise { - const response = await this.request(signal, JSON.parse(body), id); - return JSON.stringify(response); + nativeRequest(signal: string, body: string, id: number = 0) { + taoyaoSignal.request(signal, JSON.parse(body), id).then(response => { + taoyaoModule.callback(JSON.stringify(response)); + }); } } -const signal = new TaoyaoSignal(); +const taoyaoSignal = new TaoyaoSignal(); + +if(setting.initOnLoad) { + taoyaoSignal.connect(); +} export { - signal + taoyaoSignal }