From 739538b34c2064ff3e56510b410a27f7afd30d46 Mon Sep 17 00:00:00 2001 From: acgist <289547414@qq.com> Date: Sat, 6 May 2023 23:57:25 +0800 Subject: [PATCH] =?UTF-8?q?[+]=20=E6=B7=B7=E9=9F=B3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/build.yml | 5 +- .github/workflows/codeql.yml | 30 +++++ docs/AOSP.md | 4 +- docs/Deploy.md | 14 +-- docs/MI5S.md | 2 +- docs/TODO.md | 3 + docs/WebRTC.md | 55 ++++----- docs/etc/nginx.conf | 2 +- .../acgist/taoyao/client/MainActivity.java | 10 +- .../acgist/taoyao/client/MediaService.java | 16 ++- .../client/src/main/res/values/settings.xml | 6 +- .../com/acgist/taoyao/media/Broadcaster.java | 53 --------- .../com/acgist/taoyao/media/MediaManager.java | 104 +++++++++++++++--- .../taoyao/media/audio/MixerProcesser.java | 98 ++++++++++++++++- .../taoyao/media/client/PhotographClient.java | 9 +- .../taoyao/media/client/RecordClient.java | 66 ++++++----- .../webrtc/audio/JavaAudioDeviceModule.java | 32 ++++++ .../org/webrtc/audio/WebRtcAudioRecord.java | 24 ++-- .../org/webrtc/audio/WebRtcAudioTrack.java | 21 ++++ .../com/acgist/taoyao/AudioMixerTest.java | 31 ++++++ 20 files changed, 423 insertions(+), 162 deletions(-) create mode 100644 .github/workflows/codeql.yml delete mode 100644 taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/Broadcaster.java create mode 100644 taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/AudioMixerTest.java diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 759d93c..d447faa 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,4 +1,4 @@ -name: build all +name: build on: push: @@ -97,4 +97,5 @@ jobs: if: runner.os == 'windows' run: | cd ./taoyao-client-android/taoyao - ./gradlew.bat --no-daemon assembleRelease \ No newline at end of file + ./gradlew.bat --no-daemon assembleRelease + \ No newline at end of file diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 0000000..6745923 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,30 @@ +name: codeql + +on: + push: + branches: [ master ] + +jobs: + analyze: + name: Analyze + strategy: + fail-fast: false + matrix: + language: [ "cpp", "java", "javascript" ] + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v2 + - name: Set up JDK + uses: actions/setup-java@v1 + with: + java-version: 17 + - name: Initialize CodeQL + uses: github/codeql-action/init@v1 + with: + languages: ${{ matrix.language }} + - name: Autobuild + uses: github/codeql-action/autobuild@v1 + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v1 + \ No newline at end of file diff --git a/docs/AOSP.md b/docs/AOSP.md index 29e3fc1..efa65e3 100644 --- a/docs/AOSP.md +++ b/docs/AOSP.md @@ -12,10 +12,10 @@ ## 机器配置 * 内存`32G` +* 16核`CPU` * 硬盘`300G` -* 十六核`CPU` * 系统`Ubuntu 18.xx` -* 公司网络`1000Mbps/s` +* 网络带宽`1000MB/s` * 整个下载过程大概需要三到五个小时 * 整个编译过程大概需要半到一个小时 diff --git a/docs/Deploy.md b/docs/Deploy.md index 5f1b0d8..18b06bc 100644 --- a/docs/Deploy.md +++ b/docs/Deploy.md @@ -120,7 +120,7 @@ cmake -v mkdir -p /data/dev/nodejs cd /data/dev/nodejs wget https://nodejs.org/dist/v16.19.0/node-v16.19.0-linux-x64.tar.xz -tar -xvJf node-v16.19.0-linux-x64.tar.xz +tar -Jxvf node-v16.19.0-linux-x64.tar.xz # 连接 ln -sf /data/dev/nodejs/node-v16.19.0-linux-x64/bin/npm /usr/local/bin/ @@ -213,7 +213,7 @@ mkdir -p /data/dev/python cd /data/dev/python #wget https://www.python.org/ftp/python/3.8.16/Python-3.8.16.tar.xz wget https://mirrors.huaweicloud.com/python/3.8.16/Python-3.8.16.tar.xz -tar -xvJf Python-3.8.16.tar.xz +tar -Jxvf Python-3.8.16.tar.xz # 安装 cd Python-3.8.16 @@ -265,7 +265,7 @@ yum install nginx systemctl enable nginx # 管理服务 -systemctl start | stop | restart nginx +systemctl start|stop|restart nginx # 加载配置 nginx -s reload @@ -307,7 +307,7 @@ systemctl enable taoyao-signal-server ./deploy.sh # 管理服务 -systemctl start | stop | restart taoyao-signal-server +systemctl start|stop|restart taoyao-signal-server ``` ## 安装媒体 @@ -318,11 +318,11 @@ cd /data/taoyao/taoyao-client-media npm install # 配置ecosystem -pm2 start | reload ecosystem.config.json +pm2 start|reload ecosystem.config.json pm2 save # 管理服务：服务名称必须和配置终端标识一致否则不能执行重启和关闭信令 -pm2 start | stop | restart taoyao-client-media +pm2 start|stop|restart taoyao-client-media ``` ### Mediasoup编译失败 @@ -359,7 +359,7 @@ pm2 start npm --name "taoyao-client-web" -- run dev pm2 save # 管理服务 -pm2 start | stop | restart taoyao-client-web +pm2 start|stop|restart taoyao-client-web # 打包代码 npm run build diff --git a/docs/MI5S.md b/docs/MI5S.md index 5718c66..35829b4 100644 --- a/docs/MI5S.md +++ b/docs/MI5S.md @@ -1,6 +1,6 @@ # 小米5S -由于使用小米5S作为测试机，没有适合的`LineageOS`版本，所有这里选择了`PixelExperience`作为测试系统，如果其他机器建议使用`LineageOS`，`ROM`下载地址： +由于使用小米5S作为测试机，没有适合的`LineageOS`版本，所有这里选择了`PixelExperience`作为测试系统。如果其他机器建议使用`LineageOS`，当然最好还是使用`Pixel`作为测试机，可以直接使用原生系统，或者自己定制编译`AOSP`。 ## TWRP diff --git a/docs/TODO.md b/docs/TODO.md index 3f06300..13adf6f 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -14,7 +14,10 @@ ## 计划任务 * 混音 +* 音频视频时间对齐 * 分辨率调整 * 查询消费者生产者信息 ## 完成任务 + +* 屏幕共享 diff --git a/docs/WebRTC.md b/docs/WebRTC.md index c4b1b37..00fff9c 100644 --- a/docs/WebRTC.md +++ b/docs/WebRTC.md @@ -1,6 +1,6 @@ # WebRTC -本文档内容旨在独立编译`WebRTC`项目，非必需使用。 +本文档内容旨在独立编译`WebRTC`项目，并非必需使用。 ## libwebrtc @@ -15,22 +15,24 @@ * 四核`CPU` * 硬盘`100G` * 系统`Ubuntu 20.xx` -* 宽带按需`100Mbps/s` +* 宽带按需`100MB/s` * 整个下载过程大概需要半到一个小时 * 整个编译过程大概需要一到两个小时 +## 代码编译 + ``` # 编译工具 mkdir -p /data git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git -# 源码 +# 下载源码 mkdir -p /data/webrtc cd /data/webrtc /data/depot_tools/fetch --nohooks webrtc_android /data/depot_tools/gclient sync -# 分支 +# 切换分支 cd src git checkout -b m94 branch-heads/4606 /data/depot_tools/gclient sync @@ -42,28 +44,32 @@ source ./build/android/envsetup.sh # 编译配置：./tools_webrtc/android/build_aar.py --- -'target_os': 'android', -'is_clang': True, -'is_debug': False, -'use_rtti': True, -'rtc_use_h264': True, -'use_custom_libcxx': False, -'rtc_include_tests': False, -'is_component_build': False, +'target_os' : 'android', +'is_clang' : True, +'is_debug' : False, +'use_rtti' : True, +'rtc_use_h264' : True, +'use_custom_libcxx' : False, +'rtc_include_tests' : False, +'is_component_build' : False, 'treat_warnings_as_errors': False, -'use_goma': use_goma, -'target_cpu': _GetTargetCpu(arch) +'use_goma' : use_goma, +'target_cpu' : _GetTargetCpu(arch) --- # 编译项目 ./tools_webrtc/android/build_aar.py --build-dir ./out/release-build/ --arch x86 x86_64 arm64-v8a armeabi-v7a -# 安装工具 +# 安装re2c +#sudo apt-get install re2c cd /data wget https://github.com/skvadrik/re2c/releases/download/3.0/re2c-3.0.tar.xz -tar -xJf re2c-3.0.tar.xz +tar -Jxvf re2c-3.0.tar.xz cd re2c-3.0 ./configure +make && make install + +# 安装ninja cd /data git clone https://github.com/ninja-build/ninja.git cd ninja @@ -90,7 +96,8 @@ out/release-build/arm64-v8a/gen/sdk/android/video_api_java/generated_java/input_ out/release-build/arm64-v8a/gen/sdk/android/peerconnection_java/generated_java/input_srcjars/ # 提取头文件 -mkdir linux-include +mkdir src +vim header.sh --- #!/bin/bash @@ -98,24 +105,18 @@ src=`find ./ -name "*.h"` for header in $src do echo "cp header file $header" - cp --parents $header linux-include + cp --parents $header src done src=`find ./ -name "*.hpp"` for header in $src do echo "cp header file $header" - cp --parents $header linux-include -done - -src=`find ./ -name "*.hxx"` -for header in $src -do - echo "cp header file $header" - cp --parents $header linux-include + cp --parents $header src done --- -zip -r src.zip linux-include +sh header.sh +zip -r src.zip src ``` [WebRTC](https://pan.baidu.com/s/1E_DXv32D9ODyj5J-o-ji_g?pwd=hudc) diff --git a/docs/etc/nginx.conf b/docs/etc/nginx.conf index bb3810d..373719c 100644 --- a/docs/etc/nginx.conf +++ b/docs/etc/nginx.conf @@ -11,11 +11,11 @@ events { http { log_format main '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main buffer=32k flush=10s; include /etc/nginx/mime.types; include /etc/nginx/conf.d/*.conf; + default_type application/octet-stream; gzip on; diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java index 532c472..69de06e 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java @@ -2,6 +2,7 @@ package com.acgist.taoyao.client; import android.Manifest; import android.app.Activity; +import android.content.Context; import android.content.Intent; import android.content.pm.PackageManager; import android.content.res.ColorStateList; @@ -134,9 +135,11 @@ public class MainActivity extends AppCompatActivity { return; } this.mainHandler = new MainHandler(); + final Context context = this.getApplicationContext(); final Resources resources = this.getResources(); - MediaManager.getInstance().initContext( - this.mainHandler, this.getApplicationContext(), + final MediaManager mediaManager = MediaManager.getInstance(); + mediaManager.initContext( + this.mainHandler, context, resources.getInteger(R.integer.imageQuantity), resources.getString(R.string.audioQuantity), resources.getString(R.string.videoQuantity), @@ -147,6 +150,9 @@ public class MainActivity extends AppCompatActivity { resources.getString(R.string.watermark), VideoSourceType.valueOf(resources.getString(R.string.videoSourceType)) ); + if(resources.getBoolean(R.bool.broadcaster)) { + mediaManager.initTTS(context); + } // 注意：不能使用intent传递 MediaService.mainHandler = this.mainHandler; Log.i(MainActivity.class.getSimpleName(), "拉起媒体服务"); diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java index 6080153..f1a3ae1 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java @@ -76,8 +76,9 @@ public class MediaService extends Service { :: https://gitee.com/acgist/taoyao """); super.onCreate(); - this.mkdir(R.string.storagePathImage); - this.mkdir(R.string.storagePathVideo); + final Resources resources = this.getResources(); + this.mkdir(resources.getString(R.string.storagePathImage), Environment.DIRECTORY_PICTURES); + this.mkdir(resources.getString(R.string.storagePathVideo), Environment.DIRECTORY_MOVIES); this.buildNotificationChannel(); } @@ -192,13 +193,16 @@ public class MediaService extends Service { MediaManager.getInstance().initScreen(intent.getParcelableExtra("data")); } - private void mkdir(int id) { + private void mkdir(String path, String type) { final Path imagePath = Paths.get( - Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath(), - this.getResources().getString(id) + Environment.getExternalStoragePublicDirectory(type).getAbsolutePath(), + path ); final File file = imagePath.toFile(); - if(!file.exists()) { + if(file.exists()) { + Log.d(MediaService.class.getSimpleName(), "目录已经存在：" + imagePath); + } else { + Log.d(MediaService.class.getSimpleName(), "新建文件目录：" + imagePath); file.mkdirs(); } } diff --git a/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml b/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml index 45473aa..e6c8214 100644 --- a/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml +++ b/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml @@ -17,9 +17,9 @@ 2SPWy+TF1zM= - /taoyao/image + /taoyao - /taoyao/video + /taoyao BACK @@ -34,6 +34,8 @@ true true + + false 100 diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/Broadcaster.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/Broadcaster.java deleted file mode 100644 index dc1948a..0000000 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/Broadcaster.java +++ /dev/null @@ -1,53 +0,0 @@ -package com.acgist.taoyao.media; - -import android.content.Context; -import android.speech.tts.TextToSpeech; -import android.util.Log; - -import java.util.Locale; -import java.util.UUID; - -/** - * 声音播报 - * - * @author acgist - */ -public final class Broadcaster { - - private static final Broadcaster INSTANCE = new Broadcaster(); - - public static final Broadcaster getInstance() { - return INSTANCE; - } - - private TextToSpeech textToSpeech; - - private Broadcaster() { - } - - public void init(Context context) { - this.textToSpeech = new TextToSpeech(context, new TextToSpeechInitListener()); - } - - public void broadcast(String text) { -// this.textToSpeech.stop(); - this.textToSpeech.speak(text, TextToSpeech.QUEUE_FLUSH, null, UUID.randomUUID().toString()); - } - - public void shutdown() { - this.textToSpeech.shutdown(); - } - - private class TextToSpeechInitListener implements TextToSpeech.OnInitListener { - @Override - public void onInit(int status) { - Log.i(Broadcaster.class.getSimpleName(), "加载TTS：" + status); - if(status == TextToSpeech.SUCCESS) { - Broadcaster.this.textToSpeech.setLanguage(Locale.CANADA); - Broadcaster.this.textToSpeech.setPitch(1.0F); - Broadcaster.this.textToSpeech.setSpeechRate(1.0F); - } - } - } - -} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java index a92532a..8026264 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java @@ -4,6 +4,7 @@ import android.content.Context; import android.content.Intent; import android.media.projection.MediaProjection; import android.os.Handler; +import android.speech.tts.TextToSpeech; import android.util.Log; import com.acgist.taoyao.media.client.PhotographClient; @@ -42,6 +43,8 @@ import org.webrtc.VideoTrack; import org.webrtc.audio.JavaAudioDeviceModule; import java.util.Arrays; +import java.util.Locale; +import java.util.UUID; /** * 媒体来源管理器 @@ -154,10 +157,18 @@ public final class MediaManager { * PeerConnectionFactory */ private PeerConnectionFactory peerConnectionFactory; + /** + * JavaAudioDeviceModule + */ + private JavaAudioDeviceModule javaAudioDeviceModule; /** * 视频处理 */ private VideoProcesser videoProcesser; + /** + * TTS + */ + private TextToSpeech textToSpeech; /** * 录屏等待锁 */ @@ -263,6 +274,28 @@ public final class MediaManager { this.taoyao = taoyao; } + public void initTTS(Context context) { + if(this.textToSpeech != null) { + return; + } + this.textToSpeech = new TextToSpeech(context, new MediaManager.TextToSpeechInitListener()); + } + + public void broadcast(String text) { + if(this.textToSpeech == null) { + return; + } +// this.textToSpeech.stop(); + this.textToSpeech.speak(text, TextToSpeech.QUEUE_FLUSH, null, UUID.randomUUID().toString()); + } + + public void closeTTS() { + if(this.textToSpeech == null) { + return; + } + this.textToSpeech.shutdown(); + } + /** * 新建终端 * @@ -339,11 +372,11 @@ public final class MediaManager { Log.i(MediaManager.class.getSimpleName(), "加载媒体：" + this.videoSourceType); final VideoDecoderFactory videoDecoderFactory = new DefaultVideoDecoderFactory(this.eglContext); final VideoEncoderFactory videoEncoderFactory = new DefaultVideoEncoderFactory(this.eglContext, true, true); - final JavaAudioDeviceModule javaAudioDeviceModule = this.javaAudioDeviceModule(); + this.javaAudioDeviceModule = this.javaAudioDeviceModule(); this.peerConnectionFactory = PeerConnectionFactory.builder() .setVideoDecoderFactory(videoDecoderFactory) .setVideoEncoderFactory(videoEncoderFactory) - .setAudioDeviceModule(javaAudioDeviceModule) + .setAudioDeviceModule(this.javaAudioDeviceModule) // .setAudioProcessingFactory() // .setAudioEncoderFactoryFactory(new BuiltinAudioEncoderFactoryFactory()) // .setAudioDecoderFactoryFactory(new BuiltinAudioDecoderFactoryFactory()) @@ -372,33 +405,58 @@ public final class MediaManager { // .setAudioSource(MediaRecorder.AudioSource.MIC) // .setAudioFormat(AudioFormat.ENCODING_PCM_16BIT) // .setAudioAttributes(audioAttributes) -// .setUseStereoInput() -// .setUseStereoOutput() // 超低延迟 // .setUseLowLatency() - .setSamplesReadyCallback(audioSamples -> { - if(this.recordClient != null) { - this.recordClient.onWebRtcAudioRecordSamplesReady(audioSamples); +// .setUseStereoInput() +// .setUseStereoOutput() + // 本地声音 +// .setSamplesReadyCallback() + .setAudioTrackErrorCallback(new JavaAudioDeviceModule.AudioTrackErrorCallback() { + @Override + public void onWebRtcAudioTrackInitError(String errorMessage) { + Log.e(MediaManager.class.getSimpleName(), "WebRTC远程音频Track加载异常：" + errorMessage); + } + @Override + public void onWebRtcAudioTrackStartError(JavaAudioDeviceModule.AudioTrackStartErrorCode errorCode, String errorMessage) { + Log.e(MediaManager.class.getSimpleName(), "WebRTC远程音频Track开始异常：" + errorMessage); + } + @Override + public void onWebRtcAudioTrackError(String errorMessage) { + Log.e(MediaManager.class.getSimpleName(), "WebRTC远程音频Track异常：" + errorMessage); } }) .setAudioTrackStateCallback(new JavaAudioDeviceModule.AudioTrackStateCallback() { @Override public void onWebRtcAudioTrackStart() { - Log.i(MediaManager.class.getSimpleName(), "WebRTC声音Track开始"); + Log.i(MediaManager.class.getSimpleName(), "WebRTC远程音频Track开始"); } @Override public void onWebRtcAudioTrackStop() { - Log.i(MediaManager.class.getSimpleName(), "WebRTC声音Track结束"); + Log.i(MediaManager.class.getSimpleName(), "WebRTC远程音频Track结束"); + } + }) + .setAudioRecordErrorCallback(new JavaAudioDeviceModule.AudioRecordErrorCallback() { + @Override + public void onWebRtcAudioRecordInitError(String errorMessage) { + Log.e(MediaManager.class.getSimpleName(), "WebRTC本地音频录制加载异常：" + errorMessage); + } + @Override + public void onWebRtcAudioRecordStartError(JavaAudioDeviceModule.AudioRecordStartErrorCode errorCode, String errorMessage) { + Log.e(MediaManager.class.getSimpleName(), "WebRTC本地音频录制开始异常：" + errorMessage); + } + @Override + public void onWebRtcAudioRecordError(String errorMessage) { + Log.e(MediaManager.class.getSimpleName(), "WebRTC本地音频录制异常：" + errorMessage); } }) .setAudioRecordStateCallback(new JavaAudioDeviceModule.AudioRecordStateCallback() { @Override public void onWebRtcAudioRecordStart() { - Log.i(MediaManager.class.getSimpleName(), "WebRTC声音录制开始"); + Log.i(MediaManager.class.getSimpleName(), "WebRTC本地音频录制开始"); } @Override public void onWebRtcAudioRecordStop() { - Log.i(MediaManager.class.getSimpleName(), "WebRTC声音录制结束"); + Log.i(MediaManager.class.getSimpleName(), "WebRTC本地音频录制结束"); } }) // .setUseHardwareNoiseSuppressor(true) @@ -689,7 +747,7 @@ public final class MediaManager { this.videoPath, this.taoyao, this.mainHandler ); this.recordClient.start(); - this.recordClient.record(this.mainVideoSource, this.peerConnectionFactory); + this.recordClient.record(this.mainVideoSource, this.javaAudioDeviceModule, this.peerConnectionFactory); this.mainHandler.obtainMessage(Config.WHAT_RECORD, Boolean.TRUE).sendToTarget(); return this.recordClient; } @@ -758,6 +816,10 @@ public final class MediaManager { this.surfaceTextureHelper.dispose(); this.surfaceTextureHelper = null; } + if(this.javaAudioDeviceModule != null) { + this.javaAudioDeviceModule.release(); + this.javaAudioDeviceModule = null; + } if (this.peerConnectionFactory != null) { this.peerConnectionFactory.dispose(); this.peerConnectionFactory = null; @@ -806,7 +868,11 @@ public final class MediaManager { } else { final VideoFrame.I420Buffer i420Buffer = videoFrame.getBuffer().toI420(); MediaManager.this.videoProcesser.process(i420Buffer); - final VideoFrame processVideoFrame = new VideoFrame(i420Buffer.cropAndScale(0, 0, i420Buffer.getWidth(), i420Buffer.getHeight(), i420Buffer.getWidth(), i420Buffer.getHeight()), videoFrame.getRotation(), videoFrame.getTimestampNs()); + final VideoFrame processVideoFrame = new VideoFrame( + i420Buffer.cropAndScale(0, 0, i420Buffer.getWidth(), i420Buffer.getHeight(), i420Buffer.getWidth(), i420Buffer.getHeight()), + videoFrame.getRotation(), + videoFrame.getTimestampNs() + ); i420Buffer.release(); this.mainObserver.onFrameCaptured(processVideoFrame); this.shareObserver.onFrameCaptured(processVideoFrame); @@ -869,6 +935,18 @@ public final class MediaManager { } + private class TextToSpeechInitListener implements TextToSpeech.OnInitListener { + @Override + public void onInit(int status) { + Log.i(MediaManager.class.getSimpleName(), "加载TTS：" + status); + if(status == TextToSpeech.SUCCESS) { + MediaManager.this.textToSpeech.setLanguage(Locale.CANADA); + MediaManager.this.textToSpeech.setPitch(1.0F); + MediaManager.this.textToSpeech.setSpeechRate(1.0F); + } + } + } + private native void nativeInit(); private native void nativeStop(); diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java index b8f73c8..2910a78 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java @@ -1,13 +1,109 @@ package com.acgist.taoyao.media.audio; +import android.util.Log; + +import com.acgist.taoyao.media.client.RecordClient; + +import org.webrtc.AudioSource; +import org.webrtc.audio.JavaAudioDeviceModule; + +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + /** * 混音处理器 * + * JavaAudioDeviceModule : 音频 * WebRtcAudioTrack#AudioTrackThread ：远程音频 * WebRtcAudioRecord#AudioRecordThread：本地音频 * + * 注意：只能远程终端拉取才能采集音频数据，如果需要离线采集自己使用AudioRecord实现。 + * * @author acgist */ -public class MixerProcesser { +public class MixerProcesser extends Thread implements JavaAudioDeviceModule.SamplesReadyCallback { + + private boolean close; + private final RecordClient recordClient; + private final BlockingQueue local; + private final BlockingQueue remote; + + public MixerProcesser(RecordClient recordClient) { + this.setDaemon(true); + this.setName("AudioMixer"); + this.close = false; + this.recordClient = recordClient; + this.local = new LinkedBlockingQueue<>(1024); + this.remote = new LinkedBlockingQueue<>(1024); + } + + @Override + public void onWebRtcAudioTrackSamplesReady(JavaAudioDeviceModule.AudioSamples samples) { +// Log.d(MixerProcesser.class.getSimpleName(), "远程音频信息：" + samples.getAudioFormat()); + if(!this.remote.offer(samples)) { + Log.e(MixerProcesser.class.getSimpleName(), "远程音频队列阻塞"); + } + } + + @Override + public void onWebRtcAudioRecordSamplesReady(JavaAudioDeviceModule.AudioSamples samples) { +// Log.d(MixerProcesser.class.getSimpleName(), "本地音频信息：" + samples.getAudioFormat()); + if(!this.local.offer(samples)) { + Log.e(MixerProcesser.class.getSimpleName(), "本地音频队列阻塞"); + } + } + + @Override + public void run() { + long pts = System.nanoTime(); +// final byte[] target = new byte[length]; + // PCM时间计算：1000000 microseconds / 48000 hz / 2 bytes + JavaAudioDeviceModule.AudioSamples local; + JavaAudioDeviceModule.AudioSamples remote; + int localValue; + int remoteValue; + byte[] localData; + byte[] remoteData; + byte[] data = null; + // TODO：固定长度采样率等等 + while(!this.close) { + try { + local = this.local.poll(100, TimeUnit.MILLISECONDS); + remote = this.remote.poll(); + if(local != null && remote != null) { + localData = local.getData(); + remoteData = remote.getData(); + Log.d(MixerProcesser.class.getSimpleName(), String.format(""" + 混音长度：%d - %d + 混音采样：%d - %d + 混音格式：%d - %d + """, localData.length, remoteData.length, local.getSampleRate(), remote.getSampleRate(), local.getAudioFormat(), remote.getAudioFormat())); + data = new byte[localData.length]; + for (int index = 0; index < localData.length; index++) { + localValue = localData[index]; + remoteValue = remoteData[index]; + data[index] = (byte) ((localValue +remoteValue) / 2); + } + pts += data.length * (1_000_000 / local.getSampleRate() / 2); + } else if(local != null && remote == null) { + data = local.getData(); + pts += data.length * (1_000_000 / local.getSampleRate() / 2); + } else if(local == null && remote != null) { + data = remote.getData(); + pts += data.length * (1_000_000 / remote.getSampleRate() / 2); + } else { + continue; + } + this.recordClient.onPcm(pts, data); + } catch (Exception e) { + Log.e(MixerProcesser.class.getSimpleName(), "音频处理异常", e); + } + } + } + + public void close() { + this.close = true; + } } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java index 443fbe0..256f90f 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java @@ -65,7 +65,7 @@ public class PhotographClient implements VideoSink { public PhotographClient(int quantity, String path) { this.quantity = quantity; this.filename = DateUtils.format(LocalDateTime.now(), DateUtils.DateTimeStyle.YYYYMMDDHH24MMSS) + ".jpg"; - this.filepath = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath(), path, this.filename).toString(); + this.filepath = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_PICTURES).getAbsolutePath(), path, this.filename).toString(); this.done = false; this.finish = false; Log.i(RecordClient.class.getSimpleName(), "拍摄照片文件：" + this.filepath); @@ -97,13 +97,6 @@ public class PhotographClient implements VideoSink { } public void photograph(VideoSource videoSource, PeerConnectionFactory peerConnectionFactory) { - if(this.videoTrack != null) { - return; - } - if(videoSource == null || peerConnectionFactory == null) { - Log.e(PhotographClient.class.getSimpleName(), "数据采集无效"); - return; - } this.videoTrack = peerConnectionFactory.createVideoTrack("TaoyaoVP", videoSource); this.videoTrack.setEnabled(true); this.videoTrack.addSink(this); diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java index 4939f64..9d02cff 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java @@ -11,6 +11,7 @@ import android.util.Log; import com.acgist.taoyao.boot.utils.DateUtils; import com.acgist.taoyao.media.MediaManager; +import com.acgist.taoyao.media.audio.MixerProcesser; import com.acgist.taoyao.media.signal.ITaoyao; import org.webrtc.PeerConnectionFactory; @@ -32,7 +33,7 @@ import java.time.LocalDateTime; * * @author acgist */ -public class RecordClient extends Client implements VideoSink, JavaAudioDeviceModule.SamplesReadyCallback { +public class RecordClient extends Client implements VideoSink { /** * 等待时间（毫秒） @@ -126,6 +127,8 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo */ private MediaMuxer mediaMuxer; private VideoTrack videoTrack; + private MixerProcesser mixerProcesser; + private JavaAudioDeviceModule javaAudioDeviceModule; public RecordClient( int audioBitRate, int sampleRate, int channelCount, @@ -144,7 +147,9 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo this.height = height; this.yuvSize = width * height * 3 / 2; this.filename = DateUtils.format(LocalDateTime.now(), DateUtils.DateTimeStyle.YYYYMMDDHH24MMSS) + ".mp4"; - this.filepath = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath(), path, this.filename).toString(); + this.filepath = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_MOVIES).getAbsolutePath(), path, this.filename).toString(); + this.audioActive = false; + this.videoActive = false; } public void start() { @@ -191,15 +196,12 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo this.audioHandler.post(this::audioCodec); } - private volatile long audioPts = 0; - private void audioCodec() { long pts = 0L; int trackIndex = -1; int outputIndex; this.audioCodec.start(); this.audioActive = true; - JavaAudioDeviceModule.AudioSamples audioSamples = null; final MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); while (!this.close) { outputIndex = this.audioCodec.dequeueOutputBuffer(bufferInfo, WAIT_TIME_US); @@ -369,17 +371,20 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo } } - public void record(VideoSource videoSource, PeerConnectionFactory peerConnectionFactory) { - if(this.videoTrack != null) { - return; + public void record(VideoSource videoSource, JavaAudioDeviceModule javaAudioDeviceModule, PeerConnectionFactory peerConnectionFactory) { + // 音频 + if(javaAudioDeviceModule != null) { + this.mixerProcesser = new MixerProcesser(this); + this.mixerProcesser.start(); + javaAudioDeviceModule.setMixerProcesser(this.mixerProcesser); + this.javaAudioDeviceModule = javaAudioDeviceModule; } - if(videoSource == null || peerConnectionFactory == null) { - Log.e(RecordClient.class.getSimpleName(), "数据采集无效"); - return; + // 视频 + if(videoSource != null && peerConnectionFactory != null) { + this.videoTrack = peerConnectionFactory.createVideoTrack("TaoyaoVR", videoSource); + this.videoTrack.setEnabled(true); + this.videoTrack.addSink(this); } - this.videoTrack = peerConnectionFactory.createVideoTrack("TaoyaoVR", videoSource); - this.videoTrack.setEnabled(true); - this.videoTrack.addSink(this); } @Override @@ -390,6 +395,13 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo } super.close(); Log.i(RecordClient.class.getSimpleName(), "结束录制：" + this.filepath); + if(this.javaAudioDeviceModule != null) { + this.javaAudioDeviceModule.removeMixerProcesser(); + } + if(this.mixerProcesser != null) { + this.mixerProcesser.close(); + this.mixerProcesser = null; + } if(this.videoTrack != null) { this.videoTrack.removeSink(this); this.videoTrack.dispose(); @@ -422,26 +434,20 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo } /** - * @param audioSamples PCM数据 + * @param pts PTS时间偏移 + * @param data PCM数据 */ - @Override - public void onWebRtcAudioRecordSamplesReady(JavaAudioDeviceModule.AudioSamples audioSamples) { + public void onPcm(long pts, byte[] data) { if(this.close || !this.audioActive) { return; } - Log.i(RecordClient.class.getSimpleName(), "音频信息：" + audioSamples.getAudioFormat()); - int index = this.audioCodec.dequeueInputBuffer(WAIT_TIME_US); - if (index >= 0) { - final byte[] data = audioSamples.getData(); - final ByteBuffer buffer = this.audioCodec.getInputBuffer(index); - buffer.put(data); - this.audioCodec.queueInputBuffer(index, 0, data.length, this.audioPts, 0); - // 1000000 microseconds / 48000 hz / 2 bytes - this.audioPts += data.length * (1_000_000 / audioSamples.getSampleRate() / 2); - } else { - // WARN + final int index = this.audioCodec.dequeueInputBuffer(WAIT_TIME_US); + if (index < 0) { + return; } - audioSamples = null; + final ByteBuffer buffer = this.audioCodec.getInputBuffer(index); + buffer.put(data); + this.audioCodec.queueInputBuffer(index, 0, data.length, pts, 0); } @Override @@ -449,7 +455,7 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo if (this.close || !this.videoActive) { return; } - Log.i(RecordClient.class.getSimpleName(), "视频信息：" + videoFrame.getRotatedWidth() + " - " + videoFrame.getRotatedHeight()); +// Log.d(RecordClient.class.getSimpleName(), "视频信息：" + videoFrame.getRotatedWidth() + " - " + videoFrame.getRotatedHeight()); final int index = this.videoCodec.dequeueInputBuffer(WAIT_TIME_US); if(index < 0) { return; diff --git a/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/JavaAudioDeviceModule.java b/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/JavaAudioDeviceModule.java index 48402e4..bfc41d3 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/JavaAudioDeviceModule.java +++ b/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/JavaAudioDeviceModule.java @@ -316,6 +316,17 @@ public class JavaAudioDeviceModule implements AudioDeviceModule { /** Called when new audio samples are ready. This should only be set for debug purposes */ public static interface SamplesReadyCallback { + /** + * 远程音频 + * + * @param samples 音频采样 + */ + void onWebRtcAudioTrackSamplesReady(AudioSamples samples); + /** + * 本地音频 + * + * @param samples 音频采样 + */ void onWebRtcAudioRecordSamplesReady(AudioSamples samples); } @@ -379,6 +390,26 @@ public class JavaAudioDeviceModule implements AudioDeviceModule { this.useStereoOutput = useStereoOutput; } + /** + * 设置录音工具 + * + * @param samplesReadyCallback 录音回调 + * + * @Taoyao + */ + public void setMixerProcesser(SamplesReadyCallback samplesReadyCallback) { + this.audioInput.setMixerProcesser(samplesReadyCallback); + this.audioOutput.setMixerProcesser(samplesReadyCallback); + } + + /** + * 删除录音工具 + */ + public void removeMixerProcesser() { + this.audioInput.setMixerProcesser(null); + this.audioOutput.setMixerProcesser(null); + } + @Override public long getNativeAudioDeviceModulePointer() { synchronized (nativeLock) { @@ -427,4 +458,5 @@ public class JavaAudioDeviceModule implements AudioDeviceModule { private static native long nativeCreateAudioDeviceModule(Context context, AudioManager audioManager, WebRtcAudioRecord audioInput, WebRtcAudioTrack audioOutput, int inputSampleRate, int outputSampleRate, boolean useStereoInput, boolean useStereoOutput); + } diff --git a/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioRecord.java b/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioRecord.java index 623b529..ac6b049 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioRecord.java +++ b/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioRecord.java @@ -105,10 +105,21 @@ class WebRtcAudioRecord { private final @Nullable AudioRecordErrorCallback errorCallback; private final @Nullable AudioRecordStateCallback stateCallback; - private final @Nullable SamplesReadyCallback audioSamplesReadyCallback; + private @Nullable SamplesReadyCallback audioSamplesReadyCallback; private final boolean isAcousticEchoCancelerSupported; private final boolean isNoiseSuppressorSupported; + /** + * 设置录音工具 + * + * @param samplesReadyCallback 录音回调 + * + * @Taoyao + */ + public void setMixerProcesser(SamplesReadyCallback samplesReadyCallback) { + this.audioSamplesReadyCallback = samplesReadyCallback; + } + /** * Audio thread which keeps calling ByteBuffer.read() waiting for audio * to be recorded. Feeds recorded data to the native counterpart as a @@ -131,7 +142,6 @@ class WebRtcAudioRecord { // Audio recording has started and the client is informed about it. doAudioRecordStateCallback(AUDIO_RECORD_START); - long lastTime = System.nanoTime(); while (keepAlive) { int bytesRead = audioRecord.read(byteBuffer, byteBuffer.capacity()); if (bytesRead == byteBuffer.capacity()) { @@ -148,11 +158,11 @@ class WebRtcAudioRecord { if (audioSamplesReadyCallback != null) { // Copy the entire byte buffer array. The start of the byteBuffer is not necessarily // at index 0. - byte[] data = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.arrayOffset(), - byteBuffer.capacity() + byteBuffer.arrayOffset()); - audioSamplesReadyCallback.onWebRtcAudioRecordSamplesReady( - new JavaAudioDeviceModule.AudioSamples(audioRecord.getAudioFormat(), - audioRecord.getChannelCount(), audioRecord.getSampleRate(), data)); + SamplesReadyCallback nullable = audioSamplesReadyCallback; + if(nullable != null) { + final byte[] data = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.arrayOffset(), byteBuffer.capacity() + byteBuffer.arrayOffset()); + nullable.onWebRtcAudioRecordSamplesReady(new JavaAudioDeviceModule.AudioSamples(audioRecord.getAudioFormat(), audioRecord.getChannelCount(), audioRecord.getSampleRate(), data)); + } } } else { String errorMessage = "AudioRecord.read failed: " + bytesRead; diff --git a/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioTrack.java b/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioTrack.java index c313f9c..52d1cb2 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioTrack.java +++ b/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioTrack.java @@ -27,8 +27,10 @@ import org.webrtc.ThreadUtils; import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackErrorCallback; import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackStartErrorCode; import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackStateCallback; +import org.webrtc.audio.JavaAudioDeviceModule.SamplesReadyCallback; import java.nio.ByteBuffer; +import java.util.Arrays; class WebRtcAudioTrack { private static final String TAG = "WebRtcAudioTrackExternal"; @@ -87,6 +89,18 @@ class WebRtcAudioTrack { private final @Nullable AudioTrackErrorCallback errorCallback; private final @Nullable AudioTrackStateCallback stateCallback; + private @Nullable SamplesReadyCallback audioSamplesReadyCallback; + + /** + * 设置录音工具 + * + * @param samplesReadyCallback 录音回调 + * + * @Taoyao + */ + public void setMixerProcesser(JavaAudioDeviceModule.SamplesReadyCallback samplesReadyCallback) { + this.audioSamplesReadyCallback = samplesReadyCallback; + } /** * Audio thread which keeps calling AudioTrack.write() to stream audio. @@ -131,6 +145,13 @@ class WebRtcAudioTrack { byteBuffer.position(0); } int bytesWritten = writeBytes(audioTrack, byteBuffer, sizeInBytes); + if (audioSamplesReadyCallback != null) { + SamplesReadyCallback nullable = audioSamplesReadyCallback; + if(nullable != null) { + final byte[] data = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.arrayOffset(), byteBuffer.capacity() + byteBuffer.arrayOffset()); + nullable.onWebRtcAudioTrackSamplesReady(new JavaAudioDeviceModule.AudioSamples(audioTrack.getAudioFormat(), audioTrack.getChannelCount(), audioTrack.getSampleRate(), data)); + } + } if (bytesWritten != sizeInBytes) { Logging.e(TAG, "AudioTrack.write played invalid number of bytes: " + bytesWritten); // If a write() returns a negative value, an error has occurred. diff --git a/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/AudioMixerTest.java b/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/AudioMixerTest.java new file mode 100644 index 0000000..faaac82 --- /dev/null +++ b/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/AudioMixerTest.java @@ -0,0 +1,31 @@ +package com.acgist.taoyao; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Paths; + +import org.junit.jupiter.api.Test; + +public class AudioMixerTest { + + @Test + public void testMixer() throws IOException { +// ffmpeg -i audio.mp3 -f s32le audio.pcm +// ffplay -i audio.pcm -f s32le -ar 48000 -ac 1 + final File fileA = new File("D:\\tmp\\mixer\\1.pcm"); + final File fileB = new File("D:\\tmp\\mixer\\2.pcm"); + final byte[] bytesA = Files.readAllBytes(fileA.toPath()); + final byte[] bytesB = Files.readAllBytes(fileB.toPath()); + final int length = Math.min(bytesA.length, bytesB.length); + final byte[] target = new byte[length]; + int a, b; + for (int i = 0; i < length; i++) { + a = bytesA[i]; + b = bytesB[i]; + target[i] = (byte) ((a + b) / 2); + } + Files.write(Paths.get("D:\\tmp\\mixer\\3.pcm"), target); + } + +}