[+] 混音

.github/workflows/build.yml

@@ -1,4 +1,4 @@
-name: build all
+name: build
 
 on:
   push:
@@ -98,3 +98,4 @@ jobs:
         run: |
           cd ./taoyao-client-android/taoyao
           ./gradlew.bat --no-daemon assembleRelease
+          

.github/workflows/codeql.yml

@@ -0,0 +1,30 @@
+name: codeql
+
+on:
+  push:
+    branches: [ master ]
+
+jobs:
+  analyze:
+    name: Analyze
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ "cpp", "java", "javascript" ]
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v2
+    - name: Set up JDK
+      uses: actions/setup-java@v1
+      with:
+        java-version: 17
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v1
+      with:
+        languages: ${{ matrix.language }}
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v1
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v1
+      

docs/AOSP.md

@@ -12,10 +12,10 @@
 ## 机器配置
 
 * 内存`32G`
+* 16核`CPU`
 * 硬盘`300G`
-* 十六核`CPU`
 * 系统`Ubuntu 18.xx`
-* 公司网络`1000Mbps/s`
+* 网络带宽`1000MB/s`
 * 整个下载过程大概需要三到五个小时
 * 整个编译过程大概需要半到一个小时
 

docs/Deploy.md

@@ -120,7 +120,7 @@ cmake -v
 mkdir -p /data/dev/nodejs
 cd /data/dev/nodejs
 wget https://nodejs.org/dist/v16.19.0/node-v16.19.0-linux-x64.tar.xz
-tar -xvJf node-v16.19.0-linux-x64.tar.xz
+tar -Jxvf node-v16.19.0-linux-x64.tar.xz
 
 # 连接
 ln -sf /data/dev/nodejs/node-v16.19.0-linux-x64/bin/npm /usr/local/bin/
@@ -213,7 +213,7 @@ mkdir -p /data/dev/python
 cd /data/dev/python
 #wget https://www.python.org/ftp/python/3.8.16/Python-3.8.16.tar.xz
 wget https://mirrors.huaweicloud.com/python/3.8.16/Python-3.8.16.tar.xz
-tar -xvJf Python-3.8.16.tar.xz
+tar -Jxvf Python-3.8.16.tar.xz
 
 # 安装
 cd Python-3.8.16
@@ -265,7 +265,7 @@ yum install nginx
 systemctl enable nginx
 
 # 管理服务
-systemctl start | stop | restart nginx
+systemctl start|stop|restart nginx
 
 # 加载配置
 nginx -s reload
@@ -307,7 +307,7 @@ systemctl enable taoyao-signal-server
 ./deploy.sh
 
 # 管理服务
-systemctl start | stop | restart taoyao-signal-server
+systemctl start|stop|restart taoyao-signal-server
 ```
 
 ## 安装媒体
@@ -318,11 +318,11 @@ cd /data/taoyao/taoyao-client-media
 npm install
 
 # 配置ecosystem
-pm2 start | reload ecosystem.config.json
+pm2 start|reload ecosystem.config.json
 pm2 save
 
 # 管理服务：服务名称必须和配置终端标识一致否则不能执行重启和关闭信令
-pm2 start | stop | restart taoyao-client-media
+pm2 start|stop|restart taoyao-client-media
 ```
 
 ### Mediasoup编译失败
@@ -359,7 +359,7 @@ pm2 start npm --name "taoyao-client-web" -- run dev
 pm2 save
 
 # 管理服务
-pm2 start | stop | restart taoyao-client-web
+pm2 start|stop|restart taoyao-client-web
 
 # 打包代码
 npm run build

docs/MI5S.md

@@ -1,6 +1,6 @@
 # 小米5S
 
-由于使用小米5S作为测试机，没有适合的`LineageOS`版本，所有这里选择了`PixelExperience`作为测试系统，如果其他机器建议使用`LineageOS`，`ROM`下载地址：
+由于使用小米5S作为测试机，没有适合的`LineageOS`版本，所有这里选择了`PixelExperience`作为测试系统。如果其他机器建议使用`LineageOS`，当然最好还是使用`Pixel`作为测试机，可以直接使用原生系统，或者自己定制编译`AOSP`。
 
 ## TWRP
 

docs/TODO.md

@@ -14,7 +14,10 @@
 ## 计划任务
 
 * 混音
+* 音频视频时间对齐
 * 分辨率调整
 * 查询消费者生产者信息
 
 ## 完成任务
+
+* 屏幕共享

docs/WebRTC.md

@@ -1,6 +1,6 @@
 # WebRTC
 
-本文档内容旨在独立编译`WebRTC`项目，非必需使用。
+本文档内容旨在独立编译`WebRTC`项目，并非必需使用。
 
 ## libwebrtc
 
@@ -15,22 +15,24 @@
 * 四核`CPU`
 * 硬盘`100G`
 * 系统`Ubuntu 20.xx`
-* 宽带按需`100Mbps/s`
+* 宽带按需`100MB/s`
 * 整个下载过程大概需要半到一个小时
 * 整个编译过程大概需要一到两个小时
 
+## 代码编译
+
 ```
 # 编译工具
 mkdir -p /data
 git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git
 
-# 源码
+# 下载源码
 mkdir -p /data/webrtc
 cd /data/webrtc
 /data/depot_tools/fetch --nohooks webrtc_android
 /data/depot_tools/gclient sync
 
-# 分支
+# 切换分支
 cd src
 git checkout -b m94 branch-heads/4606
 /data/depot_tools/gclient sync
@@ -42,28 +44,32 @@ source ./build/android/envsetup.sh
 
 # 编译配置：./tools_webrtc/android/build_aar.py
 ---
-'target_os': 'android',
+'target_os'               : 'android',
-'is_clang': True,
+'is_clang'                : True,
-'is_debug': False,
+'is_debug'                : False,
-'use_rtti': True,
+'use_rtti'                : True,
-'rtc_use_h264': True,
+'rtc_use_h264'            : True,
-'use_custom_libcxx': False,
+'use_custom_libcxx'       : False,
-'rtc_include_tests': False,
+'rtc_include_tests'       : False,
-'is_component_build': False,
+'is_component_build'      : False,
 'treat_warnings_as_errors': False,
-'use_goma': use_goma,
+'use_goma'                : use_goma,
-'target_cpu': _GetTargetCpu(arch)
+'target_cpu'              : _GetTargetCpu(arch)
 ---
 
 # 编译项目
 ./tools_webrtc/android/build_aar.py --build-dir ./out/release-build/ --arch x86 x86_64 arm64-v8a armeabi-v7a
 
-# 安装工具
+# 安装re2c
+#sudo apt-get install re2c
 cd /data
 wget https://github.com/skvadrik/re2c/releases/download/3.0/re2c-3.0.tar.xz
-tar -xJf re2c-3.0.tar.xz
+tar -Jxvf re2c-3.0.tar.xz
 cd re2c-3.0
 ./configure
+make && make install
+
+# 安装ninja
 cd /data
 git clone https://github.com/ninja-build/ninja.git
 cd ninja
@@ -90,7 +96,8 @@ out/release-build/arm64-v8a/gen/sdk/android/video_api_java/generated_java/input_
 out/release-build/arm64-v8a/gen/sdk/android/peerconnection_java/generated_java/input_srcjars/
 
 # 提取头文件
-mkdir linux-include
+mkdir src
+vim header.sh
 ---
 #!/bin/bash
  
@@ -98,24 +105,18 @@ src=`find ./ -name "*.h"`
 for header in $src
 do
     echo "cp header file $header"
-    cp --parents $header linux-include
+    cp --parents $header src
 done
 
 src=`find ./ -name "*.hpp"`
 for header in $src
 do
     echo "cp header file $header"
-    cp --parents $header linux-include
+    cp --parents $header src
-done
-
-src=`find ./ -name "*.hxx"`
-for header in $src
-do
-    echo "cp header file $header"
-    cp --parents $header linux-include
 done
 ---
-zip -r src.zip linux-include
+sh header.sh
+zip -r src.zip src
 ```
 
 [WebRTC](https://pan.baidu.com/s/1E_DXv32D9ODyj5J-o-ji_g?pwd=hudc)

docs/etc/nginx.conf

@@ -11,11 +11,11 @@ events {
 
 http {
     log_format main '$remote_addr - $remote_user [$time_local] "$request" $status $body_bytes_sent "$http_referer" "$http_user_agent" "$http_x_forwarded_for"';
-    
     access_log /var/log/nginx/access.log main buffer=32k flush=10s;
 
     include      /etc/nginx/mime.types;
     include      /etc/nginx/conf.d/*.conf;
+    
     default_type application/octet-stream;
 
     gzip            on;

taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java

@@ -2,6 +2,7 @@ package com.acgist.taoyao.client;
 
 import android.Manifest;
 import android.app.Activity;
+import android.content.Context;
 import android.content.Intent;
 import android.content.pm.PackageManager;
 import android.content.res.ColorStateList;
@@ -134,9 +135,11 @@ public class MainActivity extends AppCompatActivity {
             return;
         }
         this.mainHandler = new MainHandler();
+        final Context context = this.getApplicationContext();
         final Resources resources = this.getResources();
-        MediaManager.getInstance().initContext(
+        final MediaManager mediaManager = MediaManager.getInstance();
-            this.mainHandler, this.getApplicationContext(),
+        mediaManager.initContext(
+            this.mainHandler, context,
             resources.getInteger(R.integer.imageQuantity),
             resources.getString(R.string.audioQuantity),
             resources.getString(R.string.videoQuantity),
@@ -147,6 +150,9 @@ public class MainActivity extends AppCompatActivity {
             resources.getString(R.string.watermark),
             VideoSourceType.valueOf(resources.getString(R.string.videoSourceType))
         );
+        if(resources.getBoolean(R.bool.broadcaster)) {
+            mediaManager.initTTS(context);
+        }
         // 注意：不能使用intent传递
         MediaService.mainHandler = this.mainHandler;
         Log.i(MainActivity.class.getSimpleName(), "拉起媒体服务");

taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java

@@ -76,8 +76,9 @@ public class MediaService extends Service {
         :: https://gitee.com/acgist/taoyao
         """);
         super.onCreate();
-        this.mkdir(R.string.storagePathImage);
+        final Resources resources = this.getResources();
-        this.mkdir(R.string.storagePathVideo);
+        this.mkdir(resources.getString(R.string.storagePathImage), Environment.DIRECTORY_PICTURES);
+        this.mkdir(resources.getString(R.string.storagePathVideo), Environment.DIRECTORY_MOVIES);
         this.buildNotificationChannel();
     }
 
@@ -192,13 +193,16 @@ public class MediaService extends Service {
         MediaManager.getInstance().initScreen(intent.getParcelableExtra("data"));
     }
 
-    private void mkdir(int id) {
+    private void mkdir(String path, String type) {
         final Path imagePath = Paths.get(
-            Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath(),
+            Environment.getExternalStoragePublicDirectory(type).getAbsolutePath(),
-            this.getResources().getString(id)
+            path
         );
         final File file = imagePath.toFile();
-        if(!file.exists()) {
+        if(file.exists()) {
+            Log.d(MediaService.class.getSimpleName(), "目录已经存在：" + imagePath);
+        } else {
+            Log.d(MediaService.class.getSimpleName(), "新建文件目录：" + imagePath);
             file.mkdirs();
         }
     }

taoyao-client-android/taoyao/client/src/main/res/values/settings.xml

@@ -17,9 +17,9 @@
     <!-- 信令加密密钥 -->
     <string name="encryptSecret">2SPWy+TF1zM=</string>
     <!-- 图片存储目录 -->
-    <string name="storagePathImage">/taoyao/image</string>
+    <string name="storagePathImage">/taoyao</string>
     <!-- 视频存储目录 -->
-    <string name="storagePathVideo">/taoyao/video</string>
+    <string name="storagePathVideo">/taoyao</string>
     <!-- 视频来源：FILE|BACK|FRONT|SCREEN -->
     <string name="videoSourceType">BACK</string>
     <!-- 媒体配置：是否消费数据 -->
@@ -34,6 +34,8 @@
     <bool name="audioProduce">true</bool>
     <!-- 媒体配置：是否生产视频 -->
     <bool name="videoProduce">true</bool>
+    <!-- 语音播报 -->
+    <bool name="broadcaster">false</bool>
     <!-- 图片质量 -->
     <integer name="imageQuantity">100</integer>
     <!-- 音频质量 -->

taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/Broadcaster.java

@@ -1,53 +0,0 @@
-package com.acgist.taoyao.media;
-
-import android.content.Context;
-import android.speech.tts.TextToSpeech;
-import android.util.Log;
-
-import java.util.Locale;
-import java.util.UUID;
-
-/**
- * 声音播报
- *
- * @author acgist
- */
-public final class Broadcaster {
-
-    private static final Broadcaster INSTANCE = new Broadcaster();
-
-    public static final Broadcaster getInstance() {
-        return INSTANCE;
-    }
-
-    private TextToSpeech textToSpeech;
-
-    private Broadcaster() {
-    }
-
-    public void init(Context context) {
-        this.textToSpeech = new TextToSpeech(context, new TextToSpeechInitListener());
-    }
-
-    public void broadcast(String text) {
-//      this.textToSpeech.stop();
-        this.textToSpeech.speak(text, TextToSpeech.QUEUE_FLUSH, null, UUID.randomUUID().toString());
-    }
-
-    public void shutdown() {
-        this.textToSpeech.shutdown();
-    }
-
-    private class TextToSpeechInitListener implements TextToSpeech.OnInitListener {
-        @Override
-        public void onInit(int status) {
-            Log.i(Broadcaster.class.getSimpleName(), "加载TTS：" + status);
-            if(status == TextToSpeech.SUCCESS) {
-                Broadcaster.this.textToSpeech.setLanguage(Locale.CANADA);
-                Broadcaster.this.textToSpeech.setPitch(1.0F);
-                Broadcaster.this.textToSpeech.setSpeechRate(1.0F);
-            }
-        }
-    }
-
-}

taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java

@@ -4,6 +4,7 @@ import android.content.Context;
 import android.content.Intent;
 import android.media.projection.MediaProjection;
 import android.os.Handler;
+import android.speech.tts.TextToSpeech;
 import android.util.Log;
 
 import com.acgist.taoyao.media.client.PhotographClient;
@@ -42,6 +43,8 @@ import org.webrtc.VideoTrack;
 import org.webrtc.audio.JavaAudioDeviceModule;
 
 import java.util.Arrays;
+import java.util.Locale;
+import java.util.UUID;
 
 /**
  * 媒体来源管理器
@@ -154,10 +157,18 @@ public final class MediaManager {
      * PeerConnectionFactory
      */
     private PeerConnectionFactory peerConnectionFactory;
+    /**
+     * JavaAudioDeviceModule
+     */
+    private JavaAudioDeviceModule javaAudioDeviceModule;
     /**
      * 视频处理
      */
     private VideoProcesser videoProcesser;
+    /**
+     * TTS
+     */
+    private TextToSpeech textToSpeech;
     /**
      * 录屏等待锁
      */
@@ -263,6 +274,28 @@ public final class MediaManager {
         this.taoyao = taoyao;
     }
 
+    public void initTTS(Context context) {
+        if(this.textToSpeech != null) {
+            return;
+        }
+        this.textToSpeech = new TextToSpeech(context, new MediaManager.TextToSpeechInitListener());
+    }
+
+    public void broadcast(String text) {
+        if(this.textToSpeech == null) {
+            return;
+        }
+//      this.textToSpeech.stop();
+        this.textToSpeech.speak(text, TextToSpeech.QUEUE_FLUSH, null, UUID.randomUUID().toString());
+    }
+
+    public void closeTTS() {
+        if(this.textToSpeech == null) {
+            return;
+        }
+        this.textToSpeech.shutdown();
+    }
+
     /**
      * 新建终端
      *
@@ -339,11 +372,11 @@ public final class MediaManager {
         Log.i(MediaManager.class.getSimpleName(), "加载媒体：" + this.videoSourceType);
         final VideoDecoderFactory videoDecoderFactory = new DefaultVideoDecoderFactory(this.eglContext);
         final VideoEncoderFactory videoEncoderFactory = new DefaultVideoEncoderFactory(this.eglContext, true, true);
-        final JavaAudioDeviceModule javaAudioDeviceModule = this.javaAudioDeviceModule();
+        this.javaAudioDeviceModule = this.javaAudioDeviceModule();
         this.peerConnectionFactory = PeerConnectionFactory.builder()
             .setVideoDecoderFactory(videoDecoderFactory)
             .setVideoEncoderFactory(videoEncoderFactory)
-            .setAudioDeviceModule(javaAudioDeviceModule)
+            .setAudioDeviceModule(this.javaAudioDeviceModule)
 //          .setAudioProcessingFactory()
 //          .setAudioEncoderFactoryFactory(new BuiltinAudioEncoderFactoryFactory())
 //          .setAudioDecoderFactoryFactory(new BuiltinAudioDecoderFactoryFactory())
@@ -372,33 +405,58 @@ public final class MediaManager {
 //          .setAudioSource(MediaRecorder.AudioSource.MIC)
 //          .setAudioFormat(AudioFormat.ENCODING_PCM_16BIT)
 //          .setAudioAttributes(audioAttributes)
-//          .setUseStereoInput()
-//          .setUseStereoOutput()
             // 超低延迟
 //          .setUseLowLatency()
-            .setSamplesReadyCallback(audioSamples -> {
+//          .setUseStereoInput()
-                if(this.recordClient != null) {
+//          .setUseStereoOutput()
-                    this.recordClient.onWebRtcAudioRecordSamplesReady(audioSamples);
+            // 本地声音
+//          .setSamplesReadyCallback()
+            .setAudioTrackErrorCallback(new JavaAudioDeviceModule.AudioTrackErrorCallback() {
+                @Override
+                public void onWebRtcAudioTrackInitError(String errorMessage) {
+                    Log.e(MediaManager.class.getSimpleName(), "WebRTC远程音频Track加载异常：" + errorMessage);
+                }
+                @Override
+                public void onWebRtcAudioTrackStartError(JavaAudioDeviceModule.AudioTrackStartErrorCode errorCode, String errorMessage) {
+                    Log.e(MediaManager.class.getSimpleName(), "WebRTC远程音频Track开始异常：" + errorMessage);
+                }
+                @Override
+                public void onWebRtcAudioTrackError(String errorMessage) {
+                    Log.e(MediaManager.class.getSimpleName(), "WebRTC远程音频Track异常：" + errorMessage);
                 }
             })
             .setAudioTrackStateCallback(new JavaAudioDeviceModule.AudioTrackStateCallback() {
                 @Override
                 public void onWebRtcAudioTrackStart() {
-                    Log.i(MediaManager.class.getSimpleName(), "WebRTC声音Track开始");
+                    Log.i(MediaManager.class.getSimpleName(), "WebRTC远程音频Track开始");
                 }
                 @Override
                 public void onWebRtcAudioTrackStop() {
-                    Log.i(MediaManager.class.getSimpleName(), "WebRTC声音Track结束");
+                    Log.i(MediaManager.class.getSimpleName(), "WebRTC远程音频Track结束");
+                }
+            })
+            .setAudioRecordErrorCallback(new JavaAudioDeviceModule.AudioRecordErrorCallback() {
+                @Override
+                public void onWebRtcAudioRecordInitError(String errorMessage) {
+                    Log.e(MediaManager.class.getSimpleName(), "WebRTC本地音频录制加载异常：" + errorMessage);
+                }
+                @Override
+                public void onWebRtcAudioRecordStartError(JavaAudioDeviceModule.AudioRecordStartErrorCode errorCode, String errorMessage) {
+                    Log.e(MediaManager.class.getSimpleName(), "WebRTC本地音频录制开始异常：" + errorMessage);
+                }
+                @Override
+                public void onWebRtcAudioRecordError(String errorMessage) {
+                    Log.e(MediaManager.class.getSimpleName(), "WebRTC本地音频录制异常：" + errorMessage);
                 }
             })
             .setAudioRecordStateCallback(new JavaAudioDeviceModule.AudioRecordStateCallback() {
                 @Override
                 public void onWebRtcAudioRecordStart() {
-                    Log.i(MediaManager.class.getSimpleName(), "WebRTC声音录制开始");
+                    Log.i(MediaManager.class.getSimpleName(), "WebRTC本地音频录制开始");
                 }
                 @Override
                 public void onWebRtcAudioRecordStop() {
-                    Log.i(MediaManager.class.getSimpleName(), "WebRTC声音录制结束");
+                    Log.i(MediaManager.class.getSimpleName(), "WebRTC本地音频录制结束");
                 }
             })
 //          .setUseHardwareNoiseSuppressor(true)
@@ -689,7 +747,7 @@ public final class MediaManager {
                 this.videoPath, this.taoyao, this.mainHandler
             );
             this.recordClient.start();
-            this.recordClient.record(this.mainVideoSource, this.peerConnectionFactory);
+            this.recordClient.record(this.mainVideoSource, this.javaAudioDeviceModule, this.peerConnectionFactory);
             this.mainHandler.obtainMessage(Config.WHAT_RECORD, Boolean.TRUE).sendToTarget();
             return this.recordClient;
         }
@@ -758,6 +816,10 @@ public final class MediaManager {
             this.surfaceTextureHelper.dispose();
             this.surfaceTextureHelper = null;
         }
+        if(this.javaAudioDeviceModule != null) {
+            this.javaAudioDeviceModule.release();
+            this.javaAudioDeviceModule = null;
+        }
         if (this.peerConnectionFactory != null) {
             this.peerConnectionFactory.dispose();
             this.peerConnectionFactory = null;
@@ -806,7 +868,11 @@ public final class MediaManager {
             } else {
                 final VideoFrame.I420Buffer i420Buffer = videoFrame.getBuffer().toI420();
                 MediaManager.this.videoProcesser.process(i420Buffer);
-                final VideoFrame processVideoFrame = new VideoFrame(i420Buffer.cropAndScale(0, 0, i420Buffer.getWidth(), i420Buffer.getHeight(), i420Buffer.getWidth(), i420Buffer.getHeight()), videoFrame.getRotation(), videoFrame.getTimestampNs());
+                final VideoFrame processVideoFrame = new VideoFrame(
+                    i420Buffer.cropAndScale(0, 0, i420Buffer.getWidth(), i420Buffer.getHeight(), i420Buffer.getWidth(), i420Buffer.getHeight()),
+                    videoFrame.getRotation(),
+                    videoFrame.getTimestampNs()
+                );
                 i420Buffer.release();
                 this.mainObserver.onFrameCaptured(processVideoFrame);
                 this.shareObserver.onFrameCaptured(processVideoFrame);
@@ -869,6 +935,18 @@ public final class MediaManager {
 
     }
 
+    private class TextToSpeechInitListener implements TextToSpeech.OnInitListener {
+        @Override
+        public void onInit(int status) {
+            Log.i(MediaManager.class.getSimpleName(), "加载TTS：" + status);
+            if(status == TextToSpeech.SUCCESS) {
+                MediaManager.this.textToSpeech.setLanguage(Locale.CANADA);
+                MediaManager.this.textToSpeech.setPitch(1.0F);
+                MediaManager.this.textToSpeech.setSpeechRate(1.0F);
+            }
+        }
+    }
+
     private native void nativeInit();
     private native void nativeStop();
 

taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java

@@ -1,13 +1,109 @@
 package com.acgist.taoyao.media.audio;
 
+import android.util.Log;
+
+import com.acgist.taoyao.media.client.RecordClient;
+
+import org.webrtc.AudioSource;
+import org.webrtc.audio.JavaAudioDeviceModule;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
 /**
  * 混音处理器
  *
+ * JavaAudioDeviceModule              : 音频
  * WebRtcAudioTrack#AudioTrackThread  ：远程音频
  * WebRtcAudioRecord#AudioRecordThread：本地音频
  *
+ * 注意：只能远程终端拉取才能采集音频数据，如果需要离线采集自己使用AudioRecord实现。
+ *
  * @author acgist
  */
-public class MixerProcesser {
+public class MixerProcesser extends Thread implements JavaAudioDeviceModule.SamplesReadyCallback {
+
+    private boolean close;
+    private final RecordClient recordClient;
+    private final BlockingQueue<JavaAudioDeviceModule.AudioSamples> local;
+    private final BlockingQueue<JavaAudioDeviceModule.AudioSamples> remote;
+
+    public MixerProcesser(RecordClient recordClient) {
+        this.setDaemon(true);
+        this.setName("AudioMixer");
+        this.close = false;
+        this.recordClient = recordClient;
+        this.local  = new LinkedBlockingQueue<>(1024);
+        this.remote = new LinkedBlockingQueue<>(1024);
+    }
+
+    @Override
+    public void onWebRtcAudioTrackSamplesReady(JavaAudioDeviceModule.AudioSamples samples) {
+//        Log.d(MixerProcesser.class.getSimpleName(), "远程音频信息：" + samples.getAudioFormat());
+        if(!this.remote.offer(samples)) {
+            Log.e(MixerProcesser.class.getSimpleName(), "远程音频队列阻塞");
+        }
+    }
+
+    @Override
+    public void onWebRtcAudioRecordSamplesReady(JavaAudioDeviceModule.AudioSamples samples) {
+//        Log.d(MixerProcesser.class.getSimpleName(), "本地音频信息：" + samples.getAudioFormat());
+        if(!this.local.offer(samples)) {
+            Log.e(MixerProcesser.class.getSimpleName(), "本地音频队列阻塞");
+        }
+    }
+
+    @Override
+    public void run() {
+        long pts = System.nanoTime();
+//        final byte[] target = new byte[length];
+        // PCM时间计算：1000000 microseconds / 48000 hz / 2 bytes
+        JavaAudioDeviceModule.AudioSamples local;
+        JavaAudioDeviceModule.AudioSamples remote;
+        int localValue;
+        int remoteValue;
+        byte[] localData;
+        byte[] remoteData;
+        byte[] data = null;
+        // TODO：固定长度采样率等等
+        while(!this.close) {
+            try {
+                local  = this.local.poll(100, TimeUnit.MILLISECONDS);
+                remote = this.remote.poll();
+                if(local != null && remote != null) {
+                    localData  = local.getData();
+                    remoteData = remote.getData();
+                    Log.d(MixerProcesser.class.getSimpleName(), String.format("""
+                        混音长度：%d - %d
+                        混音采样：%d - %d
+                        混音格式：%d - %d
+                        """, localData.length, remoteData.length, local.getSampleRate(), remote.getSampleRate(), local.getAudioFormat(), remote.getAudioFormat()));
+                    data = new byte[localData.length];
+                    for (int index = 0; index < localData.length; index++) {
+                        localValue  = localData[index];
+                        remoteValue = remoteData[index];
+                        data[index] = (byte) ((localValue +remoteValue) / 2);
+                    }
+                    pts += data.length * (1_000_000 / local.getSampleRate() / 2);
+                } else if(local != null && remote == null) {
+                    data = local.getData();
+                    pts += data.length * (1_000_000 / local.getSampleRate() / 2);
+                } else if(local == null && remote != null) {
+                    data = remote.getData();
+                    pts += data.length * (1_000_000 / remote.getSampleRate() / 2);
+                } else {
+                    continue;
+                }
+                this.recordClient.onPcm(pts, data);
+            } catch (Exception e) {
+                Log.e(MixerProcesser.class.getSimpleName(), "音频处理异常", e);
+            }
+        }
+    }
+
+    public void close() {
+        this.close = true;
+    }
 
 }

taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java

@@ -65,7 +65,7 @@ public class PhotographClient implements VideoSink {
     public PhotographClient(int quantity, String path) {
         this.quantity = quantity;
         this.filename = DateUtils.format(LocalDateTime.now(), DateUtils.DateTimeStyle.YYYYMMDDHH24MMSS) + ".jpg";
-        this.filepath = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath(), path, this.filename).toString();
+        this.filepath = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_PICTURES).getAbsolutePath(), path, this.filename).toString();
         this.done     = false;
         this.finish   = false;
         Log.i(RecordClient.class.getSimpleName(), "拍摄照片文件：" + this.filepath);
@@ -97,13 +97,6 @@ public class PhotographClient implements VideoSink {
     }
 
     public void photograph(VideoSource videoSource, PeerConnectionFactory peerConnectionFactory) {
-        if(this.videoTrack != null) {
-            return;
-        }
-        if(videoSource == null || peerConnectionFactory == null) {
-            Log.e(PhotographClient.class.getSimpleName(), "数据采集无效");
-            return;
-        }
         this.videoTrack = peerConnectionFactory.createVideoTrack("TaoyaoVP", videoSource);
         this.videoTrack.setEnabled(true);
         this.videoTrack.addSink(this);

taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java

@@ -11,6 +11,7 @@ import android.util.Log;
 
 import com.acgist.taoyao.boot.utils.DateUtils;
 import com.acgist.taoyao.media.MediaManager;
+import com.acgist.taoyao.media.audio.MixerProcesser;
 import com.acgist.taoyao.media.signal.ITaoyao;
 
 import org.webrtc.PeerConnectionFactory;
@@ -32,7 +33,7 @@ import java.time.LocalDateTime;
  *
  * @author acgist
  */
-public class RecordClient extends Client implements VideoSink, JavaAudioDeviceModule.SamplesReadyCallback {
+public class RecordClient extends Client implements VideoSink {
 
     /**
      * 等待时间（毫秒）
@@ -126,6 +127,8 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
      */
     private MediaMuxer mediaMuxer;
     private VideoTrack videoTrack;
+    private MixerProcesser mixerProcesser;
+    private JavaAudioDeviceModule javaAudioDeviceModule;
 
     public RecordClient(
         int audioBitRate, int sampleRate, int channelCount,
@@ -144,7 +147,9 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
         this.height         = height;
         this.yuvSize        = width * height * 3 / 2;
         this.filename       = DateUtils.format(LocalDateTime.now(), DateUtils.DateTimeStyle.YYYYMMDDHH24MMSS) + ".mp4";
-        this.filepath       = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath(), path, this.filename).toString();
+        this.filepath       = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_MOVIES).getAbsolutePath(), path, this.filename).toString();
+        this.audioActive    = false;
+        this.videoActive    = false;
     }
 
     public void start() {
@@ -191,15 +196,12 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
         this.audioHandler.post(this::audioCodec);
     }
 
-    private volatile long audioPts = 0;
-
     private void audioCodec() {
         long pts       = 0L;
         int trackIndex = -1;
         int outputIndex;
         this.audioCodec.start();
         this.audioActive = true;
-        JavaAudioDeviceModule.AudioSamples audioSamples = null;
         final MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
         while (!this.close) {
             outputIndex = this.audioCodec.dequeueOutputBuffer(bufferInfo, WAIT_TIME_US);
@@ -369,17 +371,20 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
         }
     }
 
-    public void record(VideoSource videoSource, PeerConnectionFactory peerConnectionFactory) {
+    public void record(VideoSource videoSource, JavaAudioDeviceModule javaAudioDeviceModule, PeerConnectionFactory peerConnectionFactory) {
-        if(this.videoTrack != null) {
+        // 音频
-            return;
+        if(javaAudioDeviceModule != null) {
+            this.mixerProcesser = new MixerProcesser(this);
+            this.mixerProcesser.start();
+            javaAudioDeviceModule.setMixerProcesser(this.mixerProcesser);
+            this.javaAudioDeviceModule = javaAudioDeviceModule;
         }
-        if(videoSource == null || peerConnectionFactory == null) {
+        // 视频
-            Log.e(RecordClient.class.getSimpleName(), "数据采集无效");
+        if(videoSource != null && peerConnectionFactory != null) {
-            return;
+            this.videoTrack = peerConnectionFactory.createVideoTrack("TaoyaoVR", videoSource);
+            this.videoTrack.setEnabled(true);
+            this.videoTrack.addSink(this);
         }
-        this.videoTrack = peerConnectionFactory.createVideoTrack("TaoyaoVR", videoSource);
-        this.videoTrack.setEnabled(true);
-        this.videoTrack.addSink(this);
     }
 
     @Override
@@ -390,6 +395,13 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
             }
             super.close();
             Log.i(RecordClient.class.getSimpleName(), "结束录制：" + this.filepath);
+            if(this.javaAudioDeviceModule != null) {
+                this.javaAudioDeviceModule.removeMixerProcesser();
+            }
+            if(this.mixerProcesser != null) {
+                this.mixerProcesser.close();
+                this.mixerProcesser = null;
+            }
             if(this.videoTrack != null) {
                 this.videoTrack.removeSink(this);
                 this.videoTrack.dispose();
@@ -422,26 +434,20 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
     }
 
     /**
-     * @param audioSamples PCM数据
+     * @param pts  PTS时间偏移
+     * @param data PCM数据
      */
-    @Override
+    public void onPcm(long pts, byte[] data) {
-    public void onWebRtcAudioRecordSamplesReady(JavaAudioDeviceModule.AudioSamples audioSamples) {
         if(this.close || !this.audioActive) {
             return;
         }
-        Log.i(RecordClient.class.getSimpleName(), "音频信息：" + audioSamples.getAudioFormat());
+        final int index = this.audioCodec.dequeueInputBuffer(WAIT_TIME_US);
-        int index = this.audioCodec.dequeueInputBuffer(WAIT_TIME_US);
+        if (index < 0) {
-        if (index >= 0) {
+            return;
-            final byte[] data = audioSamples.getData();
-            final ByteBuffer buffer = this.audioCodec.getInputBuffer(index);
-            buffer.put(data);
-            this.audioCodec.queueInputBuffer(index, 0, data.length, this.audioPts, 0);
-            // 1000000 microseconds / 48000 hz / 2 bytes
-            this.audioPts += data.length * (1_000_000 / audioSamples.getSampleRate() / 2);
-        } else {
-            // WARN
         }
-        audioSamples = null;
+        final ByteBuffer buffer = this.audioCodec.getInputBuffer(index);
+        buffer.put(data);
+        this.audioCodec.queueInputBuffer(index, 0, data.length, pts, 0);
     }
 
     @Override
@@ -449,7 +455,7 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
         if (this.close || !this.videoActive) {
             return;
         }
-        Log.i(RecordClient.class.getSimpleName(), "视频信息：" + videoFrame.getRotatedWidth() + " - " + videoFrame.getRotatedHeight());
+//      Log.d(RecordClient.class.getSimpleName(), "视频信息：" + videoFrame.getRotatedWidth() + " - " + videoFrame.getRotatedHeight());
         final int index = this.videoCodec.dequeueInputBuffer(WAIT_TIME_US);
         if(index < 0) {
             return;

taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/JavaAudioDeviceModule.java

@@ -316,6 +316,17 @@ public class JavaAudioDeviceModule implements AudioDeviceModule {
 
   /** Called when new audio samples are ready. This should only be set for debug purposes */
   public static interface SamplesReadyCallback {
+    /**
+     * 远程音频
+     *
+     * @param samples 音频采样
+     */
+    void onWebRtcAudioTrackSamplesReady(AudioSamples samples);
+    /**
+     * 本地音频
+     *
+     * @param samples 音频采样
+     */
     void onWebRtcAudioRecordSamplesReady(AudioSamples samples);
   }
 
@@ -379,6 +390,26 @@ public class JavaAudioDeviceModule implements AudioDeviceModule {
     this.useStereoOutput = useStereoOutput;
   }
 
+  /**
+   * 设置录音工具
+   *
+   * @param samplesReadyCallback 录音回调
+   *
+   * @Taoyao
+   */
+  public void setMixerProcesser(SamplesReadyCallback samplesReadyCallback) {
+    this.audioInput.setMixerProcesser(samplesReadyCallback);
+    this.audioOutput.setMixerProcesser(samplesReadyCallback);
+  }
+
+  /**
+   * 删除录音工具
+   */
+  public void removeMixerProcesser() {
+    this.audioInput.setMixerProcesser(null);
+    this.audioOutput.setMixerProcesser(null);
+  }
+
   @Override
   public long getNativeAudioDeviceModulePointer() {
     synchronized (nativeLock) {
@@ -427,4 +458,5 @@ public class JavaAudioDeviceModule implements AudioDeviceModule {
   private static native long nativeCreateAudioDeviceModule(Context context,
       AudioManager audioManager, WebRtcAudioRecord audioInput, WebRtcAudioTrack audioOutput,
       int inputSampleRate, int outputSampleRate, boolean useStereoInput, boolean useStereoOutput);
+
 }

taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioRecord.java

@@ -105,10 +105,21 @@ class WebRtcAudioRecord {
 
   private final @Nullable AudioRecordErrorCallback errorCallback;
   private final @Nullable AudioRecordStateCallback stateCallback;
-  private final @Nullable SamplesReadyCallback audioSamplesReadyCallback;
+  private @Nullable SamplesReadyCallback audioSamplesReadyCallback;
   private final boolean isAcousticEchoCancelerSupported;
   private final boolean isNoiseSuppressorSupported;
 
+  /**
+   * 设置录音工具
+   *
+   * @param samplesReadyCallback 录音回调
+   *
+   * @Taoyao
+   */
+  public void setMixerProcesser(SamplesReadyCallback samplesReadyCallback) {
+    this.audioSamplesReadyCallback = samplesReadyCallback;
+  }
+
   /**
    * Audio thread which keeps calling ByteBuffer.read() waiting for audio
    * to be recorded. Feeds recorded data to the native counterpart as a
@@ -131,7 +142,6 @@ class WebRtcAudioRecord {
       // Audio recording has started and the client is informed about it.
       doAudioRecordStateCallback(AUDIO_RECORD_START);
 
-      long lastTime = System.nanoTime();
       while (keepAlive) {
         int bytesRead = audioRecord.read(byteBuffer, byteBuffer.capacity());
         if (bytesRead == byteBuffer.capacity()) {
@@ -148,11 +158,11 @@ class WebRtcAudioRecord {
           if (audioSamplesReadyCallback != null) {
             // Copy the entire byte buffer array. The start of the byteBuffer is not necessarily
             // at index 0.
-            byte[] data = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.arrayOffset(),
+            SamplesReadyCallback nullable = audioSamplesReadyCallback;
-                byteBuffer.capacity() + byteBuffer.arrayOffset());
+            if(nullable != null) {
-            audioSamplesReadyCallback.onWebRtcAudioRecordSamplesReady(
+              final byte[] data = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.arrayOffset(), byteBuffer.capacity() + byteBuffer.arrayOffset());
-                new JavaAudioDeviceModule.AudioSamples(audioRecord.getAudioFormat(),
+              nullable.onWebRtcAudioRecordSamplesReady(new JavaAudioDeviceModule.AudioSamples(audioRecord.getAudioFormat(), audioRecord.getChannelCount(), audioRecord.getSampleRate(), data));
-                    audioRecord.getChannelCount(), audioRecord.getSampleRate(), data));
+            }
           }
         } else {
           String errorMessage = "AudioRecord.read failed: " + bytesRead;

taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioTrack.java

@@ -27,8 +27,10 @@ import org.webrtc.ThreadUtils;
 import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackErrorCallback;
 import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackStartErrorCode;
 import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackStateCallback;
+import org.webrtc.audio.JavaAudioDeviceModule.SamplesReadyCallback;
 
 import java.nio.ByteBuffer;
+import java.util.Arrays;
 
 class WebRtcAudioTrack {
   private static final String TAG = "WebRtcAudioTrackExternal";
@@ -87,6 +89,18 @@ class WebRtcAudioTrack {
 
   private final @Nullable AudioTrackErrorCallback errorCallback;
   private final @Nullable AudioTrackStateCallback stateCallback;
+  private @Nullable SamplesReadyCallback audioSamplesReadyCallback;
+
+  /**
+   * 设置录音工具
+   *
+   * @param samplesReadyCallback 录音回调
+   *
+   * @Taoyao
+   */
+  public void setMixerProcesser(JavaAudioDeviceModule.SamplesReadyCallback samplesReadyCallback) {
+    this.audioSamplesReadyCallback = samplesReadyCallback;
+  }
 
   /**
    * Audio thread which keeps calling AudioTrack.write() to stream audio.
@@ -131,6 +145,13 @@ class WebRtcAudioTrack {
           byteBuffer.position(0);
         }
         int bytesWritten = writeBytes(audioTrack, byteBuffer, sizeInBytes);
+        if (audioSamplesReadyCallback != null) {
+          SamplesReadyCallback nullable = audioSamplesReadyCallback;
+          if(nullable != null) {
+            final byte[] data = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.arrayOffset(), byteBuffer.capacity() + byteBuffer.arrayOffset());
+            nullable.onWebRtcAudioTrackSamplesReady(new JavaAudioDeviceModule.AudioSamples(audioTrack.getAudioFormat(), audioTrack.getChannelCount(), audioTrack.getSampleRate(), data));
+          }
+        }
         if (bytesWritten != sizeInBytes) {
           Logging.e(TAG, "AudioTrack.write played invalid number of bytes: " + bytesWritten);
           // If a write() returns a negative value, an error has occurred.

taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/AudioMixerTest.java

@@ -0,0 +1,31 @@
+package com.acgist.taoyao;
+
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+
+import org.junit.jupiter.api.Test;
+
+public class AudioMixerTest {
+
+    @Test
+    public void testMixer() throws IOException {
+//      ffmpeg -i audio.mp3 -f s32le audio.pcm
+//      ffplay -i audio.pcm -f s32le -ar 48000 -ac 1
+        final File fileA = new File("D:\\tmp\\mixer\\1.pcm");
+        final File fileB = new File("D:\\tmp\\mixer\\2.pcm");
+        final byte[] bytesA = Files.readAllBytes(fileA.toPath());
+        final byte[] bytesB = Files.readAllBytes(fileB.toPath());
+        final int length = Math.min(bytesA.length, bytesB.length);
+        final byte[] target = new byte[length];
+        int a, b;
+        for (int i = 0; i < length; i++) {
+            a = bytesA[i];
+            b = bytesB[i];
+            target[i] = (byte) ((a + b) / 2);
+        }
+        Files.write(Paths.get("D:\\tmp\\mixer\\3.pcm"), target);
+    }
+    
+}