[+] 混音

2023-05-06 23:57:25 +08:00
parent f59840e2c6
commit 739538b34c
20 changed files with 423 additions and 162 deletions
--- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java
+++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java
@@ -2,6 +2,7 @@ package com.acgist.taoyao.client;

 import android.Manifest;
 import android.app.Activity;
+import android.content.Context;
 import android.content.Intent;
 import android.content.pm.PackageManager;
 import android.content.res.ColorStateList;
@@ -134,9 +135,11 @@ public class MainActivity extends AppCompatActivity {
            return;
        }
        this.mainHandler = new MainHandler();
+        final Context context = this.getApplicationContext();
        final Resources resources = this.getResources();
-        MediaManager.getInstance().initContext(
-            this.mainHandler, this.getApplicationContext(),
+        final MediaManager mediaManager = MediaManager.getInstance();
+        mediaManager.initContext(
+            this.mainHandler, context,
            resources.getInteger(R.integer.imageQuantity),
            resources.getString(R.string.audioQuantity),
            resources.getString(R.string.videoQuantity),
@@ -147,6 +150,9 @@ public class MainActivity extends AppCompatActivity {
            resources.getString(R.string.watermark),
            VideoSourceType.valueOf(resources.getString(R.string.videoSourceType))
        );
+        if(resources.getBoolean(R.bool.broadcaster)) {
+            mediaManager.initTTS(context);
+        }
        // 注意：不能使用intent传递
        MediaService.mainHandler = this.mainHandler;
        Log.i(MainActivity.class.getSimpleName(), "拉起媒体服务");
--- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java
+++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java
@@ -76,8 +76,9 @@ public class MediaService extends Service {
        :: https://gitee.com/acgist/taoyao
        """);
        super.onCreate();
-        this.mkdir(R.string.storagePathImage);
-        this.mkdir(R.string.storagePathVideo);
+        final Resources resources = this.getResources();
+        this.mkdir(resources.getString(R.string.storagePathImage), Environment.DIRECTORY_PICTURES);
+        this.mkdir(resources.getString(R.string.storagePathVideo), Environment.DIRECTORY_MOVIES);
        this.buildNotificationChannel();
    }

@@ -192,13 +193,16 @@ public class MediaService extends Service {
        MediaManager.getInstance().initScreen(intent.getParcelableExtra("data"));
    }

-    private void mkdir(int id) {
+    private void mkdir(String path, String type) {
        final Path imagePath = Paths.get(
-            Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath(),
-            this.getResources().getString(id)
+            Environment.getExternalStoragePublicDirectory(type).getAbsolutePath(),
+            path
        );
        final File file = imagePath.toFile();
-        if(!file.exists()) {
+        if(file.exists()) {
+            Log.d(MediaService.class.getSimpleName(), "目录已经存在：" + imagePath);
+        } else {
+            Log.d(MediaService.class.getSimpleName(), "新建文件目录：" + imagePath);
            file.mkdirs();
        }
    }
--- a/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml
+++ b/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml
@@ -17,9 +17,9 @@
    <!-- 信令加密密钥 -->
    <string name="encryptSecret">2SPWy+TF1zM=</string>
    <!-- 图片存储目录 -->
-    <string name="storagePathImage">/taoyao/image</string>
+    <string name="storagePathImage">/taoyao</string>
    <!-- 视频存储目录 -->
-    <string name="storagePathVideo">/taoyao/video</string>
+    <string name="storagePathVideo">/taoyao</string>
    <!-- 视频来源：FILE|BACK|FRONT|SCREEN -->
    <string name="videoSourceType">BACK</string>
    <!-- 媒体配置：是否消费数据 -->
@@ -34,6 +34,8 @@
    <bool name="audioProduce">true</bool>
    <!-- 媒体配置：是否生产视频 -->
    <bool name="videoProduce">true</bool>
+    <!-- 语音播报 -->
+    <bool name="broadcaster">false</bool>
    <!-- 图片质量 -->
    <integer name="imageQuantity">100</integer>
    <!-- 音频质量 -->
--- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/Broadcaster.java
+++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/Broadcaster.java
@@ -1,53 +0,0 @@
-package com.acgist.taoyao.media;
-
-import android.content.Context;
-import android.speech.tts.TextToSpeech;
-import android.util.Log;
-
-import java.util.Locale;
-import java.util.UUID;
-
-/**
- * 声音播报
- *
- * @author acgist
- */
-public final class Broadcaster {
-
-    private static final Broadcaster INSTANCE = new Broadcaster();
-
-    public static final Broadcaster getInstance() {
-        return INSTANCE;
-    }
-
-    private TextToSpeech textToSpeech;
-
-    private Broadcaster() {
-    }
-
-    public void init(Context context) {
-        this.textToSpeech = new TextToSpeech(context, new TextToSpeechInitListener());
-    }
-
-    public void broadcast(String text) {
-//      this.textToSpeech.stop();
-        this.textToSpeech.speak(text, TextToSpeech.QUEUE_FLUSH, null, UUID.randomUUID().toString());
-    }
-
-    public void shutdown() {
-        this.textToSpeech.shutdown();
-    }
-
-    private class TextToSpeechInitListener implements TextToSpeech.OnInitListener {
-        @Override
-        public void onInit(int status) {
-            Log.i(Broadcaster.class.getSimpleName(), "加载TTS：" + status);
-            if(status == TextToSpeech.SUCCESS) {
-                Broadcaster.this.textToSpeech.setLanguage(Locale.CANADA);
-                Broadcaster.this.textToSpeech.setPitch(1.0F);
-                Broadcaster.this.textToSpeech.setSpeechRate(1.0F);
-            }
-        }
-    }
-
-}
--- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java
+++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java
@@ -4,6 +4,7 @@ import android.content.Context;
 import android.content.Intent;
 import android.media.projection.MediaProjection;
 import android.os.Handler;
+import android.speech.tts.TextToSpeech;
 import android.util.Log;

 import com.acgist.taoyao.media.client.PhotographClient;
@@ -42,6 +43,8 @@ import org.webrtc.VideoTrack;
 import org.webrtc.audio.JavaAudioDeviceModule;

 import java.util.Arrays;
+import java.util.Locale;
+import java.util.UUID;

 /**
 * 媒体来源管理器
@@ -154,10 +157,18 @@ public final class MediaManager {
     * PeerConnectionFactory
     */
    private PeerConnectionFactory peerConnectionFactory;
+    /**
+     * JavaAudioDeviceModule
+     */
+    private JavaAudioDeviceModule javaAudioDeviceModule;
    /**
     * 视频处理
     */
    private VideoProcesser videoProcesser;
+    /**
+     * TTS
+     */
+    private TextToSpeech textToSpeech;
    /**
     * 录屏等待锁
     */
@@ -263,6 +274,28 @@ public final class MediaManager {
        this.taoyao = taoyao;
    }

+    public void initTTS(Context context) {
+        if(this.textToSpeech != null) {
+            return;
+        }
+        this.textToSpeech = new TextToSpeech(context, new MediaManager.TextToSpeechInitListener());
+    }
+
+    public void broadcast(String text) {
+        if(this.textToSpeech == null) {
+            return;
+        }
+//      this.textToSpeech.stop();
+        this.textToSpeech.speak(text, TextToSpeech.QUEUE_FLUSH, null, UUID.randomUUID().toString());
+    }
+
+    public void closeTTS() {
+        if(this.textToSpeech == null) {
+            return;
+        }
+        this.textToSpeech.shutdown();
+    }
+
    /**
     * 新建终端
     *
@@ -339,11 +372,11 @@ public final class MediaManager {
        Log.i(MediaManager.class.getSimpleName(), "加载媒体：" + this.videoSourceType);
        final VideoDecoderFactory videoDecoderFactory = new DefaultVideoDecoderFactory(this.eglContext);
        final VideoEncoderFactory videoEncoderFactory = new DefaultVideoEncoderFactory(this.eglContext, true, true);
-        final JavaAudioDeviceModule javaAudioDeviceModule = this.javaAudioDeviceModule();
+        this.javaAudioDeviceModule = this.javaAudioDeviceModule();
        this.peerConnectionFactory = PeerConnectionFactory.builder()
            .setVideoDecoderFactory(videoDecoderFactory)
            .setVideoEncoderFactory(videoEncoderFactory)
-            .setAudioDeviceModule(javaAudioDeviceModule)
+            .setAudioDeviceModule(this.javaAudioDeviceModule)
 //          .setAudioProcessingFactory()
 //          .setAudioEncoderFactoryFactory(new BuiltinAudioEncoderFactoryFactory())
 //          .setAudioDecoderFactoryFactory(new BuiltinAudioDecoderFactoryFactory())
@@ -372,33 +405,58 @@ public final class MediaManager {
 //          .setAudioSource(MediaRecorder.AudioSource.MIC)
 //          .setAudioFormat(AudioFormat.ENCODING_PCM_16BIT)
 //          .setAudioAttributes(audioAttributes)
-//          .setUseStereoInput()
-//          .setUseStereoOutput()
            // 超低延迟
 //          .setUseLowLatency()
-            .setSamplesReadyCallback(audioSamples -> {
-                if(this.recordClient != null) {
-                    this.recordClient.onWebRtcAudioRecordSamplesReady(audioSamples);
+//          .setUseStereoInput()
+//          .setUseStereoOutput()
+            // 本地声音
+//          .setSamplesReadyCallback()
+            .setAudioTrackErrorCallback(new JavaAudioDeviceModule.AudioTrackErrorCallback() {
+                @Override
+                public void onWebRtcAudioTrackInitError(String errorMessage) {
+                    Log.e(MediaManager.class.getSimpleName(), "WebRTC远程音频Track加载异常：" + errorMessage);
+                }
+                @Override
+                public void onWebRtcAudioTrackStartError(JavaAudioDeviceModule.AudioTrackStartErrorCode errorCode, String errorMessage) {
+                    Log.e(MediaManager.class.getSimpleName(), "WebRTC远程音频Track开始异常：" + errorMessage);
+                }
+                @Override
+                public void onWebRtcAudioTrackError(String errorMessage) {
+                    Log.e(MediaManager.class.getSimpleName(), "WebRTC远程音频Track异常：" + errorMessage);
                }
            })
            .setAudioTrackStateCallback(new JavaAudioDeviceModule.AudioTrackStateCallback() {
                @Override
                public void onWebRtcAudioTrackStart() {
-                    Log.i(MediaManager.class.getSimpleName(), "WebRTC声音Track开始");
+                    Log.i(MediaManager.class.getSimpleName(), "WebRTC远程音频Track开始");
                }
                @Override
                public void onWebRtcAudioTrackStop() {
-                    Log.i(MediaManager.class.getSimpleName(), "WebRTC声音Track结束");
+                    Log.i(MediaManager.class.getSimpleName(), "WebRTC远程音频Track结束");
+                }
+            })
+            .setAudioRecordErrorCallback(new JavaAudioDeviceModule.AudioRecordErrorCallback() {
+                @Override
+                public void onWebRtcAudioRecordInitError(String errorMessage) {
+                    Log.e(MediaManager.class.getSimpleName(), "WebRTC本地音频录制加载异常：" + errorMessage);
+                }
+                @Override
+                public void onWebRtcAudioRecordStartError(JavaAudioDeviceModule.AudioRecordStartErrorCode errorCode, String errorMessage) {
+                    Log.e(MediaManager.class.getSimpleName(), "WebRTC本地音频录制开始异常：" + errorMessage);
+                }
+                @Override
+                public void onWebRtcAudioRecordError(String errorMessage) {
+                    Log.e(MediaManager.class.getSimpleName(), "WebRTC本地音频录制异常：" + errorMessage);
                }
            })
            .setAudioRecordStateCallback(new JavaAudioDeviceModule.AudioRecordStateCallback() {
                @Override
                public void onWebRtcAudioRecordStart() {
-                    Log.i(MediaManager.class.getSimpleName(), "WebRTC声音录制开始");
+                    Log.i(MediaManager.class.getSimpleName(), "WebRTC本地音频录制开始");
                }
                @Override
                public void onWebRtcAudioRecordStop() {
-                    Log.i(MediaManager.class.getSimpleName(), "WebRTC声音录制结束");
+                    Log.i(MediaManager.class.getSimpleName(), "WebRTC本地音频录制结束");
                }
            })
 //          .setUseHardwareNoiseSuppressor(true)
@@ -689,7 +747,7 @@ public final class MediaManager {
                this.videoPath, this.taoyao, this.mainHandler
            );
            this.recordClient.start();
-            this.recordClient.record(this.mainVideoSource, this.peerConnectionFactory);
+            this.recordClient.record(this.mainVideoSource, this.javaAudioDeviceModule, this.peerConnectionFactory);
            this.mainHandler.obtainMessage(Config.WHAT_RECORD, Boolean.TRUE).sendToTarget();
            return this.recordClient;
        }
@@ -758,6 +816,10 @@ public final class MediaManager {
            this.surfaceTextureHelper.dispose();
            this.surfaceTextureHelper = null;
        }
+        if(this.javaAudioDeviceModule != null) {
+            this.javaAudioDeviceModule.release();
+            this.javaAudioDeviceModule = null;
+        }
        if (this.peerConnectionFactory != null) {
            this.peerConnectionFactory.dispose();
            this.peerConnectionFactory = null;
@@ -806,7 +868,11 @@ public final class MediaManager {
            } else {
                final VideoFrame.I420Buffer i420Buffer = videoFrame.getBuffer().toI420();
                MediaManager.this.videoProcesser.process(i420Buffer);
-                final VideoFrame processVideoFrame = new VideoFrame(i420Buffer.cropAndScale(0, 0, i420Buffer.getWidth(), i420Buffer.getHeight(), i420Buffer.getWidth(), i420Buffer.getHeight()), videoFrame.getRotation(), videoFrame.getTimestampNs());
+                final VideoFrame processVideoFrame = new VideoFrame(
+                    i420Buffer.cropAndScale(0, 0, i420Buffer.getWidth(), i420Buffer.getHeight(), i420Buffer.getWidth(), i420Buffer.getHeight()),
+                    videoFrame.getRotation(),
+                    videoFrame.getTimestampNs()
+                );
                i420Buffer.release();
                this.mainObserver.onFrameCaptured(processVideoFrame);
                this.shareObserver.onFrameCaptured(processVideoFrame);
@@ -869,6 +935,18 @@ public final class MediaManager {

    }

+    private class TextToSpeechInitListener implements TextToSpeech.OnInitListener {
+        @Override
+        public void onInit(int status) {
+            Log.i(MediaManager.class.getSimpleName(), "加载TTS：" + status);
+            if(status == TextToSpeech.SUCCESS) {
+                MediaManager.this.textToSpeech.setLanguage(Locale.CANADA);
+                MediaManager.this.textToSpeech.setPitch(1.0F);
+                MediaManager.this.textToSpeech.setSpeechRate(1.0F);
+            }
+        }
+    }
+
    private native void nativeInit();
    private native void nativeStop();

--- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java
+++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java
@@ -1,13 +1,109 @@
 package com.acgist.taoyao.media.audio;

+import android.util.Log;
+
+import com.acgist.taoyao.media.client.RecordClient;
+
+import org.webrtc.AudioSource;
+import org.webrtc.audio.JavaAudioDeviceModule;
+
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
 /**
 * 混音处理器
 *
+ * JavaAudioDeviceModule              : 音频
 * WebRtcAudioTrack#AudioTrackThread  ：远程音频
 * WebRtcAudioRecord#AudioRecordThread：本地音频
 *
+ * 注意：只能远程终端拉取才能采集音频数据，如果需要离线采集自己使用AudioRecord实现。
+ *
 * @author acgist
 */
-public class MixerProcesser {
+public class MixerProcesser extends Thread implements JavaAudioDeviceModule.SamplesReadyCallback {
+
+    private boolean close;
+    private final RecordClient recordClient;
+    private final BlockingQueue<JavaAudioDeviceModule.AudioSamples> local;
+    private final BlockingQueue<JavaAudioDeviceModule.AudioSamples> remote;
+
+    public MixerProcesser(RecordClient recordClient) {
+        this.setDaemon(true);
+        this.setName("AudioMixer");
+        this.close = false;
+        this.recordClient = recordClient;
+        this.local  = new LinkedBlockingQueue<>(1024);
+        this.remote = new LinkedBlockingQueue<>(1024);
+    }
+
+    @Override
+    public void onWebRtcAudioTrackSamplesReady(JavaAudioDeviceModule.AudioSamples samples) {
+//        Log.d(MixerProcesser.class.getSimpleName(), "远程音频信息：" + samples.getAudioFormat());
+        if(!this.remote.offer(samples)) {
+            Log.e(MixerProcesser.class.getSimpleName(), "远程音频队列阻塞");
+        }
+    }
+
+    @Override
+    public void onWebRtcAudioRecordSamplesReady(JavaAudioDeviceModule.AudioSamples samples) {
+//        Log.d(MixerProcesser.class.getSimpleName(), "本地音频信息：" + samples.getAudioFormat());
+        if(!this.local.offer(samples)) {
+            Log.e(MixerProcesser.class.getSimpleName(), "本地音频队列阻塞");
+        }
+    }
+
+    @Override
+    public void run() {
+        long pts = System.nanoTime();
+//        final byte[] target = new byte[length];
+        // PCM时间计算：1000000 microseconds / 48000 hz / 2 bytes
+        JavaAudioDeviceModule.AudioSamples local;
+        JavaAudioDeviceModule.AudioSamples remote;
+        int localValue;
+        int remoteValue;
+        byte[] localData;
+        byte[] remoteData;
+        byte[] data = null;
+        // TODO：固定长度采样率等等
+        while(!this.close) {
+            try {
+                local  = this.local.poll(100, TimeUnit.MILLISECONDS);
+                remote = this.remote.poll();
+                if(local != null && remote != null) {
+                    localData  = local.getData();
+                    remoteData = remote.getData();
+                    Log.d(MixerProcesser.class.getSimpleName(), String.format("""
+                        混音长度：%d - %d
+                        混音采样：%d - %d
+                        混音格式：%d - %d
+                        """, localData.length, remoteData.length, local.getSampleRate(), remote.getSampleRate(), local.getAudioFormat(), remote.getAudioFormat()));
+                    data = new byte[localData.length];
+                    for (int index = 0; index < localData.length; index++) {
+                        localValue  = localData[index];
+                        remoteValue = remoteData[index];
+                        data[index] = (byte) ((localValue +remoteValue) / 2);
+                    }
+                    pts += data.length * (1_000_000 / local.getSampleRate() / 2);
+                } else if(local != null && remote == null) {
+                    data = local.getData();
+                    pts += data.length * (1_000_000 / local.getSampleRate() / 2);
+                } else if(local == null && remote != null) {
+                    data = remote.getData();
+                    pts += data.length * (1_000_000 / remote.getSampleRate() / 2);
+                } else {
+                    continue;
+                }
+                this.recordClient.onPcm(pts, data);
+            } catch (Exception e) {
+                Log.e(MixerProcesser.class.getSimpleName(), "音频处理异常", e);
+            }
+        }
+    }
+
+    public void close() {
+        this.close = true;
+    }

 }
--- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java
+++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java
@@ -65,7 +65,7 @@ public class PhotographClient implements VideoSink {
    public PhotographClient(int quantity, String path) {
        this.quantity = quantity;
        this.filename = DateUtils.format(LocalDateTime.now(), DateUtils.DateTimeStyle.YYYYMMDDHH24MMSS) + ".jpg";
-        this.filepath = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath(), path, this.filename).toString();
+        this.filepath = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_PICTURES).getAbsolutePath(), path, this.filename).toString();
        this.done     = false;
        this.finish   = false;
        Log.i(RecordClient.class.getSimpleName(), "拍摄照片文件：" + this.filepath);
@@ -97,13 +97,6 @@ public class PhotographClient implements VideoSink {
    }

    public void photograph(VideoSource videoSource, PeerConnectionFactory peerConnectionFactory) {
-        if(this.videoTrack != null) {
-            return;
-        }
-        if(videoSource == null || peerConnectionFactory == null) {
-            Log.e(PhotographClient.class.getSimpleName(), "数据采集无效");
-            return;
-        }
        this.videoTrack = peerConnectionFactory.createVideoTrack("TaoyaoVP", videoSource);
        this.videoTrack.setEnabled(true);
        this.videoTrack.addSink(this);
--- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java
+++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java
@@ -11,6 +11,7 @@ import android.util.Log;

 import com.acgist.taoyao.boot.utils.DateUtils;
 import com.acgist.taoyao.media.MediaManager;
+import com.acgist.taoyao.media.audio.MixerProcesser;
 import com.acgist.taoyao.media.signal.ITaoyao;

 import org.webrtc.PeerConnectionFactory;
@@ -32,7 +33,7 @@ import java.time.LocalDateTime;
 *
 * @author acgist
 */
-public class RecordClient extends Client implements VideoSink, JavaAudioDeviceModule.SamplesReadyCallback {
+public class RecordClient extends Client implements VideoSink {

    /**
     * 等待时间（毫秒）
@@ -126,6 +127,8 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
     */
    private MediaMuxer mediaMuxer;
    private VideoTrack videoTrack;
+    private MixerProcesser mixerProcesser;
+    private JavaAudioDeviceModule javaAudioDeviceModule;

    public RecordClient(
        int audioBitRate, int sampleRate, int channelCount,
@@ -144,7 +147,9 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
        this.height         = height;
        this.yuvSize        = width * height * 3 / 2;
        this.filename       = DateUtils.format(LocalDateTime.now(), DateUtils.DateTimeStyle.YYYYMMDDHH24MMSS) + ".mp4";
-        this.filepath       = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath(), path, this.filename).toString();
+        this.filepath       = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_MOVIES).getAbsolutePath(), path, this.filename).toString();
+        this.audioActive    = false;
+        this.videoActive    = false;
    }

    public void start() {
@@ -191,15 +196,12 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
        this.audioHandler.post(this::audioCodec);
    }

-    private volatile long audioPts = 0;
-
    private void audioCodec() {
        long pts       = 0L;
        int trackIndex = -1;
        int outputIndex;
        this.audioCodec.start();
        this.audioActive = true;
-        JavaAudioDeviceModule.AudioSamples audioSamples = null;
        final MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
        while (!this.close) {
            outputIndex = this.audioCodec.dequeueOutputBuffer(bufferInfo, WAIT_TIME_US);
@@ -369,17 +371,20 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
        }
    }

-    public void record(VideoSource videoSource, PeerConnectionFactory peerConnectionFactory) {
-        if(this.videoTrack != null) {
-            return;
+    public void record(VideoSource videoSource, JavaAudioDeviceModule javaAudioDeviceModule, PeerConnectionFactory peerConnectionFactory) {
+        // 音频
+        if(javaAudioDeviceModule != null) {
+            this.mixerProcesser = new MixerProcesser(this);
+            this.mixerProcesser.start();
+            javaAudioDeviceModule.setMixerProcesser(this.mixerProcesser);
+            this.javaAudioDeviceModule = javaAudioDeviceModule;
        }
-        if(videoSource == null || peerConnectionFactory == null) {
-            Log.e(RecordClient.class.getSimpleName(), "数据采集无效");
-            return;
+        // 视频
+        if(videoSource != null && peerConnectionFactory != null) {
+            this.videoTrack = peerConnectionFactory.createVideoTrack("TaoyaoVR", videoSource);
+            this.videoTrack.setEnabled(true);
+            this.videoTrack.addSink(this);
        }
-        this.videoTrack = peerConnectionFactory.createVideoTrack("TaoyaoVR", videoSource);
-        this.videoTrack.setEnabled(true);
-        this.videoTrack.addSink(this);
    }

    @Override
@@ -390,6 +395,13 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
            }
            super.close();
            Log.i(RecordClient.class.getSimpleName(), "结束录制：" + this.filepath);
+            if(this.javaAudioDeviceModule != null) {
+                this.javaAudioDeviceModule.removeMixerProcesser();
+            }
+            if(this.mixerProcesser != null) {
+                this.mixerProcesser.close();
+                this.mixerProcesser = null;
+            }
            if(this.videoTrack != null) {
                this.videoTrack.removeSink(this);
                this.videoTrack.dispose();
@@ -422,26 +434,20 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
    }

    /**
-     * @param audioSamples PCM数据
+     * @param pts  PTS时间偏移
+     * @param data PCM数据
     */
-    @Override
-    public void onWebRtcAudioRecordSamplesReady(JavaAudioDeviceModule.AudioSamples audioSamples) {
+    public void onPcm(long pts, byte[] data) {
        if(this.close || !this.audioActive) {
            return;
        }
-        Log.i(RecordClient.class.getSimpleName(), "音频信息：" + audioSamples.getAudioFormat());
-        int index = this.audioCodec.dequeueInputBuffer(WAIT_TIME_US);
-        if (index >= 0) {
-            final byte[] data = audioSamples.getData();
-            final ByteBuffer buffer = this.audioCodec.getInputBuffer(index);
-            buffer.put(data);
-            this.audioCodec.queueInputBuffer(index, 0, data.length, this.audioPts, 0);
-            // 1000000 microseconds / 48000 hz / 2 bytes
-            this.audioPts += data.length * (1_000_000 / audioSamples.getSampleRate() / 2);
-        } else {
-            // WARN
+        final int index = this.audioCodec.dequeueInputBuffer(WAIT_TIME_US);
+        if (index < 0) {
+            return;
        }
-        audioSamples = null;
+        final ByteBuffer buffer = this.audioCodec.getInputBuffer(index);
+        buffer.put(data);
+        this.audioCodec.queueInputBuffer(index, 0, data.length, pts, 0);
    }

    @Override
@@ -449,7 +455,7 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
        if (this.close || !this.videoActive) {
            return;
        }
-        Log.i(RecordClient.class.getSimpleName(), "视频信息：" + videoFrame.getRotatedWidth() + " - " + videoFrame.getRotatedHeight());
+//      Log.d(RecordClient.class.getSimpleName(), "视频信息：" + videoFrame.getRotatedWidth() + " - " + videoFrame.getRotatedHeight());
        final int index = this.videoCodec.dequeueInputBuffer(WAIT_TIME_US);
        if(index < 0) {
            return;
--- a/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/JavaAudioDeviceModule.java
+++ b/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/JavaAudioDeviceModule.java
@@ -316,6 +316,17 @@ public class JavaAudioDeviceModule implements AudioDeviceModule {

  /** Called when new audio samples are ready. This should only be set for debug purposes */
  public static interface SamplesReadyCallback {
+    /**
+     * 远程音频
+     *
+     * @param samples 音频采样
+     */
+    void onWebRtcAudioTrackSamplesReady(AudioSamples samples);
+    /**
+     * 本地音频
+     *
+     * @param samples 音频采样
+     */
    void onWebRtcAudioRecordSamplesReady(AudioSamples samples);
  }

@@ -379,6 +390,26 @@ public class JavaAudioDeviceModule implements AudioDeviceModule {
    this.useStereoOutput = useStereoOutput;
  }

+  /**
+   * 设置录音工具
+   *
+   * @param samplesReadyCallback 录音回调
+   *
+   * @Taoyao
+   */
+  public void setMixerProcesser(SamplesReadyCallback samplesReadyCallback) {
+    this.audioInput.setMixerProcesser(samplesReadyCallback);
+    this.audioOutput.setMixerProcesser(samplesReadyCallback);
+  }
+
+  /**
+   * 删除录音工具
+   */
+  public void removeMixerProcesser() {
+    this.audioInput.setMixerProcesser(null);
+    this.audioOutput.setMixerProcesser(null);
+  }
+
  @Override
  public long getNativeAudioDeviceModulePointer() {
    synchronized (nativeLock) {
@@ -427,4 +458,5 @@ public class JavaAudioDeviceModule implements AudioDeviceModule {
  private static native long nativeCreateAudioDeviceModule(Context context,
      AudioManager audioManager, WebRtcAudioRecord audioInput, WebRtcAudioTrack audioOutput,
      int inputSampleRate, int outputSampleRate, boolean useStereoInput, boolean useStereoOutput);
+
 }
--- a/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioRecord.java
+++ b/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioRecord.java
@@ -105,10 +105,21 @@ class WebRtcAudioRecord {

  private final @Nullable AudioRecordErrorCallback errorCallback;
  private final @Nullable AudioRecordStateCallback stateCallback;
-  private final @Nullable SamplesReadyCallback audioSamplesReadyCallback;
+  private @Nullable SamplesReadyCallback audioSamplesReadyCallback;
  private final boolean isAcousticEchoCancelerSupported;
  private final boolean isNoiseSuppressorSupported;

+  /**
+   * 设置录音工具
+   *
+   * @param samplesReadyCallback 录音回调
+   *
+   * @Taoyao
+   */
+  public void setMixerProcesser(SamplesReadyCallback samplesReadyCallback) {
+    this.audioSamplesReadyCallback = samplesReadyCallback;
+  }
+
  /**
   * Audio thread which keeps calling ByteBuffer.read() waiting for audio
   * to be recorded. Feeds recorded data to the native counterpart as a
@@ -131,7 +142,6 @@ class WebRtcAudioRecord {
      // Audio recording has started and the client is informed about it.
      doAudioRecordStateCallback(AUDIO_RECORD_START);

-      long lastTime = System.nanoTime();
      while (keepAlive) {
        int bytesRead = audioRecord.read(byteBuffer, byteBuffer.capacity());
        if (bytesRead == byteBuffer.capacity()) {
@@ -148,11 +158,11 @@ class WebRtcAudioRecord {
          if (audioSamplesReadyCallback != null) {
            // Copy the entire byte buffer array. The start of the byteBuffer is not necessarily
            // at index 0.
-            byte[] data = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.arrayOffset(),
-                byteBuffer.capacity() + byteBuffer.arrayOffset());
-            audioSamplesReadyCallback.onWebRtcAudioRecordSamplesReady(
-                new JavaAudioDeviceModule.AudioSamples(audioRecord.getAudioFormat(),
-                    audioRecord.getChannelCount(), audioRecord.getSampleRate(), data));
+            SamplesReadyCallback nullable = audioSamplesReadyCallback;
+            if(nullable != null) {
+              final byte[] data = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.arrayOffset(), byteBuffer.capacity() + byteBuffer.arrayOffset());
+              nullable.onWebRtcAudioRecordSamplesReady(new JavaAudioDeviceModule.AudioSamples(audioRecord.getAudioFormat(), audioRecord.getChannelCount(), audioRecord.getSampleRate(), data));
+            }
          }
        } else {
          String errorMessage = "AudioRecord.read failed: " + bytesRead;
--- a/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioTrack.java
+++ b/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioTrack.java
@@ -27,8 +27,10 @@ import org.webrtc.ThreadUtils;
 import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackErrorCallback;
 import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackStartErrorCode;
 import org.webrtc.audio.JavaAudioDeviceModule.AudioTrackStateCallback;
+import org.webrtc.audio.JavaAudioDeviceModule.SamplesReadyCallback;

 import java.nio.ByteBuffer;
+import java.util.Arrays;

 class WebRtcAudioTrack {
  private static final String TAG = "WebRtcAudioTrackExternal";
@@ -87,6 +89,18 @@ class WebRtcAudioTrack {

  private final @Nullable AudioTrackErrorCallback errorCallback;
  private final @Nullable AudioTrackStateCallback stateCallback;
+  private @Nullable SamplesReadyCallback audioSamplesReadyCallback;
+
+  /**
+   * 设置录音工具
+   *
+   * @param samplesReadyCallback 录音回调
+   *
+   * @Taoyao
+   */
+  public void setMixerProcesser(JavaAudioDeviceModule.SamplesReadyCallback samplesReadyCallback) {
+    this.audioSamplesReadyCallback = samplesReadyCallback;
+  }

  /**
   * Audio thread which keeps calling AudioTrack.write() to stream audio.
@@ -131,6 +145,13 @@ class WebRtcAudioTrack {
          byteBuffer.position(0);
        }
        int bytesWritten = writeBytes(audioTrack, byteBuffer, sizeInBytes);
+        if (audioSamplesReadyCallback != null) {
+          SamplesReadyCallback nullable = audioSamplesReadyCallback;
+          if(nullable != null) {
+            final byte[] data = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.arrayOffset(), byteBuffer.capacity() + byteBuffer.arrayOffset());
+            nullable.onWebRtcAudioTrackSamplesReady(new JavaAudioDeviceModule.AudioSamples(audioTrack.getAudioFormat(), audioTrack.getChannelCount(), audioTrack.getSampleRate(), data));
+          }
+        }
        if (bytesWritten != sizeInBytes) {
          Logging.e(TAG, "AudioTrack.write played invalid number of bytes: " + bytesWritten);
          // If a write() returns a negative value, an error has occurred.