diff --git a/docs/TODO.md b/docs/TODO.md index 13adf6f..eecb97d 100644 --- a/docs/TODO.md +++ b/docs/TODO.md @@ -13,11 +13,10 @@ ## 计划任务 -* 混音 -* 音频视频时间对齐 * 分辨率调整 * 查询消费者生产者信息 ## 完成任务 +* 混音 * 屏幕共享 diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java index 8026264..1fe2d9a 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java @@ -401,7 +401,7 @@ public final class MediaManager { // } // }); final JavaAudioDeviceModule javaAudioDeviceModule = JavaAudioDeviceModule.builder(this.context) -// .setSampleRate() +// .setSampleRate(48000) // .setAudioSource(MediaRecorder.AudioSource.MIC) // .setAudioFormat(AudioFormat.ENCODING_PCM_16BIT) // .setAudioAttributes(audioAttributes) @@ -573,6 +573,22 @@ public final class MediaManager { } } + public void muteAllRemote() { + this.javaAudioDeviceModule.setSpeakerMute(true); + } + + public void unmuteAllRemote() { + this.javaAudioDeviceModule.setSpeakerMute(false); + } + + public void muteAllLocal() { + this.javaAudioDeviceModule.setMicrophoneMute(true); + } + + public void unmuteAllLocal() { + this.javaAudioDeviceModule.setMicrophoneMute(false); + } + /** * 更新配置 * diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java index 2910a78..ee862b8 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java @@ -1,12 +1,17 @@ package com.acgist.taoyao.media.audio; +import android.annotation.SuppressLint; +import android.media.AudioFormat; +import android.media.AudioRecord; +import android.media.MediaRecorder; import android.util.Log; import com.acgist.taoyao.media.client.RecordClient; -import org.webrtc.AudioSource; import org.webrtc.audio.JavaAudioDeviceModule; +import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.TimeUnit; @@ -18,21 +23,61 @@ import java.util.concurrent.TimeUnit; * WebRtcAudioTrack#AudioTrackThread ：远程音频 * WebRtcAudioRecord#AudioRecordThread：本地音频 * - * 注意：只能远程终端拉取才能采集音频数据，如果需要离线采集自己使用AudioRecord实现。 + * AudioFormat.ENCODING_PCM_16BIT = 2KB + * + * PCM时间计算：1_000_000 microseconds / 48000 hz / 2 bytes * * @author acgist */ public class MixerProcesser extends Thread implements JavaAudioDeviceModule.SamplesReadyCallback { + /** + * 音频数据来源 + * 其实不用切换可以两个同时录制，但是有点浪费资源。 + * + * @author acgist + */ + public enum Source { + // 本地 + NATIVE, + // WebRTC + WEBRTC; + } + private boolean close; + private Source source; + private final int sampleRate; + private final int audioFormat; + private final int audioSource; + private final int channelCount; + private final int channelConfig; + private final AudioRecord audioRecord; private final RecordClient recordClient; private final BlockingQueue local; private final BlockingQueue remote; - public MixerProcesser(RecordClient recordClient) { + @SuppressLint("MissingPermission") + public MixerProcesser(int sampleRate, int channelCount, RecordClient recordClient) { this.setDaemon(true); this.setName("AudioMixer"); - this.close = false; + this.close = false; + this.source = Source.WEBRTC; + this.sampleRate = sampleRate; + this.audioFormat = AudioFormat.ENCODING_PCM_16BIT; + this.audioSource = MediaRecorder.AudioSource.MIC; + this.channelCount = channelCount; + this.channelConfig = AudioFormat.CHANNEL_IN_MONO; + this.audioRecord = new AudioRecord.Builder() + .setAudioFormat( + new AudioFormat.Builder() + .setEncoding(this.audioFormat) + .setSampleRate(this.sampleRate) + .setChannelMask(this.channelConfig) + .build() + ) + .setAudioSource(this.audioSource) + .setBufferSizeInBytes(AudioRecord.getMinBufferSize(this.sampleRate, this.channelConfig, this.audioFormat)) + .build(); this.recordClient = recordClient; this.local = new LinkedBlockingQueue<>(1024); this.remote = new LinkedBlockingQueue<>(1024); @@ -40,7 +85,7 @@ public class MixerProcesser extends Thread implements JavaAudioDeviceModule.Samp @Override public void onWebRtcAudioTrackSamplesReady(JavaAudioDeviceModule.AudioSamples samples) { -// Log.d(MixerProcesser.class.getSimpleName(), "远程音频信息：" + samples.getAudioFormat()); +// Log.d(MixerProcesser.class.getSimpleName(), "远程音频信息：" + samples.getAudioFormat()); if(!this.remote.offer(samples)) { Log.e(MixerProcesser.class.getSimpleName(), "远程音频队列阻塞"); } @@ -48,7 +93,7 @@ public class MixerProcesser extends Thread implements JavaAudioDeviceModule.Samp @Override public void onWebRtcAudioRecordSamplesReady(JavaAudioDeviceModule.AudioSamples samples) { -// Log.d(MixerProcesser.class.getSimpleName(), "本地音频信息：" + samples.getAudioFormat()); +// Log.d(MixerProcesser.class.getSimpleName(), "本地音频信息：" + samples.getAudioFormat()); if(!this.local.offer(samples)) { Log.e(MixerProcesser.class.getSimpleName(), "本地音频队列阻塞"); } @@ -57,53 +102,107 @@ public class MixerProcesser extends Thread implements JavaAudioDeviceModule.Samp @Override public void run() { long pts = System.nanoTime(); -// final byte[] target = new byte[length]; - // PCM时间计算：1000000 microseconds / 48000 hz / 2 bytes - JavaAudioDeviceModule.AudioSamples local; - JavaAudioDeviceModule.AudioSamples remote; - int localValue; - int remoteValue; - byte[] localData; - byte[] remoteData; - byte[] data = null; - // TODO：固定长度采样率等等 + byte[] mixData = null; + byte[] localData = null; + byte[] remoteData = null; + byte[] recordData = null; + int mixDataLength = 0; + JavaAudioDeviceModule.AudioSamples local = null; + JavaAudioDeviceModule.AudioSamples remote = null; + int recordSize = 0; + // 采集数据大小：采样频率 / (一秒 / 回调频率) * 通道数量 * 采样数据大小 + final ByteBuffer byteBuffer = ByteBuffer.allocateDirect(this.sampleRate / (1000 / 10) * this.channelCount * 2); while(!this.close) { try { - local = this.local.poll(100, TimeUnit.MILLISECONDS); - remote = this.remote.poll(); - if(local != null && remote != null) { - localData = local.getData(); - remoteData = remote.getData(); - Log.d(MixerProcesser.class.getSimpleName(), String.format(""" - 混音长度：%d - %d - 混音采样：%d - %d - 混音格式：%d - %d - """, localData.length, remoteData.length, local.getSampleRate(), remote.getSampleRate(), local.getAudioFormat(), remote.getAudioFormat())); - data = new byte[localData.length]; - for (int index = 0; index < localData.length; index++) { - localValue = localData[index]; - remoteValue = remoteData[index]; - data[index] = (byte) ((localValue +remoteValue) / 2); + if(this.source == Source.NATIVE) { + recordSize = this.audioRecord.read(byteBuffer, byteBuffer.capacity()); + if(recordSize != byteBuffer.capacity()) { + continue; + } + recordData = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.arrayOffset(), byteBuffer.capacity() + byteBuffer.arrayOffset()); + pts += recordData.length * (1_000_000 / this.sampleRate / 2); + this.recordClient.onPcm(pts, recordData); + } else if(this.source == Source.WEBRTC) { + local = this.local.poll(100, TimeUnit.MILLISECONDS); + remote = this.remote.poll(); + if(local != null && remote != null) { +// Log.d(MixerProcesser.class.getSimpleName(), String.format(""" +// 混音长度：%d - %d +// 混音采样：%d - %d +// 混音格式：%d - %d +// 混音数量：%d - %d""", +// local.getData().length, remote.getData().length, +// local.getSampleRate(), remote.getSampleRate(), +// local.getAudioFormat(), remote.getAudioFormat(), +// local.getChannelCount(), remote.getChannelCount() +// )); + localData = local.getData(); + remoteData = remote.getData(); + if(mixDataLength != localData.length) { +// if(mixDataLength != localData.length && mixDataLength != remoteData.length) { + mixDataLength = localData.length; + mixData = new byte[mixDataLength]; + } + // 如果多路远程声音变小：(remote * 远程路数 + local) / (远程路数 + 1) + for (int index = 0; index < mixDataLength; index++) { +// -0x8000 ~ 0x7FFF; + mixData[index] = (byte) (((localData[index] + remoteData[index]) & 0x7FFF) / 2); +// mixData[index] = (byte) (((localData[index] + remoteData[index]) & 0xFFFF) / 2); +// mixData[index] = (byte) (((localData[index] + remoteData[index] * remoteCount) & 0xFFFF) / (1 + remoteCount)); + } + pts += mixData.length * (1_000_000 / local.getSampleRate() / 2); + this.recordClient.onPcm(pts, mixData); + } else if(local != null && remote == null) { + localData = local.getData(); + pts += localData.length * (1_000_000 / local.getSampleRate() / 2); + this.recordClient.onPcm(pts, localData); + } else if(local == null && remote != null) { + remoteData = remote.getData(); + pts += remoteData.length * (1_000_000 / remote.getSampleRate() / 2); + this.recordClient.onPcm(pts, remoteData); + } else { + continue; } - pts += data.length * (1_000_000 / local.getSampleRate() / 2); - } else if(local != null && remote == null) { - data = local.getData(); - pts += data.length * (1_000_000 / local.getSampleRate() / 2); - } else if(local == null && remote != null) { - data = remote.getData(); - pts += data.length * (1_000_000 / remote.getSampleRate() / 2); } else { - continue; } - this.recordClient.onPcm(pts, data); } catch (Exception e) { Log.e(MixerProcesser.class.getSimpleName(), "音频处理异常", e); } } + if(this.audioRecord != null) { + this.audioRecord.stop(); + this.audioRecord.release(); + } + } + + @Override + public void startNative() { + synchronized (this) { + if(this.source == Source.NATIVE) { + return; + } + this.audioRecord.startRecording(); + this.source = Source.NATIVE; + Log.i(MixerProcesser.class.getSimpleName(), "混音切换来源：" + this.source); + } + } + + @Override + public void startWebRTC() { + synchronized (this) { + if(this.source == Source.WEBRTC) { + return; + } + this.audioRecord.stop(); + this.source = Source.WEBRTC; + Log.i(MixerProcesser.class.getSimpleName(), "混音切换来源：" + this.source); + } } public void close() { - this.close = true; + synchronized (this) { + this.close = true; + } } } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java index d1c7599..68853ef 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java @@ -52,6 +52,7 @@ public class LocalClient extends RoomClient { return; } ListUtils.getOnlyOne(this.mediaStream.audioTracks, audioTrack -> { + audioTrack.setVolume(Config.DEFAULT_VOLUME); audioTrack.setEnabled(true); return audioTrack; }); diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java index 9d02cff..271ea1b 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java @@ -374,10 +374,10 @@ public class RecordClient extends Client implements VideoSink { public void record(VideoSource videoSource, JavaAudioDeviceModule javaAudioDeviceModule, PeerConnectionFactory peerConnectionFactory) { // 音频 if(javaAudioDeviceModule != null) { - this.mixerProcesser = new MixerProcesser(this); - this.mixerProcesser.start(); - javaAudioDeviceModule.setMixerProcesser(this.mixerProcesser); this.javaAudioDeviceModule = javaAudioDeviceModule; + this.mixerProcesser = new MixerProcesser(this.sampleRate, this.channelCount, this); + this.mixerProcesser.start(); + this.javaAudioDeviceModule.setMixerProcesser(this.mixerProcesser); } // 视频 if(videoSource != null && peerConnectionFactory != null) { diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java index ae1c64b..8315654 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java @@ -7,6 +7,7 @@ import com.acgist.taoyao.boot.utils.ListUtils; import com.acgist.taoyao.media.config.Config; import com.acgist.taoyao.media.signal.ITaoyao; +import org.webrtc.AudioTrack; import org.webrtc.MediaStreamTrack; import org.webrtc.VideoTrack; @@ -39,8 +40,12 @@ public class RemoteClient extends RoomClient { public void playAudio() { super.playAudio(); ListUtils.getOnlyOne( - this.tracks.values().stream().filter(v -> MediaStreamTrack.AUDIO_TRACK_KIND.equals(v.kind())).collect(Collectors.toList()), + this.tracks.values().stream() + .filter(v -> MediaStreamTrack.AUDIO_TRACK_KIND.equals(v.kind())) + .map(v -> (AudioTrack) v) + .collect(Collectors.toList()), audioTrack -> { + audioTrack.setVolume(Config.DEFAULT_VOLUME); audioTrack.setEnabled(true); return audioTrack; } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java index 4878590..51558d4 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java @@ -230,6 +230,7 @@ public class SessionClient extends Client { return; } ListUtils.getOnlyOne(this.remoteMediaStream.audioTracks, audioTrack -> { + audioTrack.setVolume(Config.DEFAULT_VOLUME); audioTrack.setEnabled(true); return audioTrack; }); @@ -239,6 +240,7 @@ public class SessionClient extends Client { public void pauseAudio() { super.pauseAudio(); ListUtils.getOnlyOne(this.remoteMediaStream.audioTracks, audioTrack -> { + audioTrack.setVolume(0); audioTrack.setEnabled(false); return audioTrack; }); @@ -248,6 +250,7 @@ public class SessionClient extends Client { public void resumeAudio() { super.resumeAudio(); ListUtils.getOnlyOne(this.remoteMediaStream.audioTracks, audioTrack -> { + audioTrack.setVolume(Config.DEFAULT_VOLUME); audioTrack.setEnabled(true); return audioTrack; }); @@ -296,6 +299,7 @@ public class SessionClient extends Client { public void pause(String type) { if(MediaStreamTrack.AUDIO_TRACK_KIND.equals(type)) { ListUtils.getOnlyOne(this.mediaStream.audioTracks, audioTrack -> { + audioTrack.setVolume(0); audioTrack.setEnabled(false); return audioTrack; }); @@ -318,6 +322,7 @@ public class SessionClient extends Client { public void resume(String type) { if(MediaStreamTrack.AUDIO_TRACK_KIND.equals(type)) { ListUtils.getOnlyOne(this.mediaStream.audioTracks, audioTrack -> { + audioTrack.setVolume(Config.DEFAULT_VOLUME); audioTrack.setEnabled(true); return audioTrack; }); diff --git a/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/JavaAudioDeviceModule.java b/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/JavaAudioDeviceModule.java index bfc41d3..a165268 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/JavaAudioDeviceModule.java +++ b/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/JavaAudioDeviceModule.java @@ -14,6 +14,7 @@ import android.content.Context; import android.media.AudioAttributes; import android.media.AudioDeviceInfo; import android.media.AudioManager; +import android.media.AudioRecord; import android.os.Build; import androidx.annotation.RequiresApi; @@ -316,6 +317,14 @@ public class JavaAudioDeviceModule implements AudioDeviceModule { /** Called when new audio samples are ready. This should only be set for debug purposes */ public static interface SamplesReadyCallback { + /** + * 本地录制 + */ + void startNative(); + /** + * 远程录制 + */ + void startWebRTC(); /** * 远程音频 * diff --git a/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioRecord.java b/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioRecord.java index ac6b049..89acb28 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioRecord.java +++ b/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioRecord.java @@ -117,7 +117,18 @@ class WebRtcAudioRecord { * @Taoyao */ public void setMixerProcesser(SamplesReadyCallback samplesReadyCallback) { + // 不用处理这个逻辑：设置为空表示关闭录制 +// if(this.audioSamplesReadyCallback != null && samplesReadyCallback == null) { +// this.audioSamplesReadyCallback.startNative(); +// } this.audioSamplesReadyCallback = samplesReadyCallback; + if(this.audioSamplesReadyCallback != null) { + if(this.audioThread == null) { + this.audioSamplesReadyCallback.startNative(); + } else { + this.audioSamplesReadyCallback.startWebRTC(); + } + } } /** @@ -158,7 +169,8 @@ class WebRtcAudioRecord { if (audioSamplesReadyCallback != null) { // Copy the entire byte buffer array. The start of the byteBuffer is not necessarily // at index 0. - SamplesReadyCallback nullable = audioSamplesReadyCallback; + // 注意不能定义其他地方否则不能回收 + final SamplesReadyCallback nullable = audioSamplesReadyCallback; if(nullable != null) { final byte[] data = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.arrayOffset(), byteBuffer.capacity() + byteBuffer.arrayOffset()); nullable.onWebRtcAudioRecordSamplesReady(new JavaAudioDeviceModule.AudioSamples(audioRecord.getAudioFormat(), audioRecord.getChannelCount(), audioRecord.getSampleRate(), data)); @@ -182,6 +194,9 @@ class WebRtcAudioRecord { } catch (IllegalStateException e) { Logging.e(TAG, "AudioRecord.stop failed: " + e.getMessage()); } + if(audioSamplesReadyCallback != null) { + audioSamplesReadyCallback.startNative(); + } } // Stops the inner thread loop and also calls AudioRecord.stop(). @@ -376,6 +391,9 @@ class WebRtcAudioRecord { Logging.d(TAG, "startRecording"); assertTrue(audioRecord != null); assertTrue(audioThread == null); + if(audioSamplesReadyCallback != null) { + audioSamplesReadyCallback.startWebRTC(); + } try { audioRecord.startRecording(); } catch (IllegalStateException e) { diff --git a/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioTrack.java b/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioTrack.java index 52d1cb2..5da5c16 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioTrack.java +++ b/taoyao-client-android/taoyao/media/src/main/java/org/webrtc/audio/WebRtcAudioTrack.java @@ -146,7 +146,8 @@ class WebRtcAudioTrack { } int bytesWritten = writeBytes(audioTrack, byteBuffer, sizeInBytes); if (audioSamplesReadyCallback != null) { - SamplesReadyCallback nullable = audioSamplesReadyCallback; + // 注意不能定义其他地方否则不能回收 + final SamplesReadyCallback nullable = audioSamplesReadyCallback; if(nullable != null) { final byte[] data = Arrays.copyOfRange(byteBuffer.array(), byteBuffer.arrayOffset(), byteBuffer.capacity() + byteBuffer.arrayOffset()); nullable.onWebRtcAudioTrackSamplesReady(new JavaAudioDeviceModule.AudioSamples(audioTrack.getAudioFormat(), audioTrack.getChannelCount(), audioTrack.getSampleRate(), data)); diff --git a/taoyao-client-web/src/components/Config.js b/taoyao-client-web/src/components/Config.js index b7b38e9..ad87b07 100644 --- a/taoyao-client-web/src/components/Config.js +++ b/taoyao-client-web/src/components/Config.js @@ -5,7 +5,7 @@ const defaultAudioConfig = { // 设备 // deviceId : '', // 音量：0~1 - volume: 0.5, + volume: 1.0, // 延迟大小（单位毫秒）：500毫秒以内较好 latency: 0.4, // 采样位数：8|16|32 diff --git a/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/AudioMixerTest.java b/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/AudioMixerTest.java index faaac82..dfea486 100644 --- a/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/AudioMixerTest.java +++ b/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/AudioMixerTest.java @@ -19,11 +19,8 @@ public class AudioMixerTest { final byte[] bytesB = Files.readAllBytes(fileB.toPath()); final int length = Math.min(bytesA.length, bytesB.length); final byte[] target = new byte[length]; - int a, b; for (int i = 0; i < length; i++) { - a = bytesA[i]; - b = bytesB[i]; - target[i] = (byte) ((a + b) / 2); + target[i] = (byte) (((bytesA[i] + bytesB[i]) & 0xFFFF) / 2); } Files.write(Paths.get("D:\\tmp\\mixer\\3.pcm"), target); }