qq27800734/taoyao
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[*] 日常优化

Browse Source
This commit is contained in:
acgist
2023-05-20 20:09:31 +08:00
parent aceeae4ede
commit 59b683af24
10 changed files with 267 additions and 140 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
taoyao-client-android/taoyao/boot/src/main/java/com/acgist/taoyao/boot/utils/ListUtils.java
View File

@@ -1,36 +0,0 @@
package com.acgist.taoyao.boot.utils;
import android.util.Log;
import java.util.List;
import java.util.function.Function;
/**
* 集合工具
*
* @author acgist
*/
public final class ListUtils {
private ListUtils() {
}
/**
* @param <T> 集合类型
* @param list 集合
* @param function 执行函数
*
* @return 集合首个元素执行函数返回结果
*/
public static final <T> T getOnlyOne(List<T> list, Function<T, T> function) {
if(list == null || list.isEmpty()) {
return null;
}
final int size = list.size();
if(size > 1) {
Log.w(ListUtils.class.getSimpleName(), "集合不止一条数据：" + size);
}
return function.apply(list.get(0));
}
}

62
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java
View File

@@ -19,7 +19,7 @@ import java.util.concurrent.TimeUnit;
/**
* 混音处理器
*
* JavaAudioDeviceModule : 音频
* JavaAudioDeviceModule : 音频模块
* WebRtcAudioTrack#AudioTrackThread ：远程音频
* WebRtcAudioRecord#AudioRecordThread：本地音频
*
@@ -33,7 +33,7 @@ public class MixerProcesser extends Thread implements JavaAudioDeviceModule.Samp
/**
* 音频数据来源
* 其实不用切换可以两个同时录制，但是有点浪费资源。
* 其实可以不用切换可以两个同时录制，但是有点浪费资源。
*
* @author acgist
*/
@@ -44,18 +44,56 @@ public class MixerProcesser extends Thread implements JavaAudioDeviceModule.Samp
WEBRTC;
}
/**
* 是否关闭
*/
private boolean close;
/**
* 音频数据来源
*/
private Source source;
/**
* 采样率
*/
private final int sampleRate;
/**
* 音频格式
*/
private final int audioFormat;
/**
* 音频来源
*/
private final int audioSource;
/**
* 通道数量
*/
private final int channelCount;
/**
* 通道配置
*/
private final int channelConfig;
/**
* 录音器
*/
private final AudioRecord audioRecord;
/**
* 录像机
*/
private final RecordClient recordClient;
/**
* 本地音频队列
*/
private final BlockingQueue<JavaAudioDeviceModule.AudioSamples> local;
/**
* 远程音频队列
*/
private final BlockingQueue<JavaAudioDeviceModule.AudioSamples> remote;
/**
* @param sampleRate 采样率
* @param channelCount 通道数量
* @param recordClient 录像机
*/
@SuppressLint("MissingPermission")
public MixerProcesser(int sampleRate, int channelCount, RecordClient recordClient) {
this.setDaemon(true);
@@ -79,8 +117,8 @@ public class MixerProcesser extends Thread implements JavaAudioDeviceModule.Samp
.setBufferSizeInBytes(AudioRecord.getMinBufferSize(this.sampleRate, this.channelConfig, this.audioFormat))
.build();
this.recordClient = recordClient;
this.local = new LinkedBlockingQueue<>(1024);
this.remote = new LinkedBlockingQueue<>(1024);
this.local = new LinkedBlockingQueue<>(1024);
this.remote = new LinkedBlockingQueue<>(1024);
}
@Override
@@ -101,16 +139,16 @@ public class MixerProcesser extends Thread implements JavaAudioDeviceModule.Samp
@Override
public void run() {
long pts = System.nanoTime();
long pts = System.nanoTime();
int recordSize = 0;
int mixDataLength = 0;
byte[] mixData = null;
byte[] localData = null;
byte[] remoteData = null;
byte[] recordData = null;
int mixDataLength = 0;
JavaAudioDeviceModule.AudioSamples local = null;
JavaAudioDeviceModule.AudioSamples remote = null;
int recordSize = 0;
// 采集数据大小：采样频率 / (一秒 / 回调频率) * 通道数量 * 采样数据大小
// 采集数据大小：采样频率 / (一千毫秒 / 回调频率毫秒) * 通道数量 * 采样数据大小
final ByteBuffer byteBuffer = ByteBuffer.allocateDirect(this.sampleRate / (1000 / 10) * this.channelCount * 2);
while(!this.close) {
try {
@@ -126,6 +164,7 @@ public class MixerProcesser extends Thread implements JavaAudioDeviceModule.Samp
} else if(this.source == Source.WEBRTC) {
// 平均10毫秒
local = this.local.poll(64, TimeUnit.MILLISECONDS);
// 远程数据不要等待
remote = this.remote.poll();
if(local != null && remote != null) {
// Log.d(MixerProcesser.class.getSimpleName(), String.format("""
@@ -140,8 +179,7 @@ public class MixerProcesser extends Thread implements JavaAudioDeviceModule.Samp
// ));
localData = local.getData();
remoteData = remote.getData();
if(mixDataLength != localData.length) {
// if(mixDataLength != localData.length && mixDataLength != remoteData.length) {
if(mixDataLength != localData.length && mixDataLength != remoteData.length) {
mixDataLength = localData.length;
mixData = new byte[mixDataLength];
}
@@ -164,7 +202,6 @@ public class MixerProcesser extends Thread implements JavaAudioDeviceModule.Samp
this.recordClient.onPcm(pts, remoteData);
} else {
Thread.yield();
continue;
}
} else {
Thread.yield();
@@ -203,6 +240,9 @@ public class MixerProcesser extends Thread implements JavaAudioDeviceModule.Samp
}
}
/**
* 关闭混音
*/
public void close() {
synchronized (this) {
this.close = true;

37
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Client.java
View File

@@ -28,9 +28,15 @@ public abstract class Client extends CloseableClient {
*/
protected SurfaceViewRenderer surfaceViewRenderer;
/**
* @param name 终端名称
* @param clientId 终端ID
* @param taoyao 信令
* @param mainHandler MainHandler
*/
public Client(String name, String clientId, ITaoyao taoyao, Handler mainHandler) {
super(taoyao, mainHandler);
this.name = name;
this.name = name;
this.clientId = clientId;
}
@@ -40,9 +46,15 @@ public abstract class Client extends CloseableClient {
public void playAudio() {
}
/**
* 暂停音频
*/
public void pauseAudio() {
}
/**
* 恢复音频
*/
public void resumeAudio() {
}
@@ -52,23 +64,32 @@ public abstract class Client extends CloseableClient {
public void playVideo() {
}
/**
* 暂停视频
*/
public void pauseVideo() {
if(this.surfaceViewRenderer != null) {
this.surfaceViewRenderer.pauseVideo();
}
}
/**
* 恢复视频
*/
public void resumeVideo() {
if(this.surfaceViewRenderer != null) {
// TODO：验证是否正确
this.surfaceViewRenderer.disableFpsReduction();
}
}
/**
* 暂停音视频
*/
public void pause() {
this.pauseAudio();
this.pauseVideo();
}
/**
* 恢复音视频
*/
public void resume() {
this.resumeAudio();
this.resumeVideo();
}
@Override

16
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/CloseableClient.java
View File

@@ -16,7 +16,6 @@ public abstract class CloseableClient implements Closeable {
/**
* 是否加载
* 防止重复加载
*/
protected volatile boolean init;
/**
@@ -28,7 +27,7 @@ public abstract class CloseableClient implements Closeable {
*/
protected final ITaoyao taoyao;
/**
* Handler
* MainHandler
*/
protected final Handler mainHandler;
/**
@@ -36,11 +35,15 @@ public abstract class CloseableClient implements Closeable {
*/
protected final MediaManager mediaManager;
/**
* @param taoyao 信令
* @param mainHandler MainHandler
*/
public CloseableClient(ITaoyao taoyao, Handler mainHandler) {
this.init = false;
this.close = false;
this.taoyao = taoyao;
this.mainHandler = mainHandler;
this.init = false;
this.close = false;
this.taoyao = taoyao;
this.mainHandler = mainHandler;
this.mediaManager = MediaManager.getInstance();
}
@@ -56,4 +59,5 @@ public abstract class CloseableClient implements Closeable {
public void close() {
this.close = true;
}
}

80
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java
View File

@@ -3,7 +3,6 @@ package com.acgist.taoyao.media.client;
import android.os.Handler;
import android.util.Log;
import com.acgist.taoyao.boot.utils.ListUtils;
import com.acgist.taoyao.media.config.Config;
import com.acgist.taoyao.media.signal.ITaoyao;
@@ -29,18 +28,36 @@ public class LocalClient extends RoomClient {
* 生产者ID = 媒体流Track指针
*/
protected final Map<String, Long> tracks;
/**
* 音频媒体生产者指针
*/
protected long audioProducerPointer;
/**
* 视频媒体生产者指针
*/
protected long videoProducerPointer;
/**
* @param name 终端名称
* @param clientId 终端ID
* @param taoyao 信令
* @param mainHandler MainHandler
*/
public LocalClient(String name, String clientId, ITaoyao taoyao, Handler mainHandler) {
super(name, clientId, taoyao, mainHandler);
this.tracks = new ConcurrentHashMap<>();
}
/**
* @return 媒体流
*/
public MediaStream getMediaStream() {
return this.mediaStream;
}
/**
* @param mediaStream 媒体流
*/
public void setMediaStream(MediaStream mediaStream) {
this.mediaStream = mediaStream;
}
@@ -51,10 +68,31 @@ public class LocalClient extends RoomClient {
if(this.mediaStream == null) {
return;
}
ListUtils.getOnlyOne(this.mediaStream.audioTracks, audioTrack -> {
this.mediaStream.audioTracks.forEach(audioTrack -> {
audioTrack.setVolume(Config.DEFAULT_VOLUME);
audioTrack.setEnabled(true);
return audioTrack;
});
}
@Override
public void pauseAudio() {
super.pauseAudio();
if(this.mediaStream == null) {
return;
}
this.mediaStream.audioTracks.forEach(audioTrack -> {
audioTrack.setEnabled(false);
});
}
@Override
public void resumeAudio() {
super.resumeAudio();
if(this.mediaStream == null) {
return;
}
this.mediaStream.audioTracks.forEach(audioTrack -> {
audioTrack.setEnabled(true);
});
}
@@ -64,12 +102,33 @@ public class LocalClient extends RoomClient {
if(this.mediaStream == null) {
return;
}
ListUtils.getOnlyOne(this.mediaStream.videoTracks, videoTrack -> {
this.mediaStream.videoTracks.forEach(videoTrack -> {
videoTrack.setEnabled(true);
if(this.surfaceViewRenderer == null) {
this.surfaceViewRenderer = this.mediaManager.buildSurfaceViewRenderer(Config.WHAT_NEW_LOCAL_VIDEO, videoTrack);
}
return videoTrack;
});
}
@Override
public void pauseVideo() {
super.pauseVideo();
if(this.mediaStream == null) {
return;
}
this.mediaStream.videoTracks.forEach(videoTrack -> {
videoTrack.setEnabled(false);
});
}
@Override
public void resumeVideo() {
super.resumeVideo();
if(this.mediaStream == null) {
return;
}
this.mediaStream.videoTracks.forEach(videoTrack -> {
videoTrack.setEnabled(true);
});
}
@@ -96,12 +155,17 @@ public class LocalClient extends RoomClient {
* @param producerId 生产者ID
*/
public void close(String producerId) {
Log.i(RemoteClient.class.getSimpleName(), "关闭本地终端生产者者：" + this.clientId + " - " + producerId);
final Long pointer = this.tracks.get(producerId);
if(pointer == null || this.mediaStream == null) {
if(this.mediaStream == null) {
return;
}
Log.i(RemoteClient.class.getSimpleName(), "关闭本地终端生产者：" + this.clientId + " - " + producerId);
synchronized (this.mediaStream) {
final Long pointer = this.tracks.get(producerId);
// TODO：测试remove方法
// final Long pointer = this.tracks.remove(producerId);
if(pointer == null) {
return;
}
if(pointer.equals(this.audioProducerPointer)) {
this.mediaStream.audioTracks.forEach(MediaStreamTrack::dispose);
this.mediaStream.audioTracks.clear();

85
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java
View File

@@ -3,7 +3,6 @@ package com.acgist.taoyao.media.client;
import android.os.Handler;
import android.util.Log;
import com.acgist.taoyao.boot.utils.ListUtils;
import com.acgist.taoyao.media.config.Config;
import com.acgist.taoyao.media.signal.ITaoyao;
@@ -13,10 +12,10 @@ import org.webrtc.VideoTrack;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.Collectors;
/**
* 房间远程终端
* 注意：这里媒体MediaStreamTrack使用Mediasoup方法释放不要直接调用MediaStreamTrack.dispose()释放
*
* @author acgist
*/
@@ -39,35 +38,67 @@ public class RemoteClient extends RoomClient {
@Override
public void playAudio() {
super.playAudio();
ListUtils.getOnlyOne(
this.tracks.values().stream()
.filter(v -> MediaStreamTrack.AUDIO_TRACK_KIND.equals(v.kind()))
.map(v -> (AudioTrack) v)
.collect(Collectors.toList()),
audioTrack -> {
audioTrack.setVolume(Config.DEFAULT_VOLUME);
audioTrack.setEnabled(true);
return audioTrack;
}
);
this.tracks.values().stream()
.filter(v -> MediaStreamTrack.AUDIO_TRACK_KIND.equals(v.kind()))
.map(v -> (AudioTrack) v)
.forEach(audioTrack -> {
audioTrack.setVolume(Config.DEFAULT_VOLUME);
audioTrack.setEnabled(true);
});
}
@Override
public void pauseAudio() {
super.pauseAudio();
this.tracks.values().stream()
.filter(v -> MediaStreamTrack.AUDIO_TRACK_KIND.equals(v.kind()))
.forEach(audioTrack -> {
audioTrack.setEnabled(false);
});
}
@Override
public void resumeAudio() {
super.resumeAudio();
this.tracks.values().stream()
.filter(v -> MediaStreamTrack.AUDIO_TRACK_KIND.equals(v.kind()))
.forEach(audioTrack -> {
audioTrack.setEnabled(true);
});
}
@Override
public void playVideo() {
super.playVideo();
ListUtils.getOnlyOne(
this.tracks.values().stream()
.filter(v -> MediaStreamTrack.VIDEO_TRACK_KIND.equals(v.kind()))
.map(v -> (VideoTrack) v)
.collect(Collectors.toList()),
videoTrack -> {
videoTrack.setEnabled(true);
if(this.surfaceViewRenderer == null) {
this.surfaceViewRenderer = this.mediaManager.buildSurfaceViewRenderer(Config.WHAT_NEW_REMOTE_VIDEO, videoTrack);
}
return videoTrack;
this.tracks.values().stream()
.filter(v -> MediaStreamTrack.VIDEO_TRACK_KIND.equals(v.kind()))
.map(v -> (VideoTrack) v)
.forEach(videoTrack -> {
videoTrack.setEnabled(true);
if(this.surfaceViewRenderer == null) {
this.surfaceViewRenderer = this.mediaManager.buildSurfaceViewRenderer(Config.WHAT_NEW_REMOTE_VIDEO, videoTrack);
}
);
});
}
@Override
public void pauseVideo() {
super.pauseVideo();
this.tracks.values().stream()
.filter(v -> MediaStreamTrack.VIDEO_TRACK_KIND.equals(v.kind()))
.forEach(audioTrack -> {
audioTrack.setEnabled(false);
});
}
@Override
public void resumeVideo() {
super.resumeVideo();
this.tracks.values().stream()
.filter(v -> MediaStreamTrack.VIDEO_TRACK_KIND.equals(v.kind()))
.forEach(audioTrack -> {
audioTrack.setEnabled(true);
});
}
@Override
@@ -79,7 +110,7 @@ public class RemoteClient extends RoomClient {
super.close();
Log.i(RemoteClient.class.getSimpleName(), "关闭远程终端：" + this.clientId);
synchronized (this.tracks) {
// 注意：使用nativeMediaConsumerClose释放
// 注意：使用nativeMediaConsumerClose释放资源
this.tracks.clear();
}
}
@@ -93,7 +124,7 @@ public class RemoteClient extends RoomClient {
public void close(String consumerId) {
Log.i(RemoteClient.class.getSimpleName(), "关闭远程终端消费者：" + this.clientId + " - " + consumerId);
synchronized (this.tracks) {
// 注意：使用nativeMediaConsumerClose释放
// 注意：使用nativeMediaConsumerClose释放资源
this.tracks.remove(consumerId);
}
}

65
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java
View File

@@ -4,7 +4,6 @@ import android.os.Handler;
import android.util.Log;
import com.acgist.taoyao.boot.model.Message;
import com.acgist.taoyao.boot.utils.ListUtils;
import com.acgist.taoyao.boot.utils.MapUtils;
import com.acgist.taoyao.media.config.Config;
import com.acgist.taoyao.media.config.MediaProperties;
@@ -231,30 +230,33 @@ public class SessionClient extends Client {
if(this.remoteMediaStream == null) {
return;
}
ListUtils.getOnlyOne(this.remoteMediaStream.audioTracks, audioTrack -> {
this.remoteMediaStream.audioTracks.forEach(audioTrack -> {
audioTrack.setVolume(Config.DEFAULT_VOLUME);
audioTrack.setEnabled(true);
return audioTrack;
});
}
@Override
public void pauseAudio() {
super.pauseAudio();
ListUtils.getOnlyOne(this.remoteMediaStream.audioTracks, audioTrack -> {
if(this.remoteMediaStream == null) {
return;
}
this.remoteMediaStream.audioTracks.forEach(audioTrack -> {
audioTrack.setVolume(0);
audioTrack.setEnabled(false);
return audioTrack;
});
}
@Override
public void resumeAudio() {
super.resumeAudio();
ListUtils.getOnlyOne(this.remoteMediaStream.audioTracks, audioTrack -> {
if(this.remoteMediaStream == null) {
return;
}
this.remoteMediaStream.audioTracks.forEach(audioTrack -> {
audioTrack.setVolume(Config.DEFAULT_VOLUME);
audioTrack.setEnabled(true);
return audioTrack;
});
}
@@ -264,74 +266,67 @@ public class SessionClient extends Client {
if(this.remoteMediaStream == null) {
return;
}
ListUtils.getOnlyOne(this.remoteMediaStream.videoTracks, videoTrack -> {
this.remoteMediaStream.videoTracks.forEach(videoTrack -> {
videoTrack.setEnabled(true);
if(this.surfaceViewRenderer == null) {
this.surfaceViewRenderer = this.mediaManager.buildSurfaceViewRenderer(Config.WHAT_NEW_REMOTE_VIDEO, videoTrack);
}
return videoTrack;
});
}
@Override
public void pauseVideo() {
super.pauseVideo();
ListUtils.getOnlyOne(this.remoteMediaStream.videoTracks, videoTrack -> {
if(this.remoteMediaStream == null) {
return;
}
if(this.surfaceViewRenderer != null) {
// TODO：测试
this.surfaceViewRenderer.pauseVideo();
}
this.remoteMediaStream.videoTracks.forEach(videoTrack -> {
videoTrack.setEnabled(false);
return videoTrack;
});
}
@Override
public void resumeVideo() {
super.resumeVideo();
ListUtils.getOnlyOne(this.remoteMediaStream.videoTracks, videoTrack -> {
if(this.remoteMediaStream == null) {
return;
}
if(this.surfaceViewRenderer != null) {
// TODO：测试
this.surfaceViewRenderer.disableFpsReduction();
}
this.remoteMediaStream.videoTracks.forEach(videoTrack -> {
videoTrack.setEnabled(true);
return videoTrack;
});
}
@Override
public void pause() {
super.pause();
this.pauseAudio();
this.pauseVideo();
}
public void pause(String type) {
if(MediaStreamTrack.AUDIO_TRACK_KIND.equals(type)) {
ListUtils.getOnlyOne(this.mediaStream.audioTracks, audioTrack -> {
this.mediaStream.audioTracks.forEach(audioTrack -> {
audioTrack.setVolume(0);
audioTrack.setEnabled(false);
return audioTrack;
});
} else if(MediaStreamTrack.VIDEO_TRACK_KIND.equals(type)) {
ListUtils.getOnlyOne(this.mediaStream.videoTracks, videoTrack -> {
this.mediaStream.videoTracks.forEach(videoTrack -> {
videoTrack.setEnabled(false);
return videoTrack;
});
} else {
}
}
@Override
public void resume() {
super.resume();
this.resumeAudio();
this.resumeVideo();
}
public void resume(String type) {
if(MediaStreamTrack.AUDIO_TRACK_KIND.equals(type)) {
ListUtils.getOnlyOne(this.mediaStream.audioTracks, audioTrack -> {
this.mediaStream.audioTracks.forEach(audioTrack -> {
audioTrack.setVolume(Config.DEFAULT_VOLUME);
audioTrack.setEnabled(true);
return audioTrack;
});
} else if(MediaStreamTrack.VIDEO_TRACK_KIND.equals(type)) {
ListUtils.getOnlyOne(this.mediaStream.videoTracks, videoTrack -> {
this.mediaStream.videoTracks.forEach(videoTrack -> {
videoTrack.setEnabled(true);
return videoTrack;
});
} else {
}