qq27800734/taoyao
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[+] Web端P2P监控

Browse Source
This commit is contained in:
acgist
2023-04-01 23:21:16 +08:00
parent ed66875a47
commit 1d79de3ef7
27 changed files with 922 additions and 265 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
README.md
View File

@@ -29,16 +29,17 @@
### Web终端功能
|功能|是否支持|是否实现|描述|
|:--|:--|:--|:--|
|P2P|支持|暂未实现|P2P监控模式|
|WebRTC|支持|实现|Web终端不能同时进入多个房间|
### 安卓终端功能
|功能|是否支持|是否实现|描述|
|:--|:--|:--|:--|
|P2P|支持|暂未实现|P2P监控模式|
|WebRTC|支持|暂未实现|安卓终端支持同时进入多个房间|
|RTP|支持|暂未实现|支持房间RTP推流（不会拉流）|
||||
## 证书

12
taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java
View File

@@ -4,7 +4,6 @@ import android.Manifest;
import android.app.Activity;
import android.content.Intent;
import android.content.pm.PackageManager;
import android.media.projection.MediaProjection;
import android.media.projection.MediaProjectionManager;
import android.os.Bundle;
import android.os.Handler;
@@ -16,12 +15,11 @@ import android.view.SurfaceView;
import android.view.View;
import android.view.ViewGroup;
import android.view.WindowManager;
import android.widget.Toast;
import android.widget.LinearLayout;
import androidx.activity.result.ActivityResultLauncher;
import androidx.activity.result.contract.ActivityResultContracts;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.appcompat.app.AppCompatActivity;
import androidx.core.app.ActivityCompat;
@@ -61,6 +59,7 @@ public class MainActivity extends AppCompatActivity implements Serializable {
this.launchMediaService();
// 布局
this.binding = ActivityMainBinding.inflate(this.getLayoutInflater());
this.binding.getRoot().setZ(100F);
this.setContentView(this.binding.getRoot());
this.registerMediaProjection();
this.binding.record.setOnClickListener(this::switchRecord);
@@ -161,7 +160,7 @@ public class MainActivity extends AppCompatActivity implements Serializable {
MediaManager.getInstance().init(this.mainHandler, this.getApplicationContext());
MediaManager.getInstance().initAudio();
MediaManager.getInstance().initVideo();
mediaRecorder.init(System.currentTimeMillis() + ".mp4", null, null, 1, 1);
MediaManager.getInstance().record();
}
}
@@ -216,7 +215,10 @@ public class MainActivity extends AppCompatActivity implements Serializable {
*/
private void newLocalVideo(Message message) {
final SurfaceView surfaceView = (SurfaceView) message.obj;
this.addContentView(surfaceView, new ViewGroup.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT));
final LinearLayout.LayoutParams layoutParams = new LinearLayout.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT);
layoutParams.weight = 1;
surfaceView.setZ(0F);
this.addContentView(surfaceView, layoutParams);
}
}

91
taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java
View File

@@ -10,6 +10,7 @@ import android.net.wifi.WifiInfo;
import android.net.wifi.WifiManager;
import android.os.BatteryManager;
import android.os.Handler;
import android.os.HandlerThread;
import android.util.Log;
import androidx.core.app.ActivityCompat;
@@ -20,9 +21,9 @@ import com.acgist.taoyao.boot.model.MessageCode;
import com.acgist.taoyao.boot.model.MessageCodeException;
import com.acgist.taoyao.boot.utils.CloseableUtils;
import com.acgist.taoyao.boot.utils.JSONUtils;
import com.acgist.taoyao.media.MediaRecorder;
import com.acgist.taoyao.client.utils.IdUtils;
import com.acgist.taoyao.media.P2PClient;
import com.acgist.taoyao.media.MediaRecorder;
import com.acgist.taoyao.media.SessionClient;
import com.acgist.taoyao.media.Room;
import org.apache.commons.lang3.ArrayUtils;
@@ -40,10 +41,6 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import javax.crypto.BadPaddingException;
import javax.crypto.Cipher;
@@ -58,8 +55,6 @@ import javax.crypto.spec.SecretKeySpec;
*/
public final class Taoyao {
private static final long MAX_TIMEOUT = 60L * 1000;
/**
* 端口
*/
@@ -104,10 +99,6 @@ public final class Taoyao {
* 超时时间
*/
private final int timeout;
/**
* 重试次数
*/
private int connectRetryTimes;
/**
* Socket
*/
@@ -131,7 +122,7 @@ public final class Taoyao {
/**
* Handler
*/
private final Handler handler;
private final Handler mainHandler;
/**
* 服务上下文
*/
@@ -152,14 +143,12 @@ public final class Taoyao {
* 请求消息：同步消息
*/
private final Map<Long, Message> requestMessage;
/**
* 线程池
*/
private final ExecutorService executor;
/**
* 定时任务线程池
*/
private final ScheduledExecutorService scheduled;
private final Handler loopMessageHandler;
private final HandlerThread loopMessageThread;
private final Handler heartbeatHandler;
private final HandlerThread heartbeatThread;
private final Handler executeMessageHandler;
private final HandlerThread executeMessageThread;
/**
* 房间列表
*/
@@ -167,13 +156,13 @@ public final class Taoyao {
/**
* P2P终端列表
*/
private final List<P2PClient> p2pClientList;
private final List<SessionClient> p2pClientList;
public Taoyao(
int port, String host, String version,
String name, String clientId, String clientType, String username, String password,
int timeout, String algo, String secret,
Handler handler, Context context,
Handler mainHandler, Context context,
WifiManager wifiManager, BatteryManager batteryManager, LocationManager locationManager
) {
this.close = false;
@@ -187,22 +176,26 @@ public final class Taoyao {
this.username = username;
this.password = password;
this.timeout = timeout;
this.connectRetryTimes = 1;
final boolean plaintext = algo == null || algo.isEmpty() || algo.equals("PLAINTEXT");
this.encrypt = plaintext ? null : this.buildCipher(Cipher.ENCRYPT_MODE, algo, secret);
this.decrypt = plaintext ? null : this.buildCipher(Cipher.DECRYPT_MODE, algo, secret);
this.handler = handler;
this.mainHandler = mainHandler;
this.context = context;
this.wifiManager = wifiManager;
this.batteryManager = batteryManager;
this.locationManager = locationManager;
this.requestMessage = new ConcurrentHashMap<>();
// 读取线程 + 两条处理线程
this.executor = Executors.newFixedThreadPool(3);
// 心跳线程
this.scheduled = Executors.newScheduledThreadPool(1);
this.executor.submit(this::loopMessage);
this.scheduled.scheduleWithFixedDelay(this::heartbeat, 30, 30, TimeUnit.SECONDS);
this.loopMessageThread = new HandlerThread("TaoyaoLoopMessageThread");
this.loopMessageThread.start();
this.loopMessageHandler = new Handler(this.loopMessageThread.getLooper());
this.loopMessageHandler.post(this::loopMessage);
this.heartbeatThread = new HandlerThread("TaoyaoHeartbeatThread");
this.heartbeatThread.start();
this.heartbeatHandler = new Handler(this.heartbeatThread.getLooper());
this.heartbeatHandler.postDelayed(this::heartbeat, 30L * 1000);
this.executeMessageThread = new HandlerThread("TaoyaoExecuteMessageThread");
this.executeMessageThread.start();
this.executeMessageHandler = new Handler(this.executeMessageThread.getLooper());
this.roomList = new CopyOnWriteArrayList<>();
this.p2pClientList = new CopyOnWriteArrayList<>();
}
@@ -247,12 +240,18 @@ public final class Taoyao {
this.output = this.socket.getOutputStream();
this.register();
this.connect = true;
this.connectRetryTimes = 1;
synchronized (this) {
this.notifyAll();
}
} else {
this.connect = false;
synchronized (this) {
try {
this.wait(this.timeout);
} catch (InterruptedException e) {
Log.d(Taoyao.class.getSimpleName(), "信令等待异常", e);
}
}
}
} catch (Exception e) {
Log.e(Taoyao.class.getSimpleName(), "连接信令异常：" + this.host + ":" + this.port, e);
@@ -273,22 +272,9 @@ public final class Taoyao {
// 重连
while (!this.close && !this.connect) {
this.connect();
synchronized (this) {
try {
long timeout = this.timeout;
if(MAX_TIMEOUT > this.timeout * this.connectRetryTimes) {
timeout = this.timeout * this.connectRetryTimes++;
} else {
timeout = MAX_TIMEOUT;
}
this.wait(timeout);
} catch (InterruptedException e) {
Log.d(Taoyao.class.getSimpleName(), "信令等待异常", e);
}
}
}
// 读取
while ((length = this.input.read(bytes)) >= 0) {
while (this.input != null && (length = this.input.read(bytes)) >= 0) {
buffer.put(bytes, 0, length);
while (buffer.position() > 0) {
if (messageLength <= 0) {
@@ -314,9 +300,9 @@ public final class Taoyao {
buffer.get(message);
buffer.compact();
final String content = new String(this.decrypt.doFinal(message));
Log.d(Taoyao.class.getSimpleName(), "处理信令：" + content);
executor.submit(() -> {
this.executeMessageHandler.post(() -> {
try {
Log.d(Taoyao.class.getSimpleName(), "处理信令：" + content);
Taoyao.this.on(content);
} catch (Exception e) {
Log.e(Taoyao.class.getSimpleName(), "处理信令异常：" + content, e);
@@ -428,10 +414,11 @@ public final class Taoyao {
Log.d(Taoyao.class.getSimpleName(), "关闭信令：" + this.host + ":" + this.port);
this.close = true;
this.disconnect();
this.executor.shutdown();
this.scheduled.shutdown();
this.heartbeatThread.quitSafely();
this.loopMessageThread.quitSafely();
this.executeMessageThread.quitSafely();
this.roomList.forEach(Room::close);
this.p2pClientList.forEach(P2PClient::close);
this.p2pClientList.forEach(SessionClient::close);
}
/**
@@ -471,7 +458,6 @@ public final class Taoyao {
* @param content 信令消息
*/
private void on(String content) {
Log.d(Taoyao.class.getSimpleName(), "收到消息：" + content);
final Message message = JSONUtils.toJava(content, Message.class);
if (message == null) {
return;
@@ -535,6 +521,7 @@ public final class Taoyao {
* 心跳
*/
private void heartbeat() {
this.heartbeatHandler.postDelayed(this::heartbeat, 30L * 1000);
if(this.close || !this.connect) {
return;
}

4
taoyao-client-android/taoyao/media/CMakeLists.txt
View File

@@ -28,17 +28,17 @@ set(
${SOURCE_DIR}/include/LocalClient.hpp
${SOURCE_DIR}/include/MediaRecorder.hpp
${SOURCE_DIR}/include/MediasoupClient.hpp
${SOURCE_DIR}/include/P2PClient.hpp
${SOURCE_DIR}/include/RemoteClient.hpp
${SOURCE_DIR}/include/Room.hpp
${SOURCE_DIR}/include/RtpAudioPublisher.hpp
${SOURCE_DIR}/include/RtpClient.hpp
${SOURCE_DIR}/include/RtpVideoPublisher.hpp
${SOURCE_DIR}/include/SessionClient.hpp
${SOURCE_DIR}/media/LocalClient.cpp
${SOURCE_DIR}/media/MediaRecorder.cpp
${SOURCE_DIR}/media/P2PClient.cpp
${SOURCE_DIR}/media/RemoteClient.cpp
${SOURCE_DIR}/media/Room.cpp
${SOURCE_DIR}/media/SessionClient.cpp
${SOURCE_DIR}/rtp/RtpAudioPublisher.cpp
${SOURCE_DIR}/rtp/RtpClient.cpp
${SOURCE_DIR}/rtp/RtpVideoPublisher.cpp

0
taoyao-client-android/taoyao/media/src/main/cpp/include/P2PClient.hpp → taoyao-client-android/taoyao/media/src/main/cpp/include/SessionClient.hpp
View File

5
taoyao-client-android/taoyao/media/src/main/cpp/media/P2PClient.cpp
View File

@@ -1,5 +0,0 @@
#include "P2PClient.hpp"
namespace acgist {
}

5
taoyao-client-android/taoyao/media/src/main/cpp/media/SessionClient.cpp Normal file
View File

@@ -0,0 +1,5 @@
#include "SessionClient.hpp"
namespace acgist {
}

68
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java
View File

@@ -1,19 +1,26 @@
package com.acgist.taoyao.media;
import android.annotation.SuppressLint;
import android.content.Context;
import android.content.Intent;
import android.media.AudioFormat;
import android.media.AudioRecord;
import android.media.MediaFormat;
import android.media.projection.MediaProjection;
import android.media.projection.MediaProjectionManager;
import android.os.Environment;
import android.os.Handler;
import android.os.Looper;
import android.os.Message;
import android.util.Log;
import android.view.Surface;
import com.acgist.taoyao.config.Config;
import org.webrtc.AudioSource;
import org.webrtc.AudioTrack;
import org.webrtc.BuiltinAudioDecoderFactoryFactory;
import org.webrtc.BuiltinAudioEncoderFactoryFactory;
import org.webrtc.Camera2Enumerator;
import org.webrtc.CameraEnumerator;
import org.webrtc.CameraVideoCapturer;
@@ -30,15 +37,18 @@ import org.webrtc.SurfaceViewRenderer;
import org.webrtc.VideoCapturer;
import org.webrtc.VideoDecoderFactory;
import org.webrtc.VideoEncoderFactory;
import org.webrtc.VideoFileRenderer;
import org.webrtc.VideoFrame;
import org.webrtc.VideoSink;
import org.webrtc.VideoSource;
import org.webrtc.VideoTrack;
import org.webrtc.audio.AudioDeviceModule;
import org.webrtc.audio.JavaAudioDeviceModule;
import org.webrtc.voiceengine.WebRtcAudioManager;
import org.webrtc.voiceengine.WebRtcAudioRecord;
import org.webrtc.voiceengine.WebRtcAudioUtils;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
@@ -85,7 +95,6 @@ public class MediaManager {
return this == BACK || this == FRONT;
}
}
private static final MediaManager INSTANCE = new MediaManager();
@@ -137,9 +146,11 @@ public class MediaManager {
// 设置采样
// WebRtcAudioUtils.setDefaultSampleRateHz();
// 噪声消除
WebRtcAudioUtils.setWebRtcBasedNoiseSuppressor(true);
// 回声小丑
WebRtcAudioUtils.setWebRtcBasedAcousticEchoCanceler(true);
// WebRtcAudioUtils.setWebRtcBasedNoiseSuppressor(true);
// 回声消除
// WebRtcAudioUtils.setWebRtcBasedAcousticEchoCanceler(true);
// 自动增益
// WebRtcAudioUtils.setWebRtcBasedAutomaticGainControl(true);
// 使用OpenSL ES
// WebRtcAudioManager.setBlacklistDeviceForOpenSLESUsage(true);
}
@@ -156,12 +167,13 @@ public class MediaManager {
this.eglBase = EglBase.create();
PeerConnectionFactory.initialize(
PeerConnectionFactory.InitializationOptions.builder(this.context)
.setEnableInternalTracer(true)
// .setEnableInternalTracer(true)
.createInitializationOptions()
);
final VideoDecoderFactory videoDecoderFactory = new DefaultVideoDecoderFactory(this.eglBase.getEglBaseContext());
final VideoEncoderFactory videoEncoderFactory = new DefaultVideoEncoderFactory(this.eglBase.getEglBaseContext(), true, true);
final JavaAudioDeviceModule javaAudioDeviceModule = JavaAudioDeviceModule.builder(this.context)
// .setAudioSource(android.media.MediaRecorder.AudioSource.MIC)
// 本地音频
.setSamplesReadyCallback(MediaRecorder.getInstance().audioRecoder)
// 远程音频
@@ -171,8 +183,8 @@ public class MediaManager {
.createAudioDeviceModule();
this.peerConnectionFactory = PeerConnectionFactory.builder()
// .setAudioProcessingFactory()
// .setAudioDecoderFactoryFactory()
// .setAudioEncoderFactoryFactory()
// .setAudioEncoderFactoryFactory(new BuiltinAudioEncoderFactoryFactory())
// .setAudioDecoderFactoryFactory(new BuiltinAudioDecoderFactoryFactory())
.setAudioDeviceModule(javaAudioDeviceModule)
.setVideoDecoderFactory(videoDecoderFactory)
.setVideoEncoderFactory(videoEncoderFactory)
@@ -221,26 +233,22 @@ public class MediaManager {
// 加载音频
final MediaConstraints mediaConstraints = new MediaConstraints();
// 高音过滤
// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googHighpassFilter", "true"));
// // 自动增益
// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googAutoGainControl", "true"));
// // 回声消除
// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googEchoCancellation", "true"));
// // 噪音处理
// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googNoiseSuppression", "true"));
// 更多
// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googEchoCancellation", "true"));
mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googHighpassFilter", "true"));
// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googAudioMirroring", "false"));
// 自动增益
mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googAutoGainControl", "true"));
// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googAutoGainControl2", "true"));
// 回声消除
mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googEchoCancellation", "true"));
// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googEchoCancellation2", "true"));
// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googDAEchoCancellation", "true"));
// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googTypingNoiseDetection", "true"));
// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googAutoGainControl", "true"));
// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googAutoGainControl2", "true"));
// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googNoiseSuppression", "true"));
// 噪音处理
mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googNoiseSuppression", "true"));
// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googNoiseSuppression2", "true"));
// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googAudioMirroring", "false"));
// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googHighpassFilter", "true"));
// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googTypingNoiseDetection", "true"));
final AudioSource audioSource = this.peerConnectionFactory.createAudioSource(mediaConstraints);
final AudioTrack audioTrack = this.peerConnectionFactory.createAudioTrack("ARDAMSa0", audioSource);
// audioTrack.setVolume(100);
audioTrack.setEnabled(true);
this.mediaStream.addTrack(audioTrack);
Log.i(MediaManager.class.getSimpleName(), "加载音频：" + audioTrack.id());
@@ -299,7 +307,7 @@ public class MediaManager {
final SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("MediaVideoThread", this.eglBase.getEglBaseContext());
final VideoSource videoSource = this.peerConnectionFactory.createVideoSource(this.videoCapturer.isScreencast());
this.videoCapturer.initialize(surfaceTextureHelper, this.context, videoSource.getCapturerObserver());
this.videoCapturer.startCapture(640, 480, 30);
this.videoCapturer.startCapture(480, 640, 30);
final VideoTrack videoTrack = this.peerConnectionFactory.createVideoTrack("ARDAMSv0", videoSource);
videoTrack.addSink(surfaceViewRenderer);
videoTrack.addSink(MediaRecorder.getInstance().videoRecoder);
@@ -329,6 +337,10 @@ public class MediaManager {
message.obj = surfaceViewRenderer;
message.what = Config.WHAT_NEW_LOCAL_VIDEO;
this.handler.sendMessage(message);
// 暂停
// surfaceViewRenderer.pauseVideo();
// 恢复
// surfaceViewRenderer.disableFpsReduction();
return surfaceViewRenderer;
}
@@ -362,6 +374,10 @@ public class MediaManager {
}
}
public void record() {
MediaRecorder.getInstance().init(System.currentTimeMillis() + ".mp4", null, null, 1, 1);
}
/**
* 关闭声音
*/
@@ -408,6 +424,12 @@ public class MediaManager {
* 释放资源
*/
public void close() {
this.closeAudioTrack();
this.closeVideoTrack();
if(this.eglBase != null) {
this.eglBase.release();
this.eglBase = null;
}
if(this.videoCapturer != null) {
this.videoCapturer.dispose();
this.videoCapturer = null;

219
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaRecorder.java
View File

@@ -1,5 +1,6 @@
package com.acgist.taoyao.media;
import android.graphics.YuvImage;
import android.media.AudioFormat;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
@@ -7,24 +8,35 @@ import android.media.MediaCodecList;
import android.media.MediaFormat;
import android.media.MediaMuxer;
import android.os.Environment;
import android.os.Handler;
import android.os.HandlerThread;
import android.util.Log;
import android.view.Surface;
import org.webrtc.EglBase;
import org.webrtc.GlRectDrawer;
import org.webrtc.HardwareVideoEncoderFactory;
import org.webrtc.VideoEncoderFactory;
import org.webrtc.VideoFrame;
import org.webrtc.VideoFrameDrawer;
import org.webrtc.VideoSink;
import org.webrtc.YuvConverter;
import org.webrtc.YuvHelper;
import org.webrtc.audio.JavaAudioDeviceModule;
import org.webrtc.voiceengine.WebRtcAudioRecord;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.file.Paths;
import java.util.Optional;
import java.util.concurrent.Executor;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
import java.util.stream.Stream;
/**
* 录像机
*
* <p>
* https://blog.csdn.net/m0_60259116/article/details/126875532
*
* @author acgist
@@ -33,6 +45,43 @@ public final class MediaRecorder {
private static final MediaRecorder INSTANCE = new MediaRecorder();
public static final MediaRecorder getInstance() {
return INSTANCE;
}
/**
* 是否正在录像
*/
private volatile boolean active;
private volatile boolean audioActive;
private volatile boolean videoActive;
private volatile long pts;
private String file;
/**
* 音频编码
*/
private MediaCodec audioCodec;
private HandlerThread audioThread;
/**
* 视频编码
*/
private MediaCodec videoCodec;
private HandlerThread videoThread;
private Handler videoHandler;
private ExecutorService executorService;
/**
* 媒体合成器
*/
private MediaMuxer mediaMuxer;
/**
* 音频录制
*/
public final JavaAudioDeviceModule.SamplesReadyCallback audioRecoder;
/**
* 视频录制
*/
public final VideoSink videoRecoder;
private MediaRecorder() {
final MediaCodecList mediaCodecList = new MediaCodecList(-1);
for (MediaCodecInfo mediaCodecInfo : mediaCodecList.getCodecInfos()) {
@@ -49,54 +98,36 @@ public final class MediaRecorder {
}
}
}
this.executorService = Executors.newFixedThreadPool(2);
this.audioRecoder = audioSamples -> {
Log.d(MediaRecorder.class.getSimpleName(), audioSamples + " - 音频");
};
this.videoRecoder = videoFrame -> {
// Log.d(MediaRecorder.class.getSimpleName(), videoFrame + " - 视频");
if (this.active && this.videoActive) {
final VideoFrame.Buffer buffer = videoFrame.getBuffer();
final VideoFrame.I420Buffer i420Buffer = buffer.toI420();
i420Buffer.getDataU();
// this.putVideo(videoFrame.getBuffer(), videoFrame.getTimestampNs());
this.executorService.submit(() -> {
// videoFrame.retain();
final int outputFrameSize = videoFrame.getRotatedWidth() * videoFrame.getRotatedHeight() * 3 / 2;
final ByteBuffer outputFrameBuffer = ByteBuffer.allocateDirect(outputFrameSize);
final int index = this.videoCodec.dequeueInputBuffer(1000L * 1000);
// YuvImage：截图
// YV12
VideoFrame.I420Buffer i420 = videoFrame.getBuffer().toI420();
// i420.retain();
Log.i(MediaRecorder.class.getSimpleName(), "视频信息：" + videoFrame.getRotatedWidth() + " - " + videoFrame.getRotatedHeight());
// YuvHelper.I420Copy(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight());
// NV12
YuvHelper.I420ToNV12(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight());
// YuvHelper.I420Rotate(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight(), videoFrame.getRotation());
final ByteBuffer x = this.videoCodec.getInputBuffer(index);
// i420.release();
x.put(outputFrameBuffer.array());
this.videoCodec.queueInputBuffer(index, 0, outputFrameSize, System.currentTimeMillis(), 0);
// this.putVideo(outputFrameBuffer, System.currentTimeMillis());
// videoFrame.release();
});
}
};
}
public static final MediaRecorder getInstance() {
return INSTANCE;
}
/**
* 是否正在录像
*/
private volatile boolean active;
private volatile boolean audioActive;
private volatile boolean videoActive;
private volatile long pts;
/**
* 音频编码
*/
private MediaCodec audioCodec;
private Thread audioThread;
/**
* 视频编码
*/
private MediaCodec videoCodec;
private Thread videoThread;
/**
* 媒体合成器
*/
private MediaMuxer mediaMuxer;
/**
* 音频录制
*/
public final JavaAudioDeviceModule.SamplesReadyCallback audioRecoder;
/**
* 视频录制
*/
public final VideoSink videoRecoder;
/**
* @return 是否正在录像
*/
@@ -106,6 +137,7 @@ public final class MediaRecorder {
public void init(String file, String audioFormat, String videoFormat, int width, int height) {
synchronized (MediaRecorder.INSTANCE) {
this.file = file;
this.active = true;
if (
this.audioThread == null || !this.audioThread.isAlive() ||
@@ -115,7 +147,6 @@ public final class MediaRecorder {
this.initAudioThread(MediaFormat.MIMETYPE_AUDIO_AAC, 96000, 44100, 1);
this.initVideoThread(MediaFormat.MIMETYPE_VIDEO_AVC, 2500 * 1000, 30, 1, 1920, 1080);
}
// this.audioCodec = MediaCodec.createByCodecName();
}
}
@@ -139,36 +170,39 @@ public final class MediaRecorder {
Log.e(MediaRecorder.class.getSimpleName(), "加载音频录制线程异常", e);
}
final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
this.audioThread = new Thread(() -> {
int trackIndex;
this.audioThread = new HandlerThread("AudioRecoderThread");
this.audioThread.start();
final Handler audioHandler = new Handler(this.audioThread.getLooper());
audioHandler.post(() -> {
int trackIndex = -1;
int outputIndex;
synchronized (MediaRecorder.INSTANCE) {
Log.i(MediaRecorder.class.getSimpleName(), "开始录制音频");
this.audioCodec.start();
this.audioActive = true;
while (this.active) {
outputIndex = this.audioCodec.dequeueOutputBuffer(info, 1000L * 1000);
if (outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
} else if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
synchronized (MediaRecorder.INSTANCE) {
trackIndex = this.mediaMuxer.addTrack(this.audioCodec.getOutputFormat());
Log.i(MediaRecorder.class.getSimpleName(), "开始录制音频：" + trackIndex);
if (this.videoActive) {
Log.i(MediaRecorder.class.getSimpleName(), "开始录制文件");
Log.i(MediaRecorder.class.getSimpleName(), "开始录制文件：" + this.file);
this.pts = System.currentTimeMillis();
this.mediaMuxer.start();
MediaRecorder.INSTANCE.notifyAll();
} else {
} else if (this.active) {
try {
MediaRecorder.INSTANCE.wait();
} catch (InterruptedException e) {
}
}
}
while(this.active) {
outputIndex = this.audioCodec.dequeueOutputBuffer(info, 1000L * 1000);
if(outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
} else if(outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
} else {
} else if (outputIndex >= 0) {
final ByteBuffer outputBuffer = this.audioCodec.getOutputBuffer(outputIndex);
outputBuffer.position(info.offset);
outputBuffer.limit(info.offset + info.size);
info.presentationTimeUs = info.presentationTimeUs - this.pts;
this.mediaMuxer.writeSampleData(trackIndex, outputBuffer, info);
info.presentationTimeUs = (info.presentationTimeUs - this.pts) * 1000;
// this.mediaMuxer.writeSampleData(trackIndex, outputBuffer, info);
this.audioCodec.releaseOutputBuffer(outputIndex, false);
}
}
@@ -181,15 +215,13 @@ public final class MediaRecorder {
}
this.audioActive = false;
if (this.mediaMuxer != null && !this.videoActive) {
Log.i(MediaRecorder.class.getSimpleName(), "结束录制文件");
Log.i(MediaRecorder.class.getSimpleName(), "结束录制文件：" + this.file);
this.mediaMuxer.stop();
this.mediaMuxer.release();
this.mediaMuxer = null;
}
}
});
this.audioThread.setName("AudioRecoder");
this.audioThread.start();
}
public void putAudio(byte[] bytes) {
@@ -207,51 +239,54 @@ public final class MediaRecorder {
private void initVideoThread(String videoType, int bitRate, int frameRate, int iFrameInterval, int width, int height) {
try {
this.videoCodec = MediaCodec.createEncoderByType(videoType);
final MediaFormat videoFormat = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, width, height);
videoFormat.setInteger(MediaFormat.KEY_BIT_RATE, 2500000);
// videoFormat.setInteger(MediaFormat.KEY_LEVEL, MediaCodecInfo.CodecProfileLevel.AVCLevel32);
final MediaFormat videoFormat = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, 360, 480);
// videoFormat.setInteger(MediaFormat.KEY_LEVEL, MediaCodecInfo.CodecProfileLevel.AVCLevel31);
// videoFormat.setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileHigh);
videoFormat.setInteger(MediaFormat.KEY_BIT_RATE, 800 * 1000);
videoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30);
// videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible);
// videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible);
videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedSemiPlanar);
videoFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1);
this.videoCodec.configure(videoFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
} catch (Exception e) {
Log.e(MediaRecorder.class.getSimpleName(), "加载视频录制线程异常", e);
}
final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
this.videoThread = new Thread(() -> {
int trackIndex;
this.videoThread = new HandlerThread("VideoRecoderThread");
this.videoThread.start();
this.videoHandler = new Handler(this.videoThread.getLooper());
this.videoHandler.post(() -> {
int trackIndex = -1;
int outputIndex;
synchronized (MediaRecorder.INSTANCE) {
Log.i(MediaRecorder.class.getSimpleName(), "开始录制视频");
this.videoCodec.start();
this.videoActive = true;
while (this.active) {
outputIndex = this.videoCodec.dequeueOutputBuffer(info, 1000L * 1000);
if (outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
} else if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
synchronized (MediaRecorder.INSTANCE) {
trackIndex = this.mediaMuxer.addTrack(this.videoCodec.getOutputFormat());
Log.i(MediaRecorder.class.getSimpleName(), "开始录制视频：" + trackIndex);
if (this.audioActive) {
Log.i(MediaRecorder.class.getSimpleName(), "开始录制文件");
Log.i(MediaRecorder.class.getSimpleName(), "开始录制文件：" + this.file);
this.pts = System.currentTimeMillis();
this.mediaMuxer.start();
MediaRecorder.INSTANCE.notifyAll();
} else {
} else if (this.active) {
try {
MediaRecorder.INSTANCE.wait();
} catch (InterruptedException e) {
}
}
}
while(this.active) {
outputIndex = this.videoCodec.dequeueOutputBuffer(info, 1000L * 1000);
if(outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
} else if(outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
} else {
Log.i(MediaRecorder.class.getSimpleName(), "======" + info.size);
final ByteBuffer outputBuffer = this.audioCodec.getOutputBuffer(outputIndex);
} else if (outputIndex >= 0) {
final ByteBuffer outputBuffer = this.videoCodec.getOutputBuffer(outputIndex);
outputBuffer.position(info.offset);
outputBuffer.limit(info.offset + info.size);
info.presentationTimeUs = info.presentationTimeUs - this.pts;
info.presentationTimeUs = (info.presentationTimeUs - this.pts) * 1000;
this.mediaMuxer.writeSampleData(trackIndex, outputBuffer, info);
this.audioCodec.releaseOutputBuffer(outputIndex, false);
this.videoCodec.releaseOutputBuffer(outputIndex, false);
Log.d(MediaRecorder.class.getSimpleName(), "录制视频帧（时间戳）：" + (info.presentationTimeUs / 1000000F));
// if(info.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME) {
// } else if(info.flags == MediaCodec.BUFFER_FLAG_CODEC_CONFIG) {
// } else if(info.flags == MediaCodec.BUFFER_FLAG_END_OF_STREAM) {
@@ -268,26 +303,24 @@ public final class MediaRecorder {
}
this.videoActive = false;
if (this.mediaMuxer != null && !this.audioActive) {
Log.i(MediaRecorder.class.getSimpleName(), "结束录制文件");
Log.i(MediaRecorder.class.getSimpleName(), "结束录制文件：" + this.file);
this.mediaMuxer.stop();
this.mediaMuxer.release();
this.mediaMuxer = null;
}
}
});
this.videoThread.setName("VideoRecoder");
this.videoThread.start();
}
public void putVideo(byte[] bytes, long pts) {
public void putVideo(ByteBuffer buffer, long pts) {
while (this.active && this.videoActive) {
final int index = this.videoCodec.dequeueInputBuffer(1000L * 1000);
if (index < 0) {
continue;
}
final ByteBuffer byteBuffer = this.videoCodec.getInputBuffer(index);
byteBuffer.put(bytes);
this.videoCodec.queueInputBuffer(index, 0, bytes.length, pts, 0);
byteBuffer.put(buffer);
this.videoCodec.queueInputBuffer(index, 0, buffer.capacity(), pts, 0);
}
}
@@ -306,10 +339,22 @@ public final class MediaRecorder {
public void stop() {
synchronized (MediaRecorder.INSTANCE) {
Log.i(MediaRecorder.class.getSimpleName(), "结束录制：" + this.file);
this.active = false;
if (audioThread != null) {
this.audioThread.quitSafely();
this.audioThread = null;
}
if (this.videoThread != null) {
this.videoThread.quitSafely();
this.videoThread = null;
}
if(this.executorService != null) {
this.executorService.shutdown();
this.executorService = null;
}
MediaRecorder.INSTANCE.notifyAll();
}
}
}

9
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/P2PClient.java → taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/SessionClient.java
View File

@@ -3,7 +3,6 @@ package com.acgist.taoyao.media;
import android.util.Log;
import java.io.Closeable;
import java.io.IOException;
/**
* P2P终端
@@ -11,11 +10,11 @@ import java.io.IOException;
*
* @author acgist
*/
public class P2PClient implements Closeable {
public class SessionClient implements Closeable {
private final String clientId;
public P2PClient(String clientId) {
public SessionClient(String clientId) {
this.clientId = clientId;
}
@@ -28,6 +27,10 @@ public class P2PClient implements Closeable {
// PeerConnectionObserver connectionObserver = getObserver();
// peerConnection = peerConnectionFactory.createPeerConnection(configuration, connectionObserver);
// pcConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true"));
//pcConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true"));
//pcConstraints.optional.add(new MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true"));
@Override
public void close() {
Log.i(Room.class.getSimpleName(), "关闭终端：" + this.clientId);

43
taoyao-client-web/src/App.vue
View File

@@ -31,10 +31,10 @@
<el-dialog center width="30%" title="房间设置" :show-close="false" v-model="roomVisible" @open="loadList">
<el-form ref="RoomSetting">
<el-tabs v-model="roomActive">
<el-tab-pane label="进入房间" name="enter">
<el-form-item label="房间标识">
<el-select v-model="room.roomId" placeholder="房间标识">
<el-option v-for="value in rooms" :key="value.roomId" :label="value.name || value.roomId" :value="value.roomId" />
<el-tab-pane label="监控终端" name="call">
<el-form-item label="终端标识">
<el-select v-model="room.callClientId" placeholder="终端标识">
<el-option v-for="value in clients" :key="value.clientId" :label="value.name || value.clientId" :value="value.clientId" />
</el-select>
</el-form-item>
</el-tab-pane>
@@ -48,6 +48,13 @@
<el-input v-model="room.name" placeholder="房间名称" />
</el-form-item>
</el-tab-pane>
<el-tab-pane label="选择房间" name="enter">
<el-form-item label="房间标识">
<el-select v-model="room.roomId" placeholder="房间标识">
<el-option v-for="value in rooms" :key="value.roomId" :label="value.name || value.roomId" :value="value.roomId" />
</el-select>
</el-form-item>
</el-tab-pane>
<el-tab-pane label="邀请终端" name="invite">
<el-form-item label="终端标识">
<el-select v-model="room.inviteClientId" placeholder="终端标识">
@@ -56,13 +63,14 @@
</el-form-item>
</el-tab-pane>
</el-tabs>
<el-form-item label="房间密码">
<el-form-item label="房间密码" v-if="roomActive !== 'call'">
<el-input v-model="room.password" placeholder="房间密码" />
</el-form-item>
</el-form>
<template #footer>
<el-button type="primary" @click="roomEnter" v-if="roomActive === 'enter'">进入</el-button>
<el-button type="primary" @click="sessionCall" v-if="roomActive === 'call'">监控</el-button>
<el-button type="primary" @click="roomCreate" v-if="roomActive === 'create'">创建</el-button>
<el-button type="primary" @click="roomEnter" v-if="roomActive === 'enter'">进入</el-button>
<el-button type="primary" @click="roomInvite" v-if="roomActive === 'invite'">邀请</el-button>
</template>
</el-dialog>
@@ -70,8 +78,9 @@
<!-- 菜单 -->
<div class="menus">
<el-button @click="signalVisible = true" type="primary" :disabled="taoyao && taoyao.connect">连接信令</el-button>
<el-button @click="roomActive = 'enter'; roomVisible = true;" type="primary" :disabled="!taoyao">选择房间</el-button>
<el-button @click="roomActive = 'call'; roomVisible = true;" :disabled="!taoyao">监控终端</el-button>
<el-button @click="roomActive = 'create'; roomVisible = true;" type="primary" :disabled="!taoyao">创建房间</el-button>
<el-button @click="roomActive = 'enter'; roomVisible = true;" type="primary" :disabled="!taoyao">选择房间</el-button>
<el-button @click="roomActive = 'invite'; roomVisible = true;" :disabled="!taoyao || !taoyao.roomId">邀请终端</el-button>
<el-button @click="roomLeave" :disabled="!taoyao || !taoyao.roomId">离开房间</el-button>
<el-button @click="roomClose" :disabled="!taoyao || !taoyao.roomId" type="danger">关闭房间</el-button>
@@ -83,6 +92,8 @@
<LocalClient v-if="taoyao && taoyao.roomId" ref="local-client" :client="taoyao" :taoyao="taoyao"></LocalClient>
<!-- 远程终端 -->
<RemoteClient v-for="(kv, index) in remoteClients" :key="index" :ref="'remote-client-' + kv[0]" :client="kv[1]" :taoyao="taoyao"></RemoteClient>
<!-- 远程会话 -->
<SessionClient v-for="(kv, index) in sessionClients" :key="index" :ref="'session-client-' + kv[0]" :client="kv[1]" :taoyao="taoyao"></SessionClient>
</div>
</div>
</template>
@@ -92,6 +103,7 @@ import { ElMessage } from 'element-plus'
import { Taoyao } from "./components/Taoyao.js";
import LocalClient from './components/LocalClient.vue';
import RemoteClient from './components/RemoteClient.vue';
import SessionClient from './components/SessionClient.vue';
export default {
name: "Taoyao",
@@ -110,10 +122,11 @@ export default {
password: "taoyao",
},
taoyao: null,
roomActive: "enter",
roomActive: "call",
roomVisible: false,
signalVisible: false,
remoteClients: new Map(),
sessionClients: new Map(),
};
},
mounted() {
@@ -131,6 +144,7 @@ export default {
await me.taoyao.connectSignal(me.callback);
me.signalVisible = false;
me.remoteClients = me.taoyao.remoteClients;
me.sessionClients = me.taoyao.sessionClients;
// 全局绑定
window.taoyao = me.taoyao;
},
@@ -145,9 +159,8 @@ export default {
async roomClose() {
this.taoyao.roomClose();
},
async roomEnter() {
await this.taoyao.roomEnter(this.room.roomId, this.room.password);
await this.taoyao.produceMedia();
async sessionCall() {
this.taoyao.sessionCall(this.room.callClientId);
this.roomVisible = false;
},
async roomCreate() {
@@ -155,6 +168,11 @@ export default {
this.room.roomId = room.roomId;
await this.roomEnter();
},
async roomEnter() {
await this.taoyao.roomEnter(this.room.roomId, this.room.password);
await this.taoyao.produceMedia();
this.roomVisible = false;
},
async roomInvite() {
this.taoyao.roomInvite(this.room.inviteClientId);
this.roomVisible = false;
@@ -190,7 +208,8 @@ export default {
},
components: {
LocalClient,
RemoteClient
RemoteClient,
SessionClient,
},
};
</script>

98
taoyao-client-web/src/components/SessionClient.vue Normal file
View File

@@ -0,0 +1,98 @@
<!-- 会话终端 -->
<template>
<div class="client">
<audio ref="audio"></audio>
<video ref="video"></video>
<p class="title">{{ client?.name || "" }}</p>
<div class="buttons">
<el-button @click="taoyao.mediaConsumerResume(audioConsumer.id)" v-show="stream" type="primary" title="打开麦克风" :icon="Microphone" circle />
<el-button @click="taoyao.mediaConsumerPause(audioConsumer.id)" v-show="stream" type="danger" title="关闭麦克风" :icon="Mute" circle />
<el-button @click="taoyao.mediaConsumerResume(videoConsumer.id)" v-show="stream" type="primary" title="打开摄像头" :icon="VideoPlay" circle />
<el-button @click="taoyao.mediaConsumerPause(videoConsumer.id)" v-show="stream" type="danger" title="关闭摄像头" :icon="VideoPause" circle />
<el-button title="拍照" :icon="Camera" circle />
<el-button title="录像" :icon="VideoCamera" circle />
<el-button @click="close" title="踢出" :icon="CircleClose" circle />
</div>
</div>
</template>
<script>
import {
Mute,
Camera,
Refresh,
VideoPlay,
VideoPause,
InfoFilled,
Microphone,
VideoCamera,
CircleClose,
} from "@element-plus/icons";
export default {
name: "SessionClient",
setup() {
return {
Mute,
Camera,
Refresh,
VideoPlay,
VideoPause,
InfoFilled,
Microphone,
VideoCamera,
CircleClose,
};
},
data() {
return {
audio: null,
video: null,
stream: null,
audioStream: null,
videoStream: null,
};
},
mounted() {
this.audio = this.$refs.audio;
this.video = this.$refs.video;
this.client.proxy = this;
},
props: {
"client": {
type: Object
},
"taoyao": {
type: Object
}
},
methods: {
close() {
this.taoyao.sessionClose(this.client.id);
},
media(track) {
console.log(track);
if(track.kind === 'audio') {
if (this.audioStream) {
// TODO：资源释放
} else {
this.audioStream = new MediaStream();
this.audioStream.addTrack(track);
this.audio.srcObject = this.audioStream;
}
this.audio.play().catch((error) => console.warn("视频播放失败", error));
} else if(track.kind === 'video') {
if (this.videoStream) {
// TODO：资源释放
} else {
this.videoStream = new MediaStream();
this.videoStream.addTrack(track);
this.video.srcObject = this.videoStream;
}
this.video.play().catch((error) => console.warn("视频播放失败", error));
} else {
}
}
}
};
</script>

201
taoyao-client-web/src/components/Taoyao.js
View File

@@ -226,6 +226,40 @@ const signalChannel = {
},
};
/**
* 会话
*/
class Session {
// 会话ID
id;
// 远程终端名称
name;
// 远程终端ID
clientId;
// 本地音频
localAudioTrack;
// 本地视频
localVideoTrack;
// 远程音频
remoteAudioTrack;
// 远程视频
remoteVideoTrack;
// PeerConnection
peerConnection;
constructor({
id,
name,
clientId
}) {
this.id = id;
this.name = name;
this.clientId = clientId;
}
}
/**
* 远程终端
*/
@@ -339,6 +373,8 @@ class Taoyao extends RemoteClient {
dataConsumers = new Map();
// 远程终端
remoteClients = new Map();
// 会话终端
sessionClients = new Map();
constructor({
name,
@@ -560,6 +596,15 @@ class Taoyao extends RemoteClient {
case "media::video::orientation::change":
me.defaultMediaVideoOrientationChange(message);
break;
case "session::call":
me.defaultSessionCall(message);
break;
case "session::close":
me.defaultSessionClose(message);
break;
case "session::exchange":
me.defaultSessionExchange(message);
break;
case "room::client::list":
me.defaultRoomClientList(message);
break;
@@ -596,6 +641,37 @@ class Taoyao extends RemoteClient {
return null;
}
}
async getStream() {
let stream;
const self = this;
if (self.videoSource === "file") {
// TODO：实现文件分享
// const stream = await this._getExternalVideoStream();
// track = stream.getVideoTracks()[0].clone();
} else if (self.videoSource === "camera") {
console.debug("enableWebcam() | calling getUserMedia()");
// TODO：参数
stream = await navigator.mediaDevices.getUserMedia({
audio: self.audioConfig,
video: self.videoConfig,
});
} else if (self.videoSource === "screen") {
stream = await navigator.mediaDevices.getDisplayMedia({
audio: self.audioConfig,
video: {
cursor: true,
width: { max: 1920 },
height: { max: 1080 },
frameRate: { max: 30 },
logicalSurface: true,
displaySurface: "monitor",
},
});
} else {
// TODO：异常
}
return stream;
}
async getVideoTrack() {
let track;
const self = this;
@@ -1696,7 +1772,7 @@ class Taoyao extends RemoteClient {
});
const tracks = stream.getAudioTracks();
if (tracks.length > 1) {
console.log("多个音频轨道");
console.warn("多个音频轨道");
}
track = tracks[0];
// TODO：验证修改API audioTrack.applyCapabilities
@@ -2006,6 +2082,129 @@ class Taoyao extends RemoteClient {
}
}
/**
* 发起会话
*
* @param {*} clientId 接收者ID
*/
async sessionCall(clientId) {
const me = this;
if (!clientId) {
this.callbackError("无效终端");
return;
}
const response = await me.request(
protocol.buildMessage("session::call", {
clientId
})
);
const { name, sessionId } = response.body;
const session = new Session(name, response.body.clientId, sessionId);
this.sessionClients.set(sessionId, session);
session.peerConnection = await me.buildPeerConnection(session, sessionId);
const localStream = await me.getStream();
session.localAudioTrack = localStream.getAudioTracks()[0];
session.localVideoTrack = localStream.getVideoTracks()[0];
session.peerConnection.addTrack(session.localAudioTrack, localStream);
session.peerConnection.addTrack(session.localVideoTrack, localStream);
}
async defaultSessionCall(message) {
const me = this;
const { name, clientId, sessionId } = message.body;
const session = new Session(name, clientId, sessionId);
this.sessionClients.set(sessionId, session);
session.peerConnection = await me.buildPeerConnection(session, sessionId);
const localStream = await me.getStream();
session.localAudioTrack = localStream.getAudioTracks()[0];
session.localVideoTrack = localStream.getVideoTracks()[0];
session.peerConnection.addTrack(session.localAudioTrack, localStream);
session.peerConnection.addTrack(session.localVideoTrack, localStream);
session.peerConnection.createOffer().then(async description => {
await session.peerConnection.setLocalDescription(description);
me.push(
protocol.buildMessage("session::exchange", {
sdp : description.sdp,
type : description.type,
sessionId: sessionId
})
);
});
}
async sessionClose() {
}
async defaultSessionClose(message) {
}
async defaultSessionExchange(message) {
const me = this;
const { type, candidate, sessionId } = message.body;
const session = this.sessionClients.get(sessionId);
if (type === "offer") {
session.peerConnection.setRemoteDescription(new RTCSessionDescription(message.body));
session.peerConnection.createAnswer().then(async description => {
await session.peerConnection.setLocalDescription(description);
me.push(
protocol.buildMessage("session::exchange", {
sdp : description.sdp,
type : description.type,
sessionId: sessionId
})
);
});
} else if (type === "answer") {
await session.peerConnection.setRemoteDescription(new RTCSessionDescription(message.body));
} else if (type === "candidate") {
if(candidate) {
await session.peerConnection.addIceCandidate(new RTCIceCandidate(candidate));
}
}
}
async buildPeerConnection(session, sessionId) {
const me = this;
const peerConnection = new RTCPeerConnection({"iceServers" : [{"url" : "stun:stun1.l.google.com:19302"}]});
peerConnection.ontrack = event => {
console.debug("buildPeerConnection ontrack", event);
const track = event.track;
if(track.kind === 'audio') {
session.remoteAudioTrack = track;
} else if(track.kind === 'video') {
session.remoteVideoTrack = track;
} else {
}
if(session.proxy && session.proxy.media) {
session.proxy.media(track);
}
};
peerConnection.onicecandidate = event => {
console.debug("buildPeerConnection onicecandidate", event);
me.push(
protocol.buildMessage("session::exchange", {
type : "candidate",
sessionId : sessionId,
candidate : event.candidate
})
);
};
peerConnection.onnegotiationneeded = event => {
console.debug("buildPeerConnection onnegotiationneeded", event);
session.peerConnection.createOffer().then(async description => {
await session.peerConnection.setLocalDescription(description);
me.push(
protocol.buildMessage("session::exchange", {
sdp : description.sdp,
type : description.type,
sessionId: sessionId
})
);
});
}
return peerConnection;
}
/**
* 关闭媒体
*/

4
taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/Constant.java
View File

@@ -161,6 +161,10 @@ public interface Constant {
* 来源终端ID
*/
String SOURCE_ID = "sourceId";
/**
* 会话ID
*/
String SESSION_ID = "sessionId";
/**
* 传输通道ID
*/

5
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/p2p/Session.java
View File

@@ -1,5 +0,0 @@
package com.acgist.taoyao.signal.party.p2p;
public class Session {
}

5
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/p2p/SessionManager.java
View File

@@ -1,5 +0,0 @@
package com.acgist.taoyao.signal.party.p2p;
public class SessionManager {
}

57
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/session/Session.java Normal file
View File

@@ -0,0 +1,57 @@
package com.acgist.taoyao.signal.party.session;
import java.io.Closeable;
import com.acgist.taoyao.boot.model.Message;
import com.acgist.taoyao.signal.client.Client;
import lombok.Getter;
/**
* P2P会话
*
* @author acgist
*/
@Getter
public class Session implements Closeable {
/**
* ID
*/
private final String id;
/**
* 发起者
*/
private final Client source;
/**
* 接收者
*/
private final Client target;
public Session(String id, Client source, Client target) {
this.id = id;
this.source = source;
this.target = target;
}
/**
* 发送消息给对方终端
*
* @param clientId 当前终端ID
* @param message 消息
*/
public void pushRemote(String clientId, Message message) {
if(this.source.clientId().equals(clientId)) {
this.target.push(message);
} else {
this.source.push(message);
}
}
@Override
public void close() {
}
}

47
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/session/SessionManager.java Normal file
View File

@@ -0,0 +1,47 @@
package com.acgist.taoyao.signal.party.session;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import com.acgist.taoyao.boot.annotation.Manager;
import com.acgist.taoyao.boot.service.IdService;
import com.acgist.taoyao.signal.client.Client;
/**
* P2P会话管理器
*
* @author acgist
*/
@Manager
public class SessionManager {
private final IdService idService;
private final Map<String, Session> sessions;
public SessionManager(IdService idService) {
this.idService = idService;
this.sessions = new ConcurrentHashMap<>();
}
/**
* @param source 发起者
* @param target 接收者
*
* @return 会话
*/
public Session call(Client source, Client target) {
final Session session = new Session(this.idService.buildUuid(), source, target);
this.sessions.put(session.getId(), session);
return session;
}
/**
* @param sessionId 会话ID
*
* @return 会话
*/
public Session get(String sessionId) {
return this.sessions.get(sessionId);
}
}

1
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolAdapter.java
View File

@@ -14,6 +14,7 @@ import com.acgist.taoyao.boot.service.IdService;
import com.acgist.taoyao.signal.client.ClientManager;
import com.acgist.taoyao.signal.event.ApplicationEventAdapter;
import com.acgist.taoyao.signal.party.media.RoomManager;
import com.acgist.taoyao.signal.party.session.SessionManager;
import lombok.extern.slf4j.Slf4j;

54
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolSessionAdapter.java Normal file
View File

@@ -0,0 +1,54 @@
package com.acgist.taoyao.signal.protocol;
import java.util.Map;
import org.springframework.beans.factory.annotation.Autowired;
import com.acgist.taoyao.boot.config.Constant;
import com.acgist.taoyao.boot.model.Message;
import com.acgist.taoyao.boot.model.MessageCodeException;
import com.acgist.taoyao.boot.utils.MapUtils;
import com.acgist.taoyao.signal.client.Client;
import com.acgist.taoyao.signal.client.ClientType;
import com.acgist.taoyao.signal.party.media.Room;
import com.acgist.taoyao.signal.party.session.Session;
import com.acgist.taoyao.signal.party.session.SessionManager;
/**
* 会话信令适配器
*
* @author acgist
*/
public abstract class ProtocolSessionAdapter extends ProtocolClientAdapter {
@Autowired
protected SessionManager sessionManager;
protected ProtocolSessionAdapter(String name, String signal) {
super(name, signal);
}
@Override
public void execute(String clientId, ClientType clientType, Client client, Message message, Map<String, Object> body) {
final String sessionId = MapUtils.get(body, Constant.SESSION_ID);
final Session session = this.sessionManager.get(sessionId);
if(session == null) {
throw MessageCodeException.of("无效会话：" + sessionId);
}
this.execute(clientId, clientType, session, client, message, body);
}
/**
* 处理终端会话信令
*
* @param clientId 终端标识
* @param clientType 终端类型
* @param session 会话
* @param client 终端
* @param message 消息
* @param body 消息主体
*/
public void execute(String clientId, ClientType clientType, Session session, Client client, Message message, Map<String, Object> body) {
}
}

5
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/p2p/P2PAnswerProtocol.java
View File

@@ -1,5 +0,0 @@
package com.acgist.taoyao.signal.protocol.p2p;
public class P2PAnswerProtocol {
}

5
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/p2p/P2PCallProtocol.java
View File

@@ -1,5 +0,0 @@
package com.acgist.taoyao.signal.protocol.p2p;
public class P2PCallProtocol {
}

5
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/p2p/P2PCandidateProtocol.java
View File

@@ -1,5 +0,0 @@
package com.acgist.taoyao.signal.protocol.p2p;
public class P2PCandidateProtocol {
}

5
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/p2p/P2POfferProtocol.java
View File

@@ -1,5 +0,0 @@
package com.acgist.taoyao.signal.protocol.p2p;
public class P2POfferProtocol {
}

65
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCallProtocol.java Normal file
View File

@@ -0,0 +1,65 @@
package com.acgist.taoyao.signal.protocol.session;
import java.util.Map;
import org.apache.tomcat.util.bcel.Const;
import com.acgist.taoyao.boot.annotation.Description;
import com.acgist.taoyao.boot.annotation.Protocol;
import com.acgist.taoyao.boot.config.Constant;
import com.acgist.taoyao.boot.model.Message;
import com.acgist.taoyao.boot.utils.MapUtils;
import com.acgist.taoyao.signal.client.Client;
import com.acgist.taoyao.signal.client.ClientType;
import com.acgist.taoyao.signal.party.session.Session;
import com.acgist.taoyao.signal.protocol.ProtocolSessionAdapter;
import lombok.extern.slf4j.Slf4j;
/**
* 发起会话信令
*
* @author acgist
*/
@Slf4j
@Protocol
@Description(
body = """
{
"clientId": "接收者ID"
}
""",
flow = {
"终端->信令服务->终端",
"终端=>信令服务->终端"
}
)
public class SessionCallProtocol extends ProtocolSessionAdapter {
public static final String SIGNAL = "session::call";
public SessionCallProtocol() {
super("发起会话信令", SIGNAL);
}
@Override
public void execute(String clientId, ClientType clientType, Client client, Message message, Map<String, Object> body) {
final String targetId = MapUtils.get(body, Constant.CLIENT_ID);
final Client target = this.clientManager.clients(targetId);
final Session session = this.sessionManager.call(client, target);
message.setBody(Map.of(
Constant.NAME, target.status().getName(),
Constant.CLIENT_ID, target.clientId(),
Constant.SESSION_ID, session.getId()
));
client.push(message);
final Message callMessage = message.cloneWithoutBody();
callMessage.setBody(Map.of(
Constant.NAME, client.status().getName(),
Constant.CLIENT_ID, client.clientId(),
Constant.SESSION_ID, session.getId()
));
target.push(callMessage);
}
}

41
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCloseProtocol.java Normal file
View File

@@ -0,0 +1,41 @@
package com.acgist.taoyao.signal.protocol.session;
import java.util.Map;
import com.acgist.taoyao.boot.annotation.Description;
import com.acgist.taoyao.boot.annotation.Protocol;
import com.acgist.taoyao.boot.model.Message;
import com.acgist.taoyao.signal.client.Client;
import com.acgist.taoyao.signal.client.ClientType;
import com.acgist.taoyao.signal.party.session.Session;
import com.acgist.taoyao.signal.protocol.ProtocolSessionAdapter;
import lombok.extern.slf4j.Slf4j;
/**
* 关闭媒体信令
*
* @author acgist
*/
@Slf4j
@Protocol
@Description(
body = """
{
}
""",
flow = "终端->信令服务->终端"
)
public class SessionCloseProtocol extends ProtocolSessionAdapter {
public static final String SIGNAL = "session::close";
public SessionCloseProtocol() {
super("关闭媒体信令", SIGNAL);
}
@Override
public void execute(String clientId, ClientType clientType, Session session, Client client, Message message, Map<String, Object> body) {
}
}

42
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionExchangeProtocol.java Normal file
View File

@@ -0,0 +1,42 @@
package com.acgist.taoyao.signal.protocol.session;
import java.util.Map;
import com.acgist.taoyao.boot.annotation.Description;
import com.acgist.taoyao.boot.annotation.Protocol;
import com.acgist.taoyao.boot.model.Message;
import com.acgist.taoyao.signal.client.Client;
import com.acgist.taoyao.signal.client.ClientType;
import com.acgist.taoyao.signal.party.session.Session;
import com.acgist.taoyao.signal.protocol.ProtocolSessionAdapter;
import lombok.extern.slf4j.Slf4j;
/**
* 媒体交换信令
*
* @author acgist
*/
@Slf4j
@Protocol
@Description(
body = """
{
}
""",
flow = "终端->信令服务->终端"
)
public class SessionExchangeProtocol extends ProtocolSessionAdapter {
public static final String SIGNAL = "session::exchange";
public SessionExchangeProtocol() {
super("媒体交换信令", SIGNAL);
}
@Override
public void execute(String clientId, ClientType clientType, Session session, Client client, Message message, Map<String, Object> body) {
session.pushRemote(clientId, message);
}
}