diff --git a/README.md b/README.md index 6e183ff..9773670 100644 --- a/README.md +++ b/README.md @@ -29,16 +29,17 @@ ### Web终端功能 |功能|是否支持|是否实现|描述| +|:--|:--|:--|:--| |P2P|支持|暂未实现|P2P监控模式| |WebRTC|支持|实现|Web终端不能同时进入多个房间| ### 安卓终端功能 |功能|是否支持|是否实现|描述| +|:--|:--|:--|:--| |P2P|支持|暂未实现|P2P监控模式| |WebRTC|支持|暂未实现|安卓终端支持同时进入多个房间| |RTP|支持|暂未实现|支持房间RTP推流（不会拉流）| -|||| ## 证书 diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java index bfd6764..3176ee6 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java @@ -4,7 +4,6 @@ import android.Manifest; import android.app.Activity; import android.content.Intent; import android.content.pm.PackageManager; -import android.media.projection.MediaProjection; import android.media.projection.MediaProjectionManager; import android.os.Bundle; import android.os.Handler; @@ -16,12 +15,11 @@ import android.view.SurfaceView; import android.view.View; import android.view.ViewGroup; import android.view.WindowManager; -import android.widget.Toast; +import android.widget.LinearLayout; import androidx.activity.result.ActivityResultLauncher; import androidx.activity.result.contract.ActivityResultContracts; import androidx.annotation.NonNull; -import androidx.annotation.Nullable; import androidx.appcompat.app.AppCompatActivity; import androidx.core.app.ActivityCompat; @@ -61,6 +59,7 @@ public class MainActivity extends AppCompatActivity implements Serializable { this.launchMediaService(); // 布局 this.binding = ActivityMainBinding.inflate(this.getLayoutInflater()); + this.binding.getRoot().setZ(100F); this.setContentView(this.binding.getRoot()); this.registerMediaProjection(); this.binding.record.setOnClickListener(this::switchRecord); @@ -161,7 +160,7 @@ public class MainActivity extends AppCompatActivity implements Serializable { MediaManager.getInstance().init(this.mainHandler, this.getApplicationContext()); MediaManager.getInstance().initAudio(); MediaManager.getInstance().initVideo(); - mediaRecorder.init(System.currentTimeMillis() + ".mp4", null, null, 1, 1); + MediaManager.getInstance().record(); } } @@ -216,7 +215,10 @@ public class MainActivity extends AppCompatActivity implements Serializable { */ private void newLocalVideo(Message message) { final SurfaceView surfaceView = (SurfaceView) message.obj; - this.addContentView(surfaceView, new ViewGroup.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT)); + final LinearLayout.LayoutParams layoutParams = new LinearLayout.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT); + layoutParams.weight = 1; + surfaceView.setZ(0F); + this.addContentView(surfaceView, layoutParams); } } \ No newline at end of file diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java index c750906..b046732 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java @@ -10,6 +10,7 @@ import android.net.wifi.WifiInfo; import android.net.wifi.WifiManager; import android.os.BatteryManager; import android.os.Handler; +import android.os.HandlerThread; import android.util.Log; import androidx.core.app.ActivityCompat; @@ -20,9 +21,9 @@ import com.acgist.taoyao.boot.model.MessageCode; import com.acgist.taoyao.boot.model.MessageCodeException; import com.acgist.taoyao.boot.utils.CloseableUtils; import com.acgist.taoyao.boot.utils.JSONUtils; -import com.acgist.taoyao.media.MediaRecorder; import com.acgist.taoyao.client.utils.IdUtils; -import com.acgist.taoyao.media.P2PClient; +import com.acgist.taoyao.media.MediaRecorder; +import com.acgist.taoyao.media.SessionClient; import com.acgist.taoyao.media.Room; import org.apache.commons.lang3.ArrayUtils; @@ -40,10 +41,6 @@ import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; import javax.crypto.BadPaddingException; import javax.crypto.Cipher; @@ -58,8 +55,6 @@ import javax.crypto.spec.SecretKeySpec; */ public final class Taoyao { - private static final long MAX_TIMEOUT = 60L * 1000; - /** * 端口 */ @@ -104,10 +99,6 @@ public final class Taoyao { * 超时时间 */ private final int timeout; - /** - * 重试次数 - */ - private int connectRetryTimes; /** * Socket */ @@ -131,7 +122,7 @@ public final class Taoyao { /** * Handler */ - private final Handler handler; + private final Handler mainHandler; /** * 服务上下文 */ @@ -152,14 +143,12 @@ public final class Taoyao { * 请求消息：同步消息 */ private final Map requestMessage; - /** - * 线程池 - */ - private final ExecutorService executor; - /** - * 定时任务线程池 - */ - private final ScheduledExecutorService scheduled; + private final Handler loopMessageHandler; + private final HandlerThread loopMessageThread; + private final Handler heartbeatHandler; + private final HandlerThread heartbeatThread; + private final Handler executeMessageHandler; + private final HandlerThread executeMessageThread; /** * 房间列表 */ @@ -167,13 +156,13 @@ public final class Taoyao { /** * P2P终端列表 */ - private final List p2pClientList; + private final List p2pClientList; public Taoyao( int port, String host, String version, String name, String clientId, String clientType, String username, String password, int timeout, String algo, String secret, - Handler handler, Context context, + Handler mainHandler, Context context, WifiManager wifiManager, BatteryManager batteryManager, LocationManager locationManager ) { this.close = false; @@ -187,22 +176,26 @@ public final class Taoyao { this.username = username; this.password = password; this.timeout = timeout; - this.connectRetryTimes = 1; final boolean plaintext = algo == null || algo.isEmpty() || algo.equals("PLAINTEXT"); this.encrypt = plaintext ? null : this.buildCipher(Cipher.ENCRYPT_MODE, algo, secret); this.decrypt = plaintext ? null : this.buildCipher(Cipher.DECRYPT_MODE, algo, secret); - this.handler = handler; + this.mainHandler = mainHandler; this.context = context; this.wifiManager = wifiManager; this.batteryManager = batteryManager; this.locationManager = locationManager; this.requestMessage = new ConcurrentHashMap<>(); - // 读取线程 + 两条处理线程 - this.executor = Executors.newFixedThreadPool(3); - // 心跳线程 - this.scheduled = Executors.newScheduledThreadPool(1); - this.executor.submit(this::loopMessage); - this.scheduled.scheduleWithFixedDelay(this::heartbeat, 30, 30, TimeUnit.SECONDS); + this.loopMessageThread = new HandlerThread("TaoyaoLoopMessageThread"); + this.loopMessageThread.start(); + this.loopMessageHandler = new Handler(this.loopMessageThread.getLooper()); + this.loopMessageHandler.post(this::loopMessage); + this.heartbeatThread = new HandlerThread("TaoyaoHeartbeatThread"); + this.heartbeatThread.start(); + this.heartbeatHandler = new Handler(this.heartbeatThread.getLooper()); + this.heartbeatHandler.postDelayed(this::heartbeat, 30L * 1000); + this.executeMessageThread = new HandlerThread("TaoyaoExecuteMessageThread"); + this.executeMessageThread.start(); + this.executeMessageHandler = new Handler(this.executeMessageThread.getLooper()); this.roomList = new CopyOnWriteArrayList<>(); this.p2pClientList = new CopyOnWriteArrayList<>(); } @@ -247,12 +240,18 @@ public final class Taoyao { this.output = this.socket.getOutputStream(); this.register(); this.connect = true; - this.connectRetryTimes = 1; synchronized (this) { this.notifyAll(); } } else { this.connect = false; + synchronized (this) { + try { + this.wait(this.timeout); + } catch (InterruptedException e) { + Log.d(Taoyao.class.getSimpleName(), "信令等待异常", e); + } + } } } catch (Exception e) { Log.e(Taoyao.class.getSimpleName(), "连接信令异常：" + this.host + ":" + this.port, e); @@ -273,22 +272,9 @@ public final class Taoyao { // 重连 while (!this.close && !this.connect) { this.connect(); - synchronized (this) { - try { - long timeout = this.timeout; - if(MAX_TIMEOUT > this.timeout * this.connectRetryTimes) { - timeout = this.timeout * this.connectRetryTimes++; - } else { - timeout = MAX_TIMEOUT; - } - this.wait(timeout); - } catch (InterruptedException e) { - Log.d(Taoyao.class.getSimpleName(), "信令等待异常", e); - } - } } // 读取 - while ((length = this.input.read(bytes)) >= 0) { + while (this.input != null && (length = this.input.read(bytes)) >= 0) { buffer.put(bytes, 0, length); while (buffer.position() > 0) { if (messageLength <= 0) { @@ -314,9 +300,9 @@ public final class Taoyao { buffer.get(message); buffer.compact(); final String content = new String(this.decrypt.doFinal(message)); - Log.d(Taoyao.class.getSimpleName(), "处理信令：" + content); - executor.submit(() -> { + this.executeMessageHandler.post(() -> { try { + Log.d(Taoyao.class.getSimpleName(), "处理信令：" + content); Taoyao.this.on(content); } catch (Exception e) { Log.e(Taoyao.class.getSimpleName(), "处理信令异常：" + content, e); @@ -428,10 +414,11 @@ public final class Taoyao { Log.d(Taoyao.class.getSimpleName(), "关闭信令：" + this.host + ":" + this.port); this.close = true; this.disconnect(); - this.executor.shutdown(); - this.scheduled.shutdown(); + this.heartbeatThread.quitSafely(); + this.loopMessageThread.quitSafely(); + this.executeMessageThread.quitSafely(); this.roomList.forEach(Room::close); - this.p2pClientList.forEach(P2PClient::close); + this.p2pClientList.forEach(SessionClient::close); } /** @@ -471,7 +458,6 @@ public final class Taoyao { * @param content 信令消息 */ private void on(String content) { - Log.d(Taoyao.class.getSimpleName(), "收到消息：" + content); final Message message = JSONUtils.toJava(content, Message.class); if (message == null) { return; @@ -535,6 +521,7 @@ public final class Taoyao { * 心跳 */ private void heartbeat() { + this.heartbeatHandler.postDelayed(this::heartbeat, 30L * 1000); if(this.close || !this.connect) { return; } diff --git a/taoyao-client-android/taoyao/media/CMakeLists.txt b/taoyao-client-android/taoyao/media/CMakeLists.txt index f52016f..8a4e274 100644 --- a/taoyao-client-android/taoyao/media/CMakeLists.txt +++ b/taoyao-client-android/taoyao/media/CMakeLists.txt @@ -28,17 +28,17 @@ set( ${SOURCE_DIR}/include/LocalClient.hpp ${SOURCE_DIR}/include/MediaRecorder.hpp ${SOURCE_DIR}/include/MediasoupClient.hpp - ${SOURCE_DIR}/include/P2PClient.hpp ${SOURCE_DIR}/include/RemoteClient.hpp ${SOURCE_DIR}/include/Room.hpp ${SOURCE_DIR}/include/RtpAudioPublisher.hpp ${SOURCE_DIR}/include/RtpClient.hpp ${SOURCE_DIR}/include/RtpVideoPublisher.hpp + ${SOURCE_DIR}/include/SessionClient.hpp ${SOURCE_DIR}/media/LocalClient.cpp ${SOURCE_DIR}/media/MediaRecorder.cpp - ${SOURCE_DIR}/media/P2PClient.cpp ${SOURCE_DIR}/media/RemoteClient.cpp ${SOURCE_DIR}/media/Room.cpp + ${SOURCE_DIR}/media/SessionClient.cpp ${SOURCE_DIR}/rtp/RtpAudioPublisher.cpp ${SOURCE_DIR}/rtp/RtpClient.cpp ${SOURCE_DIR}/rtp/RtpVideoPublisher.cpp diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/include/P2PClient.hpp b/taoyao-client-android/taoyao/media/src/main/cpp/include/SessionClient.hpp similarity index 100% rename from taoyao-client-android/taoyao/media/src/main/cpp/include/P2PClient.hpp rename to taoyao-client-android/taoyao/media/src/main/cpp/include/SessionClient.hpp diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/media/P2PClient.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/media/P2PClient.cpp deleted file mode 100644 index 7a80561..0000000 --- a/taoyao-client-android/taoyao/media/src/main/cpp/media/P2PClient.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "P2PClient.hpp" - -namespace acgist { - -} \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/media/SessionClient.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/media/SessionClient.cpp new file mode 100644 index 0000000..4883440 --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/cpp/media/SessionClient.cpp @@ -0,0 +1,5 @@ +#include "SessionClient.hpp" + +namespace acgist { + +} \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java index 85f4ca5..c3d267a 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java @@ -1,19 +1,26 @@ package com.acgist.taoyao.media; +import android.annotation.SuppressLint; import android.content.Context; import android.content.Intent; +import android.media.AudioFormat; import android.media.AudioRecord; +import android.media.MediaFormat; import android.media.projection.MediaProjection; import android.media.projection.MediaProjectionManager; +import android.os.Environment; import android.os.Handler; import android.os.Looper; import android.os.Message; import android.util.Log; +import android.view.Surface; import com.acgist.taoyao.config.Config; import org.webrtc.AudioSource; import org.webrtc.AudioTrack; +import org.webrtc.BuiltinAudioDecoderFactoryFactory; +import org.webrtc.BuiltinAudioEncoderFactoryFactory; import org.webrtc.Camera2Enumerator; import org.webrtc.CameraEnumerator; import org.webrtc.CameraVideoCapturer; @@ -30,15 +37,18 @@ import org.webrtc.SurfaceViewRenderer; import org.webrtc.VideoCapturer; import org.webrtc.VideoDecoderFactory; import org.webrtc.VideoEncoderFactory; +import org.webrtc.VideoFileRenderer; import org.webrtc.VideoFrame; import org.webrtc.VideoSink; import org.webrtc.VideoSource; import org.webrtc.VideoTrack; +import org.webrtc.audio.AudioDeviceModule; import org.webrtc.audio.JavaAudioDeviceModule; import org.webrtc.voiceengine.WebRtcAudioManager; import org.webrtc.voiceengine.WebRtcAudioRecord; import org.webrtc.voiceengine.WebRtcAudioUtils; +import java.nio.file.Paths; import java.util.Arrays; import java.util.Iterator; import java.util.List; @@ -85,7 +95,6 @@ public class MediaManager { return this == BACK || this == FRONT; } - } private static final MediaManager INSTANCE = new MediaManager(); @@ -137,9 +146,11 @@ public class MediaManager { // 设置采样 // WebRtcAudioUtils.setDefaultSampleRateHz(); // 噪声消除 - WebRtcAudioUtils.setWebRtcBasedNoiseSuppressor(true); - // 回声小丑 - WebRtcAudioUtils.setWebRtcBasedAcousticEchoCanceler(true); +// WebRtcAudioUtils.setWebRtcBasedNoiseSuppressor(true); + // 回声消除 +// WebRtcAudioUtils.setWebRtcBasedAcousticEchoCanceler(true); + // 自动增益 +// WebRtcAudioUtils.setWebRtcBasedAutomaticGainControl(true); // 使用OpenSL ES // WebRtcAudioManager.setBlacklistDeviceForOpenSLESUsage(true); } @@ -156,12 +167,13 @@ public class MediaManager { this.eglBase = EglBase.create(); PeerConnectionFactory.initialize( PeerConnectionFactory.InitializationOptions.builder(this.context) - .setEnableInternalTracer(true) +// .setEnableInternalTracer(true) .createInitializationOptions() ); final VideoDecoderFactory videoDecoderFactory = new DefaultVideoDecoderFactory(this.eglBase.getEglBaseContext()); final VideoEncoderFactory videoEncoderFactory = new DefaultVideoEncoderFactory(this.eglBase.getEglBaseContext(), true, true); final JavaAudioDeviceModule javaAudioDeviceModule = JavaAudioDeviceModule.builder(this.context) +// .setAudioSource(android.media.MediaRecorder.AudioSource.MIC) // 本地音频 .setSamplesReadyCallback(MediaRecorder.getInstance().audioRecoder) // 远程音频 @@ -171,8 +183,8 @@ public class MediaManager { .createAudioDeviceModule(); this.peerConnectionFactory = PeerConnectionFactory.builder() // .setAudioProcessingFactory() -// .setAudioDecoderFactoryFactory() -// .setAudioEncoderFactoryFactory() +// .setAudioEncoderFactoryFactory(new BuiltinAudioEncoderFactoryFactory()) +// .setAudioDecoderFactoryFactory(new BuiltinAudioDecoderFactoryFactory()) .setAudioDeviceModule(javaAudioDeviceModule) .setVideoDecoderFactory(videoDecoderFactory) .setVideoEncoderFactory(videoEncoderFactory) @@ -221,26 +233,22 @@ public class MediaManager { // 加载音频 final MediaConstraints mediaConstraints = new MediaConstraints(); // 高音过滤 -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googHighpassFilter", "true")); -// // 自动增益 -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googAutoGainControl", "true")); -// // 回声消除 -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googEchoCancellation", "true")); -// // 噪音处理 -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googNoiseSuppression", "true")); - // 更多 -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googEchoCancellation", "true")); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googEchoCancellation2", "true")); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googDAEchoCancellation", "true")); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googTypingNoiseDetection", "true")); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googAutoGainControl", "true")); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googAutoGainControl2", "true")); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googNoiseSuppression", "true")); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googNoiseSuppression2", "true")); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googAudioMirroring", "false")); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googHighpassFilter", "true")); + mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googHighpassFilter", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googAudioMirroring", "false")); + // 自动增益 + mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googAutoGainControl", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googAutoGainControl2", "true")); + // 回声消除 + mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googEchoCancellation", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googEchoCancellation2", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googDAEchoCancellation", "true")); + // 噪音处理 + mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googNoiseSuppression", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googNoiseSuppression2", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googTypingNoiseDetection", "true")); final AudioSource audioSource = this.peerConnectionFactory.createAudioSource(mediaConstraints); final AudioTrack audioTrack = this.peerConnectionFactory.createAudioTrack("ARDAMSa0", audioSource); +// audioTrack.setVolume(100); audioTrack.setEnabled(true); this.mediaStream.addTrack(audioTrack); Log.i(MediaManager.class.getSimpleName(), "加载音频：" + audioTrack.id()); @@ -299,7 +307,7 @@ public class MediaManager { final SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("MediaVideoThread", this.eglBase.getEglBaseContext()); final VideoSource videoSource = this.peerConnectionFactory.createVideoSource(this.videoCapturer.isScreencast()); this.videoCapturer.initialize(surfaceTextureHelper, this.context, videoSource.getCapturerObserver()); - this.videoCapturer.startCapture(640, 480, 30); + this.videoCapturer.startCapture(480, 640, 30); final VideoTrack videoTrack = this.peerConnectionFactory.createVideoTrack("ARDAMSv0", videoSource); videoTrack.addSink(surfaceViewRenderer); videoTrack.addSink(MediaRecorder.getInstance().videoRecoder); @@ -329,6 +337,10 @@ public class MediaManager { message.obj = surfaceViewRenderer; message.what = Config.WHAT_NEW_LOCAL_VIDEO; this.handler.sendMessage(message); + // 暂停 +// surfaceViewRenderer.pauseVideo(); + // 恢复 +// surfaceViewRenderer.disableFpsReduction(); return surfaceViewRenderer; } @@ -362,6 +374,10 @@ public class MediaManager { } } + public void record() { + MediaRecorder.getInstance().init(System.currentTimeMillis() + ".mp4", null, null, 1, 1); + } + /** * 关闭声音 */ @@ -408,6 +424,12 @@ public class MediaManager { * 释放资源 */ public void close() { + this.closeAudioTrack(); + this.closeVideoTrack(); + if(this.eglBase != null) { + this.eglBase.release(); + this.eglBase = null; + } if(this.videoCapturer != null) { this.videoCapturer.dispose(); this.videoCapturer = null; diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaRecorder.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaRecorder.java index a4e379f..8721514 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaRecorder.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaRecorder.java @@ -1,5 +1,6 @@ package com.acgist.taoyao.media; +import android.graphics.YuvImage; import android.media.AudioFormat; import android.media.MediaCodec; import android.media.MediaCodecInfo; @@ -7,24 +8,35 @@ import android.media.MediaCodecList; import android.media.MediaFormat; import android.media.MediaMuxer; import android.os.Environment; +import android.os.Handler; +import android.os.HandlerThread; import android.util.Log; +import android.view.Surface; +import org.webrtc.EglBase; +import org.webrtc.GlRectDrawer; +import org.webrtc.HardwareVideoEncoderFactory; +import org.webrtc.VideoEncoderFactory; import org.webrtc.VideoFrame; +import org.webrtc.VideoFrameDrawer; import org.webrtc.VideoSink; +import org.webrtc.YuvConverter; +import org.webrtc.YuvHelper; import org.webrtc.audio.JavaAudioDeviceModule; -import org.webrtc.voiceengine.WebRtcAudioRecord; -import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Paths; +import java.util.Optional; +import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.stream.Collectors; import java.util.stream.IntStream; -import java.util.stream.Stream; /** * 录像机 - * + *

* https://blog.csdn.net/m0_60259116/article/details/126875532 * * @author acgist @@ -33,6 +45,43 @@ public final class MediaRecorder { private static final MediaRecorder INSTANCE = new MediaRecorder(); + public static final MediaRecorder getInstance() { + return INSTANCE; + } + + /** + * 是否正在录像 + */ + private volatile boolean active; + private volatile boolean audioActive; + private volatile boolean videoActive; + private volatile long pts; + private String file; + /** + * 音频编码 + */ + private MediaCodec audioCodec; + private HandlerThread audioThread; + /** + * 视频编码 + */ + private MediaCodec videoCodec; + private HandlerThread videoThread; + private Handler videoHandler; + private ExecutorService executorService; + /** + * 媒体合成器 + */ + private MediaMuxer mediaMuxer; + /** + * 音频录制 + */ + public final JavaAudioDeviceModule.SamplesReadyCallback audioRecoder; + /** + * 视频录制 + */ + public final VideoSink videoRecoder; + private MediaRecorder() { final MediaCodecList mediaCodecList = new MediaCodecList(-1); for (MediaCodecInfo mediaCodecInfo : mediaCodecList.getCodecInfos()) { @@ -49,54 +98,36 @@ public final class MediaRecorder { } } } + this.executorService = Executors.newFixedThreadPool(2); this.audioRecoder = audioSamples -> { - Log.d(MediaRecorder.class.getSimpleName(), audioSamples + " - 音频"); }; this.videoRecoder = videoFrame -> { -// Log.d(MediaRecorder.class.getSimpleName(), videoFrame + " - 视频"); - if(this.active && this.videoActive) { - final VideoFrame.Buffer buffer = videoFrame.getBuffer(); - final VideoFrame.I420Buffer i420Buffer = buffer.toI420(); - i420Buffer.getDataU(); -// this.putVideo(videoFrame.getBuffer(), videoFrame.getTimestampNs()); + if (this.active && this.videoActive) { + this.executorService.submit(() -> { +// videoFrame.retain(); + final int outputFrameSize = videoFrame.getRotatedWidth() * videoFrame.getRotatedHeight() * 3 / 2; + final ByteBuffer outputFrameBuffer = ByteBuffer.allocateDirect(outputFrameSize); + final int index = this.videoCodec.dequeueInputBuffer(1000L * 1000); +// YuvImage：截图 + // YV12 + VideoFrame.I420Buffer i420 = videoFrame.getBuffer().toI420(); +// i420.retain(); + Log.i(MediaRecorder.class.getSimpleName(), "视频信息：" + videoFrame.getRotatedWidth() + " - " + videoFrame.getRotatedHeight()); +// YuvHelper.I420Copy(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight()); + // NV12 + YuvHelper.I420ToNV12(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight()); +// YuvHelper.I420Rotate(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight(), videoFrame.getRotation()); + final ByteBuffer x = this.videoCodec.getInputBuffer(index); +// i420.release(); + x.put(outputFrameBuffer.array()); + this.videoCodec.queueInputBuffer(index, 0, outputFrameSize, System.currentTimeMillis(), 0); +// this.putVideo(outputFrameBuffer, System.currentTimeMillis()); +// videoFrame.release(); + }); } }; } - public static final MediaRecorder getInstance() { - return INSTANCE; - } - - /** - * 是否正在录像 - */ - private volatile boolean active; - private volatile boolean audioActive; - private volatile boolean videoActive; - private volatile long pts; - /** - * 音频编码 - */ - private MediaCodec audioCodec; - private Thread audioThread; - /** - * 视频编码 - */ - private MediaCodec videoCodec; - private Thread videoThread; - /** - * 媒体合成器 - */ - private MediaMuxer mediaMuxer; - /** - * 音频录制 - */ - public final JavaAudioDeviceModule.SamplesReadyCallback audioRecoder; - /** - * 视频录制 - */ - public final VideoSink videoRecoder; - /** * @return 是否正在录像 */ @@ -106,24 +137,24 @@ public final class MediaRecorder { public void init(String file, String audioFormat, String videoFormat, int width, int height) { synchronized (MediaRecorder.INSTANCE) { + this.file = file; this.active = true; - if( + if ( this.audioThread == null || !this.audioThread.isAlive() || - this.videoThread == null || !this.videoThread.isAlive() + this.videoThread == null || !this.videoThread.isAlive() ) { this.initMediaMuxer(file); this.initAudioThread(MediaFormat.MIMETYPE_AUDIO_AAC, 96000, 44100, 1); this.initVideoThread(MediaFormat.MIMETYPE_VIDEO_AVC, 2500 * 1000, 30, 1, 1920, 1080); } -// this.audioCodec = MediaCodec.createByCodecName(); } } /** - * @param audioType 类型 - * @param bitRate 比特率：96 * 1000 | 128 * 1000 | 256 * 1000 - * @param sampleRate 采样率：32000 | 44100 | 48000 - * @param channelCount 通道数量 + * @param audioType 类型 + * @param bitRate 比特率：96 * 1000 | 128 * 1000 | 256 * 1000 + * @param sampleRate 采样率：32000 | 44100 | 48000 + * @param channelCount 通道数量 */ private void initAudioThread(String audioType, int bitRate, int sampleRate, int channelCount) { try { @@ -139,57 +170,58 @@ public final class MediaRecorder { Log.e(MediaRecorder.class.getSimpleName(), "加载音频录制线程异常", e); } final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); - this.audioThread = new Thread(() -> { - int trackIndex; + this.audioThread = new HandlerThread("AudioRecoderThread"); + this.audioThread.start(); + final Handler audioHandler = new Handler(this.audioThread.getLooper()); + audioHandler.post(() -> { + int trackIndex = -1; int outputIndex; - synchronized (MediaRecorder.INSTANCE) { - Log.i(MediaRecorder.class.getSimpleName(), "开始录制音频"); - this.audioCodec.start(); - this.audioActive = true; - trackIndex = this.mediaMuxer.addTrack(this.audioCodec.getOutputFormat()); - if(this.videoActive) { - Log.i(MediaRecorder.class.getSimpleName(), "开始录制文件"); - this.pts = System.currentTimeMillis(); - this.mediaMuxer.start(); - MediaRecorder.INSTANCE.notifyAll(); - } else { - try { - MediaRecorder.INSTANCE.wait(); - } catch (InterruptedException e) { - } - } - } - while(this.active) { + this.audioCodec.start(); + this.audioActive = true; + while (this.active) { outputIndex = this.audioCodec.dequeueOutputBuffer(info, 1000L * 1000); - if(outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { - } else if(outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { - } else { + if (outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { + } else if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + synchronized (MediaRecorder.INSTANCE) { + trackIndex = this.mediaMuxer.addTrack(this.audioCodec.getOutputFormat()); + Log.i(MediaRecorder.class.getSimpleName(), "开始录制音频：" + trackIndex); + if (this.videoActive) { + Log.i(MediaRecorder.class.getSimpleName(), "开始录制文件：" + this.file); + this.pts = System.currentTimeMillis(); + this.mediaMuxer.start(); + MediaRecorder.INSTANCE.notifyAll(); + } else if (this.active) { + try { + MediaRecorder.INSTANCE.wait(); + } catch (InterruptedException e) { + } + } + } + } else if (outputIndex >= 0) { final ByteBuffer outputBuffer = this.audioCodec.getOutputBuffer(outputIndex); outputBuffer.position(info.offset); outputBuffer.limit(info.offset + info.size); - info.presentationTimeUs = info.presentationTimeUs - this.pts; - this.mediaMuxer.writeSampleData(trackIndex, outputBuffer, info); + info.presentationTimeUs = (info.presentationTimeUs - this.pts) * 1000; +// this.mediaMuxer.writeSampleData(trackIndex, outputBuffer, info); this.audioCodec.releaseOutputBuffer(outputIndex, false); } } synchronized (MediaRecorder.INSTANCE) { - if(this.audioCodec != null) { + if (this.audioCodec != null) { Log.i(MediaRecorder.class.getSimpleName(), "结束录制音频"); this.audioCodec.stop(); this.audioCodec.release(); this.audioCodec = null; } this.audioActive = false; - if(this.mediaMuxer != null && !this.videoActive) { - Log.i(MediaRecorder.class.getSimpleName(), "结束录制文件"); + if (this.mediaMuxer != null && !this.videoActive) { + Log.i(MediaRecorder.class.getSimpleName(), "结束录制文件：" + this.file); this.mediaMuxer.stop(); this.mediaMuxer.release(); this.mediaMuxer = null; } } }); - this.audioThread.setName("AudioRecoder"); - this.audioThread.start(); } public void putAudio(byte[] bytes) { @@ -207,51 +239,54 @@ public final class MediaRecorder { private void initVideoThread(String videoType, int bitRate, int frameRate, int iFrameInterval, int width, int height) { try { this.videoCodec = MediaCodec.createEncoderByType(videoType); - final MediaFormat videoFormat = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, width, height); - videoFormat.setInteger(MediaFormat.KEY_BIT_RATE, 2500000); -// videoFormat.setInteger(MediaFormat.KEY_LEVEL, MediaCodecInfo.CodecProfileLevel.AVCLevel32); -// videoFormat.setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileHigh); + final MediaFormat videoFormat = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, 360, 480); +// videoFormat.setInteger(MediaFormat.KEY_LEVEL, MediaCodecInfo.CodecProfileLevel.AVCLevel31); +// videoFormat.setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileHigh); + videoFormat.setInteger(MediaFormat.KEY_BIT_RATE, 800 * 1000); videoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30); -// videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); - videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible); +// videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible); + videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420PackedSemiPlanar); videoFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1); this.videoCodec.configure(videoFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); } catch (Exception e) { Log.e(MediaRecorder.class.getSimpleName(), "加载视频录制线程异常", e); } final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); - this.videoThread = new Thread(() -> { - int trackIndex; + this.videoThread = new HandlerThread("VideoRecoderThread"); + this.videoThread.start(); + this.videoHandler = new Handler(this.videoThread.getLooper()); + this.videoHandler.post(() -> { + int trackIndex = -1; int outputIndex; - synchronized (MediaRecorder.INSTANCE) { - Log.i(MediaRecorder.class.getSimpleName(), "开始录制视频"); - this.videoCodec.start(); - this.videoActive = true; - trackIndex = this.mediaMuxer.addTrack(this.videoCodec.getOutputFormat()); - if(this.audioActive) { - Log.i(MediaRecorder.class.getSimpleName(), "开始录制文件"); - this.pts = System.currentTimeMillis(); - this.mediaMuxer.start(); - MediaRecorder.INSTANCE.notifyAll(); - } else { - try { - MediaRecorder.INSTANCE.wait(); - } catch (InterruptedException e) { - } - } - } - while(this.active) { + this.videoCodec.start(); + this.videoActive = true; + while (this.active) { outputIndex = this.videoCodec.dequeueOutputBuffer(info, 1000L * 1000); - if(outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { - } else if(outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { - } else { - Log.i(MediaRecorder.class.getSimpleName(), "======" + info.size); - final ByteBuffer outputBuffer = this.audioCodec.getOutputBuffer(outputIndex); + if (outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { + } else if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + synchronized (MediaRecorder.INSTANCE) { + trackIndex = this.mediaMuxer.addTrack(this.videoCodec.getOutputFormat()); + Log.i(MediaRecorder.class.getSimpleName(), "开始录制视频：" + trackIndex); + if (this.audioActive) { + Log.i(MediaRecorder.class.getSimpleName(), "开始录制文件：" + this.file); + this.pts = System.currentTimeMillis(); + this.mediaMuxer.start(); + MediaRecorder.INSTANCE.notifyAll(); + } else if (this.active) { + try { + MediaRecorder.INSTANCE.wait(); + } catch (InterruptedException e) { + } + } + } + } else if (outputIndex >= 0) { + final ByteBuffer outputBuffer = this.videoCodec.getOutputBuffer(outputIndex); outputBuffer.position(info.offset); outputBuffer.limit(info.offset + info.size); - info.presentationTimeUs = info.presentationTimeUs - this.pts; + info.presentationTimeUs = (info.presentationTimeUs - this.pts) * 1000; this.mediaMuxer.writeSampleData(trackIndex, outputBuffer, info); - this.audioCodec.releaseOutputBuffer(outputIndex, false); + this.videoCodec.releaseOutputBuffer(outputIndex, false); + Log.d(MediaRecorder.class.getSimpleName(), "录制视频帧（时间戳）：" + (info.presentationTimeUs / 1000000F)); // if(info.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME) { // } else if(info.flags == MediaCodec.BUFFER_FLAG_CODEC_CONFIG) { // } else if(info.flags == MediaCodec.BUFFER_FLAG_END_OF_STREAM) { @@ -260,34 +295,32 @@ public final class MediaRecorder { } } synchronized (MediaRecorder.INSTANCE) { - if(this.videoCodec != null) { + if (this.videoCodec != null) { Log.i(MediaRecorder.class.getSimpleName(), "结束录制视频"); this.videoCodec.stop(); this.videoCodec.release(); this.videoCodec = null; } this.videoActive = false; - if(this.mediaMuxer != null && !this.audioActive) { - Log.i(MediaRecorder.class.getSimpleName(), "结束录制文件"); + if (this.mediaMuxer != null && !this.audioActive) { + Log.i(MediaRecorder.class.getSimpleName(), "结束录制文件：" + this.file); this.mediaMuxer.stop(); this.mediaMuxer.release(); this.mediaMuxer = null; } } }); - this.videoThread.setName("VideoRecoder"); - this.videoThread.start(); } - public void putVideo(byte[] bytes, long pts) { - while(this.active && this.videoActive) { + public void putVideo(ByteBuffer buffer, long pts) { + while (this.active && this.videoActive) { final int index = this.videoCodec.dequeueInputBuffer(1000L * 1000); - if(index < 0) { + if (index < 0) { continue; } final ByteBuffer byteBuffer = this.videoCodec.getInputBuffer(index); - byteBuffer.put(bytes); - this.videoCodec.queueInputBuffer(index, 0, bytes.length, pts, 0); + byteBuffer.put(buffer); + this.videoCodec.queueInputBuffer(index, 0, buffer.capacity(), pts, 0); } } @@ -305,10 +338,22 @@ public final class MediaRecorder { } public void stop() { - synchronized(MediaRecorder.INSTANCE) { + synchronized (MediaRecorder.INSTANCE) { + Log.i(MediaRecorder.class.getSimpleName(), "结束录制：" + this.file); this.active = false; - this.audioThread = null; - this.videoThread = null; + if (audioThread != null) { + this.audioThread.quitSafely(); + this.audioThread = null; + } + if (this.videoThread != null) { + this.videoThread.quitSafely(); + this.videoThread = null; + } + if(this.executorService != null) { + this.executorService.shutdown(); + this.executorService = null; + } + MediaRecorder.INSTANCE.notifyAll(); } } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/P2PClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/SessionClient.java similarity index 70% rename from taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/P2PClient.java rename to taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/SessionClient.java index 7a42958..cec8f4b 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/P2PClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/SessionClient.java @@ -3,7 +3,6 @@ package com.acgist.taoyao.media; import android.util.Log; import java.io.Closeable; -import java.io.IOException; /** * P2P终端 @@ -11,11 +10,11 @@ import java.io.IOException; * * @author acgist */ -public class P2PClient implements Closeable { +public class SessionClient implements Closeable { private final String clientId; - public P2PClient(String clientId) { + public SessionClient(String clientId) { this.clientId = clientId; } @@ -28,6 +27,10 @@ public class P2PClient implements Closeable { // PeerConnectionObserver connectionObserver = getObserver(); // peerConnection = peerConnectionFactory.createPeerConnection(configuration, connectionObserver); +// pcConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")); +//pcConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")); +//pcConstraints.optional.add(new MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true")); + @Override public void close() { Log.i(Room.class.getSimpleName(), "关闭终端：" + this.clientId); diff --git a/taoyao-client-web/src/App.vue b/taoyao-client-web/src/App.vue index b82ede7..05b3f4c 100644 --- a/taoyao-client-web/src/App.vue +++ b/taoyao-client-web/src/App.vue @@ -31,10 +31,10 @@ - - - - + + + + @@ -48,6 +48,13 @@ + + + + + + + @@ -56,13 +63,14 @@ - + @@ -70,8 +78,9 @@

连接信令 - 选择房间 + 监控终端 创建房间 + 选择房间 邀请终端 离开房间 关闭房间 @@ -83,6 +92,8 @@ + +
@@ -92,6 +103,7 @@ import { ElMessage } from 'element-plus' import { Taoyao } from "./components/Taoyao.js"; import LocalClient from './components/LocalClient.vue'; import RemoteClient from './components/RemoteClient.vue'; +import SessionClient from './components/SessionClient.vue'; export default { name: "Taoyao", @@ -110,10 +122,11 @@ export default { password: "taoyao", }, taoyao: null, - roomActive: "enter", + roomActive: "call", roomVisible: false, signalVisible: false, remoteClients: new Map(), + sessionClients: new Map(), }; }, mounted() { @@ -131,6 +144,7 @@ export default { await me.taoyao.connectSignal(me.callback); me.signalVisible = false; me.remoteClients = me.taoyao.remoteClients; + me.sessionClients = me.taoyao.sessionClients; // 全局绑定 window.taoyao = me.taoyao; }, @@ -145,9 +159,8 @@ export default { async roomClose() { this.taoyao.roomClose(); }, - async roomEnter() { - await this.taoyao.roomEnter(this.room.roomId, this.room.password); - await this.taoyao.produceMedia(); + async sessionCall() { + this.taoyao.sessionCall(this.room.callClientId); this.roomVisible = false; }, async roomCreate() { @@ -155,6 +168,11 @@ export default { this.room.roomId = room.roomId; await this.roomEnter(); }, + async roomEnter() { + await this.taoyao.roomEnter(this.room.roomId, this.room.password); + await this.taoyao.produceMedia(); + this.roomVisible = false; + }, async roomInvite() { this.taoyao.roomInvite(this.room.inviteClientId); this.roomVisible = false; @@ -190,7 +208,8 @@ export default { }, components: { LocalClient, - RemoteClient + RemoteClient, + SessionClient, }, }; diff --git a/taoyao-client-web/src/components/SessionClient.vue b/taoyao-client-web/src/components/SessionClient.vue new file mode 100644 index 0000000..de0176b --- /dev/null +++ b/taoyao-client-web/src/components/SessionClient.vue @@ -0,0 +1,98 @@ + + + + diff --git a/taoyao-client-web/src/components/Taoyao.js b/taoyao-client-web/src/components/Taoyao.js index 89c8c8a..6c25fba 100644 --- a/taoyao-client-web/src/components/Taoyao.js +++ b/taoyao-client-web/src/components/Taoyao.js @@ -226,6 +226,40 @@ const signalChannel = { }, }; +/** + * 会话 + */ +class Session { + + // 会话ID + id; + // 远程终端名称 + name; + // 远程终端ID + clientId; + // 本地音频 + localAudioTrack; + // 本地视频 + localVideoTrack; + // 远程音频 + remoteAudioTrack; + // 远程视频 + remoteVideoTrack; + // PeerConnection + peerConnection; + + constructor({ + id, + name, + clientId + }) { + this.id = id; + this.name = name; + this.clientId = clientId; + } + +} + /** * 远程终端 */ @@ -339,6 +373,8 @@ class Taoyao extends RemoteClient { dataConsumers = new Map(); // 远程终端 remoteClients = new Map(); + // 会话终端 + sessionClients = new Map(); constructor({ name, @@ -560,6 +596,15 @@ class Taoyao extends RemoteClient { case "media::video::orientation::change": me.defaultMediaVideoOrientationChange(message); break; + case "session::call": + me.defaultSessionCall(message); + break; + case "session::close": + me.defaultSessionClose(message); + break; + case "session::exchange": + me.defaultSessionExchange(message); + break; case "room::client::list": me.defaultRoomClientList(message); break; @@ -596,6 +641,37 @@ class Taoyao extends RemoteClient { return null; } } + async getStream() { + let stream; + const self = this; + if (self.videoSource === "file") { + // TODO：实现文件分享 + // const stream = await this._getExternalVideoStream(); + // track = stream.getVideoTracks()[0].clone(); + } else if (self.videoSource === "camera") { + console.debug("enableWebcam() | calling getUserMedia()"); + // TODO：参数 + stream = await navigator.mediaDevices.getUserMedia({ + audio: self.audioConfig, + video: self.videoConfig, + }); + } else if (self.videoSource === "screen") { + stream = await navigator.mediaDevices.getDisplayMedia({ + audio: self.audioConfig, + video: { + cursor: true, + width: { max: 1920 }, + height: { max: 1080 }, + frameRate: { max: 30 }, + logicalSurface: true, + displaySurface: "monitor", + }, + }); + } else { + // TODO：异常 + } + return stream; + } async getVideoTrack() { let track; const self = this; @@ -1696,7 +1772,7 @@ class Taoyao extends RemoteClient { }); const tracks = stream.getAudioTracks(); if (tracks.length > 1) { - console.log("多个音频轨道"); + console.warn("多个音频轨道"); } track = tracks[0]; // TODO：验证修改API audioTrack.applyCapabilities @@ -2006,6 +2082,129 @@ class Taoyao extends RemoteClient { } } + /** + * 发起会话 + * + * @param {*} clientId 接收者ID + */ + async sessionCall(clientId) { + const me = this; + if (!clientId) { + this.callbackError("无效终端"); + return; + } + const response = await me.request( + protocol.buildMessage("session::call", { + clientId + }) + ); + const { name, sessionId } = response.body; + const session = new Session(name, response.body.clientId, sessionId); + this.sessionClients.set(sessionId, session); + session.peerConnection = await me.buildPeerConnection(session, sessionId); + const localStream = await me.getStream(); + session.localAudioTrack = localStream.getAudioTracks()[0]; + session.localVideoTrack = localStream.getVideoTracks()[0]; + session.peerConnection.addTrack(session.localAudioTrack, localStream); + session.peerConnection.addTrack(session.localVideoTrack, localStream); + } + + async defaultSessionCall(message) { + const me = this; + const { name, clientId, sessionId } = message.body; + const session = new Session(name, clientId, sessionId); + this.sessionClients.set(sessionId, session); + session.peerConnection = await me.buildPeerConnection(session, sessionId); + const localStream = await me.getStream(); + session.localAudioTrack = localStream.getAudioTracks()[0]; + session.localVideoTrack = localStream.getVideoTracks()[0]; + session.peerConnection.addTrack(session.localAudioTrack, localStream); + session.peerConnection.addTrack(session.localVideoTrack, localStream); + session.peerConnection.createOffer().then(async description => { + await session.peerConnection.setLocalDescription(description); + me.push( + protocol.buildMessage("session::exchange", { + sdp : description.sdp, + type : description.type, + sessionId: sessionId + }) + ); + }); + } + + async sessionClose() { + } + + async defaultSessionClose(message) { + } + + async defaultSessionExchange(message) { + const me = this; + const { type, candidate, sessionId } = message.body; + const session = this.sessionClients.get(sessionId); + if (type === "offer") { + session.peerConnection.setRemoteDescription(new RTCSessionDescription(message.body)); + session.peerConnection.createAnswer().then(async description => { + await session.peerConnection.setLocalDescription(description); + me.push( + protocol.buildMessage("session::exchange", { + sdp : description.sdp, + type : description.type, + sessionId: sessionId + }) + ); + }); + } else if (type === "answer") { + await session.peerConnection.setRemoteDescription(new RTCSessionDescription(message.body)); + } else if (type === "candidate") { + if(candidate) { + await session.peerConnection.addIceCandidate(new RTCIceCandidate(candidate)); + } + } + } + + async buildPeerConnection(session, sessionId) { + const me = this; + const peerConnection = new RTCPeerConnection({"iceServers" : [{"url" : "stun:stun1.l.google.com:19302"}]}); + peerConnection.ontrack = event => { + console.debug("buildPeerConnection ontrack", event); + const track = event.track; + if(track.kind === 'audio') { + session.remoteAudioTrack = track; + } else if(track.kind === 'video') { + session.remoteVideoTrack = track; + } else { + } + if(session.proxy && session.proxy.media) { + session.proxy.media(track); + } + }; + peerConnection.onicecandidate = event => { + console.debug("buildPeerConnection onicecandidate", event); + me.push( + protocol.buildMessage("session::exchange", { + type : "candidate", + sessionId : sessionId, + candidate : event.candidate + }) + ); + }; + peerConnection.onnegotiationneeded = event => { + console.debug("buildPeerConnection onnegotiationneeded", event); + session.peerConnection.createOffer().then(async description => { + await session.peerConnection.setLocalDescription(description); + me.push( + protocol.buildMessage("session::exchange", { + sdp : description.sdp, + type : description.type, + sessionId: sessionId + }) + ); + }); + } + return peerConnection; + } + /** * 关闭媒体 */ diff --git a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/Constant.java b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/Constant.java index d298eaf..19b819d 100644 --- a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/Constant.java +++ b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/Constant.java @@ -161,6 +161,10 @@ public interface Constant { * 来源终端ID */ String SOURCE_ID = "sourceId"; + /** + * 会话ID + */ + String SESSION_ID = "sessionId"; /** * 传输通道ID */ diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/p2p/Session.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/p2p/Session.java deleted file mode 100644 index 0567d31..0000000 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/p2p/Session.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.acgist.taoyao.signal.party.p2p; - -public class Session { - -} diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/p2p/SessionManager.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/p2p/SessionManager.java deleted file mode 100644 index 5ebd6c4..0000000 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/p2p/SessionManager.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.acgist.taoyao.signal.party.p2p; - -public class SessionManager { - -} diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/session/Session.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/session/Session.java new file mode 100644 index 0000000..c6f9eaa --- /dev/null +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/session/Session.java @@ -0,0 +1,57 @@ +package com.acgist.taoyao.signal.party.session; + +import java.io.Closeable; + +import com.acgist.taoyao.boot.model.Message; +import com.acgist.taoyao.signal.client.Client; + +import lombok.Getter; + +/** + * P2P会话 + * + * @author acgist + */ +@Getter +public class Session implements Closeable { + + /** + * ID + */ + private final String id; + /** + * 发起者 + */ + private final Client source; + /** + * 接收者 + */ + private final Client target; + + public Session(String id, Client source, Client target) { + this.id = id; + this.source = source; + this.target = target; + } + + /** + * 发送消息给对方终端 + * + * @param clientId 当前终端ID + * @param message 消息 + */ + public void pushRemote(String clientId, Message message) { + if(this.source.clientId().equals(clientId)) { + this.target.push(message); + } else { + this.source.push(message); + } + } + + @Override + public void close() { + + } + + +} diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/session/SessionManager.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/session/SessionManager.java new file mode 100644 index 0000000..b0ba260 --- /dev/null +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/session/SessionManager.java @@ -0,0 +1,47 @@ +package com.acgist.taoyao.signal.party.session; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import com.acgist.taoyao.boot.annotation.Manager; +import com.acgist.taoyao.boot.service.IdService; +import com.acgist.taoyao.signal.client.Client; + +/** + * P2P会话管理器 + * + * @author acgist + */ +@Manager +public class SessionManager { + + private final IdService idService; + private final Map sessions; + + public SessionManager(IdService idService) { + this.idService = idService; + this.sessions = new ConcurrentHashMap<>(); + } + + /** + * @param source 发起者 + * @param target 接收者 + * + * @return 会话 + */ + public Session call(Client source, Client target) { + final Session session = new Session(this.idService.buildUuid(), source, target); + this.sessions.put(session.getId(), session); + return session; + } + + /** + * @param sessionId 会话ID + * + * @return 会话 + */ + public Session get(String sessionId) { + return this.sessions.get(sessionId); + } + +} diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolAdapter.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolAdapter.java index ae60b42..e070d94 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolAdapter.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolAdapter.java @@ -14,6 +14,7 @@ import com.acgist.taoyao.boot.service.IdService; import com.acgist.taoyao.signal.client.ClientManager; import com.acgist.taoyao.signal.event.ApplicationEventAdapter; import com.acgist.taoyao.signal.party.media.RoomManager; +import com.acgist.taoyao.signal.party.session.SessionManager; import lombok.extern.slf4j.Slf4j; diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolSessionAdapter.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolSessionAdapter.java new file mode 100644 index 0000000..bff01b3 --- /dev/null +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolSessionAdapter.java @@ -0,0 +1,54 @@ +package com.acgist.taoyao.signal.protocol; + +import java.util.Map; + +import org.springframework.beans.factory.annotation.Autowired; + +import com.acgist.taoyao.boot.config.Constant; +import com.acgist.taoyao.boot.model.Message; +import com.acgist.taoyao.boot.model.MessageCodeException; +import com.acgist.taoyao.boot.utils.MapUtils; +import com.acgist.taoyao.signal.client.Client; +import com.acgist.taoyao.signal.client.ClientType; +import com.acgist.taoyao.signal.party.media.Room; +import com.acgist.taoyao.signal.party.session.Session; +import com.acgist.taoyao.signal.party.session.SessionManager; + +/** + * 会话信令适配器 + * + * @author acgist + */ +public abstract class ProtocolSessionAdapter extends ProtocolClientAdapter { + + @Autowired + protected SessionManager sessionManager; + + protected ProtocolSessionAdapter(String name, String signal) { + super(name, signal); + } + + @Override + public void execute(String clientId, ClientType clientType, Client client, Message message, Map body) { + final String sessionId = MapUtils.get(body, Constant.SESSION_ID); + final Session session = this.sessionManager.get(sessionId); + if(session == null) { + throw MessageCodeException.of("无效会话：" + sessionId); + } + this.execute(clientId, clientType, session, client, message, body); + } + + /** + * 处理终端会话信令 + * + * @param clientId 终端标识 + * @param clientType 终端类型 + * @param session 会话 + * @param client 终端 + * @param message 消息 + * @param body 消息主体 + */ + public void execute(String clientId, ClientType clientType, Session session, Client client, Message message, Map body) { + } + +} diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/p2p/P2PAnswerProtocol.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/p2p/P2PAnswerProtocol.java deleted file mode 100644 index 4553fcc..0000000 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/p2p/P2PAnswerProtocol.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.acgist.taoyao.signal.protocol.p2p; - -public class P2PAnswerProtocol { - -} diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/p2p/P2PCallProtocol.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/p2p/P2PCallProtocol.java deleted file mode 100644 index cb256fe..0000000 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/p2p/P2PCallProtocol.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.acgist.taoyao.signal.protocol.p2p; - -public class P2PCallProtocol { - -} diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/p2p/P2PCandidateProtocol.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/p2p/P2PCandidateProtocol.java deleted file mode 100644 index c5b37c4..0000000 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/p2p/P2PCandidateProtocol.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.acgist.taoyao.signal.protocol.p2p; - -public class P2PCandidateProtocol { - -} diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/p2p/P2POfferProtocol.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/p2p/P2POfferProtocol.java deleted file mode 100644 index 6278a3c..0000000 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/p2p/P2POfferProtocol.java +++ /dev/null @@ -1,5 +0,0 @@ -package com.acgist.taoyao.signal.protocol.p2p; - -public class P2POfferProtocol { - -} diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCallProtocol.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCallProtocol.java new file mode 100644 index 0000000..0f3b854 --- /dev/null +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCallProtocol.java @@ -0,0 +1,65 @@ +package com.acgist.taoyao.signal.protocol.session; + +import java.util.Map; + +import org.apache.tomcat.util.bcel.Const; + +import com.acgist.taoyao.boot.annotation.Description; +import com.acgist.taoyao.boot.annotation.Protocol; +import com.acgist.taoyao.boot.config.Constant; +import com.acgist.taoyao.boot.model.Message; +import com.acgist.taoyao.boot.utils.MapUtils; +import com.acgist.taoyao.signal.client.Client; +import com.acgist.taoyao.signal.client.ClientType; +import com.acgist.taoyao.signal.party.session.Session; +import com.acgist.taoyao.signal.protocol.ProtocolSessionAdapter; + +import lombok.extern.slf4j.Slf4j; + +/** + * 发起会话信令 + * + * @author acgist + */ +@Slf4j +@Protocol +@Description( + body = """ + { + "clientId": "接收者ID" + } + """, + flow = { + "终端->信令服务->终端", + "终端=>信令服务->终端" + } +) +public class SessionCallProtocol extends ProtocolSessionAdapter { + + public static final String SIGNAL = "session::call"; + + public SessionCallProtocol() { + super("发起会话信令", SIGNAL); + } + + @Override + public void execute(String clientId, ClientType clientType, Client client, Message message, Map body) { + final String targetId = MapUtils.get(body, Constant.CLIENT_ID); + final Client target = this.clientManager.clients(targetId); + final Session session = this.sessionManager.call(client, target); + message.setBody(Map.of( + Constant.NAME, target.status().getName(), + Constant.CLIENT_ID, target.clientId(), + Constant.SESSION_ID, session.getId() + )); + client.push(message); + final Message callMessage = message.cloneWithoutBody(); + callMessage.setBody(Map.of( + Constant.NAME, client.status().getName(), + Constant.CLIENT_ID, client.clientId(), + Constant.SESSION_ID, session.getId() + )); + target.push(callMessage); + } + +} diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCloseProtocol.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCloseProtocol.java new file mode 100644 index 0000000..58180c3 --- /dev/null +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCloseProtocol.java @@ -0,0 +1,41 @@ +package com.acgist.taoyao.signal.protocol.session; + +import java.util.Map; + +import com.acgist.taoyao.boot.annotation.Description; +import com.acgist.taoyao.boot.annotation.Protocol; +import com.acgist.taoyao.boot.model.Message; +import com.acgist.taoyao.signal.client.Client; +import com.acgist.taoyao.signal.client.ClientType; +import com.acgist.taoyao.signal.party.session.Session; +import com.acgist.taoyao.signal.protocol.ProtocolSessionAdapter; + +import lombok.extern.slf4j.Slf4j; + +/** + * 关闭媒体信令 + * + * @author acgist + */ +@Slf4j +@Protocol +@Description( + body = """ + { + } + """, + flow = "终端->信令服务->终端" +) +public class SessionCloseProtocol extends ProtocolSessionAdapter { + + public static final String SIGNAL = "session::close"; + + public SessionCloseProtocol() { + super("关闭媒体信令", SIGNAL); + } + + @Override + public void execute(String clientId, ClientType clientType, Session session, Client client, Message message, Map body) { + } + +} diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionExchangeProtocol.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionExchangeProtocol.java new file mode 100644 index 0000000..8e61005 --- /dev/null +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionExchangeProtocol.java @@ -0,0 +1,42 @@ +package com.acgist.taoyao.signal.protocol.session; + +import java.util.Map; + +import com.acgist.taoyao.boot.annotation.Description; +import com.acgist.taoyao.boot.annotation.Protocol; +import com.acgist.taoyao.boot.model.Message; +import com.acgist.taoyao.signal.client.Client; +import com.acgist.taoyao.signal.client.ClientType; +import com.acgist.taoyao.signal.party.session.Session; +import com.acgist.taoyao.signal.protocol.ProtocolSessionAdapter; + +import lombok.extern.slf4j.Slf4j; + +/** + * 媒体交换信令 + * + * @author acgist + */ +@Slf4j +@Protocol +@Description( + body = """ + { + } + """, + flow = "终端->信令服务->终端" +) +public class SessionExchangeProtocol extends ProtocolSessionAdapter { + + public static final String SIGNAL = "session::exchange"; + + public SessionExchangeProtocol() { + super("媒体交换信令", SIGNAL); + } + + @Override + public void execute(String clientId, ClientType clientType, Session session, Client client, Message message, Map body) { + session.pushRemote(clientId, message); + } + +}