diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java index 84cd1f2..cdc3f66 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java @@ -1178,7 +1178,6 @@ public final class Taoyao implements ITaoyao { this.mediaManager.getWebrtcProperties() ); this.sessions.put(sessionId, sessionClient); - sessionClient.init(); sessionClient.offer(); } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Client.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Client.java index 41e0951..9798a4b 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Client.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Client.java @@ -7,6 +7,7 @@ import com.acgist.taoyao.media.config.Config; import com.acgist.taoyao.media.signal.ITaoyao; import org.webrtc.SurfaceViewRenderer; +import org.webrtc.VideoTrack; /** * 终端 @@ -98,6 +99,20 @@ public abstract class Client extends CloseableClient { this.resumeVideo(); } + /** + * 创建视频预览 + * + * @param flag Config.WHAT_* + * @param videoTrack 视频媒体流Track + */ + protected void buildSurfaceViewRenderer(final int flag, final VideoTrack videoTrack) { + if(this.surfaceViewRenderer == null) { + this.surfaceViewRenderer = this.mediaManager.buildSurfaceViewRenderer(flag, videoTrack); + } else { + Log.w(Client.class.getSimpleName(), "视频预览已经存在"); + } + } + @Override public void close() { super.close(); diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java index cfafe90..160a3d0 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java @@ -104,9 +104,7 @@ public class LocalClient extends RoomClient { } this.mediaStream.videoTracks.forEach(videoTrack -> { videoTrack.setEnabled(true); - if(this.surfaceViewRenderer == null) { - this.surfaceViewRenderer = this.mediaManager.buildSurfaceViewRenderer(Config.WHAT_NEW_LOCAL_VIDEO, videoTrack); - } + this.buildSurfaceViewRenderer(Config.WHAT_NEW_LOCAL_VIDEO, videoTrack); }); } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java index c3c77c1..9a974d7 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java @@ -89,9 +89,7 @@ public class RemoteClient extends RoomClient { .map(v -> (VideoTrack) v) .forEach(videoTrack -> { videoTrack.setEnabled(true); - if(this.surfaceViewRenderer == null) { - this.surfaceViewRenderer = this.mediaManager.buildSurfaceViewRenderer(Config.WHAT_NEW_REMOTE_VIDEO, videoTrack); - } + this.buildSurfaceViewRenderer(Config.WHAT_NEW_REMOTE_VIDEO, videoTrack); }); } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java index 36a3bf6..8a54365 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java @@ -50,7 +50,7 @@ public class Room extends CloseableClient implements RouterCallback { */ private final String clientId; /** - * 是否预览 + * 是否预览视频 */ private final boolean preview; /** diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java index 601c587..32b4c37 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java @@ -12,7 +12,6 @@ import com.acgist.taoyao.media.signal.ITaoyao; import org.webrtc.DataChannel; import org.webrtc.IceCandidate; -import org.webrtc.MediaConstraints; import org.webrtc.MediaStream; import org.webrtc.MediaStreamTrack; import org.webrtc.PeerConnection; @@ -44,26 +43,58 @@ public class SessionClient extends Client { * 会话ID */ private final String sessionId; - private final boolean preview; - private final boolean playAudio; - private final boolean playVideo; - private final boolean dataConsume; - private final boolean audioConsume; - private final boolean videoConsume; - private final boolean dataProduce; - private final boolean audioProduce; - private final boolean videoProduce; - private final MediaProperties mediaProperties; - private final WebrtcProperties webrtcProperties; - private SurfaceViewRenderer localSurfaceViewRenderer; /** - * 本地媒体 + * 是否预览视频 */ - private MediaStream mediaStream; + private final boolean preview; + /** + * 是否播放音频 + */ + private final boolean playAudio; + /** + * 是否播放视频 + */ + private final boolean playVideo; + /** + * 是否消费数据 + */ + private final boolean dataConsume; + /** + * 是否消费音频 + */ + private final boolean audioConsume; + /** + * 是否消费视频 + */ + private final boolean videoConsume; + /** + * 是否生产数据 + */ + private final boolean dataProduce; + /** + * 是否生产音频 + */ + private final boolean audioProduce; + /** + * 是否生产视频 + */ + private final boolean videoProduce; + /** + * 媒体配置 + */ + private final MediaProperties mediaProperties; + /** + * WebRTC配置 + */ + private final WebrtcProperties webrtcProperties; /** * 数据通道 */ private DataChannel dataChannel; + /** + * 本地媒体 + */ + private MediaStream localMediaStream; /** * 远程媒体 */ @@ -77,12 +108,15 @@ public class SessionClient extends Client { */ private PeerConnection.Observer observer; /** - * Peer连接工厂 + * PeerConnectionFactory */ private PeerConnectionFactory peerConnectionFactory; + /** + * 本地视频预览 + */ + private SurfaceViewRenderer localSurfaceViewRenderer; /** - * * @param sessionId 会话ID * @param name 远程终端名称 * @param clientId 远程终端ID @@ -109,7 +143,7 @@ public class SessionClient extends Client { ) { super(name, clientId, taoyao, mainHandler); this.sessionId = sessionId; - this.preview = preview; + this.preview = preview; this.playAudio = playAudio; this.playVideo = playVideo; this.dataConsume = dataConsume; @@ -134,25 +168,28 @@ public class SessionClient extends Client { final List iceServers = this.webrtcProperties.getIceServers(); final PeerConnection.RTCConfiguration configuration = new PeerConnection.RTCConfiguration(iceServers); this.observer = this.observer(); - this.mediaStream = this.mediaManager.buildLocalMediaStream(this.audioProduce, this.videoProduce); + this.localMediaStream = this.mediaManager.buildLocalMediaStream(this.audioProduce, this.videoProduce); this.peerConnection = this.peerConnectionFactory.createPeerConnection(configuration, this.observer); - this.peerConnection.addStream(this.mediaStream); - if(this.preview) { - this.previewLocal(); - } - // 设置streamId同步 + this.peerConnection.addStream(this.localMediaStream); // final List streamIds = new ArrayList<>(); -// ListUtils.getOnlyOne(this.mediaStream.audioTracks, audioTrack -> { +// this.localMediaStream.audioTracks.forEach(audioTrack -> { // this.peerConnection.addTrack(audioTrack, streamIds); -// return audioTrack; // }); -// ListUtils.getOnlyOne(this.mediaStream.videoTracks, videoTrack -> { +// this.localMediaStream.videoTracks.forEach(videoTrack -> { // this.peerConnection.addTrack(videoTrack, streamIds); -// return videoTrack; // }); + if(this.preview) { + this.previewLocalVideo(); + } } } + /** + * 媒体交换 + * + * @param message 信令消息 + * @param body 消息主体 + */ public void exchange(Message message, Map body) { final String type = MapUtils.get(body, "type"); switch(type) { @@ -164,10 +201,10 @@ public class SessionClient extends Client { } /** - * 提供媒体服务 + * 提供本地终端媒体 */ public synchronized void offer() { - final MediaConstraints mediaConstraints = this.mediaManager.buildMediaConstraints(); + this.init(); this.peerConnection.createOffer(this.sdpObserver( "主动Offer", sessionDescription -> { @@ -178,9 +215,15 @@ public class SessionClient extends Client { ), sessionDescription); }, null - ), mediaConstraints); + ), this.mediaManager.buildMediaConstraints()); } + /** + * 远程终端提供媒体 + * + * @param message 信令消息 + * @param body 消息主体 + */ private void offer(Message message, Map body) { this.init(); final String sdp = MapUtils.get(body, "sdp"); @@ -203,6 +246,12 @@ public class SessionClient extends Client { ), new SessionDescription(sdpType, sdp)); } + /** + * 远程终端响应媒体 + * + * @param message 信令消息 + * @param body 消息主体 + */ private void answer(Message message, Map body) { final String sdp = MapUtils.get(body, "sdp"); final String type = MapUtils.get(body, "type"); @@ -214,6 +263,12 @@ public class SessionClient extends Client { ), new SessionDescription(sdpType, sdp)); } + /** + * 远程终端媒体协商 + * + * @param message 信令消息 + * @param body 消息主体 + */ private void candidate(Message message, Map body) { final Map candidate = MapUtils.get(body, "candidate"); final String sdp = MapUtils.get(candidate, "candidate"); @@ -268,9 +323,7 @@ public class SessionClient extends Client { } this.remoteMediaStream.videoTracks.forEach(videoTrack -> { videoTrack.setEnabled(true); - if(this.surfaceViewRenderer == null) { - this.surfaceViewRenderer = this.mediaManager.buildSurfaceViewRenderer(Config.WHAT_NEW_REMOTE_VIDEO, videoTrack); - } + this.buildSurfaceViewRenderer(Config.WHAT_NEW_REMOTE_VIDEO, videoTrack); }); } @@ -296,45 +349,69 @@ public class SessionClient extends Client { }); } + /** + * 暂停本地媒体 + * + * @param type 媒体类型 + */ public void pauseLocal(String type) { + if(this.localMediaStream == null) { + return; + } if(MediaStreamTrack.AUDIO_TRACK_KIND.equals(type)) { - this.mediaStream.audioTracks.forEach(audioTrack -> { + this.localMediaStream.audioTracks.forEach(audioTrack -> { audioTrack.setEnabled(false); }); } else if(MediaStreamTrack.VIDEO_TRACK_KIND.equals(type)) { - this.mediaStream.videoTracks.forEach(videoTrack -> { + this.localMediaStream.videoTracks.forEach(videoTrack -> { videoTrack.setEnabled(false); }); } else { } } + /** + * 恢复本地媒体 + * + * @param type 媒体类型 + */ public void resumeLocal(String type) { + if(this.localMediaStream == null) { + return; + } if(MediaStreamTrack.AUDIO_TRACK_KIND.equals(type)) { - this.mediaStream.audioTracks.forEach(audioTrack -> { + this.localMediaStream.audioTracks.forEach(audioTrack -> { audioTrack.setEnabled(true); }); } else if(MediaStreamTrack.VIDEO_TRACK_KIND.equals(type)) { - this.mediaStream.videoTracks.forEach(videoTrack -> { + this.localMediaStream.videoTracks.forEach(videoTrack -> { videoTrack.setEnabled(true); }); } else { } } - private void previewLocal() { - if(this.mediaStream == null) { + /** + * 预览本地视频 + */ + private void previewLocalVideo() { + if(this.localMediaStream == null) { return; } - this.mediaStream.videoTracks.forEach(videoTrack -> { + this.localMediaStream.videoTracks.forEach(videoTrack -> { videoTrack.setEnabled(true); if(this.localSurfaceViewRenderer == null) { this.localSurfaceViewRenderer = this.mediaManager.buildSurfaceViewRenderer(Config.WHAT_NEW_LOCAL_VIDEO, videoTrack); + } else { + Log.w(SessionClient.class.getSimpleName(), "视频预览已经存在"); } }); } - private void releaseLocal() { + /** + * 释放本地媒体 + */ + private void releaseLocalVideo() { // 释放本地视频资源 if(this.localSurfaceViewRenderer != null) { // 释放资源 @@ -353,7 +430,7 @@ public class SessionClient extends Client { return; } super.close(); - this.releaseLocal(); + this.releaseLocalVideo(); try { // PeerConnection自动释放：mediaStream、remoteMediaStream if(this.peerConnection != null) { @@ -367,28 +444,28 @@ public class SessionClient extends Client { } /** - * @return 监听 + * @return PC观察者 */ private PeerConnection.Observer observer() { return new PeerConnection.Observer() { @Override public void onSignalingChange(PeerConnection.SignalingState signalingState) { - Log.d(SessionClient.class.getSimpleName(), "PC信令状态改变：" + signalingState); + Log.d(SessionClient.class.getSimpleName(), "PCSignalingState改变：" + signalingState); SessionClient.this.logState(); } @Override public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) { - Log.d(SessionClient.class.getSimpleName(), "PCIce收集状态改变：" + iceGatheringState); + Log.d(SessionClient.class.getSimpleName(), "PCIceGatheringState改变：" + iceGatheringState); SessionClient.this.logState(); } @Override public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) { - Log.d(SessionClient.class.getSimpleName(), "PCIce连接状态改变：" + iceConnectionState); - SessionClient.this.logState(); // disconnected：暂时连接不上可能自我恢复 + Log.d(SessionClient.class.getSimpleName(), "PCIceConnectionState改变：" + iceConnectionState); + SessionClient.this.logState(); } @Override @@ -397,17 +474,8 @@ public class SessionClient extends Client { @Override public void onIceCandidate(IceCandidate iceCandidate) { - Log.d(SessionClient.class.getSimpleName(), "发送媒体协商：" + SessionClient.this.sessionId); - final Map candidate = new HashMap<>(); - candidate.put("sdpMid", iceCandidate.sdpMid); - candidate.put("candidate", iceCandidate.sdp); - candidate.put("sdpMLineIndex", iceCandidate.sdpMLineIndex); - SessionClient.this.taoyao.push(SessionClient.this.taoyao.buildMessage( - "session::exchange", - "type", "candidate", - "candidate", candidate, - "sessionId", SessionClient.this.sessionId - )); + Log.d(SessionClient.class.getSimpleName(), "媒体协商：" + SessionClient.this.sessionId); + SessionClient.this.exchangeIceCandidate(iceCandidate); } @Override @@ -429,6 +497,7 @@ public class SessionClient extends Client { public void onRemoveStream(MediaStream mediaStream) { Log.i(SessionClient.class.getSimpleName(), "删除远程媒体：" + SessionClient.this.clientId); mediaStream.dispose(); + SessionClient.this.remoteMediaStream = null; } @Override @@ -444,9 +513,8 @@ public class SessionClient extends Client { public void onRenegotiationNeeded() { Log.d(SessionClient.class.getSimpleName(), "重新协商媒体：" + SessionClient.this.sessionId); if(SessionClient.this.peerConnection.connectionState() == PeerConnection.PeerConnectionState.CONNECTED) { -// SessionClient.this.peerConnection.restartIce(); - // TODO：重新协商 -// SessionClient.this.offer(); + // TODO：验证 + SessionClient.this.peerConnection.restartIce(); } } @@ -496,19 +564,44 @@ public class SessionClient extends Client { }; } + /** + * @param iceCandidate 媒体协商 + */ + private void exchangeIceCandidate(IceCandidate iceCandidate) { + if(iceCandidate == null) { + return; + } + final Map candidate = new HashMap<>(); + candidate.put("sdpMid", iceCandidate.sdpMid); + candidate.put("candidate", iceCandidate.sdp); + candidate.put("sdpMLineIndex", iceCandidate.sdpMLineIndex); + this.taoyao.push(this.taoyao.buildMessage( + "session::exchange", + "type", "candidate", + "candidate", candidate, + "sessionId", this.sessionId + )); + } + + /** + * @param sessionDescription SDP + */ private void exchangeSessionDescription(SessionDescription sessionDescription) { if(sessionDescription == null) { return; } final String type = sessionDescription.type.toString().toLowerCase(); - SessionClient.this.taoyao.push(SessionClient.this.taoyao.buildMessage( + this.taoyao.push(this.taoyao.buildMessage( "session::exchange", "sdp", sessionDescription.description, "type", type, - "sessionId", SessionClient.this.sessionId + "sessionId", this.sessionId )); } + /** + * 记录PC状态 + */ private void logState() { Log.d(SessionClient.class.getSimpleName(), String.format( """ diff --git a/taoyao-client-web/src/components/Taoyao.js b/taoyao-client-web/src/components/Taoyao.js index 1c83a4c..600867d 100644 --- a/taoyao-client-web/src/components/Taoyao.js +++ b/taoyao-client-web/src/components/Taoyao.js @@ -2449,6 +2449,7 @@ class Taoyao extends RemoteClient { console.debug("buildPeerConnection onnegotiationneeded", event); if(peerConnection.connectionState === "connected") { // TODO：重连 + peerConnection.restartIce(); } } const localStream = await me.getStream();