From 08b86b5702137f99c51d0113aeaf6726f7cb5101 Mon Sep 17 00:00:00 2001 From: acgist <289547414@qq.com> Date: Mon, 17 Apr 2023 08:27:01 +0800 Subject: [PATCH] =?UTF-8?q?[*]=20=E8=A7=86=E9=A2=91=E4=BC=9A=E8=AF=9D?= =?UTF-8?q?=E7=AE=A1=E7=90=86=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../acgist/taoyao/client/MainActivity.java | 4 +- .../com/acgist/taoyao/media/MediaManager.java | 9 ++++ .../src/components/SessionClient.vue | 1 - taoyao-signal-server/README.md | 4 ++ .../taoyao/boot/config/WebrtcProperties.java | 1 - .../signal/event/SessionEventAdapter.java | 43 +++++++++++++++++++ .../event/session/SessionCloseEvent.java | 19 ++++++++ .../taoyao/signal/party/media/Room.java | 1 + .../taoyao/signal/party/session/Session.java | 23 ++++++---- .../signal/party/session/SessionManager.java | 25 +++++++++-- .../signal/protocol/ProtocolAdapter.java | 3 ++ .../protocol/ProtocolSessionAdapter.java | 6 --- .../protocol/client/ClientCloseProtocol.java | 2 + .../protocol/session/SessionCallProtocol.java | 7 +++ .../session/SessionCloseProtocol.java | 14 +++++- 15 files changed, 139 insertions(+), 23 deletions(-) create mode 100644 taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/event/SessionEventAdapter.java create mode 100644 taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/event/session/SessionCloseEvent.java diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java index 92ce942..4679b9d 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java @@ -244,8 +244,8 @@ public class MainActivity extends AppCompatActivity implements Serializable { private void previewVideo(Message message) { final GridLayout video = this.binding.video; final int count = video.getChildCount(); - final GridLayout.Spec rowSpec = GridLayout.spec(count / 2); - final GridLayout.Spec columnSpec = GridLayout.spec(count % 2); + final GridLayout.Spec rowSpec = GridLayout.spec(count / 2, 1, 0); + final GridLayout.Spec columnSpec = GridLayout.spec(count % 2, 1, 0); GridLayout.LayoutParams layoutParams = new GridLayout.LayoutParams(rowSpec, columnSpec); layoutParams.width = 0; layoutParams.height = 0; diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java index 9a8d29b..a123703 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java @@ -2,6 +2,7 @@ package com.acgist.taoyao.media; import android.content.Context; import android.content.Intent; +import android.media.AudioRecord; import android.media.MediaCodecInfo; import android.media.MediaCodecList; import android.media.projection.MediaProjection; @@ -41,6 +42,9 @@ import org.webrtc.VideoFrame; import org.webrtc.VideoSource; import org.webrtc.VideoTrack; import org.webrtc.audio.JavaAudioDeviceModule; +import org.webrtc.voiceengine.WebRtcAudioManager; +import org.webrtc.voiceengine.WebRtcAudioRecord; +import org.webrtc.voiceengine.WebRtcAudioUtils; import java.util.Arrays; @@ -361,6 +365,11 @@ public final class MediaManager { // .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION) // .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) // .build(); +// WebRtcAudioRecord.setOnAudioSamplesReady(audioSamples -> { +// if(this.recordClient != null) { +// this.recordClient.onWebRtcAudioRecordSamplesReady(audioSamples); +// } +// }); final JavaAudioDeviceModule javaAudioDeviceModule = JavaAudioDeviceModule.builder(this.context) // .setSampleRate() // .setAudioSource(MediaRecorder.AudioSource.MIC) diff --git a/taoyao-client-web/src/components/SessionClient.vue b/taoyao-client-web/src/components/SessionClient.vue index 08d6fd5..0b6d82a 100644 --- a/taoyao-client-web/src/components/SessionClient.vue +++ b/taoyao-client-web/src/components/SessionClient.vue @@ -70,7 +70,6 @@ export default { this.taoyao.sessionClose(this.client.id); }, media(track) { - console.log(track); if(track.kind === 'audio') { if (this.audioStream) { // TODO：资源释放 diff --git a/taoyao-signal-server/README.md b/taoyao-signal-server/README.md index 6f9c815..b011e1f 100644 --- a/taoyao-signal-server/README.md +++ b/taoyao-signal-server/README.md @@ -12,3 +12,7 @@ ## 信令格式 [信令格式](https://localhost:8888/protocol/list) + +## STUN/TURN + +视频房间不用`STUN/TURN`服务，视频会话需要自己搭建`coturn`服务。 diff --git a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/WebrtcProperties.java b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/WebrtcProperties.java index 5b51b82..1f63195 100644 --- a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/WebrtcProperties.java +++ b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/WebrtcProperties.java @@ -13,7 +13,6 @@ import lombok.Setter; /** * WebRTC配置 - * P2P视频监控会用，正常会议不会使用，需要自己搭建`coturn`服务。 * * @author acgist */ diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/event/SessionEventAdapter.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/event/SessionEventAdapter.java new file mode 100644 index 0000000..d096249 --- /dev/null +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/event/SessionEventAdapter.java @@ -0,0 +1,43 @@ +package com.acgist.taoyao.signal.event; + +import java.util.Map; + +import com.acgist.taoyao.boot.model.Message; +import com.acgist.taoyao.signal.party.session.Session; + +import lombok.Getter; + +/** + * 视频会话事件适配器 + * + * @author acgist + */ +@Getter +public abstract class SessionEventAdapter extends ApplicationEventAdapter { + + private static final long serialVersionUID = 1L; + + /** + * 视频会话 + */ + private final Session session; + /** + * 视频会话ID + */ + private final String sessionId; + + public SessionEventAdapter(Session session) { + this(session, null, null); + } + + public SessionEventAdapter(Session session, Message message) { + this(session, message, null); + } + + public SessionEventAdapter(Session session, Message message, Map body) { + super(session, message, body); + this.session = session; + this.sessionId = session.getId(); + } + +} diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/event/session/SessionCloseEvent.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/event/session/SessionCloseEvent.java new file mode 100644 index 0000000..b0c5d0e --- /dev/null +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/event/session/SessionCloseEvent.java @@ -0,0 +1,19 @@ +package com.acgist.taoyao.signal.event.session; + +import com.acgist.taoyao.signal.event.SessionEventAdapter; +import com.acgist.taoyao.signal.party.session.Session; + +/** + * 关闭视频会话事件 + * + * @author acgist + */ +public class SessionCloseEvent extends SessionEventAdapter { + + private static final long serialVersionUID = 1L; + + public SessionCloseEvent(Session session) { + super(session); + } + +} diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/media/Room.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/media/Room.java index 115e2d9..e90a704 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/media/Room.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/media/Room.java @@ -139,6 +139,7 @@ public class Room extends OperatorAdapter { synchronized (this.clients) { final ClientWrapper wrapper = this.clients.remove(client); if(wrapper != null) { + log.info("终端离开房间：{} - {}", this.roomId, client.clientId()); try { wrapper.close(); } catch (Exception e) { diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/session/Session.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/session/Session.java index 568fa94..b1c46e2 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/session/Session.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/session/Session.java @@ -4,11 +4,13 @@ import java.io.Closeable; import com.acgist.taoyao.boot.model.Message; import com.acgist.taoyao.signal.client.Client; +import com.acgist.taoyao.signal.event.EventPublisher; +import com.acgist.taoyao.signal.event.session.SessionCloseEvent; import lombok.Getter; /** - * P2P会话 + * 视频会话 * * @author acgist */ @@ -27,16 +29,11 @@ public class Session implements Closeable { * 接收者 */ private final Client target; - /** - * P2P会话管理器 - */ - private final SessionManager sessionManager; - public Session(String id, Client source, Client target, SessionManager sessionManager) { + public Session(String id, Client source, Client target) { this.id = id; this.source = source; this.target = target; - this.sessionManager = sessionManager; } /** @@ -63,10 +60,18 @@ public class Session implements Closeable { } } + /** + * @param client 终端 + * + * @return 是否含有终端 + */ + public boolean hasClient(Client client) { + return this.source == client || this.target == client; + } + @Override public void close() { - this.sessionManager.remove(this.id); + EventPublisher.publishEvent(new SessionCloseEvent(this)); } - } diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/session/SessionManager.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/session/SessionManager.java index 1e42059..ea0084d 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/session/SessionManager.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/session/SessionManager.java @@ -7,11 +7,14 @@ import com.acgist.taoyao.boot.annotation.Manager; import com.acgist.taoyao.boot.service.IdService; import com.acgist.taoyao.signal.client.Client; +import lombok.extern.slf4j.Slf4j; + /** - * P2P会话管理器 + * 视频会话管理器 * * @author acgist */ +@Slf4j @Manager public class SessionManager { @@ -30,8 +33,9 @@ public class SessionManager { * @return 会话 */ public Session call(Client source, Client target) { - final Session session = new Session(this.idService.buildUuid(), source, target, this); + final Session session = new Session(this.idService.buildUuid(), source, target); this.sessions.put(session.getId(), session); + log.info("创建视频会话：{} - {} - {}", session.getId(), session.getSource().clientId(), session.getTarget().clientId()); return session; } @@ -50,7 +54,22 @@ public class SessionManager { * @return 会话 */ public Session remove(String sessionId) { - return this.sessions.remove(sessionId); + final Session session = this.sessions.remove(sessionId); + if(session != null) { + log.info("视频会话关闭：{} - {} - {}", sessionId, session.getSource().clientId(), session.getTarget().clientId()); + } + return session; + } + + /** + * 关闭所有资源 + * + * @param client 终端 + */ + public void close(Client client) { + this.sessions.values().stream() + .filter(v -> v.hasClient(client)) + .forEach(Session::close); } } diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolAdapter.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolAdapter.java index ae60b42..e278ffc 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolAdapter.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolAdapter.java @@ -14,6 +14,7 @@ import com.acgist.taoyao.boot.service.IdService; import com.acgist.taoyao.signal.client.ClientManager; import com.acgist.taoyao.signal.event.ApplicationEventAdapter; import com.acgist.taoyao.signal.party.media.RoomManager; +import com.acgist.taoyao.signal.party.session.SessionManager; import lombok.extern.slf4j.Slf4j; @@ -32,6 +33,8 @@ public abstract class ProtocolAdapter implements Protocol { @Autowired protected ClientManager clientManager; @Autowired + protected SessionManager sessionManager; + @Autowired protected TaoyaoProperties taoyaoProperties; @Autowired protected ApplicationContext applicationContext; diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolSessionAdapter.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolSessionAdapter.java index 34b2d44..7c45563 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolSessionAdapter.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolSessionAdapter.java @@ -2,8 +2,6 @@ package com.acgist.taoyao.signal.protocol; import java.util.Map; -import org.springframework.beans.factory.annotation.Autowired; - import com.acgist.taoyao.boot.config.Constant; import com.acgist.taoyao.boot.model.Message; import com.acgist.taoyao.boot.model.MessageCodeException; @@ -11,7 +9,6 @@ import com.acgist.taoyao.boot.utils.MapUtils; import com.acgist.taoyao.signal.client.Client; import com.acgist.taoyao.signal.client.ClientType; import com.acgist.taoyao.signal.party.session.Session; -import com.acgist.taoyao.signal.party.session.SessionManager; /** * 会话信令适配器 @@ -20,9 +17,6 @@ import com.acgist.taoyao.signal.party.session.SessionManager; */ public abstract class ProtocolSessionAdapter extends ProtocolClientAdapter { - @Autowired - protected SessionManager sessionManager; - protected ProtocolSessionAdapter(String name, String signal) { super(name, signal); } diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/client/ClientCloseProtocol.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/client/ClientCloseProtocol.java index 6649008..fd9d099 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/client/ClientCloseProtocol.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/client/ClientCloseProtocol.java @@ -69,6 +69,8 @@ public class ClientCloseProtocol extends ProtocolClientAdapter implements Applic log.info("关闭终端：{}", clientId); // 释放房间终端 this.roomManager.leave(client); + // 释放会话终端 + this.sessionManager.close(client); // 终端下线事件 this.publishEvent(new ClientOfflineEvent(client)); } diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCallProtocol.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCallProtocol.java index ed51752..628f86a 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCallProtocol.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCallProtocol.java @@ -12,11 +12,14 @@ import com.acgist.taoyao.signal.client.ClientType; import com.acgist.taoyao.signal.party.session.Session; import com.acgist.taoyao.signal.protocol.ProtocolSessionAdapter; +import lombok.extern.slf4j.Slf4j; + /** * 发起会话信令 * * @author acgist */ +@Slf4j @Protocol @Description( body = """ @@ -43,6 +46,10 @@ public class SessionCallProtocol extends ProtocolSessionAdapter { public void execute(String clientId, ClientType clientType, Client client, Message message, Map body) { final String targetId = MapUtils.get(body, Constant.CLIENT_ID); final Client target = this.clientManager.clients(targetId); + if(target == null) { + log.warn("邀请对象无效：{}", clientId); + return; + } final Session session = this.sessionManager.call(client, target); message.setBody(Map.of( Constant.NAME, target.status().getName(), diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCloseProtocol.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCloseProtocol.java index bc782b5..04b215e 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCloseProtocol.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCloseProtocol.java @@ -2,11 +2,15 @@ package com.acgist.taoyao.signal.protocol.session; import java.util.Map; +import org.springframework.context.ApplicationListener; + import com.acgist.taoyao.boot.annotation.Description; import com.acgist.taoyao.boot.annotation.Protocol; +import com.acgist.taoyao.boot.config.Constant; import com.acgist.taoyao.boot.model.Message; import com.acgist.taoyao.signal.client.Client; import com.acgist.taoyao.signal.client.ClientType; +import com.acgist.taoyao.signal.event.session.SessionCloseEvent; import com.acgist.taoyao.signal.party.session.Session; import com.acgist.taoyao.signal.protocol.ProtocolSessionAdapter; @@ -23,7 +27,7 @@ import com.acgist.taoyao.signal.protocol.ProtocolSessionAdapter; """, flow = "终端->信令服务+)终端" ) -public class SessionCloseProtocol extends ProtocolSessionAdapter { +public class SessionCloseProtocol extends ProtocolSessionAdapter implements ApplicationListener { public static final String SIGNAL = "session::close"; @@ -31,6 +35,14 @@ public class SessionCloseProtocol extends ProtocolSessionAdapter { super("关闭媒体信令", SIGNAL); } + @Override + public void onApplicationEvent(SessionCloseEvent event) { + final Session session = event.getSession(); + final Map body = Map.of(Constant.SESSION_ID, event.getSessionId()); + session.push(this.build(body)); + this.sessionManager.remove(session.getId()); + } + @Override public void execute(String clientId, ClientType clientType, Session session, Client client, Message message, Map body) { session.push(message);