From 3cbbc8f936bf65230ea0b10f3f257c3f9aa429f7 Mon Sep 17 00:00:00 2001 From: acgist <289547414@qq.com> Date: Sat, 29 Apr 2023 16:28:45 +0800 Subject: [PATCH] =?UTF-8?q?[+]=20=E6=8B=8D=E7=85=A7=E3=80=81=E5=BD=95?= =?UTF-8?q?=E5=83=8F=E6=8E=A7=E5=88=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../acgist/taoyao/client/MainActivity.java | 16 ++- .../acgist/taoyao/client/signal/Taoyao.java | 23 +++- .../com/acgist/taoyao/media/MediaManager.java | 2 + .../taoyao/media/client/SessionClient.java | 31 +++++ .../acgist/taoyao/media/config/Config.java | 4 + .../src/components/RemoteClient.vue | 9 +- .../src/components/SessionClient.vue | 14 +- taoyao-client-web/src/components/Taoyao.js | 124 ++++++++++++++++-- .../taoyao/signal/client/ClientAdapter.java | 6 +- 9 files changed, 201 insertions(+), 28 deletions(-) diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java index eec658a..45c3b5e 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java @@ -4,7 +4,9 @@ import android.Manifest; import android.app.Activity; import android.content.Intent; import android.content.pm.PackageManager; +import android.content.res.ColorStateList; import android.content.res.Resources; +import android.graphics.Color; import android.media.projection.MediaProjectionManager; import android.os.Bundle; import android.os.Handler; @@ -29,6 +31,7 @@ import com.acgist.taoyao.client.signal.Taoyao; import com.acgist.taoyao.media.MediaManager; import com.acgist.taoyao.media.VideoSourceType; import com.acgist.taoyao.media.config.Config; +import com.google.android.material.floatingactionbutton.FloatingActionButton; import java.io.Serializable; import java.util.stream.Stream; @@ -180,7 +183,7 @@ public class MainActivity extends AppCompatActivity implements Serializable { } this.threadHandler.post(() -> { // 进入房间 - Taoyao.taoyao.roomEnter("4f19f6fc-1763-499b-a352-d8c955af5a6e", null); +// Taoyao.taoyao.roomEnter("4f19f6fc-1763-499b-a352-d8c955af5a6e", null); // Taoyao.taoyao.sessionCall("taoyao"); }); } @@ -216,6 +219,7 @@ public class MainActivity extends AppCompatActivity implements Serializable { Log.d(MainHandler.class.getSimpleName(), "Handler消息：" + message.what + " - " + message.obj); switch (message.what) { case Config.WHAT_SCREEN_CAPTURE -> MainActivity.this.screenCapture(message); + case Config.WHAT_RECORD -> MainActivity.this.record(message); case Config.WHAT_NEW_LOCAL_VIDEO, Config.WHAT_NEW_REMOTE_VIDEO -> MainActivity.this.previewVideo(message); case Config.WHAT_REMOVE_VIDEO -> MainActivity.this.removeVideo(message); @@ -233,6 +237,16 @@ public class MainActivity extends AppCompatActivity implements Serializable { this.activityResultLauncher.launch(this.mediaProjectionManager.createScreenCaptureIntent()); } + private void record(Message message) { + final Resources resources = this.getResources(); + final FloatingActionButton record = this.binding.record; + if(Boolean.TRUE.equals(message.obj)) { + record.setBackgroundTintList(ColorStateList.valueOf(resources.getColor(R.color.purple_500, this.getTheme()))); + } else { + record.setBackgroundTintList(ColorStateList.valueOf(resources.getColor(R.color.teal_200, this.getTheme()))); + } + } + /** * 预览用户视频 * diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java index 33317ce..e164218 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java @@ -537,6 +537,8 @@ public final class Taoyao implements ITaoyao { switch (header.getSignal()) { case "control::config::audio" -> this.controlConfigAudio(message, message.body()); case "control::config::video" -> this.controlConfigVideo(message, message.body()); + case "control::photograph" -> this.controlPhotograph(message, message.body()); + case "control::record" -> this.controlRecord(message, message.body()); case "client::config" -> this.clientConfig(message, message.body()); case "client::register" -> this.clientRegister(message, message.body()); case "client::reboot" -> this.clientReboot(message, message.body()); @@ -601,6 +603,21 @@ public final class Taoyao implements ITaoyao { this.mediaManager.updateVideoConfig(mediaVideoProperties); } + private void controlPhotograph(Message message, Map body) { + this.mediaManager.photograph(); + } + + private void controlRecord(Message message, Map body) { + final Boolean enabled = MapUtils.getBoolean(body, "enabled"); + if(Boolean.TRUE.equals(enabled)) { + this.mediaManager.startRecord(); + } else { + this.mediaManager.stopRecord(); + } + body.put("enabled", enabled); + this.push(message); + } + /** * @param message 消息 * @param body 消息主体 @@ -890,7 +907,8 @@ public final class Taoyao implements ITaoyao { if(sessionClient == null) { return; } - sessionClient.pause(); + final String type = MapUtils.get(body, "type"); + sessionClient.pause(type); } private void sessionResume(Message message, Map body) { @@ -899,7 +917,8 @@ public final class Taoyao implements ITaoyao { if(sessionClient == null) { return; } - sessionClient.resume(); + final String type = MapUtils.get(body, "type"); + sessionClient.resume(type); } /** diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java index 4991a57..76a9618 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java @@ -632,6 +632,7 @@ public final class MediaManager { this.recordClient.start(); final VideoTrack videoTrack = this.peerConnectionFactory.createVideoTrack("TaoyaoVR", this.mainVideoSource); this.recordClient.source(videoTrack); + this.mainHandler.obtainMessage(Config.WHAT_RECORD, Boolean.TRUE).sendToTarget(); return this.recordClient; } } @@ -643,6 +644,7 @@ public final class MediaManager { } else { this.recordClient.close(); this.recordClient = null; + this.mainHandler.obtainMessage(Config.WHAT_RECORD, Boolean.FALSE).sendToTarget(); } } } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java index be430f1..b0f7be7 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java @@ -15,6 +15,7 @@ import org.webrtc.DataChannel; import org.webrtc.IceCandidate; import org.webrtc.MediaConstraints; import org.webrtc.MediaStream; +import org.webrtc.MediaStreamTrack; import org.webrtc.PeerConnection; import org.webrtc.PeerConnectionFactory; import org.webrtc.SdpObserver; @@ -292,6 +293,21 @@ public class SessionClient extends Client { this.pauseVideo(); } + public void pause(String type) { + if(MediaStreamTrack.AUDIO_TRACK_KIND.equals(type)) { + ListUtils.getOnlyOne(this.mediaStream.audioTracks, audioTrack -> { + audioTrack.setEnabled(false); + return audioTrack; + }); + } else if(MediaStreamTrack.VIDEO_TRACK_KIND.equals(type)) { + ListUtils.getOnlyOne(this.mediaStream.videoTracks, videoTrack -> { + videoTrack.setEnabled(false); + return videoTrack; + }); + } else { + } + } + @Override public void resume() { super.resume(); @@ -299,6 +315,21 @@ public class SessionClient extends Client { this.resumeVideo(); } + public void resume(String type) { + if(MediaStreamTrack.AUDIO_TRACK_KIND.equals(type)) { + ListUtils.getOnlyOne(this.mediaStream.audioTracks, audioTrack -> { + audioTrack.setEnabled(true); + return audioTrack; + }); + } else if(MediaStreamTrack.VIDEO_TRACK_KIND.equals(type)) { + ListUtils.getOnlyOne(this.mediaStream.videoTracks, videoTrack -> { + videoTrack.setEnabled(true); + return videoTrack; + }); + } else { + } + } + @Override public void close() { synchronized (this) { diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/Config.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/Config.java index cb1cd8c..6de815a 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/Config.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/Config.java @@ -11,6 +11,10 @@ public class Config { * 屏幕捕获 */ public static final int WHAT_SCREEN_CAPTURE = 1000; + /** + * 视频录制 + */ + public static final int WHAT_RECORD = 1001; /** * 新建本地音频 */ diff --git a/taoyao-client-web/src/components/RemoteClient.vue b/taoyao-client-web/src/components/RemoteClient.vue index 79d20c6..a32939d 100644 --- a/taoyao-client-web/src/components/RemoteClient.vue +++ b/taoyao-client-web/src/components/RemoteClient.vue @@ -6,11 +6,11 @@

{{ client?.name || "" }}

- + - - - + + +
@@ -48,6 +48,7 @@ export default { return { audio: null, video: null, + record: false, audioStream: null, videoStream: null, dataConsumer: null, diff --git a/taoyao-client-web/src/components/SessionClient.vue b/taoyao-client-web/src/components/SessionClient.vue index 0b6d82a..5581834 100644 --- a/taoyao-client-web/src/components/SessionClient.vue +++ b/taoyao-client-web/src/components/SessionClient.vue @@ -5,12 +5,12 @@

{{ client?.name || "" }}

- - - - - - + + + + + +
@@ -47,7 +47,7 @@ export default { return { audio: null, video: null, - stream: null, + record: false, audioStream: null, videoStream: null, }; diff --git a/taoyao-client-web/src/components/Taoyao.js b/taoyao-client-web/src/components/Taoyao.js index a880661..6d4a86f 100644 --- a/taoyao-client-web/src/components/Taoyao.js +++ b/taoyao-client-web/src/components/Taoyao.js @@ -177,6 +177,8 @@ const signalChannel = { }, /** * 重连 + * TODO：重连重建会话、房间 + * TODO：断开释放所有资源 */ reconnect() { const me = this; @@ -245,12 +247,16 @@ class Session { localStream; // 本地音频 localAudioTrack; + localAudioEnabled; // 本地视频 localVideoTrack; + localVideoEnabled; // 远程音频 remoteAudioTrack; + remoteAudioEnabled; // 远程视频 remoteVideoTrack; + remoteVideoEnabled; // PeerConnection peerConnection; @@ -266,18 +272,56 @@ class Session { this.sessionId = sessionId; } - async pause() { - this.localAudioTrack.enabled = false; - this.localVideoTrack.enabled = false; + async pause(type) { + if(type === 'audio') { + this.localAudioEnabled = false; + this.localAudioTrack.enabled = false; + } + if(type === 'video') { + this.localVideoEnabled = false; + this.localVideoTrack.enabled = false; + } } - async resume() { - this.localAudioTrack.enabled = true; - this.localVideoTrack.enabled = true; + async resume(type) { + if(type === 'audio') { + this.localAudioEnabled = true; + this.localAudioTrack.enabled = true; + } + if(type === 'video') { + this.localVideoEnabled = true; + this.localVideoTrack.enabled = true; + } } - + + async pauseRemote(type) { + if(type === 'audio') { + this.remoteAudioEnabled = false; + this.remoteAudioTrack.enabled = false; + } + if(type === 'video') { + this.remoteVideoEnabled = false; + this.remoteVideoTrack.enabled = false; + } + } + + async resumeRemote(type) { + if(type === 'audio') { + this.remoteAudioEnabled = true; + this.remoteAudioTrack.enabled = true; + } + if(type === 'video') { + this.remoteVideoEnabled = true; + this.remoteVideoTrack.enabled = true; + } + } + async close() { this.closed = true; + this.localAudioEnabled = false; + this.localVideoEnabled = false; + this.remoteAudioEnabled = false; + this.remoteVideoEnabled = false; this.localAudioTrack.stop(); this.localVideoTrack.stop(); this.remoteAudioTrack.stop(); @@ -830,6 +874,34 @@ class Taoyao extends RemoteClient { console.info("关闭终端"); window.close(); } + /** + * 拍照 + * + * @param {*} clientId + */ + controlPhotograph(clientId) { + const me = this; + me.push( + protocol.buildMessage("control::photograph", { + to: clientId + }) + ); + } + /** + * 录像 + * + * @param {*} clientId + * @param {*} enabled + */ + controlRecord(clientId, enabled) { + const me = this; + me.push( + protocol.buildMessage("control::record", { + to: clientId, + enabled: enabled + }) + ); + } /** * 终端音量信令 * @@ -2210,22 +2282,48 @@ class Taoyao extends RemoteClient { } else { } } + async sessionPause(sessionId, type) { + const me = this; + const session = me.sessionClients.get(sessionId); + if(!session) { + return; + } + me.push(protocol.buildMessage("session::pause", { + type, + sessionId + })); + session.pauseRemote(type); + } async defaultSessionPause(message) { const me = this; - const { sessionId } = message.body; + const { type, sessionId } = message.body; const session = me.sessionClients.get(sessionId); if(session) { - session.pause(); + session.pause(type); + } else { } } + async sessionResume(sessionId, type) { + const me = this; + const session = me.sessionClients.get(sessionId); + if(!session) { + return; + } + me.push(protocol.buildMessage("session::resume", { + type, + sessionId + })); + session.resumeRemote(type); + } + async defaultSessionResume(message) { const me = this; - const { sessionId } = message.body; + const { type, sessionId } = message.body; const session = me.sessionClients.get(sessionId); if(session) { - session.resume(); + session.resume(type); } } @@ -2240,8 +2338,10 @@ class Taoyao extends RemoteClient { const track = event.track; if(track.kind === 'audio') { session.remoteAudioTrack = track; + session.remoteAudioEnabled = true; } else if(track.kind === 'video') { session.remoteVideoTrack = track; + session.remoteVideoEnabled = true; } else { } if(session.proxy && session.proxy.media) { @@ -2268,7 +2368,9 @@ class Taoyao extends RemoteClient { session.localStream = localStream; session.peerConnection = peerConnection; session.localAudioTrack = localStream.getAudioTracks()[0]; + session.localAudioEnabled = true; session.localVideoTrack = localStream.getVideoTracks()[0]; + session.localVideoEnabled = true; await session.peerConnection.addTrack(session.localAudioTrack, localStream); await session.peerConnection.addTrack(session.localVideoTrack, localStream); return peerConnection; diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/client/ClientAdapter.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/client/ClientAdapter.java index 53052a7..9852e2c 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/client/ClientAdapter.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/client/ClientAdapter.java @@ -92,13 +92,13 @@ public abstract class ClientAdapter implements Client { try { request.wait(this.timeout); } catch (InterruptedException e) { - log.error("媒体服务等待响应异常：{}", request, e); + log.error("终端等待响应异常：{}", request, e); } } final Message response = this.requestMessage.remove(id); if (response == null || request.equals(response)) { - log.warn("媒体服务没有响应：{}", request); - throw MessageCodeException.of(MessageCode.CODE_2001, "媒体服务没有响应"); + log.warn("终端没有响应：{}", request); + throw MessageCodeException.of(MessageCode.CODE_2001, "终端没有响应"); } return response; }