From 71ada0a8ca3b14fb6a639b08d9196b5c176828c7 Mon Sep 17 00:00:00 2001 From: acgist <289547414@qq.com> Date: Sun, 26 Feb 2023 16:44:30 +0800 Subject: [PATCH] =?UTF-8?q?[+]=20=E8=A7=86=E9=A2=91=E6=B7=BB=E5=8A=A0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- taoyao-client-media/src/Signal.js | 3 +- taoyao-client-web/package.json | 3 +- taoyao-client-web/src/App.vue | 58 +++-- .../src/components/LocalClient.vue | 125 +++++++++-- .../src/components/RemoteClient.vue | 86 +++++++- taoyao-client-web/src/components/Taoyao.js | 201 ++++++++++++++---- .../acgist/taoyao/boot/config/Constant.java | 4 + .../protocol/media/MediaConsumeProtocol.java | 1 + .../protocol/room/RoomClientListProtocol.java | 33 ++- .../protocol/room/RoomEnterProtocol.java | 9 +- 10 files changed, 437 insertions(+), 86 deletions(-) diff --git a/taoyao-client-media/src/Signal.js b/taoyao-client-media/src/Signal.js index 1c0e87a..a714a25 100644 --- a/taoyao-client-media/src/Signal.js +++ b/taoyao-client-media/src/Signal.js @@ -573,7 +573,7 @@ class Signal { } async mediaConsume(message, body) { - const { roomId, clientId, streamId, producerId, transportId, rtpCapabilities } = body; + const { roomId, clientId, sourceId, streamId, producerId, transportId, rtpCapabilities } = body; const room = this.rooms.get(roomId); const producer = room.producers.get(producerId); const transport = room.transports.get(transportId); @@ -682,6 +682,7 @@ class Signal { type: consumer.type, roomId: roomId, clientId: clientId, + sourceId: sourceId, streamId: streamId, producerId: producerId, consumerId: consumer.id, diff --git a/taoyao-client-web/package.json b/taoyao-client-web/package.json index 264d100..457db1d 100644 --- a/taoyao-client-web/package.json +++ b/taoyao-client-web/package.json @@ -13,7 +13,8 @@ "vue": "^3.2.44", "moment": "^2.29.4", "element-plus": "^2.2.32", - "mediasoup-client": "^3.6.82" + "mediasoup-client": "^3.6.82", + "@element-plus/icons": "^0.0.11" }, "devDependencies": { "vite": "^4.0.0", diff --git a/taoyao-client-web/src/App.vue b/taoyao-client-web/src/App.vue index f30dcd3..aafc4cf 100644 --- a/taoyao-client-web/src/App.vue +++ b/taoyao-client-web/src/App.vue @@ -35,7 +35,7 @@ center width="30%" title="房间设置" - @open="init" + @open="loadList" :show-close="false" v-model="roomVisible" > @@ -80,22 +80,27 @@ -
+
连接信令 选择房间 创建房间 + 邀请终端 退出房间 关闭房间
-
+
+ +
+ + \ No newline at end of file diff --git a/taoyao-client-web/src/components/LocalClient.vue b/taoyao-client-web/src/components/LocalClient.vue index 2a1ae27..7c046a0 100644 --- a/taoyao-client-web/src/components/LocalClient.vue +++ b/taoyao-client-web/src/components/LocalClient.vue @@ -1,24 +1,113 @@ - + diff --git a/taoyao-client-web/src/components/RemoteClient.vue b/taoyao-client-web/src/components/RemoteClient.vue index d8b1ce5..af9225c 100644 --- a/taoyao-client-web/src/components/RemoteClient.vue +++ b/taoyao-client-web/src/components/RemoteClient.vue @@ -1,14 +1,88 @@ \ No newline at end of file + setup() { + return { + Mute, + Camera, + Refresh, + VideoPlay, + VideoPause, + InfoFilled, + Microphone, + VideoCamera, + CircleClose, + }; + }, + data() { + return { + taoyao: null, + audio: null, + video: null, + audioStream: null, + videoStream: null, + dataConsumer: null, + audioConsumer: null, + videoConsumer: null, + }; + }, + mounted() { + this.audio = this.$refs.audio; + this.video = this.$refs.video; + }, + methods: { + media(track, consumer) { + if(track.kind === 'audio') { + if (this.audioStream) { + // TODO：资源释放 + } else { + this.audioStream = new MediaStream(); + this.audioStream.addTrack(track); + this.audio.srcObject = this.audioStream; + } + this.audio.play().catch((error) => console.warn("视频播放失败", error)); + } else if(track.kind === 'video') { + if (this.videoStream) { + // TODO：资源释放 + } else { + this.videoStream = new MediaStream(); + this.videoStream.addTrack(track); + this.video.srcObject = this.videoStream; + } + this.video.play().catch((error) => console.warn("视频播放失败", error)); + } else { + + } + } + } +}; + diff --git a/taoyao-client-web/src/components/Taoyao.js b/taoyao-client-web/src/components/Taoyao.js index 4e6aef7..dd549ae 100644 --- a/taoyao-client-web/src/components/Taoyao.js +++ b/taoyao-client-web/src/components/Taoyao.js @@ -11,18 +11,14 @@ import { } from "./Config.js"; // Used for simulcast webcam video. -const WEBCAM_SIMULCAST_ENCODINGS = -[ - { scaleResolutionDownBy: 4, maxBitrate: 500000, scalabilityMode: 'S1T2' }, - { scaleResolutionDownBy: 2, maxBitrate: 1000000, scalabilityMode: 'S1T2' }, - { scaleResolutionDownBy: 1, maxBitrate: 5000000, scalabilityMode: 'S1T2' } +const WEBCAM_SIMULCAST_ENCODINGS = [ + { scaleResolutionDownBy: 4, maxBitrate: 500000, scalabilityMode: "S1T2" }, + { scaleResolutionDownBy: 2, maxBitrate: 1000000, scalabilityMode: "S1T2" }, + { scaleResolutionDownBy: 1, maxBitrate: 5000000, scalabilityMode: "S1T2" }, ]; // Used for VP9 webcam video. -const WEBCAM_KSVC_ENCODINGS = -[ - { scalabilityMode: 'S3T3_KEY' } -]; +const WEBCAM_KSVC_ENCODINGS = [{ scalabilityMode: "S3T3_KEY" }]; /** * 信令通道 @@ -291,8 +287,14 @@ const signalChannel = { case "client::reboot": self.defaultClientReboot(message); break; - case "client::shutdown": - self.defaultClientShutdown(message); + case "client::shutdown": + self.defaultClientShutdown(message); + break; + case "room::enter": + self.defaultRoomEnter(message); + break; + case "room::client::list": + self.defaultRoomClientList(message); break; case "platform::error": self.callbackError(message); @@ -334,6 +336,24 @@ const signalChannel = { console.info("关闭终端"); window.close(); }, + defaultRoomEnter(message) { + const { roomId, clientId } = message.body; + if(clientId === this.taoyao.clientId) { + // 忽略自己 + } else { + this.taoyao.remoteClients.set(clientId, roomId); + } + }, + defaultRoomClientList(message) { + const self = this; + message.body.forEach(v => { + if(v.clientId === self.taoyao.clientId) { + // 忽略自己 + } else { + self.taoyao.remoteClients.set(v.clientId, self.taoyao.roomId); + } + }); + }, }; /** @@ -354,6 +374,8 @@ class Taoyao { password; // 回调事件 callback; + // 媒体回调 + callbackMedia; // 音频媒体配置 audio; // 视频媒体配置 @@ -364,10 +386,6 @@ class Taoyao { push; // 请求信令 request; - // 本地终端 - localClient; - // 远程终端 - remoteClients = new Map(); // 信令通道 signalChannel; // 发送媒体通道 @@ -381,30 +399,31 @@ class Taoyao { // 是否生产 produce; // 视频来源：file | camera | screen - videoSource = "screen"; + videoSource = "camera"; + // 强制使用TCP + forceTcp; // 强制使用VP9 forceVP9; // 强制使用H264 forceH264; + // useSimulcast; + // 是否生产数据 + dataProduce; // 是否生产音频 audioProduce; // 是否生成视频 videoProduce; - // 强制使用TCP - forceTcp; - // 使用数据通道 - useDataChannel; + // 数据生产者 + dataProducer; // 音频生产者 audioProducer; // 视频生产者 videoProducer; - // 数据生产者 - dataChannnelProducer; - // 媒体消费者 + // 消费者：音频、视频、数据 consumers = new Map(); - // 数据消费者 - dataConsumers = new Map(); + // 远程终端 + remoteClients = new Map(); constructor({ roomId, @@ -418,7 +437,7 @@ class Taoyao { audioProduce = true, videoProduce = true, forceTcp = false, - useDataChannel = true, + dataProduce = true, }) { this.roomId = roomId; this.clientId = clientId; @@ -428,22 +447,24 @@ class Taoyao { this.password = password; this.consume = consume; this.produce = produce; + this.dataProduce = produce && dataProduce; this.audioProduce = produce && audioProduce; this.videoProduce = produce && videoProduce; this.forceTcp = forceTcp; - this.useDataChannel = useDataChannel; } /** * 连接信令 * - * @param {*} callback + * @param {*} callback 信令回调 + * @param {*} callbackMedia 媒体回调 * * @returns */ - async connectSignal(callback) { + async connectSignal(callback, callbackMedia) { const self = this; self.callback = callback; + self.callbackMedia = callbackMedia; self.signalChannel = signalChannel; signalChannel.taoyao = self; // 不能直接this.push = this.signalChannel.push这样导致this对象错误 @@ -469,6 +490,7 @@ class Taoyao { } else { console.warn("没有注册回调：", message); } + return; } // 错误回调 const errorMessage = protocol.buildMessage( @@ -490,6 +512,12 @@ class Taoyao { ); return response.body; } + async clientList() { + const response = await this.request( + protocol.buildMessage("client::list", { roomId: self.roomId }) + ); + return response.body; + } /** * 创建房间 */ @@ -526,7 +554,7 @@ class Taoyao { ? self.mediasoupDevice.rtpCapabilities : undefined, sctpCapabilities: - self.consume && self.useDataChannel + self.consume && self.dataProduce ? self.mediasoupDevice.sctpCapabilities : undefined, }) @@ -564,7 +592,7 @@ class Taoyao { forceTcp: self.forceTcp, producing: true, consuming: false, - sctpCapabilities: self.useDataChannel + sctpCapabilities: self.dataProduce ? self.mediasoupDevice.sctpCapabilities : undefined, }) @@ -661,7 +689,7 @@ class Taoyao { forceTcp: self.forceTcp, producing: false, consuming: true, - sctpCapabilities: self.useDataChannel + sctpCapabilities: self.dataProduce ? self.mediasoupDevice.sctpCapabilities : undefined, }) @@ -843,20 +871,24 @@ class Taoyao { track = stream.getVideoTracks()[0]; } else if (self.videoSource === "screen") { const stream = await navigator.mediaDevices.getDisplayMedia({ + // 如果需要共享声音 audio: false, video: { - displaySurface: "monitor", - logicalSurface: true, cursor: true, width: { max: 1920 }, height: { max: 1080 }, frameRate: { max: 30 }, + logicalSurface: true, + displaySurface: "monitor", }, }); track = stream.getVideoTracks()[0]; } else { // TODO：异常 } + + self.callbackMedia("local", track); + let codec; let encodings; const codecOptions = { @@ -904,7 +936,7 @@ class Taoyao { // if (this._e2eKey && e2e.isSupported()) { // e2e.setupSenderTransform(this.videoProducer.rtpSender); // } - + this.videoProducer.on("transportclose", () => { this.videoProducer = null; }); @@ -924,6 +956,69 @@ class Taoyao { } } + async closeVideoProducer() { + console.debug("disableWebcam()"); + if (!this.videoProducer) { + return; + } + this.videoProducer.close(); + try { + await this.request( + protocol.buildMessage("media::producer::close", { + producerId: this.videoProducer.id, + }) + ); + } catch (error) { + console.error(error); + } + + this._webcamProducer = null; + } + + async pauseVideoProducer() { + console.debug("关闭摄像头"); + this.videoProducer.pause(); + try { + await this.request( + protocol.buildMessage("media::producer::pause", { + producerId: this.videoProducer.id, + }) + ); + } catch (error) { + console.error("关闭摄像头异常", error); + // TODO：异常调用回调 + } + } + + async resumeVideoProducer() { + console.debug("恢复摄像头"); + this.videoProducer.resume(); + try { + await this.request( + protocol.buildMessage("media::producer::resume", { + producerId: this.videoProducer.id, + }) + ); + } catch (error) { + console.error("恢复摄像头异常", error); + } + } + + async updateVideoConfig(config) { + console.debug("更新摄像头参数"); + try { + this.videoProducer.track.stop(); + // TODO：screen、参数配置 + const stream = await navigator.mediaDevices.getUserMedia({ + video: true, + }); + const track = stream.getVideoTracks()[0]; + await this.videoProducer.replaceTrack({ track }); + } catch (error) { + console.error("changeWebcam() | failed: %o", error); + } + } + /** * 消费媒体 * @@ -941,6 +1036,7 @@ class Taoyao { type, roomId, clientId, + sourceId, streamId, producerId, consumerId, @@ -962,6 +1058,7 @@ class Taoyao { appData, // Trick. }); consumer.clientId = clientId; + consumer.sourceId = sourceId; consumer.streamId = streamId; self.consumers.set(consumer.id, consumer); consumer.on("transportclose", () => { @@ -991,18 +1088,10 @@ class Taoyao { // ) // ); self.push(message); - console.log(consumer); - - const audioElem = document.createElement("video"); - document.getElementsByTagName("body")[0].appendChild(audioElem); - - const stream = new MediaStream(); - stream.addTrack(consumer.track); - audioElem.srcObject = stream; - audioElem - .play() - .catch((error) => console.warn("audioElem.play() failed:%o", error)); + console.log("消费者", consumer); + self.callbackMedia("remote", consumer.track, consumer); + // If audio-only mode is enabled, pause it. if (consumer.kind === "video" && !self.videoProduce) { // this.pauseConsumer(consumer); @@ -1013,6 +1102,26 @@ class Taoyao { } } + async pauseConsumer(consumer) { + if (consumer.paused) return; + try { + await this._protoo.request("pauseConsumer", { consumerId: consumer.id }); + consumer.pause(); + } catch (error) { + logger.error("_pauseConsumer() | failed:%o", error); + } + } + + async resumeConsumer(consumer) { + if (!consumer.paused) return; + try { + await this._protoo.request("resumeConsumer", { consumerId: consumer.id }); + consumer.resume(); + } catch (error) { + logger.error("_resumeConsumer() | failed:%o", error); + } + } + /** * 验证设备 */ diff --git a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/Constant.java b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/Constant.java index 1b84f70..25382d6 100644 --- a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/Constant.java +++ b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/Constant.java @@ -163,6 +163,10 @@ public interface Constant { * 终端ID */ String CLIENT_ID = "clientId"; + /** + * 来源终端ID + */ + String SOURCE_ID = "sourceId"; /** * 路由ID */ diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/media/MediaConsumeProtocol.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/media/MediaConsumeProtocol.java index ea5cd41..b447bbf 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/media/MediaConsumeProtocol.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/media/MediaConsumeProtocol.java @@ -105,6 +105,7 @@ public class MediaConsumeProtocol extends ProtocolRoomAdapter implements Applica final String streamId = producer.getStreamId() + "->" + clientId; body.put(Constant.ROOM_ID, room.getRoomId()); body.put(Constant.CLIENT_ID, clientId); + body.put(Constant.SOURCE_ID, producer.getProduceClient().getClientId()); body.put(Constant.STREAM_ID, streamId); body.put(Constant.PRODUCER_ID, producer.getProducerId()); body.put(Constant.TRANSPORT_ID, recvTransport.getTransportId()); diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/room/RoomClientListProtocol.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/room/RoomClientListProtocol.java index f22db8b..8bdf57c 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/room/RoomClientListProtocol.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/room/RoomClientListProtocol.java @@ -1,5 +1,36 @@ package com.acgist.taoyao.signal.protocol.room; -public class RoomClientListProtocol { +import java.util.Map; +import com.acgist.taoyao.boot.annotation.Description; +import com.acgist.taoyao.boot.annotation.Protocol; +import com.acgist.taoyao.boot.model.Message; +import com.acgist.taoyao.signal.client.Client; +import com.acgist.taoyao.signal.client.ClientType; +import com.acgist.taoyao.signal.flute.media.Room; +import com.acgist.taoyao.signal.protocol.ProtocolRoomAdapter; + +/** + * 房间终端列表信令 + * + * @author acgist + */ +@Protocol +@Description( + flow = "终端=>信令服务->终端" +) +public class RoomClientListProtocol extends ProtocolRoomAdapter { + + public static final String SIGNAL = "room::client::list"; + + public RoomClientListProtocol() { + super("房间终端列表信令", SIGNAL); + } + + @Override + public void execute(String clientId, ClientType clientType, Room room, Client client, Client mediaClient, Message message, Map body) { + message.setBody(room.clientStatus()); + client.push(message); + } + } diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/room/RoomEnterProtocol.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/room/RoomEnterProtocol.java index 86c9b90..95cbc3b 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/room/RoomEnterProtocol.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/room/RoomEnterProtocol.java @@ -45,8 +45,11 @@ public class RoomEnterProtocol extends ProtocolRoomAdapter { public static final String SIGNAL = "room::enter"; - public RoomEnterProtocol() { + private final RoomClientListProtocol roomClientListProtocol; + + public RoomEnterProtocol(RoomClientListProtocol roomClientListProtocol) { super("进入房间信令", SIGNAL); + this.roomClientListProtocol = roomClientListProtocol; } @Override @@ -71,6 +74,10 @@ public class RoomEnterProtocol extends ProtocolRoomAdapter { )); room.broadcast(message); log.info("进入房间：{} - {}", clientId, room.getRoomId()); + // 推送房间用户信息 + final Message roomClientListMessage = this.roomClientListProtocol.build(); + roomClientListMessage.setBody(room.clientStatus()); + client.push(roomClientListMessage); } }