From d49444ba60c5725211eefbd5a57406a624367377 Mon Sep 17 00:00:00 2001 From: acgist <289547414@qq.com> Date: Fri, 2 Jun 2023 09:20:40 +0800 Subject: [PATCH] =?UTF-8?q?[*]=20=E4=BF=AE=E5=A4=8D=E9=80=9A=E9=81=93?= =?UTF-8?q?=E5=88=A0=E9=99=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../acgist/taoyao/client/signal/Taoyao.java | 79 ++++++++++--------- taoyao-client-media/src/Taoyao.js | 41 ++++++---- .../src/components/LocalClient.vue | 4 +- taoyao-client-web/src/components/Taoyao.js | 6 +- .../taoyao/boot/config/FfmpegProperties.java | 4 + .../src/main/resources/application.yml | 11 ++- .../java/com/acgist/taoyao/RecorderTest.java | 13 ++- .../taoyao/signal/party/media/Recorder.java | 11 +-- .../control/ControlServerRecordProtocol.java | 1 + 9 files changed, 96 insertions(+), 74 deletions(-) diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java index 4f33a4a..035d9fa 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java @@ -363,50 +363,48 @@ public final class Taoyao implements ITaoyao { short messageLength = 0; final byte[] bytes = new byte[1024]; final ByteBuffer buffer = ByteBuffer.allocateDirect(16 * 1024); - while (!this.close && this.connect) { - try { - while (!this.close && (length = this.input.read(bytes)) >= 0) { - buffer.put(bytes, 0, length); - while (buffer.position() > 0) { - if (messageLength <= 0) { - if (buffer.position() < Short.BYTES) { - // 不够消息长度 - break; - } else { - buffer.flip(); - messageLength = buffer.getShort(); - buffer.compact(); - if (messageLength > 16 * 1024) { - throw new RuntimeException("超过最大数据大小：" + messageLength); - } - } + try { + while (!this.close && this.connect && (length = this.input.read(bytes)) >= 0) { + buffer.put(bytes, 0, length); + while (buffer.position() > 0) { + if (messageLength <= 0) { + if (buffer.position() < Short.BYTES) { + // 不够消息长度 + break; } else { - if (buffer.position() < messageLength) { - // 不够消息长度 - break; - } else { - final byte[] message = new byte[messageLength]; - messageLength = 0; - buffer.flip(); - buffer.get(message); - buffer.compact(); - final String content = new String(this.decrypt.doFinal(message)); - try { - this.on(content); - } catch (Exception e) { - Log.e(Taoyao.class.getSimpleName(), "处理信令异常：" + content, e); - this.taoyaoListener.onError(e); - } + buffer.flip(); + messageLength = buffer.getShort(); + buffer.compact(); + if (messageLength > 16 * 1024) { + throw new RuntimeException("超过最大数据大小：" + messageLength); + } + } + } else { + if (buffer.position() < messageLength) { + // 不够消息长度 + break; + } else { + final byte[] message = new byte[messageLength]; + messageLength = 0; + buffer.flip(); + buffer.get(message); + buffer.compact(); + final String content = new String(this.decrypt.doFinal(message)); + try { + this.on(content); + } catch (Exception e) { + Log.e(Taoyao.class.getSimpleName(), "处理信令异常：" + content, e); + this.taoyaoListener.onError(e); } } } } - } catch (Exception e) { - Log.e(Taoyao.class.getSimpleName(), "接收信令异常", e); - this.disconnect(); } + } catch (Exception e) { + Log.e(Taoyao.class.getSimpleName(), "接收信令异常", e); + this.disconnect(); } - if (!this.close && !this.connect) { + if (!this.close) { this.messageHandler.post(this::connect); } } @@ -439,7 +437,11 @@ public final class Taoyao implements ITaoyao { */ @Override public void push(Message message) { - if (this.output == null) { + if (this.close) { + Log.w(Taoyao.class.getSimpleName(), "通道已经关闭：" + message); + return; + } + if (!this.connect) { Log.w(Taoyao.class.getSimpleName(), "通道没有打开：" + message); return; } @@ -448,6 +450,7 @@ public final class Taoyao implements ITaoyao { this.output.write(this.encrypt(message)); } catch (Exception e) { Log.e(Taoyao.class.getSimpleName(), "请求信令异常：" + message, e); + this.disconnect(); } } diff --git a/taoyao-client-media/src/Taoyao.js b/taoyao-client-media/src/Taoyao.js index 291c0a3..c88be1a 100644 --- a/taoyao-client-media/src/Taoyao.js +++ b/taoyao-client-media/src/Taoyao.js @@ -792,13 +792,15 @@ class Taoyao { const { roomId, clientId, host, audioPort, videoPort, rtpCapabilities, audioStreamId, videoStreamId, audioProducerId, videoProducerId } = body; const plainTransportOptions = { ...config.mediasoup.plainTransportOptions, - rtcpMux : true, - comedia : true + rtcpMux: false, + comedia: false }; let videoConsumerId; let audioConsumerId; let audioTransportId; let videoTransportId; + let audioRtpParameters; + let videoRtpParameters; if(audioProducerId) { const audioTransport = await room.mediasoupRouter.createPlainTransport(plainTransportOptions); audioTransportId = audioTransport.id; @@ -810,9 +812,9 @@ class Taoyao { room.transports.delete(audioTransport.id) }); await audioTransport.connect({ - ip : host, - port : audioPort, - rtcpPort : audioPort + ip : host, + port : audioPort, + rtcpPort: audioPort }); const audioConsumer = await audioTransport.consume({ producerId: audioProducerId, @@ -820,6 +822,7 @@ class Taoyao { paused: true }); audioConsumerId = audioConsumer.id; + audioRtpParameters = audioConsumer.rtpParameters; await audioConsumer.resume(); audioConsumer.clientId = clientId; audioConsumer.streamId = audioStreamId; @@ -828,7 +831,7 @@ class Taoyao { console.log("controlServerRecord audioConsumer close：", audioConsumer.id); room.consumers.delete(audioConsumer.id); }); - console.log("controlServerRecord audio", audioTransportId, audioConsumerId, audioTransport.tuple); + console.log("controlServerRecord audio", audioTransportId, audioConsumerId, audioTransport.tuple, audioRtpParameters); } if(videoProducerId) { const videoTransport = await room.mediasoupRouter.createPlainTransport(plainTransportOptions); @@ -841,9 +844,9 @@ class Taoyao { room.transports.delete(videoTransport.id) }); await videoTransport.connect({ - ip : host, - port : videoPort, - rtcpPort : videoPort + ip : host, + port : videoPort, + rtcpPort: videoPort }); const videoConsumer = await videoTransport.consume({ producerId: videoProducerId, @@ -851,6 +854,7 @@ class Taoyao { paused: true }); videoConsumerId = videoConsumer.id; + videoRtpParameters = videoConsumer.rtpParameters; await videoConsumer.resume(); videoConsumer.clientId = clientId; videoConsumer.streamId = videoStreamId; @@ -859,14 +863,16 @@ class Taoyao { console.log("controlServerRecord videoConsumer close：", videoConsumer.id); room.consumers.delete(videoConsumer.id); }); - console.log("controlServerRecord video：", videoTransportId, videoConsumerId, videoTransport.tuple); + console.log("controlServerRecord video：", videoTransportId, videoConsumerId, videoTransport.tuple, videoRtpParameters); } message.body = { - roomId : roomId, - audioConsumerId : audioConsumerId, - videoConsumerId : videoConsumerId, - audioTransportId : audioTransportId, - videoTransportId : videoTransportId, + roomId : roomId, + audioConsumerId : audioConsumerId, + videoConsumerId : videoConsumerId, + audioTransportId : audioTransportId, + videoTransportId : videoTransportId, + audioRtpParameters: audioRtpParameters, + videoRtpParameters: videoRtpParameters, }; me.push(message); } @@ -1528,6 +1534,11 @@ class Taoyao { */ transportEvent(type, roomId, transport) { const self = this; + const room = self.rooms.get(roomId); + if(!room) { + // TODO：提示 + return; + } /********************* 通用通道事件 *********************/ transport.on("routerclose", () => { console.info("transport routerclose：", transport.id); diff --git a/taoyao-client-web/src/components/LocalClient.vue b/taoyao-client-web/src/components/LocalClient.vue index 73e98bf..df82d12 100644 --- a/taoyao-client-web/src/components/LocalClient.vue +++ b/taoyao-client-web/src/components/LocalClient.vue @@ -10,8 +10,8 @@ - - + + diff --git a/taoyao-client-web/src/components/Taoyao.js b/taoyao-client-web/src/components/Taoyao.js index fdbac5b..8839f9c 100644 --- a/taoyao-client-web/src/components/Taoyao.js +++ b/taoyao-client-web/src/components/Taoyao.js @@ -1958,7 +1958,7 @@ class Taoyao extends RemoteClient { let track; try { console.debug("打开麦克风"); - let track = self.getAudioTrack(); + let track = await self.getAudioTrack(); this.audioProducer = await this.sendTransport.produce({ track, codecOptions: { @@ -2528,7 +2528,7 @@ class Taoyao extends RemoteClient { videoBitsPerSecond: 2400 * 1000, mimeType: 'video/webm;codecs=opus,h264', }); - mediaRecorder.onstop = function (e) { + me.mediaRecorder.onstop = function (e) { const blob = new Blob(me.mediaRecorderChunks); const objectURL = URL.createObjectURL(blob); const download = document.createElement('a'); @@ -2541,7 +2541,7 @@ class Taoyao extends RemoteClient { URL.revokeObjectURL(objectURL); me.mediaRecorderChunks = []; }; - mediaRecorder.ondataavailable = (e) => { + me.mediaRecorder.ondataavailable = (e) => { me.mediaRecorderChunks.push(e.data); }; me.mediaRecorder.start(); diff --git a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/FfmpegProperties.java b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/FfmpegProperties.java index cce65af..0d06e1c 100644 --- a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/FfmpegProperties.java +++ b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/FfmpegProperties.java @@ -31,6 +31,10 @@ public class FfmpegProperties { private String storageImagePath; @Schema(title = "视频存储目录", description = "视频存储目录") private String storageVideoPath; + @Schema(title = "预览截图时间", description = "预览截图时间") + private Integer previewTime; + @Schema(title = "预览截图时间", description = "预览截图时间") + private String durationRegex; @Schema(title = "录像录像地址", description = "录像录像地址") private String host; @Schema(title = "录像最小端口", description = "录像最小端口") diff --git a/taoyao-signal-server/taoyao-server/src/main/resources/application.yml b/taoyao-signal-server/taoyao-server/src/main/resources/application.yml index 814ec71..69548fd 100644 --- a/taoyao-signal-server/taoyao-server/src/main/resources/application.yml +++ b/taoyao-signal-server/taoyao-server/src/main/resources/application.yml @@ -235,13 +235,12 @@ taoyao: c=IN IP4 127.0.0.1 a=rtpmap:100 OPUS/48000/2 a=fmtp:100 sprop-stereo=1 - m=video %d RTP/AVP 107 + m=video %d RTP/AVP 101 c=IN IP4 127.0.0.1 - a=rtpmap:107 H264/90000 - a=fmtp:107 packetization-mode=1 + a=rtpmap:101 VP8/90000 # 录像命令 record: ffmpeg -protocol_whitelist "file,rtp,udp" -y -i %s %s - # 截图命令 + # 预览命令 preview: ffmpeg -y -i %s -ss %d -vframes 1 -f image2 %s # 时长命令 duration: ffprobe -i %s -show_entries format=duration @@ -251,6 +250,10 @@ taoyao: storage-image-path: /data/taoyao/storage/image # 视频存储目录 storage-video-path: /data/taoyao/storage/video + # 预览时间 + preview-time: 4 + # 时长提取 + duration-regex: .*duration\=([0-9\.]+).* # 录像地址 #host: 127.0.0.1 host: 192.168.8.40 diff --git a/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/RecorderTest.java b/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/RecorderTest.java index 2e0c00c..bbd847a 100644 --- a/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/RecorderTest.java +++ b/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/RecorderTest.java @@ -16,29 +16,28 @@ public class RecorderTest { ffmpegProperties.setHost("127.0.0.1"); ffmpegProperties.setSdp(""" v=0 - o=- 0 0 IN IP4 %s + o=- 0 0 IN IP4 127.0.0.1 s=TaoyaoRecord t=0 0 m=audio %d RTP/AVP 97 - c=IN IP4 %s + c=IN IP4 127.0.0.1 a=rtpmap:97 OPUS/48000/2 a=fmtp:97 sprop-stereo=1 m=video %d RTP/AVP 96 - c=IN IP4 %s + c=IN IP4 127.0.0.1 a=rtpmap:96 VP8/90000 - a=fmtp:96 packetization-mode=1 """); // ffmpegProperties.setSdp(""" // v=0 -// o=- 0 0 IN IP4 %s +// o=- 0 0 IN IP4 127.0.0.1 // s=TaoyaoRecord // t=0 0 // m=audio %d RTP/AVP 97 -// c=IN IP4 %s +// c=IN IP4 127.0.0.1 // a=rtpmap:97 OPUS/48000/2 // a=fmtp:97 sprop-stereo=1 // m=video %d RTP/AVP 96 -// c=IN IP4 %s +// c=IN IP4 127.0.0.1 // a=rtpmap:96 H264/90000 // a=fmtp:96 packetization-mode=1 // """); diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/media/Recorder.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/media/Recorder.java index 4d90f31..4fae2a3 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/media/Recorder.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/media/Recorder.java @@ -159,6 +159,7 @@ public class Recorder { this.thread.setDaemon(true); this.thread.setName("TaoyaoRecord"); this.thread.start(); + log.info("开始录像：{}", this.folder); } /** @@ -210,14 +211,14 @@ public class Recorder { * 视频预览截图 */ private void preview() { - int time = 2; + int time = this.ffmpegProperties.getPreviewTime(); final File file = Paths.get(this.preview).toFile(); - while(time > 0 && !(file.exists() && file.length() > 0L)) { - log.debug("视频预览截图：{}", this.preview); + do { + log.debug("视频预览：{}", this.preview); final String previewScript = String.format(this.ffmpegProperties.getPreview(), this.filepath, time, this.preview); ScriptUtils.execute(previewScript); time /= 2; - } + } while (time > 0 && !(file.exists() && file.length() > 0L)); } /** @@ -227,7 +228,7 @@ public class Recorder { log.debug("视频时长：{}", this.filepath); final String durationScript = String.format(this.ffmpegProperties.getDuration(), this.filepath); final ScriptExecutor executor = ScriptUtils.execute(durationScript); - final Pattern pattern = Pattern.compile(".*duration\\=([0-9\\.]+).*"); + final Pattern pattern = Pattern.compile(this.ffmpegProperties.getDurationRegex()); final Matcher matcher = pattern.matcher(executor.getResult()); String duration = null; if(matcher.find()) { diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/control/ControlServerRecordProtocol.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/control/ControlServerRecordProtocol.java index 2ddc3fa..8311404 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/control/ControlServerRecordProtocol.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/control/ControlServerRecordProtocol.java @@ -148,6 +148,7 @@ public class ControlServerRecordProtocol extends ProtocolControlAdapter implemen message.setBody(body); final Client mediaClient = room.getMediaClient(); mediaClient.request(message); + // TODO：回写ID，格式自动判断 return recorder.getFilepath(); }