diff --git a/README.md b/README.md index 9d78b11..81d60d6 100644 --- a/README.md +++ b/README.md @@ -30,8 +30,8 @@ |功能|是否支持|是否实现|描述| |:--|:--|:--|:--| -|P2P|支持|完成|视频会话（监控）| -|Mediasoup|支持|完成|视频房间（会议）| +|P2P|支持|完成|视频会话（监控模式）| +|Mediasoup|支持|完成|视频房间（会议模式）| |控制|支持|完成|完整控制信令| |拍照|支持|完成|拍照| |录像|支持|完成|录像| @@ -40,17 +40,17 @@ |功能|是否支持|是否实现|描述| |:--|:--|:--|:--| -|Mediasoup|支持|完成|视频房间（会议）| +|Mediasoup|支持|完成|视频房间（会议模式）| |控制|支持|完成|部分控制信令| -|拍照|支持|未完成|拍照| -|录像|支持|未完成|录像| +|拍照|支持|完成|拍照| +|录像|支持|完成|录像| ### Android终端功能 |功能|是否支持|是否实现|描述| |:--|:--|:--|:--| -|P2P|支持|完成|视频会话（监控）| -|Mediasoup|支持|完成|视频房间（会议）| +|P2P|支持|完成|视频会话（监控模式）| +|Mediasoup|支持|完成|视频房间（会议模式）| |控制|支持|完成|部分控制信令| |拍照|支持|完成|拍照| |录像|支持|完成|录像| @@ -59,8 +59,9 @@ ### 注意事项 -* Web终端不支持同时进入多个视频房间，安卓终端支持同时进入多个视频房间。 -* 服务端录制只支持视频房间（会议）模式，视频会话（监控）模式不支持服务器录制。 +* `Web`终端不支持同时进入多个视频房间，`Android`终端支持。 +* `Media`终端只支持录像之后自动生成预览图片，不支持视频直接拍照。 +* `Media`终端只支持视频房间（会议模式）录像，视频会话（监控模式）不支持。 ## Docker diff --git a/taoyao-client-media/src/Taoyao.js b/taoyao-client-media/src/Taoyao.js index 319eb4c..a1a04c7 100644 --- a/taoyao-client-media/src/Taoyao.js +++ b/taoyao-client-media/src/Taoyao.js @@ -1,3 +1,4 @@ +const fs = require("fs"); const config = require("./Config.js"); const process = require("child_process"); const WebSocket = require("ws"); @@ -789,7 +790,10 @@ class Taoyao { async controlServerRecordStart(message, body, room) { const me = this; - const { roomId, clientId, host, audioPort, videoPort, rtpCapabilities, audioStreamId, videoStreamId, audioProducerId, videoProducerId } = body; + const { + roomId, clientId, host, filepath, audioPort, audioRtcpPort, videoPort, videoRtcpPort, + rtpCapabilities, audioStreamId, videoStreamId, audioProducerId, videoProducerId + } = body; const plainTransportOptions = { ...config.mediasoup.plainTransportOptions, rtcpMux: false, @@ -816,7 +820,7 @@ class Taoyao { await audioTransport.connect({ ip : host, port : audioPort, - rtcpPort: audioPort + 1 + rtcpPort: audioRtcpPort }); audioConsumer = await audioTransport.consume({ producerId: audioProducerId, @@ -847,7 +851,7 @@ class Taoyao { await videoTransport.connect({ ip : host, port : videoPort, - rtcpPort: videoPort + 1 + rtcpPort: videoRtcpPort }); videoConsumer = await videoTransport.consume({ producerId: videoProducerId, @@ -865,13 +869,13 @@ class Taoyao { }); console.log("controlServerRecord video：", videoTransportId, videoConsumerId, videoTransport.tuple, videoRtpParameters); } - if(videoConsumer) { - await videoConsumer.resume(); - videoConsumer.requestKeyFrame(); - } if(audioConsumer) { await audioConsumer.resume(); } + if(videoConsumer) { + await videoConsumer.resume(); + } + this.requestKeyFrameForRecord(0, filepath, videoConsumer); message.body = { roomId : roomId, audioConsumerId : audioConsumerId, @@ -884,6 +888,38 @@ class Taoyao { me.push(message); } + /** + * 请求录像关键帧 + * 视频录像需要通过关键帧解析视频信息，关键帧数据太慢会丢弃视频数据包，导致录像文件只有音频没有视频。 + * + * @param {*} index 重试次数 + * @param {*} filepath 文件路径 + * @param {*} videoConsumer 视频消费者 + */ + requestKeyFrameForRecord(index, filepath, videoConsumer) { + if(!filepath || !videoConsumer) { + return; + } + if(++index >= 10) { + console.warn("请求录像关键帧次数超限", filepath, index); + return; + } + if(videoConsumer.closed) { + console.warn("请求录像关键帧视频关闭", filepath); + return; + } + // 文件开始录像同时已经开始生产数据 + if(fs.existsSync(filepath) && fs.statSync(filepath).size >= 128 * 1024) { + console.debug("请求录像关键帧已经开始录像", filepath); + return; + } + console.debug("请求录像关键帧", filepath); + videoConsumer.requestKeyFrame(); + setTimeout(() => { + this.requestKeyFrameForRecord(index, filepath, videoConsumer); + }, 1000); + } + async controlServerRecordStop(message, body, room) { const me = this; const { audioStreamId, videoStreamId, audioConsumerId, videoConsumerId, audioTransportId, videoTransportId } = body; @@ -1441,7 +1477,7 @@ class Taoyao { ...config.mediasoup.plainTransportOptions, rtcpMux : rtcpMux, comedia : comedia, - enableSctp : enableSctp || Boolean(numSctpStreams), + enableSctp : enableSctp || Boolean(numSctpStreams), numSctpStreams : numSctpStreams || 0, enableSrtp : enableSrtp, srtpCryptoSuite : srtpCryptoSuite, diff --git a/taoyao-client-web/src/components/Taoyao.js b/taoyao-client-web/src/components/Taoyao.js index 03721a0..1637865 100644 --- a/taoyao-client-web/src/components/Taoyao.js +++ b/taoyao-client-web/src/components/Taoyao.js @@ -2482,47 +2482,48 @@ class Taoyao extends RemoteClient { download.remove(); } - // 'video/webm;codecs=aac,vp8', - // 'video/webm;codecs=aac,vp9', - // 'video/webm;codecs=aac,h264', - // 'video/webm;codecs=pcm,vp8', - // 'video/webm;codecs=pcm,vp9', - // 'video/webm;codecs=pcm,h264', - // 'video/webm;codecs=opus,vp8', - // 'video/webm;codecs=opus,vp9', - // 'video/webm;codecs=opus,h264', - // 'video/mp4;codecs=aac,vp8', - // 'video/mp4;codecs=aac,vp9', - // 'video/mp4;codecs=aac,h264', - // 'video/mp4;codecs=pcm,vp8', - // 'video/mp4;codecs=pcm,vp9', - // 'video/mp4;codecs=pcm,h264', - // 'video/mp4;codecs=opus,vp8', - // 'video/mp4;codecs=opus,vp9', - // 'video/mp4;codecs=opus,h264', - // MediaRecorder.isTypeSupported(mimeType) - /** - * 本地录制 + * 本地录像 + * + * 'video/webm;codecs=aac,vp8', + * 'video/webm;codecs=aac,vp9', + * 'video/webm;codecs=aac,h264', + * 'video/webm;codecs=pcm,vp8', + * 'video/webm;codecs=pcm,vp9', + * 'video/webm;codecs=pcm,h264', + * 'video/webm;codecs=opus,vp8', + * 'video/webm;codecs=opus,vp9', + * 'video/webm;codecs=opus,h264', + * 'video/mp4;codecs=aac,vp8', + * 'video/mp4;codecs=aac,vp9', + * 'video/mp4;codecs=aac,h264', + * 'video/mp4;codecs=pcm,vp8', + * 'video/mp4;codecs=pcm,vp9', + * 'video/mp4;codecs=pcm,h264', + * 'video/mp4;codecs=opus,vp8', + * 'video/mp4;codecs=opus,vp9', + * 'video/mp4;codecs=opus,h264', + * + * MediaRecorder.isTypeSupported(mimeType) * * video.captureStream().getTracks().forEach((v) => stream.addTrack(v)); * - * @param {*} audio 音频 - * @param {*} video 视频 - * @param {*} enabled 是否录制 + * @param {*} audioStream 音频流 + * @param {*} videoStream 视频流 + * @param {*} enabled 是否录像 */ - localClientRecord(audio, video, enabled) { + localClientRecord(audioStream, videoStream, enabled) { const me = this; if (enabled) { if (me.mediaRecorder) { return; } const stream = new MediaStream(); - if(audio) { - audio.getAudioTracks().forEach(track => stream.addTrack(track)); + if(audioStream) { + audioStream.getAudioTracks().forEach(track => stream.addTrack(track)); } - if(video) { - video.getVideoTracks().forEach(track => stream.addTrack(track)); + if(videoStream) { + videoStream.getVideoTracks().forEach(track => stream.addTrack(track)); } me.mediaRecorder = new MediaRecorder(stream, { audioBitsPerSecond: 128 * 1000, diff --git a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/Constant.java b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/Constant.java index e3890f5..9c79b6d 100644 --- a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/Constant.java +++ b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/Constant.java @@ -261,6 +261,22 @@ public interface Constant { * 终端列表 */ String CLIENTS = "clients"; + /** + * 音频端口 + */ + String AUDIO_PORT = "audioPort"; + /** + * 视频端口 + */ + String VIDEO_PORT = "videoPort"; + /** + * 音频控制端口 + */ + String AUDIO_RTCP_PORT = "audioRtcpPort"; + /** + * 视频控制端口 + */ + String VIDEO_RTCP_PORT = "videoRtcpPort"; /** * 生产者ID生成器 diff --git a/taoyao-signal-server/taoyao-server/src/main/resources/application.yml b/taoyao-signal-server/taoyao-server/src/main/resources/application.yml index 98cbd30..71cac1c 100644 --- a/taoyao-signal-server/taoyao-server/src/main/resources/application.yml +++ b/taoyao-signal-server/taoyao-server/src/main/resources/application.yml @@ -153,17 +153,18 @@ taoyao: o=- 0 0 IN IP4 127.0.0.1 s=TaoyaoRecord t=0 0 - a=group:BUNDLE video audio - m=video %d RTP/AVP 101 - c=IN IP4 0.0.0.0 - a=rtpmap:101 VP8/90000 - a=recvonly m=audio %d RTP/AVP 100 c=IN IP4 0.0.0.0 + a=rtcp:%d a=rtpmap:100 OPUS/48000/2 a=recvonly + m=video %d RTP/AVP 101 + c=IN IP4 0.0.0.0 + a=rtcp:%d + a=rtpmap:101 VP8/90000 + a=recvonly # 录像命令 - record: ffmpeg -y -protocol_whitelist "file,rtp,udp" -thread_queue_size 1024 -i %s -c:a aac -c:v h264 %s + record: ffmpeg -y -protocol_whitelist "file,rtp,udp" -thread_queue_size 1024 -c:a libopus -c:v libvpx -r:v %d -i %s -c:a aac -c:v h264 %s # 预览命令 preview: ffmpeg -y -i %s -ss %d -vframes 1 -f image2 %s # 时长命令 diff --git a/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/RecorderTest.java b/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/RecorderTest.java index bbd847a..05c6ce8 100644 --- a/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/RecorderTest.java +++ b/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/RecorderTest.java @@ -3,6 +3,8 @@ package com.acgist.taoyao; import org.junit.jupiter.api.Test; import com.acgist.taoyao.boot.config.FfmpegProperties; +import com.acgist.taoyao.boot.config.MediaProperties; +import com.acgist.taoyao.boot.config.MediaVideoProperties; import com.acgist.taoyao.signal.party.media.Recorder; public class RecorderTest { @@ -20,31 +22,25 @@ public class RecorderTest { s=TaoyaoRecord t=0 0 m=audio %d RTP/AVP 97 - c=IN IP4 127.0.0.1 + c=IN IP4 0.0.0.0 + a=rtcp:%d a=rtpmap:97 OPUS/48000/2 - a=fmtp:97 sprop-stereo=1 + a=recvonly m=video %d RTP/AVP 96 - c=IN IP4 127.0.0.1 + c=IN IP4 0.0.0.0 + a=rtcp:%d a=rtpmap:96 VP8/90000 + a=recvonly """); -// ffmpegProperties.setSdp(""" -// v=0 -// o=- 0 0 IN IP4 127.0.0.1 -// s=TaoyaoRecord -// t=0 0 -// m=audio %d RTP/AVP 97 -// c=IN IP4 127.0.0.1 -// a=rtpmap:97 OPUS/48000/2 -// a=fmtp:97 sprop-stereo=1 -// m=video %d RTP/AVP 96 -// c=IN IP4 127.0.0.1 -// a=rtpmap:96 H264/90000 -// a=fmtp:96 packetization-mode=1 -// """); - ffmpegProperties.setRecord("ffmpeg -protocol_whitelist \"file,rtp,udp\" -y -i %s %s"); +// ffmpeg -re -i video.mp4 -c:a libopus -vn -f rtp rtp://192.168.1.100:50000 -c:v vp8 -an -f rtp rtp://192.168.1.100:50002 -sdp_file taoyao.sdp + ffmpegProperties.setRecord("ffmpeg -y -protocol_whitelist \"file,rtp,udp\" -thread_queue_size 1024 -c:a libopus -c:v libvpx -r:v %d -i %s -c:a aac -c:v h264 %s"); ffmpegProperties.setPreview("ffmpeg -y -i %s -ss %d -vframes 1 -f image2 %s"); ffmpegProperties.setDuration("ffprobe -i %s -show_entries format=duration"); - final Recorder recorder = new Recorder("taoyao", null, null, ffmpegProperties); + final MediaProperties mediaProperties = new MediaProperties(); + final MediaVideoProperties mediaVideoProperties = new MediaVideoProperties(); + mediaVideoProperties.setFrameRate(24); + mediaProperties.setVideo(mediaVideoProperties); + final Recorder recorder = new Recorder("taoyao", null, null, mediaProperties, ffmpegProperties); recorder.start(); Thread.sleep(20 * 1000); recorder.stop(); diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/media/Recorder.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/media/Recorder.java index 9ab63d4..a08caf3 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/media/Recorder.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/media/Recorder.java @@ -11,6 +11,8 @@ import java.util.regex.Pattern; import org.apache.commons.lang3.math.NumberUtils; import com.acgist.taoyao.boot.config.FfmpegProperties; +import com.acgist.taoyao.boot.config.MediaProperties; +import com.acgist.taoyao.boot.config.MediaVideoProperties; import com.acgist.taoyao.boot.utils.FileUtils; import com.acgist.taoyao.boot.utils.NetUtils; import com.acgist.taoyao.boot.utils.ScriptUtils; @@ -48,10 +50,18 @@ public class Recorder { * 音频端口 */ private Integer audioPort; + /** + * 音频控制端口 + */ + private Integer audioRtcpPort; /** * 视频端口 */ private Integer videoPort; + /** + * 视频控制端口 + */ + private Integer videoRtcpPort; /** * 音频流ID */ @@ -120,6 +130,10 @@ public class Recorder { * 文件路径 */ private final String filepath; + /** + * 媒体配置 + */ + private final MediaProperties mediaProperties; /** * FFmpeg配置 */ @@ -129,9 +143,13 @@ public class Recorder { * @param name 录像名称 * @param room 房间 * @param clientWrapper 终端 + * @param mediaProperties 媒体配置 * @param ffmpegProperties FFmpeg配置 */ - public Recorder(String name, Room room, ClientWrapper clientWrapper, FfmpegProperties ffmpegProperties) { + public Recorder( + String name, Room room, ClientWrapper clientWrapper, + MediaProperties mediaProperties, FfmpegProperties ffmpegProperties + ) { this.close = false; this.running = false; this.room = room; @@ -140,6 +158,7 @@ public class Recorder { this.preview = Paths.get(this.folder, "taoyao.jpg").toAbsolutePath().toString(); this.filepath = Paths.get(this.folder, "taoyao.mp4").toAbsolutePath().toString(); this.clientWrapper = clientWrapper; + this.mediaProperties = mediaProperties; this.ffmpegProperties = ffmpegProperties; FileUtils.mkdirs(this.folder); } @@ -166,7 +185,13 @@ public class Recorder { * 录制视频 */ private void record() { - final String recordScript = String.format(this.ffmpegProperties.getRecord(), this.sdpfile, this.filepath); + final MediaVideoProperties mediaVideoProperties = this.mediaProperties.getVideo(); + final String recordScript = String.format( + this.ffmpegProperties.getRecord(), + mediaVideoProperties.getFrameRate(), + this.sdpfile, + this.filepath + ); this.scriptExecutor = new ScriptExecutor(recordScript); try { log.debug(""" @@ -189,12 +214,16 @@ public class Recorder { int minPort = this.ffmpegProperties.getMinPort(); int maxPort = this.ffmpegProperties.getMaxPort(); // 预留控制端口 - this.audioPort = NetUtils.scanPort(minPort, maxPort); - this.videoPort = NetUtils.scanPort(this.audioPort + 2, maxPort); - final String sdp = String.format( + this.audioPort = NetUtils.scanPort(minPort, maxPort); + this.audioRtcpPort = NetUtils.scanPort(this.audioPort + 1, maxPort); + this.videoPort = NetUtils.scanPort(this.audioPort + 2, maxPort); + this.videoRtcpPort = NetUtils.scanPort(this.audioPort + 3, maxPort); + final String sdp = String.format( this.ffmpegProperties.getSdp(), + this.audioPort, + this.audioRtcpPort, this.videoPort, - this.audioPort + this.videoRtcpPort ); Files.write( Paths.get(this.sdpfile), @@ -225,7 +254,7 @@ public class Recorder { */ private void duration() { log.debug("视频时长：{}", this.filepath); - final String durationScript = String.format(this.ffmpegProperties.getDuration(), this.filepath); + final String durationScript = String.format(this.ffmpegProperties.getDuration(), this.filepath); final ScriptExecutor executor = ScriptUtils.execute(durationScript); final Pattern pattern = Pattern.compile(this.ffmpegProperties.getDurationRegex()); final Matcher matcher = pattern.matcher(executor.getResult()); diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/control/ControlServerRecordProtocol.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/control/ControlServerRecordProtocol.java index 8311404..1511bd1 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/control/ControlServerRecordProtocol.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/control/ControlServerRecordProtocol.java @@ -10,6 +10,7 @@ import com.acgist.taoyao.boot.annotation.Description; import com.acgist.taoyao.boot.annotation.Protocol; import com.acgist.taoyao.boot.config.Constant; import com.acgist.taoyao.boot.config.FfmpegProperties; +import com.acgist.taoyao.boot.config.MediaProperties; import com.acgist.taoyao.boot.model.Message; import com.acgist.taoyao.boot.utils.MapUtils; import com.acgist.taoyao.signal.client.Client; @@ -50,10 +51,12 @@ public class ControlServerRecordProtocol extends ProtocolControlAdapter implemen public static final String SIGNAL = "control::server::record"; + private final MediaProperties mediaProperties; private final FfmpegProperties ffmpegProperties; - public ControlServerRecordProtocol(FfmpegProperties ffmpegProperties) { + public ControlServerRecordProtocol(MediaProperties mediaProperties, FfmpegProperties ffmpegProperties) { super("服务端录像信令", SIGNAL); + this.mediaProperties = mediaProperties; this.ffmpegProperties = ffmpegProperties; } @@ -119,18 +122,21 @@ public class ControlServerRecordProtocol extends ProtocolControlAdapter implemen } final String name = UUID.randomUUID().toString(); // 打开录制线程 - final Recorder recorder = new Recorder(name, room, clientWrapper, this.ffmpegProperties); + final Recorder recorder = new Recorder(name, room, clientWrapper, this.mediaProperties, this.ffmpegProperties); recorder.start(); clientWrapper.setRecorder(recorder); // 打开媒体录制 final Message message = this.build(); final Map body = new HashMap<>(); - body.put("audioPort", recorder.getAudioPort()); - body.put("videoPort", recorder.getVideoPort()); body.put(Constant.HOST, this.ffmpegProperties.getHost()); body.put(Constant.ROOM_ID, room.getRoomId()); body.put(Constant.ENABLED, true); + body.put(Constant.FILEPATH, recorder.getFilepath()); body.put(Constant.CLIENT_ID, clientWrapper.getClientId()); + body.put(Constant.AUDIO_PORT, recorder.getAudioPort()); + body.put(Constant.VIDEO_PORT, recorder.getVideoPort()); + body.put(Constant.AUDIO_RTCP_PORT, recorder.getAudioRtcpPort()); + body.put(Constant.VIDEO_RTCP_PORT, recorder.getVideoRtcpPort()); body.put(Constant.RTP_CAPABILITIES, clientWrapper.getRtpCapabilities()); clientWrapper.getProducers().values().forEach(producer -> { if(producer.getKind() == Kind.AUDIO) {