[+] 服务端录像

README.md

@@ -30,8 +30,8 @@
 
 |功能|是否支持|是否实现|描述|
 |:--|:--|:--|:--|
-|P2P|支持|完成|视频会话（监控）|
+|P2P|支持|完成|视频会话（监控模式）|
-|Mediasoup|支持|完成|视频房间（会议）|
+|Mediasoup|支持|完成|视频房间（会议模式）|
 |控制|支持|完成|完整控制信令|
 |拍照|支持|完成|拍照|
 |录像|支持|完成|录像|
@@ -40,17 +40,17 @@
 
 |功能|是否支持|是否实现|描述|
 |:--|:--|:--|:--|
-|Mediasoup|支持|完成|视频房间（会议）|
+|Mediasoup|支持|完成|视频房间（会议模式）|
 |控制|支持|完成|部分控制信令|
-|拍照|支持|未完成|拍照|
+|拍照|支持|完成|拍照|
-|录像|支持|未完成|录像|
+|录像|支持|完成|录像|
 
 ### Android终端功能
 
 |功能|是否支持|是否实现|描述|
 |:--|:--|:--|:--|
-|P2P|支持|完成|视频会话（监控）|
+|P2P|支持|完成|视频会话（监控模式）|
-|Mediasoup|支持|完成|视频房间（会议）|
+|Mediasoup|支持|完成|视频房间（会议模式）|
 |控制|支持|完成|部分控制信令|
 |拍照|支持|完成|拍照|
 |录像|支持|完成|录像|
@@ -59,8 +59,9 @@
 
 ### 注意事项
 
-* Web终端不支持同时进入多个视频房间，安卓终端支持同时进入多个视频房间。
+* `Web`终端不支持同时进入多个视频房间，`Android`终端支持。
-* 服务端录制只支持视频房间（会议）模式，视频会话（监控）模式不支持服务器录制。
+* `Media`终端只支持录像之后自动生成预览图片，不支持视频直接拍照。
+* `Media`终端只支持视频房间（会议模式）录像，视频会话（监控模式）不支持。
 
 ## Docker
 

taoyao-client-media/src/Taoyao.js

@@ -1,3 +1,4 @@
+const fs        = require("fs");
 const config    = require("./Config.js");
 const process   = require("child_process");
 const WebSocket = require("ws");
@@ -789,7 +790,10 @@ class Taoyao {
 
   async controlServerRecordStart(message, body, room) {
     const me = this;
-    const { roomId, clientId, host, audioPort, videoPort, rtpCapabilities, audioStreamId, videoStreamId, audioProducerId, videoProducerId } = body;
+    const {
+      roomId, clientId, host, filepath, audioPort, audioRtcpPort, videoPort, videoRtcpPort,
+      rtpCapabilities, audioStreamId, videoStreamId, audioProducerId, videoProducerId
+    } = body;
     const plainTransportOptions = {
       ...config.mediasoup.plainTransportOptions,
       rtcpMux: false,
@@ -816,7 +820,7 @@ class Taoyao {
       await audioTransport.connect({
         ip      : host,
         port    : audioPort,
-        rtcpPort: audioPort + 1
+        rtcpPort: audioRtcpPort
       });
       audioConsumer = await audioTransport.consume({
         producerId: audioProducerId,
@@ -847,7 +851,7 @@ class Taoyao {
       await videoTransport.connect({
         ip      : host,
         port    : videoPort,
-        rtcpPort: videoPort + 1
+        rtcpPort: videoRtcpPort
       });
       videoConsumer = await videoTransport.consume({
         producerId: videoProducerId,
@@ -865,13 +869,13 @@ class Taoyao {
       });
       console.log("controlServerRecord video：", videoTransportId, videoConsumerId, videoTransport.tuple, videoRtpParameters);
     }
-    if(videoConsumer) {
-      await videoConsumer.resume();
-      videoConsumer.requestKeyFrame();
-    }
     if(audioConsumer) {
       await audioConsumer.resume();
     }
+    if(videoConsumer) {
+      await videoConsumer.resume();
+    }
+    this.requestKeyFrameForRecord(0, filepath, videoConsumer);
     message.body = {
       roomId            : roomId,
       audioConsumerId   : audioConsumerId,
@@ -884,6 +888,38 @@ class Taoyao {
     me.push(message);
   }
 
+  /**
+   * 请求录像关键帧
+   * 视频录像需要通过关键帧解析视频信息，关键帧数据太慢会丢弃视频数据包，导致录像文件只有音频没有视频。
+   * 
+   * @param {*} index         重试次数
+   * @param {*} filepath      文件路径
+   * @param {*} videoConsumer 视频消费者
+   */
+  requestKeyFrameForRecord(index, filepath, videoConsumer) {
+    if(!filepath || !videoConsumer) {
+      return;
+    }
+    if(++index >= 10) {
+      console.warn("请求录像关键帧次数超限", filepath, index);
+      return;
+    }
+    if(videoConsumer.closed) {
+      console.warn("请求录像关键帧视频关闭", filepath);
+      return;
+    }
+    // 文件开始录像同时已经开始生产数据
+    if(fs.existsSync(filepath) && fs.statSync(filepath).size >= 128 * 1024) {
+      console.debug("请求录像关键帧已经开始录像", filepath);
+      return;
+    }
+    console.debug("请求录像关键帧", filepath);
+    videoConsumer.requestKeyFrame();
+    setTimeout(() => {
+      this.requestKeyFrameForRecord(index, filepath, videoConsumer);
+    }, 1000);
+  }
+
   async controlServerRecordStop(message, body, room) {
     const me = this;
     const { audioStreamId, videoStreamId, audioConsumerId, videoConsumerId, audioTransportId, videoTransportId } = body;

taoyao-client-web/src/components/Taoyao.js

@@ -2482,47 +2482,48 @@ class Taoyao extends RemoteClient {
     download.remove();
   }
 
-  // 'video/webm;codecs=aac,vp8',
-  // 'video/webm;codecs=aac,vp9',
-  // 'video/webm;codecs=aac,h264',
-  // 'video/webm;codecs=pcm,vp8',
-  // 'video/webm;codecs=pcm,vp9',
-  // 'video/webm;codecs=pcm,h264',
-  // 'video/webm;codecs=opus,vp8',
-  // 'video/webm;codecs=opus,vp9',
-  // 'video/webm;codecs=opus,h264',
-  // 'video/mp4;codecs=aac,vp8',
-  // 'video/mp4;codecs=aac,vp9',
-  // 'video/mp4;codecs=aac,h264',
-  // 'video/mp4;codecs=pcm,vp8',
-  // 'video/mp4;codecs=pcm,vp9',
-  // 'video/mp4;codecs=pcm,h264',
-  // 'video/mp4;codecs=opus,vp8',
-  // 'video/mp4;codecs=opus,vp9',
-  // 'video/mp4;codecs=opus,h264',
-  // MediaRecorder.isTypeSupported(mimeType)
-
   /**
-   * 本地录制
+   * 本地录像
+   * 
+   * 'video/webm;codecs=aac,vp8',
+   * 'video/webm;codecs=aac,vp9',
+   * 'video/webm;codecs=aac,h264',
+   * 'video/webm;codecs=pcm,vp8',
+   * 'video/webm;codecs=pcm,vp9',
+   * 'video/webm;codecs=pcm,h264',
+   * 'video/webm;codecs=opus,vp8',
+   * 'video/webm;codecs=opus,vp9',
+   * 'video/webm;codecs=opus,h264',
+   * 'video/mp4;codecs=aac,vp8',
+   * 'video/mp4;codecs=aac,vp9',
+   * 'video/mp4;codecs=aac,h264',
+   * 'video/mp4;codecs=pcm,vp8',
+   * 'video/mp4;codecs=pcm,vp9',
+   * 'video/mp4;codecs=pcm,h264',
+   * 'video/mp4;codecs=opus,vp8',
+   * 'video/mp4;codecs=opus,vp9',
+   * 'video/mp4;codecs=opus,h264',
+   * 
+   * MediaRecorder.isTypeSupported(mimeType)
    * 
    * video.captureStream().getTracks().forEach((v) => stream.addTrack(v));
    * 
-   * @param {*} audio   音频
+   * @param {*} audioStream 音频流
-   * @param {*} video   视频
+   * @param {*} videoStream 视频流
-   * @param {*} enabled 是否录制
+   * @param {*} enabled     是否录像
    */
-  localClientRecord(audio, video, enabled) {
+  localClientRecord(audioStream, videoStream, enabled) {
     const me = this;
     if (enabled) {
       if (me.mediaRecorder) {
         return;
       }
       const stream = new MediaStream();
-      if(audio) {
+      if(audioStream) {
-        audio.getAudioTracks().forEach(track => stream.addTrack(track));
+        audioStream.getAudioTracks().forEach(track => stream.addTrack(track));
       }
-      if(video) {
+      if(videoStream) {
-        video.getVideoTracks().forEach(track => stream.addTrack(track));
+        videoStream.getVideoTracks().forEach(track => stream.addTrack(track));
       }
       me.mediaRecorder = new MediaRecorder(stream, {
         audioBitsPerSecond: 128  * 1000,

taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/Constant.java

@@ -261,6 +261,22 @@ public interface Constant {
      * 终端列表
      */
     String CLIENTS = "clients";
+    /**
+     * 音频端口
+     */
+    String AUDIO_PORT = "audioPort";
+    /**
+     * 视频端口
+     */
+    String VIDEO_PORT = "videoPort";
+    /**
+     * 音频控制端口
+     */
+    String AUDIO_RTCP_PORT = "audioRtcpPort";
+    /**
+     * 视频控制端口
+     */
+    String VIDEO_RTCP_PORT = "videoRtcpPort";
 
     /**
      * 生产者ID生成器

taoyao-signal-server/taoyao-server/src/main/resources/application.yml

@@ -153,17 +153,18 @@ taoyao:
       o=- 0 0 IN IP4 127.0.0.1
       s=TaoyaoRecord
       t=0 0
-      a=group:BUNDLE video audio
-      m=video %d RTP/AVP 101
-      c=IN IP4 0.0.0.0
-      a=rtpmap:101 VP8/90000
-      a=recvonly
       m=audio %d RTP/AVP 100
       c=IN IP4 0.0.0.0
+      a=rtcp:%d
       a=rtpmap:100 OPUS/48000/2
       a=recvonly
+      m=video %d RTP/AVP 101
+      c=IN IP4 0.0.0.0
+      a=rtcp:%d
+      a=rtpmap:101 VP8/90000
+      a=recvonly
     # 录像命令
-    record: ffmpeg -y -protocol_whitelist "file,rtp,udp" -thread_queue_size 1024 -i %s -c:a aac -c:v h264 %s
+    record: ffmpeg -y -protocol_whitelist "file,rtp,udp" -thread_queue_size 1024 -c:a libopus -c:v libvpx -r:v %d -i %s -c:a aac -c:v h264 %s
     # 预览命令
     preview: ffmpeg -y -i %s -ss %d -vframes 1 -f image2 %s
     # 时长命令

taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/RecorderTest.java

@@ -3,6 +3,8 @@ package com.acgist.taoyao;
 import org.junit.jupiter.api.Test;
 
 import com.acgist.taoyao.boot.config.FfmpegProperties;
+import com.acgist.taoyao.boot.config.MediaProperties;
+import com.acgist.taoyao.boot.config.MediaVideoProperties;
 import com.acgist.taoyao.signal.party.media.Recorder;
 
 public class RecorderTest {
@@ -20,31 +22,25 @@ public class RecorderTest {
         s=TaoyaoRecord
         t=0 0
         m=audio %d RTP/AVP 97
-        c=IN IP4 127.0.0.1
+        c=IN IP4 0.0.0.0
+        a=rtcp:%d
         a=rtpmap:97 OPUS/48000/2
-        a=fmtp:97 sprop-stereo=1
+        a=recvonly
         m=video %d RTP/AVP 96
-        c=IN IP4 127.0.0.1
+        c=IN IP4 0.0.0.0
+        a=rtcp:%d
         a=rtpmap:96 VP8/90000
+        a=recvonly
         """);
-//        ffmpegProperties.setSdp("""
+//      ffmpeg -re -i video.mp4 -c:a libopus -vn -f rtp rtp://192.168.1.100:50000 -c:v vp8 -an -f rtp rtp://192.168.1.100:50002 -sdp_file taoyao.sdp
-//        v=0
+        ffmpegProperties.setRecord("ffmpeg -y -protocol_whitelist \"file,rtp,udp\" -thread_queue_size 1024 -c:a libopus -c:v libvpx -r:v %d -i %s -c:a aac -c:v h264 %s");
-//        o=- 0 0 IN IP4 127.0.0.1
-//        s=TaoyaoRecord
-//        t=0 0
-//        m=audio %d RTP/AVP 97
-//        c=IN IP4 127.0.0.1
-//        a=rtpmap:97 OPUS/48000/2
-//        a=fmtp:97 sprop-stereo=1
-//        m=video %d RTP/AVP 96
-//        c=IN IP4 127.0.0.1
-//        a=rtpmap:96 H264/90000
-//        a=fmtp:96 packetization-mode=1
-//        """);
-        ffmpegProperties.setRecord("ffmpeg -protocol_whitelist \"file,rtp,udp\" -y -i %s %s");
         ffmpegProperties.setPreview("ffmpeg -y -i %s -ss %d -vframes 1 -f image2 %s");
         ffmpegProperties.setDuration("ffprobe -i %s -show_entries format=duration");
-        final Recorder recorder = new Recorder("taoyao", null, null, ffmpegProperties);
+        final MediaProperties mediaProperties = new MediaProperties();
+        final MediaVideoProperties mediaVideoProperties = new MediaVideoProperties();
+        mediaVideoProperties.setFrameRate(24);
+        mediaProperties.setVideo(mediaVideoProperties);
+        final Recorder recorder = new Recorder("taoyao", null, null, mediaProperties, ffmpegProperties);
         recorder.start();
         Thread.sleep(20 * 1000);
         recorder.stop();

taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/media/Recorder.java

@@ -11,6 +11,8 @@ import java.util.regex.Pattern;
 import org.apache.commons.lang3.math.NumberUtils;
 
 import com.acgist.taoyao.boot.config.FfmpegProperties;
+import com.acgist.taoyao.boot.config.MediaProperties;
+import com.acgist.taoyao.boot.config.MediaVideoProperties;
 import com.acgist.taoyao.boot.utils.FileUtils;
 import com.acgist.taoyao.boot.utils.NetUtils;
 import com.acgist.taoyao.boot.utils.ScriptUtils;
@@ -48,10 +50,18 @@ public class Recorder {
      * 音频端口
      */
     private Integer audioPort;
+    /**
+     * 音频控制端口
+     */
+    private Integer audioRtcpPort;
     /**
      * 视频端口
      */
     private Integer videoPort;
+    /**
+     * 视频控制端口
+     */
+    private Integer videoRtcpPort;
     /**
      * 音频流ID
      */
@@ -120,6 +130,10 @@ public class Recorder {
      * 文件路径
      */
     private final String filepath;
+    /**
+     * 媒体配置
+     */
+    private final MediaProperties mediaProperties;
     /**
      * FFmpeg配置
      */
@@ -129,9 +143,13 @@ public class Recorder {
      * @param name             录像名称
      * @param room             房间
      * @param clientWrapper    终端
+     * @param mediaProperties  媒体配置
      * @param ffmpegProperties FFmpeg配置
      */
-    public Recorder(String name, Room room, ClientWrapper clientWrapper, FfmpegProperties ffmpegProperties) {
+    public Recorder(
+        String name, Room room, ClientWrapper clientWrapper,
+        MediaProperties mediaProperties, FfmpegProperties ffmpegProperties
+    ) {
         this.close            = false;
         this.running          = false;
         this.room             = room;
@@ -140,6 +158,7 @@ public class Recorder {
         this.preview          = Paths.get(this.folder, "taoyao.jpg").toAbsolutePath().toString();
         this.filepath         = Paths.get(this.folder, "taoyao.mp4").toAbsolutePath().toString();
         this.clientWrapper    = clientWrapper;
+        this.mediaProperties  = mediaProperties;
         this.ffmpegProperties = ffmpegProperties;
         FileUtils.mkdirs(this.folder);
     }
@@ -166,7 +185,13 @@ public class Recorder {
      * 录制视频
      */
     private void record() {
-        final String recordScript = String.format(this.ffmpegProperties.getRecord(), this.sdpfile, this.filepath);
+        final MediaVideoProperties mediaVideoProperties = this.mediaProperties.getVideo();
+        final String recordScript = String.format(
+            this.ffmpegProperties.getRecord(),
+            mediaVideoProperties.getFrameRate(),
+            this.sdpfile,
+            this.filepath
+        );
         this.scriptExecutor = new ScriptExecutor(recordScript);
         try {
             log.debug("""
@@ -190,11 +215,15 @@ public class Recorder {
             int maxPort = this.ffmpegProperties.getMaxPort();
             // 预留控制端口
             this.audioPort     = NetUtils.scanPort(minPort, maxPort);
+            this.audioRtcpPort = NetUtils.scanPort(this.audioPort + 1, maxPort);
             this.videoPort     = NetUtils.scanPort(this.audioPort + 2, maxPort);
+            this.videoRtcpPort = NetUtils.scanPort(this.audioPort + 3, maxPort);
             final String sdp   = String.format(
                 this.ffmpegProperties.getSdp(),
+                this.audioPort,
+                this.audioRtcpPort,
                 this.videoPort,
-                this.audioPort
+                this.videoRtcpPort
             );
             Files.write(
                 Paths.get(this.sdpfile),

taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/control/ControlServerRecordProtocol.java

@@ -10,6 +10,7 @@ import com.acgist.taoyao.boot.annotation.Description;
 import com.acgist.taoyao.boot.annotation.Protocol;
 import com.acgist.taoyao.boot.config.Constant;
 import com.acgist.taoyao.boot.config.FfmpegProperties;
+import com.acgist.taoyao.boot.config.MediaProperties;
 import com.acgist.taoyao.boot.model.Message;
 import com.acgist.taoyao.boot.utils.MapUtils;
 import com.acgist.taoyao.signal.client.Client;
@@ -50,10 +51,12 @@ public class ControlServerRecordProtocol extends ProtocolControlAdapter implemen
 
     public static final String SIGNAL = "control::server::record";
     
+    private final MediaProperties mediaProperties;
     private final FfmpegProperties ffmpegProperties;
     
-    public ControlServerRecordProtocol(FfmpegProperties ffmpegProperties) {
+    public ControlServerRecordProtocol(MediaProperties mediaProperties, FfmpegProperties ffmpegProperties) {
         super("服务端录像信令", SIGNAL);
+        this.mediaProperties = mediaProperties;
         this.ffmpegProperties = ffmpegProperties;
     }
     
@@ -119,18 +122,21 @@ public class ControlServerRecordProtocol extends ProtocolControlAdapter implemen
         }
         final String name = UUID.randomUUID().toString();
         // 打开录制线程
-        final Recorder recorder = new Recorder(name, room, clientWrapper, this.ffmpegProperties);
+        final Recorder recorder = new Recorder(name, room, clientWrapper, this.mediaProperties, this.ffmpegProperties);
         recorder.start();
         clientWrapper.setRecorder(recorder);
         // 打开媒体录制
         final Message message = this.build();
         final Map<String, Object> body = new HashMap<>();
-        body.put("audioPort", recorder.getAudioPort());
-        body.put("videoPort", recorder.getVideoPort());
         body.put(Constant.HOST, this.ffmpegProperties.getHost());
         body.put(Constant.ROOM_ID, room.getRoomId());
         body.put(Constant.ENABLED, true);
+        body.put(Constant.FILEPATH, recorder.getFilepath());
         body.put(Constant.CLIENT_ID, clientWrapper.getClientId());
+        body.put(Constant.AUDIO_PORT, recorder.getAudioPort());
+        body.put(Constant.VIDEO_PORT, recorder.getVideoPort());
+        body.put(Constant.AUDIO_RTCP_PORT, recorder.getAudioRtcpPort());
+        body.put(Constant.VIDEO_RTCP_PORT, recorder.getVideoRtcpPort());
         body.put(Constant.RTP_CAPABILITIES, clientWrapper.getRtpCapabilities());
         clientWrapper.getProducers().values().forEach(producer -> {
             if(producer.getKind() == Kind.AUDIO) {