qq27800734/taoyao
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[+] 服务端录像

Browse Source
This commit is contained in:
acgist
2023-06-08 15:17:30 +08:00
parent 5a5bd6136f
commit cbd5661c56
8 changed files with 168 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
README.md
View File

@@ -30,8 +30,8 @@
|功能|是否支持|是否实现|描述|
|:--|:--|:--|:--|
|P2P|支持|完成|视频会话（监控）|
|Mediasoup|支持|完成|视频房间（会议）|
|P2P|支持|完成|视频会话（监控模式）|
|Mediasoup|支持|完成|视频房间（会议模式）|
|控制|支持|完成|完整控制信令|
|拍照|支持|完成|拍照|
|录像|支持|完成|录像|
@@ -40,17 +40,17 @@
|功能|是否支持|是否实现|描述|
|:--|:--|:--|:--|
|Mediasoup|支持|完成|视频房间（会议）|
|Mediasoup|支持|完成|视频房间（会议模式）|
|控制|支持|完成|部分控制信令|
|拍照|支持|未完成|拍照|
|录像|支持|未完成|录像|
|拍照|支持|完成|拍照|
|录像|支持|完成|录像|
### Android终端功能
|功能|是否支持|是否实现|描述|
|:--|:--|:--|:--|
|P2P|支持|完成|视频会话（监控）|
|Mediasoup|支持|完成|视频房间（会议）|
|P2P|支持|完成|视频会话（监控模式）|
|Mediasoup|支持|完成|视频房间（会议模式）|
|控制|支持|完成|部分控制信令|
|拍照|支持|完成|拍照|
|录像|支持|完成|录像|
@@ -59,8 +59,9 @@
### 注意事项
* Web终端不支持同时进入多个视频房间，安卓终端支持同时进入多个视频房间。
* 服务端录制只支持视频房间（会议）模式，视频会话（监控）模式不支持服务器录制。
* `Web`终端不支持同时进入多个视频房间，`Android`终端支持。
* `Media`终端只支持录像之后自动生成预览图片，不支持视频直接拍照。
* `Media`终端只支持视频房间（会议模式）录像，视频会话（监控模式）不支持。
## Docker

50
taoyao-client-media/src/Taoyao.js
View File

@@ -1,3 +1,4 @@
const fs = require("fs");
const config = require("./Config.js");
const process = require("child_process");
const WebSocket = require("ws");
@@ -789,7 +790,10 @@ class Taoyao {
async controlServerRecordStart(message, body, room) {
const me = this;
const { roomId, clientId, host, audioPort, videoPort, rtpCapabilities, audioStreamId, videoStreamId, audioProducerId, videoProducerId } = body;
const {
roomId, clientId, host, filepath, audioPort, audioRtcpPort, videoPort, videoRtcpPort,
rtpCapabilities, audioStreamId, videoStreamId, audioProducerId, videoProducerId
} = body;
const plainTransportOptions = {
...config.mediasoup.plainTransportOptions,
rtcpMux: false,
@@ -816,7 +820,7 @@ class Taoyao {
await audioTransport.connect({
ip : host,
port : audioPort,
rtcpPort: audioPort + 1
rtcpPort: audioRtcpPort
});
audioConsumer = await audioTransport.consume({
producerId: audioProducerId,
@@ -847,7 +851,7 @@ class Taoyao {
await videoTransport.connect({
ip : host,
port : videoPort,
rtcpPort: videoPort + 1
rtcpPort: videoRtcpPort
});
videoConsumer = await videoTransport.consume({
producerId: videoProducerId,
@@ -865,13 +869,13 @@ class Taoyao {
});
console.log("controlServerRecord video：", videoTransportId, videoConsumerId, videoTransport.tuple, videoRtpParameters);
}
if(videoConsumer) {
await videoConsumer.resume();
videoConsumer.requestKeyFrame();
}
if(audioConsumer) {
await audioConsumer.resume();
}
if(videoConsumer) {
await videoConsumer.resume();
}
this.requestKeyFrameForRecord(0, filepath, videoConsumer);
message.body = {
roomId : roomId,
audioConsumerId : audioConsumerId,
@@ -884,6 +888,38 @@ class Taoyao {
me.push(message);
}
/**
* 请求录像关键帧
* 视频录像需要通过关键帧解析视频信息，关键帧数据太慢会丢弃视频数据包，导致录像文件只有音频没有视频。
*
* @param {*} index 重试次数
* @param {*} filepath 文件路径
* @param {*} videoConsumer 视频消费者
*/
requestKeyFrameForRecord(index, filepath, videoConsumer) {
if(!filepath || !videoConsumer) {
return;
}
if(++index >= 10) {
console.warn("请求录像关键帧次数超限", filepath, index);
return;
}
if(videoConsumer.closed) {
console.warn("请求录像关键帧视频关闭", filepath);
return;
}
// 文件开始录像同时已经开始生产数据
if(fs.existsSync(filepath) && fs.statSync(filepath).size >= 128 * 1024) {
console.debug("请求录像关键帧已经开始录像", filepath);
return;
}
console.debug("请求录像关键帧", filepath);
videoConsumer.requestKeyFrame();
setTimeout(() => {
this.requestKeyFrameForRecord(index, filepath, videoConsumer);
}, 1000);
}
async controlServerRecordStop(message, body, room) {
const me = this;
const { audioStreamId, videoStreamId, audioConsumerId, videoConsumerId, audioTransportId, videoTransportId } = body;

59
taoyao-client-web/src/components/Taoyao.js
View File

@@ -2482,47 +2482,48 @@ class Taoyao extends RemoteClient {
download.remove();
}
// 'video/webm;codecs=aac,vp8',
// 'video/webm;codecs=aac,vp9',
// 'video/webm;codecs=aac,h264',
// 'video/webm;codecs=pcm,vp8',
// 'video/webm;codecs=pcm,vp9',
// 'video/webm;codecs=pcm,h264',
// 'video/webm;codecs=opus,vp8',
// 'video/webm;codecs=opus,vp9',
// 'video/webm;codecs=opus,h264',
// 'video/mp4;codecs=aac,vp8',
// 'video/mp4;codecs=aac,vp9',
// 'video/mp4;codecs=aac,h264',
// 'video/mp4;codecs=pcm,vp8',
// 'video/mp4;codecs=pcm,vp9',
// 'video/mp4;codecs=pcm,h264',
// 'video/mp4;codecs=opus,vp8',
// 'video/mp4;codecs=opus,vp9',
// 'video/mp4;codecs=opus,h264',
// MediaRecorder.isTypeSupported(mimeType)
/**
* 本地录制
* 本地录像
*
* 'video/webm;codecs=aac,vp8',
* 'video/webm;codecs=aac,vp9',
* 'video/webm;codecs=aac,h264',
* 'video/webm;codecs=pcm,vp8',
* 'video/webm;codecs=pcm,vp9',
* 'video/webm;codecs=pcm,h264',
* 'video/webm;codecs=opus,vp8',
* 'video/webm;codecs=opus,vp9',
* 'video/webm;codecs=opus,h264',
* 'video/mp4;codecs=aac,vp8',
* 'video/mp4;codecs=aac,vp9',
* 'video/mp4;codecs=aac,h264',
* 'video/mp4;codecs=pcm,vp8',
* 'video/mp4;codecs=pcm,vp9',
* 'video/mp4;codecs=pcm,h264',
* 'video/mp4;codecs=opus,vp8',
* 'video/mp4;codecs=opus,vp9',
* 'video/mp4;codecs=opus,h264',
*
* MediaRecorder.isTypeSupported(mimeType)
*
* video.captureStream().getTracks().forEach((v) => stream.addTrack(v));
*
* @param {*} audio 音频
* @param {*} video 视频
* @param {*} enabled 是否录制
* @param {*} audioStream 音频流
* @param {*} videoStream 视频流
* @param {*} enabled 是否录像
*/
localClientRecord(audio, video, enabled) {
localClientRecord(audioStream, videoStream, enabled) {
const me = this;
if (enabled) {
if (me.mediaRecorder) {
return;
}
const stream = new MediaStream();
if(audio) {
audio.getAudioTracks().forEach(track => stream.addTrack(track));
if(audioStream) {
audioStream.getAudioTracks().forEach(track => stream.addTrack(track));
}
if(video) {
video.getVideoTracks().forEach(track => stream.addTrack(track));
if(videoStream) {
videoStream.getVideoTracks().forEach(track => stream.addTrack(track));
}
me.mediaRecorder = new MediaRecorder(stream, {
audioBitsPerSecond: 128 * 1000,

16
taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/Constant.java
View File

@@ -261,6 +261,22 @@ public interface Constant {
* 终端列表
*/
String CLIENTS = "clients";
/**
* 音频端口
*/
String AUDIO_PORT = "audioPort";
/**
* 视频端口
*/
String VIDEO_PORT = "videoPort";
/**
* 音频控制端口
*/
String AUDIO_RTCP_PORT = "audioRtcpPort";
/**
* 视频控制端口
*/
String VIDEO_RTCP_PORT = "videoRtcpPort";
/**
* 生产者ID生成器

13
taoyao-signal-server/taoyao-server/src/main/resources/application.yml
View File

@@ -153,17 +153,18 @@ taoyao:
o=- 0 0 IN IP4 127.0.0.1
s=TaoyaoRecord
t=0 0
a=group:BUNDLE video audio
m=video %d RTP/AVP 101
c=IN IP4 0.0.0.0
a=rtpmap:101 VP8/90000
a=recvonly
m=audio %d RTP/AVP 100
c=IN IP4 0.0.0.0
a=rtcp:%d
a=rtpmap:100 OPUS/48000/2
a=recvonly
m=video %d RTP/AVP 101
c=IN IP4 0.0.0.0
a=rtcp:%d
a=rtpmap:101 VP8/90000
a=recvonly
# 录像命令
record: ffmpeg -y -protocol_whitelist "file,rtp,udp" -thread_queue_size 1024 -i %s -c:a aac -c:v h264 %s
record: ffmpeg -y -protocol_whitelist "file,rtp,udp" -thread_queue_size 1024 -c:a libopus -c:v libvpx -r:v %d -i %s -c:a aac -c:v h264 %s
# 预览命令
preview: ffmpeg -y -i %s -ss %d -vframes 1 -f image2 %s
# 时长命令

34
taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/RecorderTest.java
View File

@@ -3,6 +3,8 @@ package com.acgist.taoyao;
import org.junit.jupiter.api.Test;
import com.acgist.taoyao.boot.config.FfmpegProperties;
import com.acgist.taoyao.boot.config.MediaProperties;
import com.acgist.taoyao.boot.config.MediaVideoProperties;
import com.acgist.taoyao.signal.party.media.Recorder;
public class RecorderTest {
@@ -20,31 +22,25 @@ public class RecorderTest {
s=TaoyaoRecord
t=0 0
m=audio %d RTP/AVP 97
c=IN IP4 127.0.0.1
c=IN IP4 0.0.0.0
a=rtcp:%d
a=rtpmap:97 OPUS/48000/2
a=fmtp:97 sprop-stereo=1
a=recvonly
m=video %d RTP/AVP 96
c=IN IP4 127.0.0.1
c=IN IP4 0.0.0.0
a=rtcp:%d
a=rtpmap:96 VP8/90000
a=recvonly
""");
// ffmpegProperties.setSdp("""
// v=0
// o=- 0 0 IN IP4 127.0.0.1
// s=TaoyaoRecord
// t=0 0
// m=audio %d RTP/AVP 97
// c=IN IP4 127.0.0.1
// a=rtpmap:97 OPUS/48000/2
// a=fmtp:97 sprop-stereo=1
// m=video %d RTP/AVP 96
// c=IN IP4 127.0.0.1
// a=rtpmap:96 H264/90000
// a=fmtp:96 packetization-mode=1
// """);
ffmpegProperties.setRecord("ffmpeg -protocol_whitelist \"file,rtp,udp\" -y -i %s %s");
// ffmpeg -re -i video.mp4 -c:a libopus -vn -f rtp rtp://192.168.1.100:50000 -c:v vp8 -an -f rtp rtp://192.168.1.100:50002 -sdp_file taoyao.sdp
ffmpegProperties.setRecord("ffmpeg -y -protocol_whitelist \"file,rtp,udp\" -thread_queue_size 1024 -c:a libopus -c:v libvpx -r:v %d -i %s -c:a aac -c:v h264 %s");
ffmpegProperties.setPreview("ffmpeg -y -i %s -ss %d -vframes 1 -f image2 %s");
ffmpegProperties.setDuration("ffprobe -i %s -show_entries format=duration");
final Recorder recorder = new Recorder("taoyao", null, null, ffmpegProperties);
final MediaProperties mediaProperties = new MediaProperties();
final MediaVideoProperties mediaVideoProperties = new MediaVideoProperties();
mediaVideoProperties.setFrameRate(24);
mediaProperties.setVideo(mediaVideoProperties);
final Recorder recorder = new Recorder("taoyao", null, null, mediaProperties, ffmpegProperties);
recorder.start();
Thread.sleep(20 * 1000);
recorder.stop();

35
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/media/Recorder.java
View File

@@ -11,6 +11,8 @@ import java.util.regex.Pattern;
import org.apache.commons.lang3.math.NumberUtils;
import com.acgist.taoyao.boot.config.FfmpegProperties;
import com.acgist.taoyao.boot.config.MediaProperties;
import com.acgist.taoyao.boot.config.MediaVideoProperties;
import com.acgist.taoyao.boot.utils.FileUtils;
import com.acgist.taoyao.boot.utils.NetUtils;
import com.acgist.taoyao.boot.utils.ScriptUtils;
@@ -48,10 +50,18 @@ public class Recorder {
* 音频端口
*/
private Integer audioPort;
/**
* 音频控制端口
*/
private Integer audioRtcpPort;
/**
* 视频端口
*/
private Integer videoPort;
/**
* 视频控制端口
*/
private Integer videoRtcpPort;
/**
* 音频流ID
*/
@@ -120,6 +130,10 @@ public class Recorder {
* 文件路径
*/
private final String filepath;
/**
* 媒体配置
*/
private final MediaProperties mediaProperties;
/**
* FFmpeg配置
*/
@@ -129,9 +143,13 @@ public class Recorder {
* @param name 录像名称
* @param room 房间
* @param clientWrapper 终端
* @param mediaProperties 媒体配置
* @param ffmpegProperties FFmpeg配置
*/
public Recorder(String name, Room room, ClientWrapper clientWrapper, FfmpegProperties ffmpegProperties) {
public Recorder(
String name, Room room, ClientWrapper clientWrapper,
MediaProperties mediaProperties, FfmpegProperties ffmpegProperties
) {
this.close = false;
this.running = false;
this.room = room;
@@ -140,6 +158,7 @@ public class Recorder {
this.preview = Paths.get(this.folder, "taoyao.jpg").toAbsolutePath().toString();
this.filepath = Paths.get(this.folder, "taoyao.mp4").toAbsolutePath().toString();
this.clientWrapper = clientWrapper;
this.mediaProperties = mediaProperties;
this.ffmpegProperties = ffmpegProperties;
FileUtils.mkdirs(this.folder);
}
@@ -166,7 +185,13 @@ public class Recorder {
* 录制视频
*/
private void record() {
final String recordScript = String.format(this.ffmpegProperties.getRecord(), this.sdpfile, this.filepath);
final MediaVideoProperties mediaVideoProperties = this.mediaProperties.getVideo();
final String recordScript = String.format(
this.ffmpegProperties.getRecord(),
mediaVideoProperties.getFrameRate(),
this.sdpfile,
this.filepath
);
this.scriptExecutor = new ScriptExecutor(recordScript);
try {
log.debug("""
@@ -190,11 +215,15 @@ public class Recorder {
int maxPort = this.ffmpegProperties.getMaxPort();
// 预留控制端口
this.audioPort = NetUtils.scanPort(minPort, maxPort);
this.audioRtcpPort = NetUtils.scanPort(this.audioPort + 1, maxPort);
this.videoPort = NetUtils.scanPort(this.audioPort + 2, maxPort);
this.videoRtcpPort = NetUtils.scanPort(this.audioPort + 3, maxPort);
final String sdp = String.format(
this.ffmpegProperties.getSdp(),
this.audioPort,
this.audioRtcpPort,
this.videoPort,
this.audioPort
this.videoRtcpPort
);
Files.write(
Paths.get(this.sdpfile),

14
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/control/ControlServerRecordProtocol.java
View File

@@ -10,6 +10,7 @@ import com.acgist.taoyao.boot.annotation.Description;
import com.acgist.taoyao.boot.annotation.Protocol;
import com.acgist.taoyao.boot.config.Constant;
import com.acgist.taoyao.boot.config.FfmpegProperties;
import com.acgist.taoyao.boot.config.MediaProperties;
import com.acgist.taoyao.boot.model.Message;
import com.acgist.taoyao.boot.utils.MapUtils;
import com.acgist.taoyao.signal.client.Client;
@@ -50,10 +51,12 @@ public class ControlServerRecordProtocol extends ProtocolControlAdapter implemen
public static final String SIGNAL = "control::server::record";
private final MediaProperties mediaProperties;
private final FfmpegProperties ffmpegProperties;
public ControlServerRecordProtocol(FfmpegProperties ffmpegProperties) {
public ControlServerRecordProtocol(MediaProperties mediaProperties, FfmpegProperties ffmpegProperties) {
super("服务端录像信令", SIGNAL);
this.mediaProperties = mediaProperties;
this.ffmpegProperties = ffmpegProperties;
}
@@ -119,18 +122,21 @@ public class ControlServerRecordProtocol extends ProtocolControlAdapter implemen
}
final String name = UUID.randomUUID().toString();
// 打开录制线程
final Recorder recorder = new Recorder(name, room, clientWrapper, this.ffmpegProperties);
final Recorder recorder = new Recorder(name, room, clientWrapper, this.mediaProperties, this.ffmpegProperties);
recorder.start();
clientWrapper.setRecorder(recorder);
// 打开媒体录制
final Message message = this.build();
final Map<String, Object> body = new HashMap<>();
body.put("audioPort", recorder.getAudioPort());
body.put("videoPort", recorder.getVideoPort());
body.put(Constant.HOST, this.ffmpegProperties.getHost());
body.put(Constant.ROOM_ID, room.getRoomId());
body.put(Constant.ENABLED, true);
body.put(Constant.FILEPATH, recorder.getFilepath());
body.put(Constant.CLIENT_ID, clientWrapper.getClientId());
body.put(Constant.AUDIO_PORT, recorder.getAudioPort());
body.put(Constant.VIDEO_PORT, recorder.getVideoPort());
body.put(Constant.AUDIO_RTCP_PORT, recorder.getAudioRtcpPort());
body.put(Constant.VIDEO_RTCP_PORT, recorder.getVideoRtcpPort());
body.put(Constant.RTP_CAPABILITIES, clientWrapper.getRtpCapabilities());
clientWrapper.getProducers().values().forEach(producer -> {
if(producer.getKind() == Kind.AUDIO) {