qq27800734/taoyao
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[*] 本地录像

Browse Source
This commit is contained in:
acgist
2023-06-02 07:17:10 +08:00
parent a06f6a251f
commit becc68b05f
19 changed files with 404 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
taoyao-client-android/taoyao/client/src/main/AndroidManifest.xml
View File

@@ -8,7 +8,9 @@
android:extractNativeLibs="true"
android:fullBackupContent="@xml/backup_rules"
android:icon="@mipmap/ic_launcher"
android:killAfterRestore="true"
android:label="@string/app_name"
android:persistent="true"
android:roundIcon="@mipmap/ic_launcher_round"
android:supportsRtl="true">
<activity

42
taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java
View File

@@ -625,10 +625,10 @@ public final class Taoyao implements ITaoyao {
case "client::reboot" -> this.clientReboot(message, message.body());
case "client::register" -> this.clientRegister(message, message.body());
case "client::shutdown" -> this.clientShutdown(message, message.body());
case "control::client::record" -> this.controlClientRecord(message, message.body());
case "control::config::audio" -> this.controlConfigAudio(message, message.body());
case "control::config::video" -> this.controlConfigVideo(message, message.body());
case "control::photograph" -> this.controlPhotograph(message, message.body());
case "control::record" -> this.controlRecord(message, message.body());
case "media::audio::volume" -> this.mediaAudioVolume(message, message.body());
case "media::consume" -> this.mediaConsume(message, message.body());
case "media::consumer::close" -> this.mediaConsumerClose(message, message.body());
@@ -751,6 +751,26 @@ public final class Taoyao implements ITaoyao {
Process.killProcess(Process.myPid());
}
/**
* 录像
*
* @param message 信令消息
* @param body 信令主体
*/
private void controlClientRecord(Message message, Map<String, Object> body) {
String filepath;
final Boolean enabled = MapUtils.getBoolean(body, "enabled");
if(Boolean.TRUE.equals(enabled)) {
final RecordClient recordClient = this.mediaManager.startRecord();
filepath = recordClient.getFilepath();
} else {
filepath = this.mediaManager.stopRecord();
}
body.put("enabled", enabled);
body.put("filepath", filepath);
this.push(message);
}
/**
* 更新音频配置
*
@@ -785,26 +805,6 @@ public final class Taoyao implements ITaoyao {
this.push(message);
}
/**
* 录像
*
* @param message 信令消息
* @param body 信令主体
*/
private void controlRecord(Message message, Map<String, Object> body) {
String filepath;
final Boolean enabled = MapUtils.getBoolean(body, "enabled");
if(Boolean.TRUE.equals(enabled)) {
final RecordClient recordClient = this.mediaManager.startRecord();
filepath = recordClient.getFilepath();
} else {
filepath = this.mediaManager.stopRecord();
}
body.put("enabled", enabled);
body.put("filepath", filepath);
this.push(message);
}
/**
* 远程音量
*

35
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java
View File

@@ -2,6 +2,8 @@ package com.acgist.taoyao.media;
import android.content.Context;
import android.content.Intent;
import android.media.MediaCodecInfo;
import android.media.MediaCodecList;
import android.media.projection.MediaProjection;
import android.os.Handler;
import android.speech.tts.TextToSpeech;
@@ -45,6 +47,8 @@ import org.webrtc.audio.JavaAudioDeviceModule;
import java.util.Arrays;
import java.util.Locale;
import java.util.UUID;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
/**
* 媒体管理器
@@ -185,20 +189,23 @@ public final class MediaManager {
// WebRtcAudioUtils.setWebRtcBasedAcousticEchoCanceler(true);
// // 自动增益
// WebRtcAudioUtils.setWebRtcBasedAutomaticGainControl(true);
// // 支持的编码器
// final MediaCodecList mediaCodecList = new MediaCodecList(-1);
// for (MediaCodecInfo mediaCodecInfo : mediaCodecList.getCodecInfos()) {
// final String[] supportedTypes = mediaCodecInfo.getSupportedTypes();
// Log.d(MediaManager.class.getSimpleName(), "编码器名称：" + mediaCodecInfo.getName());
// Log.d(MediaManager.class.getSimpleName(), "编码器类型：" + String.join(", ", supportedTypes));
// for (String supportType : supportedTypes) {
// final MediaCodecInfo.CodecCapabilities codecCapabilities = mediaCodecInfo.getCapabilitiesForType(supportType);
// Log.d(MediaManager.class.getSimpleName(), "编码器支持的文件格式：" + codecCapabilities.getMimeType());
// // MediaCodecInfo.CodecCapabilities.COLOR_*
// final int[] colorFormats = codecCapabilities.colorFormats;
// Log.d(MediaManager.class.getSimpleName(), "编码器支持的色彩格式：" + IntStream.of(colorFormats).boxed().map(String::valueOf).collect(Collectors.joining(", ")));
// }
// }
// // 支持的编码解码器
final MediaCodecList mediaCodecList = new MediaCodecList(MediaCodecList.ALL_CODECS);
for (MediaCodecInfo mediaCodecInfo : mediaCodecList.getCodecInfos()) {
// OMX.google = 软编
// OMX.core = 硬编
final String[] supportedTypes = mediaCodecInfo.getSupportedTypes();
final String type = mediaCodecInfo.isEncoder() ? "编码器" : "解码器";
Log.d(MediaManager.class.getSimpleName(), type + "名称：" + mediaCodecInfo.getName());
Log.d(MediaManager.class.getSimpleName(), type + "类型：" + String.join(", ", supportedTypes));
for (String supportType : supportedTypes) {
final MediaCodecInfo.CodecCapabilities codecCapabilities = mediaCodecInfo.getCapabilitiesForType(supportType);
Log.d(MediaManager.class.getSimpleName(), type + "支持的文件格式：" + codecCapabilities.getMimeType());
// MediaCodecInfo.CodecCapabilities.COLOR_*
final int[] colorFormats = codecCapabilities.colorFormats;
Log.d(MediaManager.class.getSimpleName(), type + "支持的色彩格式：" + IntStream.of(colorFormats).boxed().map(String::valueOf).collect(Collectors.joining(", ")));
}
}
}
private MediaManager() {

29
taoyao-client-media/src/Taoyao.js
View File

@@ -455,8 +455,8 @@ class Taoyao {
case "media::producer::resume":
me.mediaProducerResume(message, body);
break;
case "media::record":
me.mediaRecord(message, body);
case "control::server::record":
me.controlServerRecord(message, body);
break;
case "media::router::rtp::capabilities":
me.mediaRouterRtpCapabilities(message, body);
@@ -776,18 +776,19 @@ class Taoyao {
* @param {*} message 消息
* @param {*} body 消息主体
*/
async mediaRecord(message, body) {
async controlServerRecord(message, body) {
const me = this;
const { enabled, roomId } = body;
const room = this.rooms.get(roomId);
if(enabled) {
await me.mediaRecordStart(message, body, room);
await me.controlServerRecordStart(message, body, room);
} else {
await me.mediaRecordStop(message, body, room);
await me.controlServerRecordStop(message, body, room);
}
}
async mediaRecordStart(message, body, room) {
async controlServerRecordStart(message, body, room) {
const me = this;
const { roomId, clientId, host, audioPort, videoPort, rtpCapabilities, audioStreamId, videoStreamId, audioProducerId, videoProducerId } = body;
const plainTransportOptions = {
...config.mediasoup.plainTransportOptions,
@@ -800,10 +801,12 @@ class Taoyao {
let videoTransportId;
if(audioProducerId) {
const audioTransport = await room.mediasoupRouter.createPlainTransport(plainTransportOptions);
audioTransportId = audioTransport.id;
me.transportEvent("plain", roomId, audioTransport);
audioTransport.clientId = clientId;
room.transports.set(audioTransport.id, audioTransport);
audioTransport.observer.on("close", () => {
console.log("controlServerRecord audioTransport close：", audioTransport.id);
room.transports.delete(audioTransport.id)
});
await audioTransport.connect({
@@ -811,7 +814,7 @@ class Taoyao {
port : audioPort,
rtcpPort : audioPort
});
const audioConsumer = await transport.consume({
const audioConsumer = await audioTransport.consume({
producerId: audioProducerId,
rtpCapabilities,
paused: true
@@ -822,15 +825,19 @@ class Taoyao {
audioConsumer.streamId = audioStreamId;
room.consumers.set(audioConsumer.id, audioConsumer);
audioConsumer.observer.on("close", () => {
console.log("controlServerRecord audioConsumer close：", audioConsumer.id);
room.consumers.delete(audioConsumer.id);
});
console.log("controlServerRecord audio", audioTransportId, audioConsumerId, audioTransport.tuple);
}
if(videoProducerId) {
const videoTransport = await room.mediasoupRouter.createPlainTransport(plainTransportOptions);
videoTransportId = videoTransport.id;
me.transportEvent("plain", roomId, videoTransport);
videoTransport.clientId = clientId;
room.transports.set(videoTransport.id, videoTransport);
videoTransport.observer.on("close", () => {
console.log("controlServerRecord videoTransport close：", videoTransport.id);
room.transports.delete(videoTransport.id)
});
await videoTransport.connect({
@@ -838,7 +845,7 @@ class Taoyao {
port : videoPort,
rtcpPort : videoPort
});
const videoConsumer = await transport.consume({
const videoConsumer = await videoTransport.consume({
producerId: videoProducerId,
rtpCapabilities,
paused: true
@@ -849,8 +856,10 @@ class Taoyao {
videoConsumer.streamId = videoStreamId;
room.consumers.set(videoConsumer.id, videoConsumer);
videoConsumer.observer.on("close", () => {
console.log("controlServerRecord videoConsumer close：", videoConsumer.id);
room.consumers.delete(videoConsumer.id);
});
console.log("controlServerRecord video：", videoTransportId, videoConsumerId, videoTransport.tuple);
}
message.body = {
roomId : roomId,
@@ -862,7 +871,8 @@ class Taoyao {
me.push(message);
}
async mediaRecordStart(message, body, room) {
async controlServerRecordStop(message, body, room) {
const me = this;
const { audioStreamId, videoStreamId, audioConsumerId, videoConsumerId, audioTransportId, videoTransportId } = body;
const audioConsumer = room.consumers.get(audioConsumerId);
if(audioConsumer) {
@@ -884,6 +894,7 @@ class Taoyao {
videoTransport.close();
room.transports.delete(videoTransportId);
}
me.push(message);
}
async mediaConsume(message, body) {

1
taoyao-client-web/src/App.vue
View File

@@ -161,7 +161,6 @@ export default {
},
async sessionCall() {
this.taoyao.sessionCall(this.room.callClientId);
// this.taoyao.sessionCall(this.room.callClientId, false, false);
this.roomVisible = false;
},
async roomCreate() {

2
taoyao-client-web/src/components/Config.js
View File

@@ -1,7 +1,7 @@
/**
* 音频默认配置
* TODO：MediaStreamTrack.applyConstraints().then().catch();
* let setting = {
* const setting = {
* autoGainControl: true,
* noiseSuppression: true
* }

30
taoyao-client-web/src/components/LocalClient.vue
View File

@@ -9,10 +9,11 @@
<el-button @click="taoyao.mediaProducerPause(audioProducer.id)" v-show="audioProducer && !audioProducer.paused" type="primary" title="关闭麦克风" :icon="Microphone" circle class="mic" :style="{'--volume': client.volume}" />
<el-button @click="taoyao.mediaProducerResume(videoProducer.id)" v-show="videoProducer && videoProducer.paused" type="danger" title="打开摄像头" :icon="VideoPlay" circle />
<el-button @click="taoyao.mediaProducerPause(videoProducer.id)" v-show="videoProducer && !videoProducer.paused" type="primary" title="关闭摄像头" :icon="VideoPause" circle />
<el-button @click="exchangeVideoSource" :icon="Refresh" circle title="交换媒体" />
<el-button :icon="Camera" circle title="拍照" />
<el-button :icon="VideoCamera" circle title="录像" />
<el-button @click="taoyao.mediaProducerStatus()" :icon="InfoFilled" circle title="媒体信息" />
<el-button @click="exchangeVideoSource" :icon="Refresh" circle title="交换媒体" />
<el-button @onclick="localPhotograph" :icon="Camera" circle title="拍照" />
<el-button @onclick="localClientRecord" :icon="VideoCamera" circle title="录像" :type="clientRecord ? 'danger' : ''" />
<el-button @click="taoyao.controlServerRecord(client.clientId, (serverRecord = !serverRecord))" :icon="MostlyCloudy" circle title="录像" :type="serverRecord ? 'danger' : ''" />
<el-button @click="taoyao.mediaProducerStatus()" :icon="InfoFilled" circle title="媒体信息" />
<el-popover placement="top" :width="240" trigger="hover">
<template #reference>
<el-button>视频质量</el-button>
@@ -37,6 +38,7 @@ import {
Microphone,
VideoCamera,
CircleClose,
MostlyCloudy,
} from "@element-plus/icons-vue";
export default {
name: "LocalClient",
@@ -51,12 +53,15 @@ export default {
Microphone,
VideoCamera,
CircleClose,
MostlyCloudy,
};
},
data() {
return {
audio: null,
video: null,
clientRecord: false,
serverRecord: false,
audioStream: null,
videoStream: null,
dataProducer: null,
@@ -78,6 +83,13 @@ export default {
}
},
methods: {
localPhotograph() {
this.taoyao.localPhotograph(this.video);
},
localClientRecord() {
this.clientRecord = !this.clientRecord;
this.taoyao.localClientRecord(this.audioStream, this.videoStream, this.clientRecord);
},
exchangeVideoSource() {
// TODO：文件支持
this.taoyao.videoSource = this.taoyao.videoSource === "camera" ? "screen" : "camera";
@@ -85,8 +97,16 @@ export default {
},
media(track, producer) {
if(track.kind === "audio") {
// 不用加载音频
this.audioProducer = producer;
if (this.audioStream) {
this.audioStream.getAudioTracks().forEach(oldTrack => {
console.debug("关闭旧的媒体：", oldTrack);
oldTrack.stop();
});
}
this.audioStream = new MediaStream();
this.audioStream.addTrack(track);
// 不用加载音频
} else if (track.kind === "video") {
this.videoProducer = producer;
if (this.videoStream) {

12
taoyao-client-web/src/components/RemoteClient.vue
View File

@@ -12,9 +12,10 @@
<el-button @click="taoyao.mediaConsumerPause(audioConsumer.id)" v-show="audioConsumer && !audioConsumer.paused" type="primary" title="关闭麦克风" :icon="Microphone" circle class="mic" :style="{'--volume': client.volume}" />
<el-button @click="taoyao.mediaConsumerResume(videoConsumer.id)" v-show="videoConsumer && videoConsumer.paused" type="danger" title="打开摄像头" :icon="VideoPlay" circle />
<el-button @click="taoyao.mediaConsumerPause(videoConsumer.id)" v-show="videoConsumer && !videoConsumer.paused" type="primary" title="关闭摄像头" :icon="VideoPause" circle />
<el-button @click="taoyao.controlPhotograph(client.clientId)" :icon="Camera" circle title="拍照" />
<el-button @click="taoyao.controlRecord(client.clientId, (record = !record))" :icon="VideoCamera" circle title="录像" :type="record ? 'danger' : ''" />
<el-button @click="taoyao.mediaConsumerStatus()" :icon="InfoFilled" circle title="媒体信息" />
<el-button @click="taoyao.controlPhotograph(client.clientId)" :icon="Camera" circle title="拍照" />
<el-button @click="taoyao.controlClientRecord(client.clientId, (clientRecord = !clientRecord))" :icon="VideoCamera" circle title="终端录像" :type="clientRecord ? 'danger' : ''" />
<el-button @click="taoyao.controlServerRecord(client.clientId, (serverRecord = !serverRecord))" :icon="MostlyCloudy" circle title="服务端录像" :type="serverRecord ? 'danger' : ''" />
<el-button @click="taoyao.mediaConsumerStatus()" :icon="InfoFilled" circle title="媒体信息" />
<el-popover placement="top" :width="240" trigger="hover">
<template #reference>
<el-button>视频质量</el-button>
@@ -40,6 +41,7 @@ import {
Microphone,
VideoCamera,
CircleClose,
MostlyCloudy,
} from "@element-plus/icons-vue";
export default {
name: "RemoteClient",
@@ -54,13 +56,15 @@ export default {
Microphone,
VideoCamera,
CircleClose,
MostlyCloudy,
};
},
data() {
return {
audio: null,
video: null,
record: false,
clientRecord: false,
serverRecord: false,
audioStream: null,
videoStream: null,
dataConsumer: null,

7
taoyao-client-web/src/components/SessionClient.vue
View File

@@ -13,8 +13,8 @@
<el-button @click="taoyao.sessionPause(client.id, 'audio')" v-show="audioStream && client.remoteAudioEnabled" type="primary" title="关闭远程麦克风" :icon="Microphone" circle class="mic" :style="{'--volume': client.volume}" />
<el-button @click="taoyao.sessionResume(client.id, 'video')" v-show="videoStream && !client.remoteVideoEnabled" type="danger" title="打开远程摄像头" :icon="VideoPlay" circle />
<el-button @click="taoyao.sessionPause(client.id, 'video')" v-show="videoStream && client.remoteVideoEnabled" type="primary" title="关闭远程摄像头" :icon="VideoPause" circle />
<el-button @click="taoyao.controlPhotograph(client.clientId)" :icon="Camera" circle title="拍照" />
<el-button @click="taoyao.controlRecord(client.clientId, (record = !record))" :icon="VideoCamera" circle title="录像" :type="record ? 'danger' : ''" />
<el-button @click="taoyao.controlPhotograph(client.clientId)" :icon="Camera" circle title="拍照" />
<el-button @click="taoyao.controlClientRecord(client.clientId, (clientRecord = !clientRecord))" :icon="VideoCamera" circle title="录像" :type="clientRecord ? 'danger' : ''" />
<el-popover placement="top" :width="240" trigger="hover">
<template #reference>
<el-button>视频质量</el-button>
@@ -64,7 +64,8 @@ export default {
return {
audio: null,
video: null,
record: false,
clientRecord: false,
serverRecord: false,
audioStream: null,
videoStream: null,
};

169
taoyao-client-web/src/components/Taoyao.js
View File

@@ -509,6 +509,10 @@ class Taoyao extends RemoteClient {
remoteClients = new Map();
// 会话终端
sessionClients = new Map();
// 本地录像机
mediaRecorder;
// 本地录像数据
mediaRecorderChunks = [];
constructor({
name,
@@ -811,6 +815,21 @@ class Taoyao extends RemoteClient {
}
return stream;
}
async getAudioTrack() {
const self = this;
const stream = await navigator.mediaDevices.getUserMedia({
audio: self.audioConfig,
});
// TODO：首个？
const track = stream.getAudioTracks()[0];
// TODO：验证修改API audioTrack.applyCapabilities
console.debug(
"音频信息：",
track.getSettings(),
track.getCapabilities()
);
return track;
}
async getVideoTrack() {
let track;
const self = this;
@@ -819,11 +838,11 @@ class Taoyao extends RemoteClient {
// const stream = await this._getExternalVideoStream();
// track = stream.getVideoTracks()[0].clone();
} else if (self.videoSource === "camera") {
console.debug("enableWebcam() | calling getUserMedia()");
// TODO：参数
const stream = await navigator.mediaDevices.getUserMedia({
video: self.videoConfig,
});
// TODO：首个？
track = stream.getVideoTracks()[0];
// TODO：验证修改API videoTrack.applyCapabilities
console.debug(
@@ -939,20 +958,36 @@ class Taoyao extends RemoteClient {
);
}
/**
* 录像
* 终端录像信令
*
* @param {*} clientId
* @param {*} enabled
*/
controlRecord(clientId, enabled) {
controlClientRecord(clientId, enabled) {
const me = this;
me.push(
protocol.buildMessage("control::record", {
protocol.buildMessage("control::client::record", {
to: clientId,
enabled: enabled
})
);
}
/**
* 服务端录像信令
*
* @param {*} clientId
* @param {*} enabled
*/
controlServerRecord(clientId, enabled) {
const me = this;
me.push(
protocol.buildMessage("control::server::record", {
to: clientId,
roomId: me.roomId,
enabled: enabled
})
);
}
/**
* 终端音量信令
*
@@ -1923,20 +1958,7 @@ class Taoyao extends RemoteClient {
let track;
try {
console.debug("打开麦克风");
const stream = await navigator.mediaDevices.getUserMedia({
audio: self.audioConfig,
});
const tracks = stream.getAudioTracks();
if (tracks.length > 1) {
console.warn("多个音频轨道");
}
track = tracks[0];
// TODO：验证修改API audioTrack.applyCapabilities
console.debug(
"音频信息：",
track.getSettings(),
track.getCapabilities()
);
let track = self.getAudioTrack();
this.audioProducer = await this.sendTransport.produce({
track,
codecOptions: {
@@ -2009,13 +2031,13 @@ class Taoyao extends RemoteClient {
* TODO：重复点击
*/
async produceVideo() {
console.debug("打开摄像头");
const self = this;
if (self.videoProduce && self.mediasoupDevice.canProduce("video")) {
if (self.videoProducer) {
return;
}
try {
console.debug("打开摄像头");
let track = await self.getVideoTrack();
let codec;
let encodings;
@@ -2215,9 +2237,13 @@ class Taoyao extends RemoteClient {
});
if (!audioEnabled && self.audioProduce) {
self.callbackError("没有音频媒体设备");
// 强制修改
self.audioProduce = false;
}
if (!videoEnabled && self.videoProduce) {
self.callbackError("没有视频媒体设备");
// 强制修改
self.videoProduce = false;
}
} else {
self.callbackError("没有媒体设备");
@@ -2251,23 +2277,22 @@ class Taoyao extends RemoteClient {
/**
* 发起会话
*
* @param {*} clientId 接收者ID
* @param {*} audioEnabled 是否打开音频
* @param {*} videoEnabled 是否打开视频
* @param {*} clientId 接收者ID
*/
async sessionCall(clientId, audioEnabled = true, videoEnabled = true) {
async sessionCall(clientId) {
const me = this;
if (!clientId) {
this.callbackError("无效终端");
return;
}
me.checkDevice();
const response = await me.request(
protocol.buildMessage("session::call", {
clientId
})
);
const { name, sessionId } = response.body;
const session = new Session({name, clientId, sessionId, audioEnabled, videoEnabled});
const session = new Session({name, clientId, sessionId, audioEnabled: me.audioProduce, videoEnabled: me.videoProduce});
this.sessionClients.set(sessionId, session);
}
@@ -2434,6 +2459,101 @@ class Taoyao extends RemoteClient {
return peerConnection;
}
/**
* 本地截图
*
* @param {*} video 视频
*/
localPhotograph(video) {
const me = this;
const canvas = document.createElement('canvas');
canvas.width = video.videoWidth;
canvas.height = video.videoHeight;
const context = canvas.getContext('2d');
context.drawImage(video, 0, 0, video.videoWidth, video.videoHeight);
const dataURL = canvas.toDataURL('images/png');
const download = document.createElement('a');
download.href = dataURL;
download.download = 'taoyao.png';
download.style.display = 'none';
document.body.appendChild(download);
download.click();
download.remove();
}
// 'video/webm;codecs=aac,vp8',
// 'video/webm;codecs=aac,vp9',
// 'video/webm;codecs=aac,h264',
// 'video/webm;codecs=pcm,vp8',
// 'video/webm;codecs=pcm,vp9',
// 'video/webm;codecs=pcm,h264',
// 'video/webm;codecs=opus,vp8',
// 'video/webm;codecs=opus,vp9',
// 'video/webm;codecs=opus,h264',
// 'video/mp4;codecs=aac,vp8',
// 'video/mp4;codecs=aac,vp9',
// 'video/mp4;codecs=aac,h264',
// 'video/mp4;codecs=pcm,vp8',
// 'video/mp4;codecs=pcm,vp9',
// 'video/mp4;codecs=pcm,h264',
// 'video/mp4;codecs=opus,vp8',
// 'video/mp4;codecs=opus,vp9',
// 'video/mp4;codecs=opus,h264',
// MediaRecorder.isTypeSupported(mimeType)
/**
* 本地录制
*
* video.captureStream().getTracks().forEach((v) => stream.addTrack(v));
*
* @param {*} audio 音频
* @param {*} video 视频
* @param {*} enabled 是否录制
*/
localClientRecord(audio, video, enabled) {
const me = this;
if (enabled) {
if (me.mediaRecorder) {
return;
}
const stream = new MediaStream();
if(audio) {
audio.getAudioTracks().forEach(track => stream.addTrack(track));
}
if(video) {
video.getVideoTracks().forEach(track => stream.addTrack(track));
}
me.mediaRecorder = new MediaRecorder(stream, {
audioBitsPerSecond: 128 * 1000,
videoBitsPerSecond: 2400 * 1000,
mimeType: 'video/webm;codecs=opus,h264',
});
mediaRecorder.onstop = function (e) {
const blob = new Blob(me.mediaRecorderChunks);
const objectURL = URL.createObjectURL(blob);
const download = document.createElement('a');
download.href = objectURL;
download.download = 'taoyao.mp4';
download.style.display = 'none';
document.body.appendChild(download);
download.click();
download.remove();
URL.revokeObjectURL(objectURL);
me.mediaRecorderChunks = [];
};
mediaRecorder.ondataavailable = (e) => {
me.mediaRecorderChunks.push(e.data);
};
me.mediaRecorder.start();
} else {
if (!me.mediaRecorder) {
return;
}
me.mediaRecorder.stop();
me.mediaRecorder = null;
}
}
/**
* 关闭视频房间媒体
*/
@@ -2504,3 +2624,4 @@ class Taoyao extends RemoteClient {
}
export { Taoyao };

18
taoyao-signal-server/taoyao-server/src/main/java/com/acgist/taoyao/controller/ControlController.java
View File

@@ -10,10 +10,11 @@ import com.acgist.taoyao.boot.config.MediaAudioProperties;
import com.acgist.taoyao.boot.config.MediaVideoProperties;
import com.acgist.taoyao.boot.model.Message;
import com.acgist.taoyao.signal.protocol.control.ControlBellProtocol;
import com.acgist.taoyao.signal.protocol.control.ControlClientRecordProtocol;
import com.acgist.taoyao.signal.protocol.control.ControlConfigAudioProtocol;
import com.acgist.taoyao.signal.protocol.control.ControlConfigVideoProtocol;
import com.acgist.taoyao.signal.protocol.control.ControlPhotographProtocol;
import com.acgist.taoyao.signal.protocol.control.ControlRecordProtocol;
import com.acgist.taoyao.signal.protocol.control.ControlServerRecordProtocol;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
@@ -34,10 +35,11 @@ import lombok.RequiredArgsConstructor;
public class ControlController {
private final ControlBellProtocol controlBellProtocol;
private final ControlRecordProtocol controlRecordProtocol;
private final ControlPhotographProtocol controlPhotographProtocol;
private final ControlConfigAudioProtocol controlConfigAudioProtocol;
private final ControlConfigVideoProtocol controlConfigVideoProtocol;
private final ControlClientRecordProtocol controlClientRecordProtocol;
private final ControlServerRecordProtocol controlServerRecordProtocol;
@Operation(summary = "响铃", description = "响铃控制")
@GetMapping("/bell/{clientId}")
@@ -45,10 +47,16 @@ public class ControlController {
return this.controlBellProtocol.execute(clientId, enabled);
}
@Operation(summary = "录像", description = "录像控制")
@GetMapping("/record/{clientId}")
@Operation(summary = "录像", description = "终端录像控制")
@GetMapping("/client/record/{clientId}")
public Message record(@PathVariable String clientId, @NotNull(message = "没有指定操作状态") Boolean enabled) {
return this.controlRecordProtocol.execute(clientId, enabled);
return this.controlClientRecordProtocol.execute(clientId, enabled);
}
@Operation(summary = "录像", description = "服务端录像控制")
@GetMapping("/server/record/{roomId}/{clientId}")
public Message record(@PathVariable String roomId, @PathVariable String clientId, @NotNull(message = "没有指定操作状态") Boolean enabled) {
return this.controlServerRecordProtocol.execute(roomId, clientId, enabled);
}
@Operation(summary = "拍照", description = "拍照控制")

37
taoyao-signal-server/taoyao-server/src/main/java/com/acgist/taoyao/controller/MediaController.java
View File

@@ -1,37 +0,0 @@
package com.acgist.taoyao.controller;
import org.springframework.validation.annotation.Validated;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import com.acgist.taoyao.boot.model.Message;
import com.acgist.taoyao.signal.protocol.media.MediaRecordProtocol;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.tags.Tag;
import jakarta.validation.constraints.NotNull;
import lombok.RequiredArgsConstructor;
/**
* 媒体
*
* @author acgist
*/
@Tag(name = "媒体", description = "媒体管理")
@Validated
@RestController
@RequestMapping("/media")
@RequiredArgsConstructor
public class MediaController {
private final MediaRecordProtocol mediaRecordProtocol;
@Operation(summary = "录像", description = "媒体录像")
@GetMapping("/record/{roomId}/{clientId}")
public Message record(@PathVariable String roomId, @PathVariable String clientId, @NotNull(message = "没有指定操作状态") Boolean enabled) {
return this.mediaRecordProtocol.execute(roomId, clientId, enabled);
}
}

21
taoyao-signal-server/taoyao-server/src/main/resources/application.yml
View File

@@ -228,17 +228,17 @@ taoyao:
# SDP：VP8 | H264
sdp: |
v=0
o=- 0 0 IN IP4 %s
o=- 0 0 IN IP4 127.0.0.1
s=TaoyaoRecord
t=0 0
m=audio %d RTP/AVP 97
c=IN IP4 %s
a=rtpmap:97 OPUS/48000/2
a=fmtp:97 sprop-stereo=1
m=video %d RTP/AVP 96
c=IN IP4 %s
a=rtpmap:96 H264/90000
a=fmtp:96 packetization-mode=1
m=audio %d RTP/AVP 100
c=IN IP4 127.0.0.1
a=rtpmap:100 OPUS/48000/2
a=fmtp:100 sprop-stereo=1
m=video %d RTP/AVP 107
c=IN IP4 127.0.0.1
a=rtpmap:107 H264/90000
a=fmtp:107 packetization-mode=1
# 录像命令
record: ffmpeg -protocol_whitelist "file,rtp,udp" -y -i %s %s
# 截图命令
@@ -252,7 +252,8 @@ taoyao:
# 视频存储目录
storage-video-path: /data/taoyao/storage/video
# 录像地址
host: 127.0.0.1
#host: 127.0.0.1
host: 192.168.8.40
# 端口范围
min-port: 50000
max-port: 59999

18
taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/RecorderTest.java
View File

@@ -25,13 +25,27 @@ public class RecorderTest {
a=fmtp:97 sprop-stereo=1
m=video %d RTP/AVP 96
c=IN IP4 %s
a=rtpmap:96 H264/90000
a=rtpmap:96 VP8/90000
a=fmtp:96 packetization-mode=1
""");
// ffmpegProperties.setSdp("""
// v=0
// o=- 0 0 IN IP4 %s
// s=TaoyaoRecord
// t=0 0
// m=audio %d RTP/AVP 97
// c=IN IP4 %s
// a=rtpmap:97 OPUS/48000/2
// a=fmtp:97 sprop-stereo=1
// m=video %d RTP/AVP 96
// c=IN IP4 %s
// a=rtpmap:96 H264/90000
// a=fmtp:96 packetization-mode=1
// """);
ffmpegProperties.setRecord("ffmpeg -protocol_whitelist \"file,rtp,udp\" -y -i %s %s");
ffmpegProperties.setPreview("ffmpeg -y -i %s -ss %d -vframes 1 -f image2 %s");
ffmpegProperties.setDuration("ffprobe -i %s -show_entries format=duration");
final Recorder recorder = new Recorder("taoyao", ffmpegProperties);
final Recorder recorder = new Recorder("taoyao", null, null, ffmpegProperties);
recorder.start();
Thread.sleep(20 * 1000);
recorder.stop();

30
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/event/room/RecorderCloseEvent.java Normal file
View File

@@ -0,0 +1,30 @@
package com.acgist.taoyao.signal.event.room;
import com.acgist.taoyao.signal.event.RoomEventAdapter;
import com.acgist.taoyao.signal.party.media.Recorder;
import lombok.Getter;
import lombok.Setter;
/**
* 服务端录像关闭事件
*
* @author acgist
*/
@Getter
@Setter
public class RecorderCloseEvent extends RoomEventAdapter {
private static final long serialVersionUID = 1L;
/**
* 媒体录像机
*/
private final Recorder recorder;
public RecorderCloseEvent(Recorder recorder) {
super(recorder.getRoom());
this.recorder = recorder;
}
}

6
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/media/ClientWrapper.java
View File

@@ -12,6 +12,7 @@ import lombok.extern.slf4j.Slf4j;
/**
* 终端包装器：Peer
* 视频房间使用
*
* @author acgist
*/
@@ -123,7 +124,10 @@ public class ClientWrapper implements AutoCloseable {
@Override
public void close() {
// 注意：不要关闭终端
// 注意：不要关闭终端（只是离开房间）
if(this.recorder != null) {
this.recorder.close();
}
this.consumers.values().forEach(Consumer::close);
this.producers.values().forEach(Producer::close);
this.dataConsumers.values().forEach(DataConsumer::close);

41
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/media/Recorder.java
View File

@@ -15,13 +15,19 @@ import com.acgist.taoyao.boot.utils.FileUtils;
import com.acgist.taoyao.boot.utils.NetUtils;
import com.acgist.taoyao.boot.utils.ScriptUtils;
import com.acgist.taoyao.boot.utils.ScriptUtils.ScriptExecutor;
import com.acgist.taoyao.signal.event.EventPublisher;
import com.acgist.taoyao.signal.event.room.RecorderCloseEvent;
import lombok.Getter;
import lombok.Setter;
import lombok.extern.slf4j.Slf4j;
/**
* 媒体录像
* 媒体录像机
*
* OPUS = 100
* VP8 = 101
* H264 = 107
*
* @author acgist
*/
@@ -90,6 +96,14 @@ public class Recorder {
* 命令执行器
*/
private ScriptExecutor scriptExecutor;
/**
* 房间
*/
private final Room room;
/**
* 终端
*/
private final ClientWrapper clientWrapper;
/**
* 文件路径
*/
@@ -113,16 +127,20 @@ public class Recorder {
/**
* @param name 录像名称
* @param room 房间
* @param clientWrapper 终端
* @param ffmpegProperties FFmpeg配置
*/
public Recorder(String name, FfmpegProperties ffmpegProperties) {
public Recorder(String name, Room room, ClientWrapper clientWrapper, FfmpegProperties ffmpegProperties) {
this.close = false;
this.running = false;
this.ffmpegProperties = ffmpegProperties;
this.room = room;
this.folder = Paths.get(ffmpegProperties.getStorageVideoPath(), name).toAbsolutePath().toString();
this.sdpfile = Paths.get(this.folder, "taoyao.sdp").toAbsolutePath().toString();
this.preview = Paths.get(this.folder, "taoyao.jpg").toAbsolutePath().toString();
this.filepath = Paths.get(this.folder, "taoyao.mp4").toAbsolutePath().toString();
this.clientWrapper = clientWrapper;
this.ffmpegProperties = ffmpegProperties;
FileUtils.mkdirs(this.folder);
}
@@ -136,6 +154,7 @@ public class Recorder {
}
this.running = true;
}
this.buildSdpfile();
this.thread = new Thread(this::record);
this.thread.setDaemon(true);
this.thread.setName("TaoyaoRecord");
@@ -146,7 +165,6 @@ public class Recorder {
* 录制视频
*/
private void record() {
this.buildSdpfile();
final String recordScript = String.format(this.ffmpegProperties.getRecord(), this.sdpfile, this.filepath);
this.scriptExecutor = new ScriptExecutor(recordScript);
try {
@@ -172,11 +190,11 @@ public class Recorder {
this.videoPort = NetUtils.scanPort(this.audioPort + 16, this.ffmpegProperties.getMaxPort());
final String sdp = String.format(
this.ffmpegProperties.getSdp(),
this.ffmpegProperties.getHost(),
// this.ffmpegProperties.getHost(),
this.audioPort,
this.ffmpegProperties.getHost(),
this.videoPort,
this.ffmpegProperties.getHost()
// this.ffmpegProperties.getHost(),
this.videoPort
// this.ffmpegProperties.getHost()
);
Files.write(
Paths.get(this.sdpfile),
@@ -242,4 +260,11 @@ public class Recorder {
this.duration();
}
/**
* 关闭录像
*/
public void close() {
EventPublisher.publishEvent(new RecorderCloseEvent(this));
}
}

10
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/control/ControlRecordProtocol.java → taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/control/ControlClientRecordProtocol.java
View File

@@ -11,7 +11,7 @@ import com.acgist.taoyao.signal.client.ClientType;
import com.acgist.taoyao.signal.protocol.ProtocolControlAdapter;
/**
* 录像信令
* 终端录像信令
*
* @author acgist
*/
@@ -36,12 +36,12 @@ import com.acgist.taoyao.signal.protocol.ProtocolControlAdapter;
"终端=>信令服务->目标终端->信令服务->终端"
}
)
public class ControlRecordProtocol extends ProtocolControlAdapter {
public class ControlClientRecordProtocol extends ProtocolControlAdapter {
public static final String SIGNAL = "control::record";
public static final String SIGNAL = "control::client::record";
public ControlRecordProtocol() {
super("录像信令", SIGNAL);
public ControlClientRecordProtocol() {
super("终端录像信令", SIGNAL);
}
@Override

73
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/media/MediaRecordProtocol.java → taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/control/ControlServerRecordProtocol.java
View File

@@ -1,9 +1,11 @@
package com.acgist.taoyao.signal.protocol.media;
package com.acgist.taoyao.signal.protocol.control;
import java.util.HashMap;
import java.util.Map;
import java.util.UUID;
import org.springframework.context.ApplicationListener;
import com.acgist.taoyao.boot.annotation.Description;
import com.acgist.taoyao.boot.annotation.Protocol;
import com.acgist.taoyao.boot.config.Constant;
@@ -12,14 +14,15 @@ import com.acgist.taoyao.boot.model.Message;
import com.acgist.taoyao.boot.utils.MapUtils;
import com.acgist.taoyao.signal.client.Client;
import com.acgist.taoyao.signal.client.ClientType;
import com.acgist.taoyao.signal.event.room.RecorderCloseEvent;
import com.acgist.taoyao.signal.party.media.ClientWrapper;
import com.acgist.taoyao.signal.party.media.Kind;
import com.acgist.taoyao.signal.party.media.Recorder;
import com.acgist.taoyao.signal.party.media.Room;
import com.acgist.taoyao.signal.protocol.ProtocolRoomAdapter;
import com.acgist.taoyao.signal.protocol.ProtocolControlAdapter;
/**
* 媒体录像
* 服务端录像信令
*
* @author acgist
*/
@@ -28,12 +31,14 @@ import com.acgist.taoyao.signal.protocol.ProtocolRoomAdapter;
body = {
"""
{
"clientId": "目标终端ID",
"to": "目标终端ID",
"roomId": "房间ID",
"enabled": 是否录像（true|false）
}
""",
"""
{
"roomId": "房间ID",
"enabled": 是否录像（true|false）,
"filepath": "视频文件路径"
}
@@ -41,23 +46,33 @@ import com.acgist.taoyao.signal.protocol.ProtocolRoomAdapter;
},
flow = "终端=>信令服务->终端"
)
public class MediaRecordProtocol extends ProtocolRoomAdapter {
public class ControlServerRecordProtocol extends ProtocolControlAdapter implements ApplicationListener<RecorderCloseEvent> {
public static final String SIGNAL = "control::server::record";
private final FfmpegProperties ffmpegProperties;
public MediaRecordProtocol(FfmpegProperties ffmpegProperties) {
super("媒体录像", "media::record");
public ControlServerRecordProtocol(FfmpegProperties ffmpegProperties) {
super("服务端录像信令", SIGNAL);
this.ffmpegProperties = ffmpegProperties;
}
@Override
public void onApplicationEvent(RecorderCloseEvent event) {
final Recorder recorder = event.getRecorder();
this.stop(recorder.getRoom(), recorder.getClientWrapper());
}
@Override
public void execute(String clientId, ClientType clientType, Room room, Client client, Client mediaClient, Message message, Map<String, Object> body) {
final Boolean enabled = MapUtils.get(body, Constant.ENABLED, Boolean.TRUE);
public void execute(String clientId, ClientType clientType, Client client, Client targetClient, Message message, Map<String, Object> body) {
String filepath;
final String roomId = MapUtils.get(body, Constant.ROOM_ID);
final Boolean enabled = MapUtils.get(body, Constant.ENABLED, Boolean.TRUE);
final Room room = this.roomManager.room(roomId);
if(enabled) {
filepath = this.start(room, client, mediaClient);
filepath = this.start(room, room.clientWrapper(client));
} else {
filepath = this.stop(room, client, mediaClient);
filepath = this.stop(room, room.clientWrapper(client));
}
body.put(Constant.FILEPATH, filepath);
client.push(message);
@@ -71,16 +86,16 @@ public class MediaRecordProtocol extends ProtocolRoomAdapter {
* @return 执行结果
*/
public Message execute(String roomId, String clientId, Boolean enabled) {
String filepath;
final Room room = this.roomManager.room(roomId);
final Client client = this.clientManager.clients(clientId);
final Client mediaClient = room.getMediaClient();
String filepath;
if(enabled) {
filepath = this.start(room, client, mediaClient);
filepath = this.start(room, room.clientWrapper(client));
} else {
filepath = this.stop(room, client, mediaClient);
filepath = this.stop(room, room.clientWrapper(client));
}
return Message.success(Map.of(
Constant.ROOM_ID, roomId,
Constant.ENABLED, enabled,
Constant.FILEPATH, filepath
));
@@ -89,22 +104,22 @@ public class MediaRecordProtocol extends ProtocolRoomAdapter {
/**
* 开始录制
*
* @param room 房间
* @param client 终端
* @param mediaClient 媒体终端
* @param room 房间
* @param clientWrapper 终端
* @param mediaClient 媒体终端
*
* @return 文件地址
*/
private String start(Room room, Client client, Client mediaClient) {
final ClientWrapper clientWrapper = room.clientWrapper(client);
private String start(Room room, ClientWrapper clientWrapper) {
synchronized (clientWrapper) {
final Recorder recorder = clientWrapper.getRecorder();
if(recorder != null) {
return recorder.getFilepath();
}
}
final String name = UUID.randomUUID().toString();
// 打开录制线程
final Recorder recorder = new Recorder(UUID.randomUUID().toString(), this.ffmpegProperties);
final Recorder recorder = new Recorder(name, room, clientWrapper, this.ffmpegProperties);
recorder.start();
clientWrapper.setRecorder(recorder);
// 打开媒体录制
@@ -115,15 +130,15 @@ public class MediaRecordProtocol extends ProtocolRoomAdapter {
body.put(Constant.HOST, this.ffmpegProperties.getHost());
body.put(Constant.ROOM_ID, room.getRoomId());
body.put(Constant.ENABLED, true);
body.put(Constant.CLIENT_ID, client.clientId());
body.put(Constant.CLIENT_ID, clientWrapper.getClientId());
body.put(Constant.RTP_CAPABILITIES, clientWrapper.getRtpCapabilities());
clientWrapper.getProducers().values().forEach(producer -> {
if(producer.getKind() == Kind.AUDIO) {
recorder.setAudioStreamId(Constant.STREAM_ID_CONSUMER.apply(producer.getStreamId(), client.clientId()));
recorder.setAudioStreamId(Constant.STREAM_ID_CONSUMER.apply(producer.getStreamId(), clientWrapper.getClientId()));
body.put("audioStreamId", recorder.getAudioStreamId());
body.put("audioProducerId", producer.getProducerId());
} else if(producer.getKind() == Kind.VIDEO) {
recorder.setAudioStreamId(Constant.STREAM_ID_CONSUMER.apply(producer.getStreamId(), client.clientId()));
recorder.setAudioStreamId(Constant.STREAM_ID_CONSUMER.apply(producer.getStreamId(), clientWrapper.getClientId()));
body.put("videoStreamId", recorder.getVideoStreamId());
body.put("videoProducerId", producer.getProducerId());
} else {
@@ -131,6 +146,7 @@ public class MediaRecordProtocol extends ProtocolRoomAdapter {
}
});
message.setBody(body);
final Client mediaClient = room.getMediaClient();
mediaClient.request(message);
return recorder.getFilepath();
}
@@ -138,15 +154,13 @@ public class MediaRecordProtocol extends ProtocolRoomAdapter {
/**
* 关闭录像
*
* @param room 房间
* @param client 终端
* @param mediaClient 媒体终端
* @param room 房间
* @param clientWrapper 终端
*
* @return 文件地址
*/
private String stop(Room room, Client client, Client mediaClient) {
private String stop(Room room, ClientWrapper clientWrapper) {
final Recorder recorder;
final ClientWrapper clientWrapper = room.clientWrapper(client);
synchronized (clientWrapper) {
recorder = clientWrapper.getRecorder();
if(recorder == null) {
@@ -168,6 +182,7 @@ public class MediaRecordProtocol extends ProtocolRoomAdapter {
body.put(Constant.ROOM_ID, room.getRoomId());
body.put(Constant.ENABLED, false);
message.setBody(body);
final Client mediaClient = room.getMediaClient();
mediaClient.request(message);
return recorder.getFilepath();
}