From becc68b05fccc04a77afa07113f33e33918355a5 Mon Sep 17 00:00:00 2001 From: acgist <289547414@qq.com> Date: Fri, 2 Jun 2023 07:17:10 +0800 Subject: [PATCH] =?UTF-8?q?[*]=20=E6=9C=AC=E5=9C=B0=E5=BD=95=E5=83=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../client/src/main/AndroidManifest.xml | 2 + .../acgist/taoyao/client/signal/Taoyao.java | 42 ++--- .../com/acgist/taoyao/media/MediaManager.java | 35 ++-- taoyao-client-media/src/Taoyao.js | 29 ++- taoyao-client-web/src/App.vue | 1 - taoyao-client-web/src/components/Config.js | 2 +- .../src/components/LocalClient.vue | 30 +++- .../src/components/RemoteClient.vue | 12 +- .../src/components/SessionClient.vue | 7 +- taoyao-client-web/src/components/Taoyao.js | 169 +++++++++++++++--- .../taoyao/controller/ControlController.java | 18 +- .../taoyao/controller/MediaController.java | 37 ---- .../src/main/resources/application.yml | 21 +-- .../java/com/acgist/taoyao/RecorderTest.java | 18 +- .../signal/event/room/RecorderCloseEvent.java | 30 ++++ .../signal/party/media/ClientWrapper.java | 6 +- .../taoyao/signal/party/media/Recorder.java | 41 ++++- ....java => ControlClientRecordProtocol.java} | 10 +- .../ControlServerRecordProtocol.java} | 73 +++++--- 19 files changed, 404 insertions(+), 179 deletions(-) delete mode 100644 taoyao-signal-server/taoyao-server/src/main/java/com/acgist/taoyao/controller/MediaController.java create mode 100644 taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/event/room/RecorderCloseEvent.java rename taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/control/{ControlRecordProtocol.java => ControlClientRecordProtocol.java} (85%) rename taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/{media/MediaRecordProtocol.java => control/ControlServerRecordProtocol.java} (69%) diff --git a/taoyao-client-android/taoyao/client/src/main/AndroidManifest.xml b/taoyao-client-android/taoyao/client/src/main/AndroidManifest.xml index 3f699a9..dcad905 100644 --- a/taoyao-client-android/taoyao/client/src/main/AndroidManifest.xml +++ b/taoyao-client-android/taoyao/client/src/main/AndroidManifest.xml @@ -8,7 +8,9 @@ android:extractNativeLibs="true" android:fullBackupContent="@xml/backup_rules" android:icon="@mipmap/ic_launcher" + android:killAfterRestore="true" android:label="@string/app_name" + android:persistent="true" android:roundIcon="@mipmap/ic_launcher_round" android:supportsRtl="true"> this.clientReboot(message, message.body()); case "client::register" -> this.clientRegister(message, message.body()); case "client::shutdown" -> this.clientShutdown(message, message.body()); + case "control::client::record" -> this.controlClientRecord(message, message.body()); case "control::config::audio" -> this.controlConfigAudio(message, message.body()); case "control::config::video" -> this.controlConfigVideo(message, message.body()); case "control::photograph" -> this.controlPhotograph(message, message.body()); - case "control::record" -> this.controlRecord(message, message.body()); case "media::audio::volume" -> this.mediaAudioVolume(message, message.body()); case "media::consume" -> this.mediaConsume(message, message.body()); case "media::consumer::close" -> this.mediaConsumerClose(message, message.body()); @@ -751,6 +751,26 @@ public final class Taoyao implements ITaoyao { Process.killProcess(Process.myPid()); } + /** + * 录像 + * + * @param message 信令消息 + * @param body 信令主体 + */ + private void controlClientRecord(Message message, Map body) { + String filepath; + final Boolean enabled = MapUtils.getBoolean(body, "enabled"); + if(Boolean.TRUE.equals(enabled)) { + final RecordClient recordClient = this.mediaManager.startRecord(); + filepath = recordClient.getFilepath(); + } else { + filepath = this.mediaManager.stopRecord(); + } + body.put("enabled", enabled); + body.put("filepath", filepath); + this.push(message); + } + /** * 更新音频配置 * @@ -785,26 +805,6 @@ public final class Taoyao implements ITaoyao { this.push(message); } - /** - * 录像 - * - * @param message 信令消息 - * @param body 信令主体 - */ - private void controlRecord(Message message, Map body) { - String filepath; - final Boolean enabled = MapUtils.getBoolean(body, "enabled"); - if(Boolean.TRUE.equals(enabled)) { - final RecordClient recordClient = this.mediaManager.startRecord(); - filepath = recordClient.getFilepath(); - } else { - filepath = this.mediaManager.stopRecord(); - } - body.put("enabled", enabled); - body.put("filepath", filepath); - this.push(message); - } - /** * 远程音量 * diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java index 65d81b9..28278ac 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java @@ -2,6 +2,8 @@ package com.acgist.taoyao.media; import android.content.Context; import android.content.Intent; +import android.media.MediaCodecInfo; +import android.media.MediaCodecList; import android.media.projection.MediaProjection; import android.os.Handler; import android.speech.tts.TextToSpeech; @@ -45,6 +47,8 @@ import org.webrtc.audio.JavaAudioDeviceModule; import java.util.Arrays; import java.util.Locale; import java.util.UUID; +import java.util.stream.Collectors; +import java.util.stream.IntStream; /** * 媒体管理器 @@ -185,20 +189,23 @@ public final class MediaManager { // WebRtcAudioUtils.setWebRtcBasedAcousticEchoCanceler(true); // // 自动增益 // WebRtcAudioUtils.setWebRtcBasedAutomaticGainControl(true); -// // 支持的编码器 -// final MediaCodecList mediaCodecList = new MediaCodecList(-1); -// for (MediaCodecInfo mediaCodecInfo : mediaCodecList.getCodecInfos()) { -// final String[] supportedTypes = mediaCodecInfo.getSupportedTypes(); -// Log.d(MediaManager.class.getSimpleName(), "编码器名称：" + mediaCodecInfo.getName()); -// Log.d(MediaManager.class.getSimpleName(), "编码器类型：" + String.join(", ", supportedTypes)); -// for (String supportType : supportedTypes) { -// final MediaCodecInfo.CodecCapabilities codecCapabilities = mediaCodecInfo.getCapabilitiesForType(supportType); -// Log.d(MediaManager.class.getSimpleName(), "编码器支持的文件格式：" + codecCapabilities.getMimeType()); -// // MediaCodecInfo.CodecCapabilities.COLOR_* -// final int[] colorFormats = codecCapabilities.colorFormats; -// Log.d(MediaManager.class.getSimpleName(), "编码器支持的色彩格式：" + IntStream.of(colorFormats).boxed().map(String::valueOf).collect(Collectors.joining(", "))); -// } -// } +// // 支持的编码解码器 + final MediaCodecList mediaCodecList = new MediaCodecList(MediaCodecList.ALL_CODECS); + for (MediaCodecInfo mediaCodecInfo : mediaCodecList.getCodecInfos()) { + // OMX.google = 软编 + // OMX.core = 硬编 + final String[] supportedTypes = mediaCodecInfo.getSupportedTypes(); + final String type = mediaCodecInfo.isEncoder() ? "编码器" : "解码器"; + Log.d(MediaManager.class.getSimpleName(), type + "名称：" + mediaCodecInfo.getName()); + Log.d(MediaManager.class.getSimpleName(), type + "类型：" + String.join(", ", supportedTypes)); + for (String supportType : supportedTypes) { + final MediaCodecInfo.CodecCapabilities codecCapabilities = mediaCodecInfo.getCapabilitiesForType(supportType); + Log.d(MediaManager.class.getSimpleName(), type + "支持的文件格式：" + codecCapabilities.getMimeType()); + // MediaCodecInfo.CodecCapabilities.COLOR_* + final int[] colorFormats = codecCapabilities.colorFormats; + Log.d(MediaManager.class.getSimpleName(), type + "支持的色彩格式：" + IntStream.of(colorFormats).boxed().map(String::valueOf).collect(Collectors.joining(", "))); + } + } } private MediaManager() { diff --git a/taoyao-client-media/src/Taoyao.js b/taoyao-client-media/src/Taoyao.js index 7b86045..291c0a3 100644 --- a/taoyao-client-media/src/Taoyao.js +++ b/taoyao-client-media/src/Taoyao.js @@ -455,8 +455,8 @@ class Taoyao { case "media::producer::resume": me.mediaProducerResume(message, body); break; - case "media::record": - me.mediaRecord(message, body); + case "control::server::record": + me.controlServerRecord(message, body); break; case "media::router::rtp::capabilities": me.mediaRouterRtpCapabilities(message, body); @@ -776,18 +776,19 @@ class Taoyao { * @param {*} message 消息 * @param {*} body 消息主体 */ - async mediaRecord(message, body) { + async controlServerRecord(message, body) { const me = this; const { enabled, roomId } = body; const room = this.rooms.get(roomId); if(enabled) { - await me.mediaRecordStart(message, body, room); + await me.controlServerRecordStart(message, body, room); } else { - await me.mediaRecordStop(message, body, room); + await me.controlServerRecordStop(message, body, room); } } - async mediaRecordStart(message, body, room) { + async controlServerRecordStart(message, body, room) { + const me = this; const { roomId, clientId, host, audioPort, videoPort, rtpCapabilities, audioStreamId, videoStreamId, audioProducerId, videoProducerId } = body; const plainTransportOptions = { ...config.mediasoup.plainTransportOptions, @@ -800,10 +801,12 @@ class Taoyao { let videoTransportId; if(audioProducerId) { const audioTransport = await room.mediasoupRouter.createPlainTransport(plainTransportOptions); + audioTransportId = audioTransport.id; me.transportEvent("plain", roomId, audioTransport); audioTransport.clientId = clientId; room.transports.set(audioTransport.id, audioTransport); audioTransport.observer.on("close", () => { + console.log("controlServerRecord audioTransport close：", audioTransport.id); room.transports.delete(audioTransport.id) }); await audioTransport.connect({ @@ -811,7 +814,7 @@ class Taoyao { port : audioPort, rtcpPort : audioPort }); - const audioConsumer = await transport.consume({ + const audioConsumer = await audioTransport.consume({ producerId: audioProducerId, rtpCapabilities, paused: true @@ -822,15 +825,19 @@ class Taoyao { audioConsumer.streamId = audioStreamId; room.consumers.set(audioConsumer.id, audioConsumer); audioConsumer.observer.on("close", () => { + console.log("controlServerRecord audioConsumer close：", audioConsumer.id); room.consumers.delete(audioConsumer.id); }); + console.log("controlServerRecord audio", audioTransportId, audioConsumerId, audioTransport.tuple); } if(videoProducerId) { const videoTransport = await room.mediasoupRouter.createPlainTransport(plainTransportOptions); + videoTransportId = videoTransport.id; me.transportEvent("plain", roomId, videoTransport); videoTransport.clientId = clientId; room.transports.set(videoTransport.id, videoTransport); videoTransport.observer.on("close", () => { + console.log("controlServerRecord videoTransport close：", videoTransport.id); room.transports.delete(videoTransport.id) }); await videoTransport.connect({ @@ -838,7 +845,7 @@ class Taoyao { port : videoPort, rtcpPort : videoPort }); - const videoConsumer = await transport.consume({ + const videoConsumer = await videoTransport.consume({ producerId: videoProducerId, rtpCapabilities, paused: true @@ -849,8 +856,10 @@ class Taoyao { videoConsumer.streamId = videoStreamId; room.consumers.set(videoConsumer.id, videoConsumer); videoConsumer.observer.on("close", () => { + console.log("controlServerRecord videoConsumer close：", videoConsumer.id); room.consumers.delete(videoConsumer.id); }); + console.log("controlServerRecord video：", videoTransportId, videoConsumerId, videoTransport.tuple); } message.body = { roomId : roomId, @@ -862,7 +871,8 @@ class Taoyao { me.push(message); } - async mediaRecordStart(message, body, room) { + async controlServerRecordStop(message, body, room) { + const me = this; const { audioStreamId, videoStreamId, audioConsumerId, videoConsumerId, audioTransportId, videoTransportId } = body; const audioConsumer = room.consumers.get(audioConsumerId); if(audioConsumer) { @@ -884,6 +894,7 @@ class Taoyao { videoTransport.close(); room.transports.delete(videoTransportId); } + me.push(message); } async mediaConsume(message, body) { diff --git a/taoyao-client-web/src/App.vue b/taoyao-client-web/src/App.vue index 03748ca..dbbb2dc 100644 --- a/taoyao-client-web/src/App.vue +++ b/taoyao-client-web/src/App.vue @@ -161,7 +161,6 @@ export default { }, async sessionCall() { this.taoyao.sessionCall(this.room.callClientId); - // this.taoyao.sessionCall(this.room.callClientId, false, false); this.roomVisible = false; }, async roomCreate() { diff --git a/taoyao-client-web/src/components/Config.js b/taoyao-client-web/src/components/Config.js index 50488a2..80fb954 100644 --- a/taoyao-client-web/src/components/Config.js +++ b/taoyao-client-web/src/components/Config.js @@ -1,7 +1,7 @@ /** * 音频默认配置 * TODO：MediaStreamTrack.applyConstraints().then().catch(); - * let setting = { + * const setting = { * autoGainControl: true, * noiseSuppression: true * } diff --git a/taoyao-client-web/src/components/LocalClient.vue b/taoyao-client-web/src/components/LocalClient.vue index 1ca30b2..73e98bf 100644 --- a/taoyao-client-web/src/components/LocalClient.vue +++ b/taoyao-client-web/src/components/LocalClient.vue @@ -9,10 +9,11 @@ - - - - + + + + +