qq27800734/taoyao
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[+] 信令优化

Browse Source
This commit is contained in:
acgist
2023-03-03 21:53:21 +08:00
parent c3dbe52d5c
commit 9c5ab2ec9f
24 changed files with 203 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
taoyao-client-web/src/components/Config.js
View File

@@ -8,10 +8,10 @@ const defaultAudioConfig = {
volume: 0.5,
// 延迟大小（单位毫秒）：500毫秒以内较好
latency: 0.4,
// 采样数：16
sampleSize: 16,
// 采样数：8|16|32
sampleSize: { min: 8, ideal: 16, max: 32 },
// 采样率：8000|16000|32000|48000
sampleRate: 48000,
sampleRate: { min: 8000, ideal: 32000, max: 48000 },
// 声道数量：1|2
channelCount: 1,
// 是否开启自动增益：true|false
@@ -35,7 +35,7 @@ const defaultVideoConfig = {
// 高度
height: { min: 480, ideal: 720, max: 2160 },
// 帧率
frameRate: 24,
frameRate: { min: 15, ideal: 24, max: 45 },
// 选摄像头：user|left|right|environment
facingMode: "environment",
};

45
taoyao-client-web/src/components/Taoyao.js
View File

@@ -249,9 +249,11 @@ class Taoyao {
// 请求回调
callbackMapping = new Map();
// 音频媒体配置
audio;
audio = defaultAudioConfig;
// 视频媒体配置
video;
video = defaultVideoConfig;
// 媒体配置
media;
// WebRTC配置
webrtc;
// 信令通道
@@ -473,11 +475,17 @@ class Taoyao {
* @param {*} message 消息
*/
defaultClientConfig(message) {
const self = this;
self.audio = { ...defaultAudioConfig, ...message.body.media.audio };
self.video = { ...defaultVideoConfig, ...message.body.media.video };
self.webrtc = message.body.webrtc;
console.debug("终端配置", self.audio, self.video, self.webrtc);
const me = this;
const { media, webrtc } = message.body;
const { audio, video} = media;
me.audio.sampleSize = { min: media.minSampleSize, ideal: audio.sampleSize, max: media.maxSampleSize };
me.audio.sampleRate = { min: media.minSampleRate, ideal: audio.sampleRate, max: media.maxSampleRate };
me.video.width = { min: media.minWidth, ideal: video.width, max: media.maxWidth };
me.video.height = { min: media.minHeight, ideal: video.height, max: media.maxHeight };
me.video.frameRate = { min: media.minFrameRate, ideal: video.frameRate, max: media.maxFrameRate };
me.media = media;
me.webrtc = webrtc;
console.debug("终端配置：", me.audio, me.video, me.media, me.webrtc);
}
/**
* 终端重启默认回调
@@ -597,12 +605,7 @@ class Taoyao {
);
}
/**
* TODO：共享 navigator.mediaDevices.getDisplayMedia();
* 生产媒体
* TODO：验证API试试修改媒体
* audioTrack.getSettings
* audioTrack.getCapabilities
* audioTrack.applyCapabilities
*/
async produceMedia() {
const self = this;
@@ -614,12 +617,15 @@ class Taoyao {
self.checkDevice();
// 释放资源
self.closeMedia();
// TODO：暂时不知道为什么？
/**
* 解决浏览器的自动播放策略问题
*/
{
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const audioTrack = stream.getAudioTracks()[0];
audioTrack.enabled = false;
setTimeout(() => audioTrack.stop(), 120000);
stream.getAudioTracks().forEach(audioTrack => {
audioTrack.enabled = false;
setTimeout(() => audioTrack.stop(), 30000);
});
}
if (self.produce) {
const response = await self.request(
@@ -791,15 +797,16 @@ class Taoyao {
let track;
try {
console.debug("打开麦克风");
// TODO：设置配置
const stream = await navigator.mediaDevices.getUserMedia({
audio: true,
audio: self.audio,
});
const tracks = stream.getAudioTracks();
if (tracks.length > 1) {
console.log("多个音频轨道");
}
track = tracks[0];
// TODO：验证修改API audioTrack.applyCapabilities
console.debug("音频信息：", track.getSettings(), track.getCapabilities());
this.audioProducer = await this.sendTransport.produce({
track,
codecOptions: {
@@ -905,6 +912,8 @@ class Taoyao {
video: true,
});
track = stream.getVideoTracks()[0];
// TODO：验证修改API videoTrack.applyCapabilities
console.debug("视频信息：", track.getSettings(), track.getCapabilities());
} else if (self.videoSource === "screen") {
const stream = await navigator.mediaDevices.getDisplayMedia({
// 如果需要共享声音

4
taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/annotation/Description.java
View File

@@ -21,12 +21,12 @@ public @interface Description {
/**
* @return 消息主体
*/
String[] body() default { "" };
String[] body() default { "{}" };
/**
* @return 数据流向
*/
String[] flow() default { "终端=>信令服务->终端" };
String[] flow() default { "终端->信令服务->终端" };
/**
* @return 描述信息

16
taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/Constant.java
View File

@@ -83,22 +83,6 @@ public interface Constant {
* 建议
*/
String IDEAL = "ideal";
/**
* 最小宽度
*/
Integer MIN_WIDTH = 720;
/**
* 最大宽度
*/
Integer MAX_WIDTH = 4096;
/**
* 最小高度
*/
Integer MIN_HEIGHT = 480;
/**
* 最大高度
*/
Integer MAX_HEIGHT = 2160;
/**
* 脚本
*/

2
taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/MediaAudioProperties.java
View File

@@ -30,7 +30,7 @@ public class MediaAudioProperties {
@Schema(title = "格式", description = "格式", example = "G722|PCMA|PCMU|OPUS")
private Format format;
@Schema(title = "采样数", description = "采样数", example = "16")
@Schema(title = "采样数", description = "采样数", example = "8|16|32")
private Integer sampleSize;
@Schema(title = "采样率", description = "采样率", example = "8000|16000|32000|48000")
private Integer sampleRate;

26
taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/MediaProperties.java
View File

@@ -16,7 +16,31 @@ import lombok.Setter;
@Schema(title = "媒体配置", description = "媒体配置")
@ConfigurationProperties(prefix = "taoyao.media")
public class MediaProperties {
@Schema(title = "最小视频宽度", description = "最小视频宽度")
private Integer minWidth;
@Schema(title = "最大视频宽度", description = "最大视频宽度")
private Integer maxWidth;
@Schema(title = "最小视频高度", description = "最小视频高度")
private Integer minHeight;
@Schema(title = "最大视频高度", description = "最大视频高度")
private Integer maxHeight;
@Schema(title = "最小视频码率", description = "最小视频码率")
private Integer minBitrate;
@Schema(title = "最大视频码率", description = "最大视频码率")
private Integer maxBitrate;
@Schema(title = "最小视频帧率", description = "最小视频帧率")
private Integer minFrameRate;
@Schema(title = "最大视频帧率", description = "最大视频帧率")
private Integer maxFrameRate;
@Schema(title = "最小音频采样数", description = "最小音频采样数")
private Integer minSampleSize;
@Schema(title = "最大音频采样数", description = "最大音频采样数")
private Integer maxSampleSize;
@Schema(title = "最小音频采样率", description = "最小音频采样率")
private Integer minSampleRate;
@Schema(title = "最大音频采样率", description = "最大音频采样率")
private Integer maxSampleRate;
@Schema(title = "音频默认配置", description = "音频默认配置")
private MediaAudioProperties audio;
@Schema(title = "视频默认配置", description = "视频默认配置")

51
taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/MediaVideoProperties.java
View File

@@ -1,8 +1,5 @@
package com.acgist.taoyao.boot.config;
import java.util.LinkedHashMap;
import java.util.Map;
import io.swagger.v3.oas.annotations.media.Schema;
import lombok.Getter;
import lombok.Setter;
@@ -35,35 +32,29 @@ public class MediaVideoProperties {
private Format format;
@Schema(title = "码率", description = "码率影响画质", example = "600|1200|1500|1800")
private Integer bitrate;
@Schema(title = "帧率", description = "帧率影响流畅", example = "20|24|30|60")
@Schema(title = "帧率", description = "帧率影响流畅", example = "15|18|20|24|30|45")
private Integer frameRate;
@Schema(title = "分辨率", description = "分辨率影响画面大小", example = "1920*1080|1280*720")
@Schema(title = "分辨率", description = "分辨率影响画面大小", example = "4096*2160|2560*1440|1920*1080|1280*720|720*480")
private String resolution;
@Schema(title = "宽度", description = "宽度", example = "{ min: 720, ideal: 1280, max: 4096 }")
private Map<String, Object> width;
@Schema(title = "高度", description = "高度", example = "{ min: 480, ideal: 720, max: 2160 }")
private Map<String, Object> height;
@Schema(title = "宽度", description = "宽度", example = "4096|2560|1920|1280|720")
private Integer width;
@Schema(title = "高度", description = "高度", example = "2160|1440|1080|720|480")
private Integer height;
public Map<String, Object> getWidth() {
if(this.width == null) {
final int index = this.resolution.indexOf('*');
this.width = new LinkedHashMap<>();
this.width.put(Constant.MIN, Constant.MIN_WIDTH);
this.width.put(Constant.IDEAL, Integer.valueOf(this.resolution.substring(0, index).strip()));
this.width.put(Constant.MAX, Constant.MAX_WIDTH);
}
return this.width;
}
public Map<String, Object> getHeight() {
if(this.height == null) {
final int index = this.resolution.indexOf('*');
this.height = new LinkedHashMap<>();
this.height.put(Constant.MIN, Constant.MIN_HEIGHT);
this.height.put(Constant.IDEAL, Integer.valueOf(this.resolution.substring(index + 1).strip()));
this.height.put(Constant.MAX, Constant.MAX_HEIGHT);
}
return this.height;
}
public Integer getWidth() {
if (this.width == null) {
final int index = this.resolution.indexOf('*');
return this.width = Integer.valueOf(this.resolution.substring(0, index).strip());
}
return this.width;
}
public Integer getHeight() {
if (this.height == null) {
final int index = this.resolution.indexOf('*');
return this.height = Integer.valueOf(this.resolution.substring(index + 1).strip());
}
return this.height;
}
}

16
taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/utils/MapUtils.java
View File

@@ -102,5 +102,21 @@ public final class MapUtils {
}
return Boolean.valueOf(object.toString());
}
/**
* @param <T> 参数泛型
*
* @param body 消息主体
* @param key 参数名称
*
* @return 参数值
*/
@SuppressWarnings("unchecked")
public static final <T> T remove(Map<?, ?> body, String key) {
if(body == null) {
return null;
}
return (T) body.remove(key);
}
}

22
taoyao-signal-server/taoyao-server/src/main/resources/application.yml
View File

@@ -67,6 +67,24 @@ taoyao:
max-client-index: 99999
# 媒体配置
media:
# 宽度
min-width: 720
max-width: 4096
# 高度
min-height: 480
max-height: 2160
# 码率
min-bitrate: 800
max-bitrate: 1600
# 帧率
min-frame-rate: 15
max-frame-rate: 45
# 采样数
min-sample-size: 8
max-sample-size: 32
# 采样率
min-sample-rate: 8000
max-sample-rate: 48000
# 默认音频
audio:
format: OPUS
@@ -82,7 +100,7 @@ taoyao:
ud-video:
format: H264
bitrate: 1600
frame-rate: 30
frame-rate: 45
resolution: 4096*2160
# 2K：QD=QHD=2K
qd-video:
@@ -106,7 +124,7 @@ taoyao:
sd-video:
format: H264
bitrate: 800
frame-rate: 16
frame-rate: 15
resolution: 720*480
# Socket信令
socket:

2
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolControlAdapter.java
View File

@@ -22,7 +22,7 @@ public class ProtocolControlAdapter extends ProtocolClientAdapter {
@Override
public void execute(String clientId, ClientType clientType, Client client, Message message, Map<String, Object> body) {
final String to = MapUtils.get(body, Constant.TO);
final String to = MapUtils.remove(body, Constant.TO);
final Client targetClient = this.clientManager.clients(to);
if(targetClient == null) {
throw MessageCodeException.of("目标终端无效：" + to);

1
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/client/ClientHeartbeatProtocol.java
View File

@@ -28,6 +28,7 @@ import com.acgist.taoyao.signal.protocol.ProtocolClientAdapter;
"alarming": 是否发生告警（true|false）,
"charging": 是否正在充电（true|false）,
"recording": 是否正在录像（true|false）,
"lastHeartbeat": "最后心跳时间",
"status": {更多状态},
"config": {更多配置}
}

11
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/client/ClientListProtocol.java
View File

@@ -21,15 +21,18 @@ import com.acgist.taoyao.signal.protocol.ProtocolClientAdapter;
@Protocol
@Description(
memo = "没有选择终端类型时返回所有类型终端状态列表",
body = """
body = {
"""
{
"clientType": "终端类型（可选）"
}
""",
"""
[
{
"ip": "终端IP",
"name": "终端名称",
"clientId": "终端标识",
"clientId": "终端ID",
"clientType": "终端类型",
"latitude": 纬度,
"longitude": 经度,
@@ -40,12 +43,14 @@ import com.acgist.taoyao.signal.protocol.ProtocolClientAdapter;
"alarming": 是否发生告警（true|false）,
"charging": 是否正在充电（true|false）,
"recording": 是否正在录像（true|false）,
"lastHeartbeat": "最后心跳时间",
"status": {更多状态},
"config": {更多配置}
},
...
]
""",
"""
},
flow = "终端->信令服务->终端"
)
public class ClientListProtocol extends ProtocolClientAdapter {

8
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/client/ClientOfflineProtocol.java
View File

@@ -19,10 +19,10 @@ import com.acgist.taoyao.signal.protocol.ProtocolClientAdapter;
@Protocol
@Description(
body = """
{
"clientId": "下线终端标识"
}
""",
{
"clientId": "下线终端ID"
}
""",
flow = "终端-[终端关闭]>信令服务-)终端"
)
public class ClientOfflineProtocol extends ProtocolClientAdapter implements ApplicationListener<ClientOfflineEvent> {

37
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/client/ClientOnlineProtocol.java
View File

@@ -17,24 +17,25 @@ import com.acgist.taoyao.signal.protocol.ProtocolClientAdapter;
@Protocol
@Description(
body = """
{
"ip": "终端IP",
"name": "终端名称",
"clientId": "终端标识",
"clientType": "终端类型",
"latitude": 纬度,
"longitude": 经度,
"humidity": 湿度,
"temperature": 温度,
"signal": 信号强度（0~100）,
"battery": 电池电量（0~100）,
"alarming": 是否发生告警（true|false）,
"charging": 是否正在充电（true|false）,
"recording": 是否正在录像（true|false）,
"status": {更多状态},
"config": {更多配置}
}
""",
{
"ip": "终端IP",
"name": "终端名称",
"clientId": "终端ID",
"clientType": "终端类型",
"latitude": 纬度,
"longitude": 经度,
"humidity": 湿度,
"temperature": 温度,
"signal": 信号强度（0~100）,
"battery": 电池电量（0~100）,
"alarming": 是否发生告警（true|false）,
"charging": 是否正在充电（true|false）,
"recording": 是否正在录像（true|false）,
"lastHeartbeat": "最后心跳时间",
"status": {更多状态},
"config": {更多配置}
}
""",
flow = "终端=[终端注册]>信令服务-)终端"
)
public class ClientOnlineProtocol extends ProtocolClientAdapter implements ApplicationListener<ClientOnlineEvent> {

2
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/client/ClientRebootProtocol.java
View File

@@ -23,7 +23,7 @@ public class ClientRebootProtocol extends ProtocolClientAdapter implements Contr
}
/**
* @param clientId 终端标识
* @param clientId 终端ID
*/
public void execute(String clientId) {
this.clientManager.unicast(clientId, this.build());

5
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/client/ClientRegisterProtocol.java
View File

@@ -34,7 +34,7 @@ import lombok.extern.slf4j.Slf4j;
"username": "信令用户",
"password": "信令密码",
"name": "终端名称",
"clientId": "终端标识",
"clientId": "终端ID",
"clientType": "终端类型",
"latitude": 纬度,
"longitude": 经度,
@@ -45,6 +45,7 @@ import lombok.extern.slf4j.Slf4j;
"alarming": 是否发生告警（true|false）,
"charging": 是否正在充电（true|false）,
"recording": 是否正在录像（true|false）,
"lastHeartbeat": "最后心跳时间",
"status": {更多状态},
"config": {更多配置}
}
@@ -101,7 +102,7 @@ public class ClientRegisterProtocol extends ProtocolClientAdapter {
}
/**
* @param clientId 终端标识
* @param clientId 终端ID
* @param clientType 终端类型
* @param client 终端
* @param body 消息主体

2
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/client/ClientShutdownProtocol.java
View File

@@ -23,7 +23,7 @@ public class ClientShutdownProtocol extends ProtocolClientAdapter implements Con
}
/**
* @param clientId 终端标识
* @param clientId 终端ID
*/
public void execute(String clientId) {
this.clientManager.unicast(clientId, this.build());

18
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/client/ClientStatusProtocol.java
View File

@@ -18,21 +18,31 @@ import com.acgist.taoyao.signal.protocol.ProtocolClientAdapter;
*/
@Protocol
@Description(
memo = "没有指定终端ID返回请求终端状态",
body = {
"""
{
"clientId": "终端标识（可选）"
"clientId": "终端ID（可选）"
}
""",
"""
{
"clientId": "终端标识",
"ip": "终端IP",
"name": "终端名称",
"clientId": "终端ID",
"clientType": "终端类型",
"latitude": 纬度,
"longitude": 经度,
"humidity": 湿度,
"temperature": 温度,
"signal": 信号强度（0~100）,
"battery": 电池电量（0~100）,
"alarming": 是否发生告警（true|false）,
"charging": 是否正在充电（true|false）,
"mediaId": "媒体服务标识",
"lastHeartbeat": "最后心跳时间"
"recording": 是否正在录像（true|false）,
"lastHeartbeat": "最后心跳时间",
"status": {更多状态},
"config": {更多配置}
}
"""
},

27
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/client/ClientUnicastProtocol.java
View File

@@ -2,32 +2,35 @@ package com.acgist.taoyao.signal.protocol.client;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import com.acgist.taoyao.boot.annotation.Description;
import com.acgist.taoyao.boot.annotation.Protocol;
import com.acgist.taoyao.boot.config.Constant;
import com.acgist.taoyao.boot.model.Message;
import com.acgist.taoyao.boot.utils.MapUtils;
import com.acgist.taoyao.signal.client.Client;
import com.acgist.taoyao.signal.client.ClientType;
import com.acgist.taoyao.signal.protocol.ProtocolClientAdapter;
import lombok.extern.slf4j.Slf4j;
/**
* 终端单播信令
*
* @author acgist
*/
@Slf4j
@Protocol
@Description(
body = """
body = {
"""
{
"to": "接收终端标识",
...自定义的主体
"to": "接收终端ID",
...
}
""",
"""
{
...
}
"""
},
flow = "终端->信令服务->终端"
)
public class ClientUnicastProtocol extends ProtocolClientAdapter {
@@ -40,12 +43,8 @@ public class ClientUnicastProtocol extends ProtocolClientAdapter {
@Override
public void execute(String clientId, ClientType clientType, Client client, Message message, Map<String, Object> body) {
final String to = (String) body.remove(Constant.TO);
if(StringUtils.isNotEmpty(to)) {
this.clientManager.unicast(to, message);
} else {
log.warn("终端单播消息没有接收终端标识：{}", to);
}
final String to = MapUtils.remove(body, Constant.TO);
this.clientManager.unicast(to, message);
}
}

2
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/client/ClientWakeupProtocol.java
View File

@@ -22,7 +22,7 @@ public class ClientWakeupProtocol extends ProtocolClientAdapter {
}
/**
* @param clientId 终端标识
* @param clientId 终端ID
*/
public void execute(String clientId) {
this.clientManager.unicast(clientId, this.build());

7
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/control/ControlBellProtocol.java
View File

@@ -16,6 +16,11 @@ import com.acgist.taoyao.signal.protocol.ProtocolControlAdapter;
*/
@Protocol
@Description(
body = """
{
"to": "目标终端ID"
}
""",
flow = {
"信令服务->终端",
"终端->信令服务->终端"
@@ -35,7 +40,7 @@ public class ControlBellProtocol extends ProtocolControlAdapter {
}
/**
* @param clientId 终端标识
* @param clientId 终端ID
*/
public void execute(String clientId) {
this.clientManager.unicast(clientId, this.build());

5
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/control/ControlPhotographProtocol.java
View File

@@ -16,6 +16,11 @@ import com.acgist.taoyao.signal.protocol.ProtocolControlAdapter;
*/
@Protocol
@Description(
body = """
{
"to": "目标终端ID"
}
""",
flow = {
"信令服务->终端",
"终端->信令服务->终端"

9
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/control/ControlPtzProtocol.java
View File

@@ -17,10 +17,11 @@ import com.acgist.taoyao.signal.protocol.ProtocolControlAdapter;
@Protocol
@Description(
body = """
{
"type": "PTZ类型（PAN|TILT|ZOOM）",
"value": PTZ参数
}
{
"to": "目标终端ID",
"type": "PTZ类型（PAN|TILT|ZOOM）",
"value": PTZ参数
}
""",
flow = {
"信令服务->终端",

5
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/control/ControlRecordProtocol.java
View File

@@ -17,6 +17,11 @@ import com.acgist.taoyao.signal.protocol.ProtocolControlAdapter;
@Protocol
@Description(
memo = "状态通过心跳回传",
body = """
{
"to": "目标终端ID"
}
""",
flow = {
"信令服务->终端",
"终端->信令服务->终端"