qq27800734/taoyao
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[+] 音频监控

Browse Source
This commit is contained in:
acgist
2023-03-05 11:56:31 +08:00
parent 3bc39b4d48
commit c49c3b19d8
15 changed files with 527 additions and 332 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
README.md
View File

@@ -49,3 +49,7 @@
* 视频：水印、美颜、AI识别
* P2P
* 反复测试推流拉流、拉人踢人、音频视频控制
* 优化JS错误回调 -> platform::error
* 24小时不关闭媒体/一秒一次推拉流十分钟测试/三十秒推拉流一小时测试
* 标识 -> ID
* 所有字段获取 -> get

49
taoyao-client-media/src/Taoyao.js
View File

@@ -211,6 +211,7 @@ const signalChannel = {
clearTimeout(me.reconnectTimer);
me.reconnection = false;
me.channel.close();
me.taoyao.connect = false;
},
};
@@ -228,9 +229,9 @@ class Room {
webRtcServer = null;
// 路由
mediasoupRouter = null;
// 音频监控
// 音量监控
audioLevelObserver = null;
// 音频监控
// 采样监控
activeSpeakerObserver = null;
// 消费者复制数量
consumerReplicas = 0;
@@ -266,24 +267,22 @@ class Room {
}
/**
* 声音监控
* 音量监控
*/
handleAudioLevelObserver() {
const self = this;
// 声音
self.audioLevelObserver.on("volumes", (volumes) => {
for (const value of volumes) {
const { producer, volume } = value;
signalChannel.push(
protocol.buildMessage("media::audio::active::speaker", {
volume: volume,
roomId: self.roomId,
clientId: producer.clientId,
volume: volume,
})
);
}
});
// 静音
self.audioLevelObserver.on("silence", () => {
signalChannel.push(
protocol.buildMessage("media::audio::active::speaker", {
@@ -294,7 +293,7 @@ class Room {
}
/**
* 说话监控
* 采样监控
*/
handleActiveSpeakerObserver() {
const self = this;
@@ -332,6 +331,8 @@ class Room {
// me.consumers.forEach(v => v.close());
// me.dataProducers.forEach(v => v.close());
// me.dataConsumers.forEach(v => v.close());
me.audioLevelObserver.close();
me.activeSpeakerObserver.close();
me.transports.forEach(v => v.close());
me.mediasoupRouter.close();
}
@@ -404,6 +405,9 @@ class Taoyao {
case "media::transport::webrtc::create":
this.mediaTransportWebrtcCreate(message, body);
break;
case "platform::error":
this.platformError(message, body);
break;
case "room::create":
this.roomCreate(message, body);
break;
@@ -737,7 +741,7 @@ class Taoyao {
}
/**
* 路由RTP能力信令
* 路由RTP协商信令
*
* @param {*} message 消息
* @param {*} body 消息主体
@@ -848,7 +852,18 @@ class Taoyao {
} catch (error) {}
}
}
/**
* 平台异常信令
*
* @param {*} message 消息
* @param {*} body 消息主体
*/
platformError(message, body) {
const { code } = message;
if(code === "3401") {
signalChannel.close();
}
}
/**
* 关闭房间信令
*
@@ -885,16 +900,18 @@ class Taoyao {
const mediasoupWorker = me.nextMediasoupWorker();
const { mediaCodecs } = config.mediasoup.routerOptions;
const mediasoupRouter = await mediasoupWorker.createRouter({ mediaCodecs });
// TODO：下面两个监控改为配置启用
// 音量监控
const audioLevelObserver = await mediasoupRouter.createAudioLevelObserver({
maxEntries: 1,
threshold: -80,
interval: 2000,
// 范围：-127~0
threshold: -80,
// 采样数量
maxEntries: 2,
});
// 采样监控
const activeSpeakerObserver = await mediasoupRouter.createActiveSpeakerObserver({
interval: 500,
});
const activeSpeakerObserver =
await mediasoupRouter.createActiveSpeakerObserver({
interval: 500,
});
room = new Room({
roomId,
webRtcServer: mediasoupWorker.appData.webRtcServer,

246
taoyao-client-web/src/App.vue
View File

@@ -1,102 +1,104 @@
<!-- 桃夭 -->
<template>
<!-- 信令 -->
<el-dialog
center
width="30%"
title="终端设置"
:show-close="false"
v-if="taoyao === null"
v-model="signalVisible"
>
<el-form ref="SignalSetting">
<el-form-item label="终端标识">
<el-input v-model="config.clientId" placeholder="终端标识" />
</el-form-item>
<el-form-item label="终端名称">
<el-input v-model="config.name" placeholder="终端名称" />
</el-form-item>
<el-form-item label="信令地址">
<el-input v-model="config.host" placeholder="信令地址" />
</el-form-item>
<el-form-item label="信令端口">
<el-input v-model="config.port" placeholder="信令端口" />
</el-form-item>
<el-form-item label="信令帐号">
<el-input v-model="config.username" placeholder="信令帐号" />
</el-form-item>
<el-form-item label="信令密码">
<el-input v-model="config.password" placeholder="信令密码" />
</el-form-item>
</el-form>
<template #footer>
<el-button type="primary" @click="connectSignal">连接信令</el-button>
</template>
</el-dialog>
<div id="taoyao">
<!-- 信令 -->
<el-dialog
center
width="30%"
title="终端设置"
:show-close="false"
v-if="taoyao === null"
v-model="signalVisible"
>
<el-form ref="SignalSetting">
<el-form-item label="终端标识">
<el-input v-model="config.clientId" placeholder="终端标识" />
</el-form-item>
<el-form-item label="终端名称">
<el-input v-model="config.name" placeholder="终端名称" />
</el-form-item>
<el-form-item label="信令地址">
<el-input v-model="config.host" placeholder="信令地址" />
</el-form-item>
<el-form-item label="信令端口">
<el-input v-model="config.port" placeholder="信令端口" />
</el-form-item>
<el-form-item label="信令帐号">
<el-input v-model="config.username" placeholder="信令帐号" />
</el-form-item>
<el-form-item label="信令密码">
<el-input v-model="config.password" placeholder="信令密码" />
</el-form-item>
</el-form>
<template #footer>
<el-button type="primary" @click="connectSignal">连接信令</el-button>
</template>
</el-dialog>
<!-- 房间 -->
<el-dialog
center
width="30%"
title="房间设置"
@open="loadList"
:show-close="false"
v-model="roomVisible"
>
<el-form ref="RoomSetting" :model="room">
<el-tabs v-model="roomActive">
<el-tab-pane label="进入房间" name="enter">
<el-form-item label="房间标识">
<el-select v-model="room.roomId" placeholder="房间标识">
<el-option
v-for="value in rooms"
:key="value.roomId"
:label="value.name || value.roomId"
:value="value.roomId"
/>
</el-select>
</el-form-item>
</el-tab-pane>
<el-tab-pane label="创建房间" name="create">
<el-form-item label="媒体服务">
<el-select v-model="room.mediaClientId" placeholder="媒体服务标识">
<el-option
v-for="value in medias"
:key="value.clientId"
:label="value.name || value.clientId"
:value="value.clientId"
/>
</el-select>
</el-form-item>
<el-form-item label="房间名称">
<el-input v-model="room.name" placeholder="房间名称" />
</el-form-item>
</el-tab-pane>
</el-tabs>
<el-form-item label="房间密码">
<el-input v-model="room.password" placeholder="房间密码" />
</el-form-item>
</el-form>
<template #footer>
<el-button type="primary" @click="enterRoom" v-if="roomActive === 'enter'">进入</el-button>
<el-button type="primary" @click="roomCreate" v-if="roomActive === 'create'">创建</el-button>
</template>
</el-dialog>
<!-- 房间 -->
<el-dialog
center
width="30%"
title="房间设置"
@open="loadList"
:show-close="false"
v-model="roomVisible"
>
<el-form ref="RoomSetting" :model="room">
<el-tabs v-model="roomActive">
<el-tab-pane label="进入房间" name="enter">
<el-form-item label="房间标识">
<el-select v-model="room.roomId" placeholder="房间标识">
<el-option
v-for="value in rooms"
:key="value.roomId"
:label="value.name || value.roomId"
:value="value.roomId"
/>
</el-select>
</el-form-item>
</el-tab-pane>
<el-tab-pane label="创建房间" name="create">
<el-form-item label="媒体服务">
<el-select v-model="room.mediaClientId" placeholder="媒体服务标识">
<el-option
v-for="value in medias"
:key="value.clientId"
:label="value.name || value.clientId"
:value="value.clientId"
/>
</el-select>
</el-form-item>
<el-form-item label="房间名称">
<el-input v-model="room.name" placeholder="房间名称" />
</el-form-item>
</el-tab-pane>
</el-tabs>
<el-form-item label="房间密码">
<el-input v-model="room.password" placeholder="房间密码" />
</el-form-item>
</el-form>
<template #footer>
<el-button type="primary" @click="roomEnter" v-if="roomActive === 'enter'">进入</el-button>
<el-button type="primary" @click="roomCreate" v-if="roomActive === 'create'">创建</el-button>
</template>
</el-dialog>
<!-- 菜单 -->
<div class="menus">
<el-button type="primary" :disabled="taoyao !== null" @click="signalVisible = true">连接信令</el-button>
<el-button type="primary" @click="roomActive = 'enter';roomVisible = true;">选择房间</el-button>
<el-button type="primary" @click="roomActive = 'create';roomVisible = true;">创建房间</el-button>
<el-button>邀请终端</el-button>
<el-button>退出房间</el-button>
<el-button @click="closeRoom()" type="danger">关闭房间</el-button>
</div>
<!-- 菜单 -->
<div class="menus">
<el-button type="primary" :disabled="taoyao && taoyao.connect" @click="signalVisible = true">连接信令</el-button>
<el-button type="primary" :disabled="!taoyao" @click="roomActive = 'enter';roomVisible = true;">选择房间</el-button>
<el-button type="primary" :disabled="!taoyao" @click="roomActive = 'create';roomVisible = true;">创建房间</el-button>
<el-button :disabled="!taoyao || !room.roomId">邀请终端</el-button>
<el-button :disabled="!taoyao || !room.roomId">退出房间</el-button>
<el-button :disabled="!taoyao || !room.roomId" @click="roomClose()" type="danger">关闭房间</el-button>
</div>
<!-- 终端 -->
<div class="clients">
<LocalClient ref="local"></LocalClient>
<RemoteClient :ref="'remote-' + kv[0]" v-for="(kv, index) in remoteClients" :key="index"></RemoteClient>
<!-- 终端 -->
<div class="clients">
<LocalClient ref="local" :client="taoyao" :taoyao="taoyao"></LocalClient>
<RemoteClient :ref="'remote-' + kv[0]" v-for="(kv, index) in remoteClients" :key="index" :client="kv[1]" :taoyao="taoyao"></RemoteClient>
</div>
</div>
</template>
@@ -148,46 +150,48 @@ export default {
this.rooms = await this.taoyao.roomList();
this.medias = await this.taoyao.mediaList();
},
async enterRoom() {
await this.taoyao.enterRoom(this.room.roomId, this.room.password);
await this.taoyao.produceMedia();
this.roomVisible = false;
async roomClose() {
this.taoyao.roomClose();
},
async roomCreate() {
const room = await this.taoyao.roomCreate(this.room);
this.room.roomId = room.roomId;
await this.enterRoom();
await this.roomEnter();
},
async closeRoom() {
this.taoyao.closeRoom();
async roomEnter() {
await this.taoyao.roomEnter(this.room.roomId, this.room.password);
await this.taoyao.produceMedia();
this.roomVisible = false;
},
audioVolume(message) {
},
/**
* 信令回调
*
* @param {*} data 消息
* @param {*} message 消息
* @param {*} error 异常
*
* @return 是否继续执行
*/
async callback(data, error) {
let self = this;
switch (data.header.signal) {
async callback(message, error) {
const me = this;
switch (message.header.signal) {
case "client::config":
self.roomVisible = true;
me.roomVisible = true;
break;
case "media::audio::active::speaker":
me.audioVolume(message);
break;
case "client::register":
self.signalVisible = data.code !== "0000";
return true;
case "platform::error":
if (error) {
console.error("发生异常：", data, error);
console.error("发生异常：", message, error);
} else {
console.warn("发生错误：", data);
console.warn("发生错误：", message);
}
ElMessage({
showClose: true,
message: data.message,
type: "error",
message: message.message,
});
return true;
}
@@ -195,14 +199,20 @@ export default {
},
/**
* 媒体回调
*
* @param {*} type 类型
* @param {*} track 媒体Track
* @param {*} consumer 消费者
*/
callbackMedia(type, track, consumer) {
const self = this;
const me = this;
return new Promise((resolve, reject) => {
if(type === 'local') {
self.$refs.local.media(track);
me.$refs.local.media(track);
} else if(type === 'remote') {
me.$refs['remote-' + consumer.sourceId][0].media(track, consumer);
} else {
this.$refs['remote-' + consumer.sourceId][0].media(track, consumer);
// 其他
}
resolve();
});
@@ -219,7 +229,9 @@ export default {
.menus{width:100%;top:1rem;left:0;text-align:center;position:fixed;z-index:1;}
.clients{width:100%;height:100%;top:0;left:0;position:fixed;}
.client{float:left;width:50vw;height:50vh;box-shadow:0 0 1px 0px rgba(0,0,0,0.4);}
.client .buttons{width:100%;bottom:1rem;left:0;text-align:center;position:absolute;padding:0.8rem 0;background:rgba(0,0,0,0.4);}
.client .buttons{width:100%;bottom:2px;left:0;text-align:center;position:absolute;padding:0.8rem 0;background:rgba(0,0,0,0.4);}
.client .buttons:after{width:0;height:2px;bottom:0;left:0;position:absolute;background:#C00;content:"";transition: all 400ms linear;}
.client audio{display:none;}
.client video{width:100%;height:100%;}
.client .title{position:absolute;top:0;left:0;text-align:center;width:100%;}
</style>

15
taoyao-client-web/src/components/LocalClient.vue
View File

@@ -3,7 +3,8 @@
<div class="client">
<audio ref="audio"></audio>
<video ref="video"></video>
<div class="buttons">
<p class="title">{{ client?.name || "" }}</p>
<div class="buttons" :style="{'--volume': client?.volume}">
<el-button type="danger" title="打开麦克风" :icon="Mute" circle />
<el-button type="primary" title="关闭麦克风" :icon="Microphone" circle />
<el-button type="danger" title="打开摄像头" :icon="VideoPause" circle />
@@ -54,7 +55,6 @@ export default {
},
data() {
return {
taoyao: null,
audio: null,
video: null,
audioStream: null,
@@ -94,6 +94,14 @@ export default {
this.audio = this.$refs.audio;
this.video = this.$refs.video;
},
props: {
"client": {
type: Object
},
"taoyao": {
type: Object
}
},
methods: {
media(track) {
if (track.kind === "video") {
@@ -112,3 +120,6 @@ export default {
},
};
</script>
<style scoped>
.client .buttons:after{width:var(--volume);}
</style>

15
taoyao-client-web/src/components/RemoteClient.vue
View File

@@ -3,7 +3,8 @@
<div class="client">
<audio ref="audio"></audio>
<video ref="video"></video>
<div class="buttons">
<p class="title">{{ client?.name || "" }}</p>
<div class="buttons" :style="{'--volume': client?.volume}">
<el-button type="danger" title="打开麦克风" :icon="Mute" circle />
<el-button type="primary" title="关闭麦克风" :icon="Microphone" circle />
<el-button type="danger" title="打开摄像头" :icon="VideoPause" circle />
@@ -45,7 +46,6 @@ export default {
},
data() {
return {
taoyao: null,
audio: null,
video: null,
audioStream: null,
@@ -59,6 +59,14 @@ export default {
this.audio = this.$refs.audio;
this.video = this.$refs.video;
},
props: {
"client": {
type: Object
},
"taoyao": {
type: Object
}
},
methods: {
media(track, consumer) {
if(track.kind === 'audio') {
@@ -86,3 +94,6 @@ export default {
}
};
</script>
<style scoped>
.client .buttons:after{width:var(--volume);}
</style>

405
taoyao-client-web/src/components/Taoyao.js
View File

@@ -219,21 +219,38 @@ const signalChannel = {
clearTimeout(me.reconnectTimer);
me.reconnection = false;
me.channel.close();
me.taoyao.connect = false;
},
};
/**
* 桃夭
* 远程终端
*/
class Taoyao {
// 信令连接
connect = false;
// 房间标识
roomId;
// 终端标识
clientId;
class RemoteClient {
// 终端名称
name;
// 终端标识
clientId;
// 音量
volume = 0;
constructor({
name,
clientId,
}) {
this.name = name;
this.clientId = clientId;
}
}
/**
* 桃夭
*/
class Taoyao extends RemoteClient {
// 信令连接
connect = false;
// 信令地址
host;
// 信令端口
@@ -242,6 +259,8 @@ class Taoyao {
username;
// 信令密码
password;
// 房间标识
roomId;
// 回调事件
callback;
// 媒体回调
@@ -249,13 +268,13 @@ class Taoyao {
// 请求回调
callbackMapping = new Map();
// 音频媒体配置
audio = defaultAudioConfig;
audioConfig = defaultAudioConfig;
// 视频媒体配置
video = defaultVideoConfig;
videoConfig = defaultVideoConfig;
// 媒体配置
media;
mediaConfig;
// WebRTC配置
webrtc;
webrtcConfig;
// 信令通道
signalChannel;
// 发送媒体通道
@@ -296,13 +315,13 @@ class Taoyao {
remoteClients = new Map();
constructor({
roomId,
clientId,
name,
clientId,
host,
port,
username,
password,
roomId,
consume = true,
produce = true,
audioProduce = true,
@@ -310,13 +329,14 @@ class Taoyao {
forceTcp = false,
dataProduce = true,
}) {
this.roomId = roomId;
this.clientId = clientId;
super({ name, clientId });
this.name = name;
this.clientId = clientId;
this.host = host;
this.port = port;
this.username = username;
this.password = password;
this.roomId = roomId;
this.consume = consume;
this.produce = produce;
this.dataProduce = produce && dataProduce;
@@ -401,6 +421,8 @@ class Taoyao {
* 2. 执行前置回调
* 3. 如果注册全局回调，同时执行结果返回true不再执行后面所有回调。
* 4. 执行后置回调
*
* @param {*} message 消息
*/
async on(message) {
const me = this;
@@ -429,19 +451,22 @@ class Taoyao {
/**
* 前置回调
*
* @param {*} message
* @param {*} message 消息
*/
async preCallback(message) {
const self = this;
const me = this;
switch (message.header.signal) {
case "client::config":
self.defaultClientConfig(message);
me.defaultClientConfig(message);
break;
case "client::register":
protocol.clientIndex = message.body.index;
me.defaultClientRegister(message);
break;
case "media::consume":
await self.consumeMedia(message);
await me.defaultMediaConsume(message);
break;
case "platform::error":
me.defaultPlatformError(message);
break;
}
}
@@ -453,15 +478,18 @@ class Taoyao {
async postCallback(message) {
const me = this;
switch (message.header.signal) {
case "room::client::list":
me.defaultRoomClientList(message);
break;
case "client::reboot":
me.defaultClientReboot(message);
break;
case "client::shutdown":
me.defaultClientShutdown(message);
break;
case "media::audio::active::speaker":
me.defaultMediaAudioActiveSpeaker(message);
break;
case "room::client::list":
me.defaultRoomClientList(message);
break;
case "room::close":
me.defaultRoomClose(message);
break;
@@ -471,32 +499,29 @@ class Taoyao {
case "platform::error":
me.callbackError(message);
break;
default:
console.warn("不支持的信令：", message);
break;
}
}
/************************ 信令 ************************/
/**
* 配置默认回调
* 终端配置信令
*
* @param {*} message 消息
*/
defaultClientConfig(message) {
const me = this;
const { media, webrtc } = message.body;
const { audio, video} = media;
me.audio.sampleSize = { min: media.minSampleSize, ideal: audio.sampleSize, max: media.maxSampleSize };
me.audio.sampleRate = { min: media.minSampleRate, ideal: audio.sampleRate, max: media.maxSampleRate };
me.video.width = { min: media.minWidth, ideal: video.width, max: media.maxWidth };
me.video.height = { min: media.minHeight, ideal: video.height, max: media.maxHeight };
me.video.frameRate = { min: media.minFrameRate, ideal: video.frameRate, max: media.maxFrameRate };
me.media = media;
me.webrtc = webrtc;
console.debug("终端配置：", me.audio, me.video, me.media, me.webrtc);
const { audio, video } = media;
me.audioConfig.sampleSize = { min: media.minSampleSize, ideal: audio.sampleSize, max: media.maxSampleSize };
me.audioConfig.sampleRate = { min: media.minSampleRate, ideal: audio.sampleRate, max: media.maxSampleRate };
me.videoConfig.width = { min: media.minWidth, ideal: video.width, max: media.maxWidth };
me.videoConfig.height = { min: media.minHeight, ideal: video.height, max: media.maxHeight };
me.videoConfig.frameRate = { min: media.minFrameRate, ideal: video.frameRate, max: media.maxFrameRate };
me.mediaConfig = media;
me.webrtcConfig = webrtc;
console.debug("终端配置：", me.audioConfig, me.videoConfig, me.mediaConfig, me.webrtcConfig);
}
/**
* 终端重启默认回调
* 重启终端信令
*
* @param {*} message 消息
*/
@@ -505,7 +530,16 @@ class Taoyao {
location.reload();
}
/**
* 终端重启默认回调
* 终端注册信令
*
* @param {*} message 消息
*/
defaultClientRegister(message) {
const { index } = message.body;
protocol.clientIndex = index;
}
/**
* 关闭终端信令
*
* @param {*} message 消息
*/
@@ -513,6 +547,118 @@ class Taoyao {
console.info("关闭终端");
window.close();
}
/**
* 当前讲话终端信令
*
* @param {*} message 消息
*/
defaultMediaAudioActiveSpeaker(message) {
const me = this;
const { volume, clientId } = message.body;
if(!clientId) {
me.volume = 0;
me.remoteClients.forEach(v => v.volume = 0);
} if(me.clientId === clientId) {
me.volume = ((volume + 127) / 127 * 100) + "%";
} else {
const remoteClient = me.remoteClients.get(clientId);
if(remoteClient) {
remoteClient.volume = ((volume + 127) / 127 * 100) + "%";
}
}
}
/**
* 消费媒体信令
*
* @param {*} message 消息
*/
async defaultMediaConsume(message) {
const self = this;
if (!self.consume) {
console.log("没有消费媒体");
return;
}
const {
kind,
type,
roomId,
clientId,
sourceId,
streamId,
producerId,
consumerId,
rtpParameters,
appData,
producerPaused,
} = message.body;
try {
const consumer = await self.recvTransport.consume({
id: consumerId,
kind,
producerId,
rtpParameters,
// NOTE: Force streamId to be same in mic and webcam and different
// in screen sharing so libwebrtc will just try to sync mic and
// webcam streams from the same remote peer.
//streamId: `${peerId}-${appData.share ? "share" : "mic-webcam"}`,
streamId: `${clientId}-${appData.share ? "share" : "mic-webcam"}`,
appData, // Trick.
});
consumer.clientId = clientId;
consumer.sourceId = sourceId;
consumer.streamId = streamId;
self.consumers.set(consumer.id, consumer);
consumer.on("transportclose", () => {
self.consumers.delete(consumer.id);
});
const { spatialLayers, temporalLayers } =
mediasoupClient.parseScalabilityMode(
consumer.rtpParameters.encodings[0].scalabilityMode
);
// store.dispatch(
// stateActions.addConsumer(
// {
// id: consumer.id,
// type: type,
// locallyPaused: false,
// remotelyPaused: producerPaused,
// rtpParameters: consumer.rtpParameters,
// spatialLayers: spatialLayers,
// temporalLayers: temporalLayers,
// preferredSpatialLayer: spatialLayers - 1,
// preferredTemporalLayer: temporalLayers - 1,
// priority: 1,
// codec: consumer.rtpParameters.codecs[0].mimeType.split("/")[1],
// track: consumer.track,
// },
// peerId
// )
// );
self.push(message);
console.log("消费者", consumer);
self.callbackMedia("remote", consumer.track, consumer);
// If audio-only mode is enabled, pause it.
if (consumer.kind === "video" && !self.videoProduce) {
// this.pauseConsumer(consumer);
// TODO：实现
}
} catch (error) {
self.callbackError("消费媒体异常", error);
}
}
/**
* 平台异常信令
*
* @param {*} message 消息
*/
defaultPlatformError(message) {
const { code } = message;
if(code === "3401") {
signalChannel.close();
}
}
/**
* 房间终端列表信令
*
@@ -524,10 +670,23 @@ class Taoyao {
if (v.clientId === me.clientId) {
// 忽略自己
} else {
me.remoteClients.set(v.clientId, me.roomId);
me.remoteClients.set(v.clientId, new RemoteClient(v));
}
});
}
/**
* 关闭房间信令
*/
async roomClose() {
const me = this;
if(!me.roomId) {
console.warn("房间无效：", me.roomId);
return;
}
me.push(protocol.buildMessage("room::close", {
roomId: me.roomId
}));
}
/**
* 关闭房间信令
*
@@ -560,12 +719,48 @@ class Taoyao {
);
return response.body;
}
/**
* 进入房间信令
*
* @param {*} roomId 房间ID
* @param {*} password 房间密码
*/
async roomEnter(roomId, password) {
const me = this;
if (!roomId) {
this.callbackError("无效房间");
return;
}
me.roomId = roomId;
me.mediasoupDevice = new mediasoupClient.Device();
const response = await me.request(
protocol.buildMessage("media::router::rtp::capabilities", {
roomId: me.roomId
})
);
const routerRtpCapabilities = response.body.rtpCapabilities;
await me.mediasoupDevice.load({ routerRtpCapabilities });
await me.request(
protocol.buildMessage("room::enter", {
roomId: roomId,
password: password,
rtpCapabilities: me.consume ? me.mediasoupDevice.rtpCapabilities : undefined,
sctpCapabilities: me.consume && me.dataProduce ? me.mediasoupDevice.sctpCapabilities : undefined,
})
);
}
/**
* 进入房间信令
*
* @param {*} message 消息
*/
defaultRoomEnter(message) {
const { roomId, clientId } = message.body;
if (clientId === this.clientId) {
const me = this;
const { roomId, clientId, status } = message.body;
if (clientId === me.clientId) {
// 忽略自己
} else {
this.remoteClients.set(clientId, roomId);
me.remoteClients.set(clientId, new RemoteClient(status));
}
}
/**
@@ -607,45 +802,6 @@ class Taoyao {
);
return response.body;
}
async enterRoom(roomId, password) {
const self = this;
if (!roomId) {
this.callbackError("无效房间");
return;
}
self.roomId = roomId;
self.mediasoupDevice = new mediasoupClient.Device();
const response = await self.request(
protocol.buildMessage("media::router::rtp::capabilities", {
roomId: self.roomId
})
);
const routerRtpCapabilities = response.body.rtpCapabilities;
await self.mediasoupDevice.load({ routerRtpCapabilities });
await self.request(
protocol.buildMessage("room::enter", {
roomId: roomId,
password: password,
rtpCapabilities: self.consume
? self.mediasoupDevice.rtpCapabilities
: undefined,
sctpCapabilities:
self.consume && self.dataProduce
? self.mediasoupDevice.sctpCapabilities
: undefined,
})
);
}
async closeRoom() {
const me = this;
if(!me.roomId) {
console.warn("房间无效：", me.roomId);
return;
}
me.push(protocol.buildMessage("room::close", {
roomId: me.roomId
}));
}
/************************ 媒体 ************************/
/**
* 生产媒体
@@ -841,7 +997,7 @@ class Taoyao {
try {
console.debug("打开麦克风");
const stream = await navigator.mediaDevices.getUserMedia({
audio: self.audio,
audio: self.audioConfig,
});
const tracks = stream.getAudioTracks();
if (tracks.length > 1) {
@@ -1107,89 +1263,6 @@ class Taoyao {
}
}
/**
* 消费媒体
*
* @param {*} message
* @returns
*/
async consumeMedia(message) {
const self = this;
if (!self.consume) {
console.log("没有消费媒体");
return;
}
const {
kind,
type,
roomId,
clientId,
sourceId,
streamId,
producerId,
consumerId,
rtpParameters,
appData,
producerPaused,
} = message.body;
try {
const consumer = await self.recvTransport.consume({
id: consumerId,
kind,
producerId,
rtpParameters,
// NOTE: Force streamId to be same in mic and webcam and different
// in screen sharing so libwebrtc will just try to sync mic and
// webcam streams from the same remote peer.
//streamId: `${peerId}-${appData.share ? "share" : "mic-webcam"}`,
streamId: `${clientId}-${appData.share ? "share" : "mic-webcam"}`,
appData, // Trick.
});
consumer.clientId = clientId;
consumer.sourceId = sourceId;
consumer.streamId = streamId;
self.consumers.set(consumer.id, consumer);
consumer.on("transportclose", () => {
self.consumers.delete(consumer.id);
});
const { spatialLayers, temporalLayers } =
mediasoupClient.parseScalabilityMode(
consumer.rtpParameters.encodings[0].scalabilityMode
);
// store.dispatch(
// stateActions.addConsumer(
// {
// id: consumer.id,
// type: type,
// locallyPaused: false,
// remotelyPaused: producerPaused,
// rtpParameters: consumer.rtpParameters,
// spatialLayers: spatialLayers,
// temporalLayers: temporalLayers,
// preferredSpatialLayer: spatialLayers - 1,
// preferredTemporalLayer: temporalLayers - 1,
// priority: 1,
// codec: consumer.rtpParameters.codecs[0].mimeType.split("/")[1],
// track: consumer.track,
// },
// peerId
// )
// );
self.push(message);
console.log("消费者", consumer);
self.callbackMedia("remote", consumer.track, consumer);
// If audio-only mode is enabled, pause it.
if (consumer.kind === "video" && !self.videoProduce) {
// this.pauseConsumer(consumer);
// TODO：实现
}
} catch (error) {
self.callbackError("消费媒体异常", error);
}
}
async pauseConsumer(consumer) {
if (consumer.paused) return;
try {

4
taoyao-signal-server/README.md
View File

@@ -13,7 +13,3 @@
[信令格式](https://localhost:8888/protocol/list)
## TODO
标识 -> ID
所有字段获取 -> get

4
taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/Constant.java
View File

@@ -196,7 +196,7 @@ public interface Constant {
*/
String RTP_PARAMETERS = "rtpParameters";
/**
* RTP能力
* RTP协商
*/
String RTP_CAPABILITIES = "rtpCapabilities";
/**
@@ -208,7 +208,7 @@ public interface Constant {
*/
String SCTP_PARAMETERS = "sctpParameters";
/**
* SCTP能力
* SCTP协商
*/
String SCTP_CAPABILITIES = "sctpCapabilities";
/**

30
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/event/RoomClientListEvent.java Normal file
View File

@@ -0,0 +1,30 @@
package com.acgist.taoyao.signal.event;
import com.acgist.taoyao.signal.client.Client;
import com.acgist.taoyao.signal.party.media.Room;
import lombok.Getter;
import lombok.Setter;
/**
* 房间终端列表事件
*
* @author acgist
*/
@Getter
@Setter
public class RoomClientListEvent extends RoomEventAdapter {
private static final long serialVersionUID = 1L;
/**
* 终端
*/
private final Client client;
public RoomClientListEvent(Room room, Client client) {
super(room);
this.client = client;
}
}

6
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/media/ClientWrapper.java
View File

@@ -76,7 +76,13 @@ public class ClientWrapper implements AutoCloseable {
* 没有订阅任何媒体时需要用户自己对媒体进行消费控制
*/
private SubscribeType subscribeType;
/**
* RTP协商
*/
private Object rtpCapabilities;
/**
* SCTP协商
*/
private Object sctpCapabilities;
/**
* 发送通道

1
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/party/media/Room.java
View File

@@ -78,6 +78,7 @@ public class Room implements Closeable {
if(clientWrapper != null) {
return clientWrapper;
}
log.info("终端进入房间：{} - {}", this.roomId, client.clientId());
clientWrapper = new ClientWrapper(this, client);
this.clients.put(client, clientWrapper);
this.roomStatus.setClientSize(this.roomStatus.getClientSize() + 1);

12
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/media/MediaAudioActiveSpeakerProtocol.java
View File

@@ -2,6 +2,7 @@ package com.acgist.taoyao.signal.protocol.media;
import java.util.Map;
import com.acgist.taoyao.boot.annotation.Description;
import com.acgist.taoyao.boot.annotation.Protocol;
import com.acgist.taoyao.boot.model.Message;
import com.acgist.taoyao.signal.client.Client;
@@ -15,6 +16,15 @@ import com.acgist.taoyao.signal.protocol.ProtocolRoomAdapter;
* @author acgist
*/
@Protocol
@Description(
body = """
{
"volume": 音量,
"clientId": "终端ID"
}
""",
flow = "媒体服务->信令服务->终端"
)
public class MediaAudioActiveSpeakerProtocol extends ProtocolRoomAdapter {
public static final String SIGNAL = "media::audio::active::speaker";
@@ -28,7 +38,7 @@ public class MediaAudioActiveSpeakerProtocol extends ProtocolRoomAdapter {
if(clientType == ClientType.MEDIA) {
room.broadcast(message);
} else {
// 忽略其他情况
this.logNoAdapter(clientType);
}
}

6
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/media/MediaRouterRtpCapabilitiesProtocol.java
View File

@@ -11,7 +11,7 @@ import com.acgist.taoyao.signal.party.media.Room;
import com.acgist.taoyao.signal.protocol.ProtocolRoomAdapter;
/**
* 路由RTP能力信令
* 路由RTP协商信令
*
* @author acgist
*/
@@ -37,7 +37,7 @@ public class MediaRouterRtpCapabilitiesProtocol extends ProtocolRoomAdapter {
public static final String SIGNAL = "media::router::rtp::capabilities";
public MediaRouterRtpCapabilitiesProtocol() {
super("路由RTP能力信令", SIGNAL);
super("路由RTP协商信令", SIGNAL);
}
@Override
@@ -45,7 +45,7 @@ public class MediaRouterRtpCapabilitiesProtocol extends ProtocolRoomAdapter {
if(clientType == ClientType.WEB || clientType == ClientType.CAMERA) {
client.push(room.request(message));
} else {
// 忽略其他情况
this.logNoAdapter(clientType);
}
}

14
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/room/RoomClientListProtocol.java
View File

@@ -2,11 +2,15 @@ package com.acgist.taoyao.signal.protocol.room;
import java.util.Map;
import org.springframework.context.ApplicationListener;
import org.springframework.scheduling.annotation.Async;
import com.acgist.taoyao.boot.annotation.Description;
import com.acgist.taoyao.boot.annotation.Protocol;
import com.acgist.taoyao.boot.model.Message;
import com.acgist.taoyao.signal.client.Client;
import com.acgist.taoyao.signal.client.ClientType;
import com.acgist.taoyao.signal.event.RoomClientListEvent;
import com.acgist.taoyao.signal.party.media.Room;
import com.acgist.taoyao.signal.protocol.ProtocolRoomAdapter;
@@ -49,13 +53,21 @@ import com.acgist.taoyao.signal.protocol.ProtocolRoomAdapter;
},
flow = "终端=>信令服务->终端"
)
public class RoomClientListProtocol extends ProtocolRoomAdapter {
public class RoomClientListProtocol extends ProtocolRoomAdapter implements ApplicationListener<RoomClientListEvent> {
public static final String SIGNAL = "room::client::list";
public RoomClientListProtocol() {
super("房间终端列表信令", SIGNAL);
}
@Async
@Override
public void onApplicationEvent(RoomClientListEvent event) {
final Room room = event.getRoom();
final Client client = event.getClient();
client.push(this.build(room.clientStatus()));
}
@Override
public void execute(String clientId, ClientType clientType, Room room, Client client, Client mediaClient, Message message, Map<String, Object> body) {

48
taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/room/RoomEnterProtocol.java
View File

@@ -13,25 +13,24 @@ import com.acgist.taoyao.boot.model.MessageCodeException;
import com.acgist.taoyao.boot.utils.MapUtils;
import com.acgist.taoyao.signal.client.Client;
import com.acgist.taoyao.signal.client.ClientType;
import com.acgist.taoyao.signal.event.RoomClientListEvent;
import com.acgist.taoyao.signal.party.media.ClientWrapper;
import com.acgist.taoyao.signal.party.media.ClientWrapper.SubscribeType;
import com.acgist.taoyao.signal.party.media.Room;
import com.acgist.taoyao.signal.protocol.ProtocolRoomAdapter;
import lombok.extern.slf4j.Slf4j;
/**
* 进入房间信令
*
* @author acgist
*/
@Slf4j
@Protocol
@Description(
body = {
"""
{
"roomId": "房间标识"
"roomId": "房间ID",
"password": "房间密码（选填）"
}
""",
"""
@@ -47,39 +46,52 @@ public class RoomEnterProtocol extends ProtocolRoomAdapter {
public static final String SIGNAL = "room::enter";
private final RoomClientListProtocol roomClientListProtocol;
public RoomEnterProtocol(RoomClientListProtocol roomClientListProtocol) {
public RoomEnterProtocol() {
super("进入房间信令", SIGNAL);
this.roomClientListProtocol = roomClientListProtocol;
}
@Override
public void execute(String clientId, ClientType clientType, Room room, Client client, Client mediaClient, Message message, Map<String, Object> body) {
if(clientType == ClientType.WEB || clientType == ClientType.CAMERA) {
this.enter(clientId, room, client, message, body);
} else {
this.logNoAdapter(clientType);
}
}
/**
* 终端进入
*
* @param clientId 终端ID
* @param room 房间
* @param client 终端
* @param message 消息
* @param body 消息主体
*/
private void enter(String clientId, Room room, Client client, Message message, Map<String, Object> body) {
final String password = MapUtils.get(body, Constant.PASSWORD);
final String subscribeType = MapUtils.get(body, Constant.SUBSCRIBE_TYPE);
final Object rtpCapabilities = MapUtils.get(body, Constant.RTP_CAPABILITIES);
final Object sctpCapabilities = MapUtils.get(body, Constant.SCTP_CAPABILITIES);
final String roomPassowrd = room.getPassword();
if(StringUtils.isNotEmpty(roomPassowrd) && !roomPassowrd.equals(password)) {
throw MessageCodeException.of(MessageCode.CODE_3401, "密码错误");
}
final String subscribeType = MapUtils.get(body, Constant.SUBSCRIBE_TYPE);
final Object rtpCapabilities = MapUtils.get(body, Constant.RTP_CAPABILITIES);
final Object sctpCapabilities = MapUtils.get(body, Constant.SCTP_CAPABILITIES);
// 进入房间
final ClientWrapper clientWrapper = room.enter(client);
// 配置参数
clientWrapper.setSubscribeType(SubscribeType.of(subscribeType));
clientWrapper.setRtpCapabilities(rtpCapabilities);
clientWrapper.setSctpCapabilities(sctpCapabilities);
// 发送通知
message.setBody(Map.of(
Constant.ROOM_ID, room.getRoomId(),
Constant.CLIENT_ID, clientId
Constant.CLIENT_ID, clientId,
Constant.STATUS, client.status()
));
room.broadcast(message);
log.info("进入房间：{} - {}", clientId, room.getRoomId());
// 推送房间用户信息：TODO event
final Message roomClientListMessage = this.roomClientListProtocol.build();
roomClientListMessage.setBody(room.clientStatus());
client.push(roomClientListMessage);
}
// 房间终端列表事件
this.publishEvent(new RoomClientListEvent(room, client));
}
}