qq27800734/taoyao
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[*] 才子词人 自是白衣卿相

Browse Source
This commit is contained in:
acgist
2023-04-08 15:14:56 +08:00
parent 9978039c9d
commit 8f2f0dd1a8
14 changed files with 673 additions and 237 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
README.md
View File

@@ -17,6 +17,8 @@
----
> 当前程序处于开发阶段，大部分功能没有实现，可以使用的功能也没有经过大量测试，建议不要用于生产。
## 模块
|模块|名称|描述|
@@ -40,6 +42,12 @@
|P2P|支持|实现|P2P监控模式|
|WebRTC|支持|暂未实现|安卓终端支持同时进入多个房间|
|RTP|支持|暂未实现|支持房间RTP推流（不会拉流）|
|录像|支持|暂未实现||
|拍照|支持|暂未实现||
|变声|支持|暂未实现||
|水印|支持|暂未实现||
|美颜|支持|暂未实现||
|AI识别|支持|暂未实现||
## 证书
@@ -60,14 +68,3 @@
#### 代理终端
将下级信令服务的终端全部使用代理终端注册到上级信令服务，上级信令服务代理终端处理信令时直接路由到下级路由服务，这样一级一级路由直到发送给真正的终端为止。
## TODO
* 标识 -> ID
* 所有字段获取 -> get
* 优化JS错误回调 -> platform::error
* 反复测试推流拉流、拉人踢人、音频视频控制
* 24小时不关闭媒体/一秒一次推拉流十分钟测试/三十秒推拉流一小时测试
* AI、美颜、水印、滤镜
* 混音、降噪、回音消除、声音特效

49
taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java
View File

@@ -499,19 +499,31 @@ public final class Taoyao implements ITaoyao {
private void dispatch(final String content, final Header header, final Message message) {
final Map<String, Object> body = message.body();
switch (header.getSignal()) {
case "client::config" -> this.clientConfig(message, body);
case "client::register" -> this.clientRegister(message, body);
case "client::reboot" -> this.clientReboot(message, body);
case "client::shutdown" -> this.clientShutdown(message, body);
case "room::close" -> this.roomClose(message, body);
case "room::enter" -> this.roomEnter(message, body);
// case "room::expel" -> this.roomExpel(message, body);
case "room::invite" -> this.roomInivte(message, body);
// case "room::leave" -> this.roomLeave(message, body);
case "session::call" -> this.sessionCall(message, body);
case "session::close" -> this.sessionClose(message, body);
case "session::exchange" -> this.sessionExchange(message, body);
default -> Log.d(Taoyao.class.getSimpleName(), "没有适配信令：" + content);
case "client::config" -> this.clientConfig(message, body);
case "client::register" -> this.clientRegister(message, body);
case "client::reboot" -> this.clientReboot(message, body);
case "client::shutdown" -> this.clientShutdown(message, body);
case "media::consume" -> this.mediaConsume(message, body);
// case "media::audio::volume" -> this.mediaAudioVolume(message, body);
// case "media::consumer::close" -> this.mediaConsumerClose(message, body);
// case "media::consumer::pause" -> this.mediaConsumerPause(message, body);
// case "media::consumer::request::key::frame" -> this.mediaConsumerRequestKeyFrame(message, body);
// case "media::consumer::resume" -> this.mediaConsumerResume(message, body);
// case "media::consumer::set::preferred::layers" -> this.mediaConsumerSetPreferredLayers(message, body);
// case "media::consumer::status" -> this.mediaConsumerStatus(message, body);
// case "media::producer::close" -> this.mediaProducerClose(message, body);
// case "media::producer::pause" -> this.mediaProducerPause(message, body);
// case "media::producer::resume" -> this.mediaProducerResume(message, body);
// case "media::producer::video::orientation:change" -> this.mediaVideoOrientationChange(message, body);
case "room::close" -> this.roomClose(message, body);
case "room::enter" -> this.roomEnter(message, body);
// case "room::expel" -> this.roomExpel(message, body);
case "room::invite" -> this.roomInivte(message, body);
// case "room::leave" -> this.roomLeave(message, body);
case "session::call" -> this.sessionCall(message, body);
case "session::close" -> this.sessionClose(message, body);
case "session::exchange" -> this.sessionExchange(message, body);
default -> Log.d(Taoyao.class.getSimpleName(), "没有适配信令：" + content);
}
}
@@ -569,6 +581,15 @@ public final class Taoyao implements ITaoyao {
Process.killProcess(Process.myPid());
}
private void mediaConsume(Message message, Map<String, Object> body) {
final String roomId = MapUtils.get(body, "roomId");
final Room room = this.rooms.get(roomId);
if(room == null) {
return;
}
room.mediaConsume(message, body);
}
private void roomClose(Message message, Map<String, Object> body) {
final String roomId = MapUtils.get(body, "roomId");
final Room room = this.rooms.remove(roomId);
@@ -610,7 +631,7 @@ public final class Taoyao implements ITaoyao {
)
);
room.enter();
room.produceMedia();
room.mediaProduce();
return room;
}

27
taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp
View File

@@ -15,13 +15,18 @@ namespace acgist {
class Room {
public:
std::string roomId;
mediasoupclient::Device *device;
mediasoupclient::PeerConnection *peerConnection;
mediasoupclient::SendTransport *sendTransport;
mediasoupclient::RecvTransport *recvTransport;
mediasoupclient::PeerConnection *peerConnection;
mediasoupclient::SendTransport::Listener *sendListener;
mediasoupclient::RecvTransport::Listener *recvListener;
std::string roomId;
mediasoupclient::Producer *audioProducer;
mediasoupclient::Producer *videoProducer;
mediasoupclient::Producer::Listener *producerListener;
mediasoupclient::Consumer::Listener *consumerListener;
std::map<std::string, mediasoupclient::Consumer*> consumers;
public:
JNIEnv *env;
jobject routerCallback;
@@ -29,16 +34,18 @@ namespace acgist {
Room(std::string roomId, JNIEnv *env, jobject routerCallback);
virtual ~Room();
public:
void enter(
std::string rtpCapabilities,
webrtc::PeerConnectionFactoryInterface *factory,
webrtc::PeerConnectionInterface::RTCConfiguration &rtcConfiguration
);
void enter(std::string rtpCapabilities, webrtc::PeerConnectionFactoryInterface *factory, webrtc::PeerConnectionInterface::RTCConfiguration &rtcConfiguration);
void createSendTransport(std::string body);
void createRecvTransport(std::string body);
void produceMedia(webrtc::MediaStreamInterface mediaStream);
void closeLocalClient();
void closeRemoteClient();
void mediaProduceAudio(webrtc::MediaStreamInterface *mediaStream);
void mediaProduceVideo(webrtc::MediaStreamInterface *mediaStream);
void mediaConsume(std::string message);
void mediaProducerPause(std::string producerId);
void mediaProducerResume(std::string producerId);
void mediaProducerClose(std::string producerId);
void mediaConsumerPause(std::string consumerId);
void mediaConsumerResume(std::string consumerId);
void mediaConsumerClose(std::string consumerId);
void close();
};

262
taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp
View File

@@ -21,20 +21,20 @@ namespace acgist {
jclass jCallbackClazz = this->env->GetObjectClass(this->routerCallback);
jmethodID sendTransportConnectCallback = this->env->GetMethodID(jCallbackClazz, "sendTransportConnectCallback", "(Ljava/lang/String;Ljava/lang/String;)V");
const char *cTransportId = transport->GetId().data();
jstring jTransportId = env->NewStringUTF(cTransportId);
jstring jTransportId = this->env->NewStringUTF(cTransportId);
const char *cDtlsParameters = dtlsParameters.dump().data();
jstring jDtlsParameters = env->NewStringUTF(cDtlsParameters);
jstring jDtlsParameters = this->env->NewStringUTF(cDtlsParameters);
this->env->CallVoidMethod(
this->routerCallback,
sendTransportConnectCallback,
jTransportId,
jDtlsParameters
);
env->DeleteLocalRef(jTransportId);
env->ReleaseStringUTFChars(jTransportId, cTransportId);
env->DeleteLocalRef(jDtlsParameters);
env->ReleaseStringUTFChars(jDtlsParameters, cDtlsParameters);
env->DeleteLocalRef(jCallbackClazz);
this->env->DeleteLocalRef(jTransportId);
this->env->ReleaseStringUTFChars(jTransportId, cTransportId);
this->env->DeleteLocalRef(jDtlsParameters);
this->env->ReleaseStringUTFChars(jDtlsParameters, cDtlsParameters);
this->env->DeleteLocalRef(jCallbackClazz);
return std::future<void>();
}
@@ -46,11 +46,12 @@ namespace acgist {
jclass jCallbackClazz = this->env->GetObjectClass(this->routerCallback);
jmethodID sendTransportProduceCallback = this->env->GetMethodID(jCallbackClazz, "sendTransportProduceCallback", "(Ljava/lang/String;Ljava/lang/String;)V");
const char *cKind = kind.data();
jstring jKind = env->NewStringUTF(cKind);
jstring jKind = this-> env->NewStringUTF(cKind);
const char *cTransportId = transport->GetId().data();
jstring jTransportId = env->NewStringUTF(cTransportId);
jstring jTransportId = this-> env->NewStringUTF(cTransportId);
const char *cRtpParameters = rtpParameters.dump().data();
jstring jRtpParameters = env->NewStringUTF(cRtpParameters);
jstring jRtpParameters = this-> env->NewStringUTF(cRtpParameters);
std::promise<std::string> promise;
jstring jResult = (jstring) this->env->CallObjectMethod(
this->routerCallback,
sendTransportProduceCallback,
@@ -58,19 +59,20 @@ namespace acgist {
jTransportId,
jRtpParameters
);
const char *cResult = env->GetStringUTFChars(jResult, 0);
const char *cResult = this-> env->GetStringUTFChars(jResult, 0);
std::string result(cResult);
env->DeleteLocalRef(jResult);
env->DeleteLocalRef(jKind);
env->ReleaseStringUTFChars(jKind, cKind);
env->DeleteLocalRef(jResult);
env->ReleaseStringUTFChars(jResult, cResult);
env->DeleteLocalRef(jTransportId);
env->ReleaseStringUTFChars(jTransportId, cTransportId);
env->DeleteLocalRef(jRtpParameters);
env->ReleaseStringUTFChars(jRtpParameters, cRtpParameters);
env->DeleteLocalRef(jCallbackClazz);
return std::future<std::string>();
promise.set_value(result);
this-> env->DeleteLocalRef(jResult);
this-> env->DeleteLocalRef(jKind);
this-> env->ReleaseStringUTFChars(jKind, cKind);
this-> env->DeleteLocalRef(jResult);
this-> env->ReleaseStringUTFChars(jResult, cResult);
this-> env->DeleteLocalRef(jTransportId);
this-> env->ReleaseStringUTFChars(jTransportId, cTransportId);
this-> env->DeleteLocalRef(jRtpParameters);
this-> env->ReleaseStringUTFChars(jRtpParameters, cRtpParameters);
this-> env->DeleteLocalRef(jCallbackClazz);
return promise.get_future();
}
std::future<std::string> OnProduceData(mediasoupclient::SendTransport *transport, const nlohmann::json &sctpStreamParameters, const std::string &label, const std::string &protocol, const nlohmann::json &appData) override {
@@ -95,23 +97,23 @@ namespace acgist {
}
std::future<void> OnConnect(mediasoupclient::Transport *transport, const nlohmann::json &dtlsParameters) override {
jclass jCallbackClazz = env->GetObjectClass(this->routerCallback);
jclass jCallbackClazz = this->env->GetObjectClass(this->routerCallback);
jmethodID recvTransportConnectCallback = this->env->GetMethodID(jCallbackClazz, "recvTransportConnectCallback", "(Ljava/lang/String;Ljava/lang/String;)V");
const char *cTransportId = transport->GetId().data();
jstring jTransportId = env->NewStringUTF(cTransportId);
jstring jTransportId = this-> env->NewStringUTF(cTransportId);
const char *cDtlsParameters = dtlsParameters.dump().data();
jstring jDtlsParameters = env->NewStringUTF(cDtlsParameters);
jstring jDtlsParameters = this-> env->NewStringUTF(cDtlsParameters);
this->env->CallVoidMethod(
this->routerCallback,
recvTransportConnectCallback,
jTransportId,
jDtlsParameters
);
env->DeleteLocalRef(jTransportId);
env->ReleaseStringUTFChars(jTransportId, cTransportId);
env->DeleteLocalRef(jDtlsParameters);
env->ReleaseStringUTFChars(jDtlsParameters, cDtlsParameters);
env->DeleteLocalRef(jCallbackClazz);
this-> env->DeleteLocalRef(jTransportId);
this-> env->ReleaseStringUTFChars(jTransportId, cTransportId);
this-> env->DeleteLocalRef(jDtlsParameters);
this-> env->ReleaseStringUTFChars(jDtlsParameters, cDtlsParameters);
this-> env->DeleteLocalRef(jCallbackClazz);
return std::future<void>();
}
@@ -121,6 +123,46 @@ namespace acgist {
};
class ProducerListener : public mediasoupclient::Producer::Listener {
public:
Room *room;
JNIEnv *env;
jobject routerCallback;
public:
ProducerListener(Room *room, JNIEnv *env, jobject routerCallback) {
this->room = room;
this->env = env;
this->routerCallback = routerCallback;
}
void OnTransportClose(mediasoupclient::Producer *producer) override {
}
};
class ConsumerListener : public mediasoupclient::Consumer::Listener {
public:
Room *room;
JNIEnv *env;
jobject routerCallback;
public:
ConsumerListener(Room *room, JNIEnv *env, jobject routerCallback) {
this->room = room;
this->env = env;
this->routerCallback = routerCallback;
}
void OnTransportClose(mediasoupclient::Consumer *consumer) override {
}
};
Room::Room(
std::string roomId,
JNIEnv *env,
@@ -132,6 +174,8 @@ namespace acgist {
this->device = new mediasoupclient::Device();
this->sendListener = new SendListener(this, env, routerCallback);
this->recvListener = new RecvListener(this, env, routerCallback);
this->producerListener = new ProducerListener(this, env, routerCallback);
this->consumerListener = new ConsumerListener(this, env, routerCallback);
}
Room::~Room() {
@@ -140,8 +184,12 @@ namespace acgist {
delete this->sendTransport;
delete this->recvListener;
delete this->recvTransport;
env->DeleteLocalRef(this->routerCallback);
env->DeleteGlobalRef(this->routerCallback);
delete this->audioProducer;
delete this->videoProducer;
delete this->producerListener;
delete this->consumerListener;
this-> env->DeleteLocalRef(this->routerCallback);
this-> env->DeleteGlobalRef(this->routerCallback);
}
void Room::enter(
@@ -184,10 +232,86 @@ namespace acgist {
);
}
void Room::produceMedia(webrtc::MediaStreamInterface mediaStream) {
// this->device->CanProduce();
void Room::mediaProduceAudio(webrtc::MediaStreamInterface *mediaStream) {
if(!this->device->CanProduce("audio")) {
return;
}
nlohmann::json codecOptions =
{
{ "opusStereo", true },
{ "opusDtx", true }
};
this->audioProducer = this->sendTransport->Produce(
this->producerListener,
mediaStream->GetAudioTracks()[0],
nullptr,
&codecOptions,
nullptr
);
}
void Room::mediaProduceVideo(webrtc::MediaStreamInterface *mediaStream) {
if(this->device->CanProduce("video")) {
return;
}
// TODO：配置读取
nlohmann::json codecOptions =
{
{ "videoGoogleStartBitrate", 400 },
{ "videoGoogleMinBitrate", 800 },
{ "videoGoogleMaxBitrate", 1600 }
};
// 设置动态码率，帧率、分辨率在摄像头初始化处设置。
// 如果需要使用`Simulcast`打开下面配置
// std::vector<webrtc::RtpEncodingParameters> encodings;
// webrtc::RtpEncodingParameters min;
// webrtc::RtpEncodingParameters mid;
// webrtc::RtpEncodingParameters max;
// min.active = true;
// min.max_framerate = 15;
// min.min_bitrate_bps = 400;
// min.max_bitrate_bps = 800;
// encodings.emplace_back(min);
// encodings.emplace_back(mid);
// encodings.emplace_back(max);
// 强制设置编码器
// nlohmann::json codec = this->device->GetRtpCapabilities()["codec"];
this->videoProducer = this->sendTransport->Produce(
this->producerListener,
mediaStream->GetVideoTracks()[0],
nullptr,
&codecOptions,
nullptr
);
}
void Room::mediaConsume(std::string message) {
nlohmann::json json = nlohmann::json::parse(message);
nlohmann::json body = json["body"];
mediasoupclient::Consumer *consumer = this->recvTransport->Consume(
this->consumerListener,
body["consumerId"],
body["producerId"],
body["kind"],
&body["rtpParameters"]
);
this->consumers.insert({ consumer->GetId(), consumer });
webrtc::MediaStreamTrackInterface* trackPointer = consumer->GetTrack();
jclass jCallbackClazz = this->env->GetObjectClass(this->routerCallback);
jmethodID consumerNewCallback = this->env->GetMethodID(jCallbackClazz, "consumerNewCallback", "(Ljava/lang/String;J;)V");
const char *cMessage = message.data();
jstring jMessage = this-> env->NewStringUTF(cMessage);
this->env->CallVoidMethod(
this->routerCallback,
consumerNewCallback,
jMessage,
(jlong) trackPointer
);
this-> env->DeleteLocalRef(jMessage);
this-> env->ReleaseStringUTFChars(jMessage, cMessage);
this-> env->DeleteLocalRef(jCallbackClazz);
};
void Room::close() {
delete this->device;
}
@@ -249,13 +373,79 @@ namespace acgist {
Room* room = (Room*) nativeRoomPointer;
const char* body = env->GetStringUTFChars(jBody, 0);
room->createRecvTransport(body);
env->DeleteLocalRef(jBody);
env->ReleaseStringUTFChars(jBody, body);
}
extern "C" JNIEXPORT void JNICALL
Java_com_acgist_taoyao_media_client_Room_nativeProduceMedia(JNIEnv *env, jobject me, jlong nativeRoomPointer, jlong mediaStreamPointer) {
Java_com_acgist_taoyao_media_client_Room_nativeMediaProduceAudio(JNIEnv *env, jobject me, jlong nativeRoomPointer, jlong mediaStreamPointer) {
Room* room = (Room*) nativeRoomPointer;
webrtc::MediaStreamInterface* mediaStream = reinterpret_cast<webrtc::MediaStreamInterface*>(mediaStreamPointer);
webrtc::MediaStreamInterface *mediaStream = reinterpret_cast<webrtc::MediaStreamInterface*>(mediaStreamPointer);
room->mediaProduceAudio(mediaStream);
}
extern "C" JNIEXPORT void JNICALL
Java_com_acgist_taoyao_media_client_Room_nativeMediaProduceVideo(JNIEnv *env, jobject me, jlong nativeRoomPointer, jlong mediaStreamPointer) {
Room* room = (Room*) nativeRoomPointer;
webrtc::MediaStreamInterface *mediaStream = reinterpret_cast<webrtc::MediaStreamInterface*>(mediaStreamPointer);
room->mediaProduceVideo(mediaStream);
}
extern "C" JNIEXPORT void JNICALL
Java_com_acgist_taoyao_media_client_Room_nativeMediaConsume(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jMessage) {
Room* room = (Room*) nativeRoomPointer;
const char *message = env->GetStringUTFChars(jMessage, 0);
room->mediaConsume(message);
env->DeleteLocalRef(jMessage);
env->ReleaseStringUTFChars(jMessage, message);
}
extern "C" JNIEXPORT void JNICALL
Java_com_acgist_taoyao_media_client_Room_nativeMediaProducerPause(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jProducerId) {
Room* room = (Room*) nativeRoomPointer;
const char *producerId = env->GetStringUTFChars(jProducerId, 0);
env->DeleteLocalRef(jProducerId);
env->ReleaseStringUTFChars(jProducerId, producerId);
}
extern "C" JNIEXPORT void JNICALL
Java_com_acgist_taoyao_media_client_Room_nativeMediaProducerResume(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jProducerId) {
Room* room = (Room*) nativeRoomPointer;
const char *producerId = env->GetStringUTFChars(jProducerId, 0);
env->DeleteLocalRef(jProducerId);
env->ReleaseStringUTFChars(jProducerId, producerId);
}
extern "C" JNIEXPORT void JNICALL
Java_com_acgist_taoyao_media_client_Room_nativeMediaProducerClose(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jProducerId) {
Room* room = (Room*) nativeRoomPointer;
const char *producerId = env->GetStringUTFChars(jProducerId, 0);
env->DeleteLocalRef(jProducerId);
env->ReleaseStringUTFChars(jProducerId, producerId);
}
extern "C" JNIEXPORT void JNICALL
Java_com_acgist_taoyao_media_client_Room_nativeMediaConsumerPause(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jConsumerId) {
Room* room = (Room*) nativeRoomPointer;
const char *consumerId = env->GetStringUTFChars(jConsumerId, 0);
env->DeleteLocalRef(jConsumerId);
env->ReleaseStringUTFChars(jConsumerId, consumerId);
}
extern "C" JNIEXPORT void JNICALL
Java_com_acgist_taoyao_media_client_Room_nativeMediaConsumerResume(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jConsumerId) {
Room* room = (Room*) nativeRoomPointer;
const char *consumerId = env->GetStringUTFChars(jConsumerId, 0);
env->DeleteLocalRef(jConsumerId);
env->ReleaseStringUTFChars(jConsumerId, consumerId);
}
extern "C" JNIEXPORT void JNICALL
Java_com_acgist_taoyao_media_client_Room_nativeMediaConsumerClose(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jConsumerId) {
Room* room = (Room*) nativeRoomPointer;
const char *consumerId = env->GetStringUTFChars(jConsumerId, 0);
env->DeleteLocalRef(jConsumerId);
env->ReleaseStringUTFChars(jConsumerId, consumerId);
}
}

80
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java
View File

@@ -2,6 +2,7 @@ package com.acgist.taoyao.media;
import android.content.Context;
import android.content.Intent;
import android.media.AudioFormat;
import android.media.MediaCodecInfo;
import android.media.MediaCodecList;
import android.media.projection.MediaProjection;
@@ -23,6 +24,7 @@ import org.webrtc.DefaultVideoEncoderFactory;
import org.webrtc.EglBase;
import org.webrtc.MediaConstraints;
import org.webrtc.MediaStream;
import org.webrtc.MediaStreamTrack;
import org.webrtc.PeerConnectionFactory;
import org.webrtc.RendererCommon;
import org.webrtc.ScreenCapturerAndroid;
@@ -168,10 +170,6 @@ public final class MediaManager {
* 视频捕获
*/
private VideoCapturer videoCapturer;
/**
* 本地视频预览
*/
private SurfaceViewRenderer localVideoRenderer;
/**
* PeerConnectionFactory
*/
@@ -339,16 +337,20 @@ public final class MediaManager {
this.recordClient.putAudio(audioSamples);
}
})
// 超低延迟
// .setUseLowLatency()
// 远程音频
// .setAudioTrackStateCallback()
// .setAudioFormat(AudioFormat.ENCODING_PCM_32BIT)
// .setUseHardwareNoiseSuppressor(true)
// .setUseHardwareAcousticEchoCanceler(true)
.createAudioDeviceModule();
this.peerConnectionFactory = PeerConnectionFactory.builder()
.setAudioDeviceModule(javaAudioDeviceModule)
// 变声
// .setAudioProcessingFactory()
// .setAudioEncoderFactoryFactory(new BuiltinAudioEncoderFactoryFactory())
// .setAudioDecoderFactoryFactory(new BuiltinAudioDecoderFactoryFactory())
.setAudioDeviceModule(javaAudioDeviceModule)
.setVideoDecoderFactory(videoDecoderFactory)
.setVideoEncoderFactory(videoEncoderFactory)
.createPeerConnectionFactory();
@@ -479,10 +481,6 @@ public final class MediaManager {
this.videoCapturer.initialize(surfaceTextureHelper, this.context, videoSource.getCapturerObserver());
this.videoCapturer.startCapture(480, 640, 30);
final VideoTrack videoTrack = this.peerConnectionFactory.createVideoTrack("ARDAMSv0", videoSource);
if(preview) {
this.localVideoRenderer = this.localVideoRenderer();
videoTrack.addSink(this.localVideoRenderer);
}
videoTrack.addSink(videoFrame -> {
if(this.recordClient != null) {
this.recordClient.putVideo(videoFrame);
@@ -493,12 +491,26 @@ public final class MediaManager {
Log.i(MediaManager.class.getSimpleName(), "加载视频：" + videoTrack.id());
}
public boolean isPreview() {
return this.preview;
}
public MediaStream getMediaStream() {
return this.mediaStream;
}
private SurfaceViewRenderer localVideoRenderer() {
// 设置预览
/**
* @param flag Config.WHAT_*
* @param forceEnabled 是否强制播放
* @param videoTrack 视频媒体流Track
*
* @return 播放控件
*/
public SurfaceViewRenderer buildSurfaceViewRenderer(
final int flag,
final VideoTrack videoTrack
) {
// 预览控件
final SurfaceViewRenderer surfaceViewRenderer = new SurfaceViewRenderer(this.context);
this.handler.post(() -> {
// 视频反转
@@ -507,52 +519,20 @@ public final class MediaManager {
surfaceViewRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL);
// 硬件拉伸
surfaceViewRenderer.setEnableHardwareScaler(true);
// 加载
// 加载OpenSL ES
surfaceViewRenderer.init(this.eglBase.getEglBaseContext(), null);
});
// 事件
// surfaceViewRenderer.setOnClickListener();
// TODO：迁移localvideo
// surfaceViewRenderer.release();
// 页面加载
final Message message = new Message();
message.obj = surfaceViewRenderer;
message.what = Config.WHAT_NEW_LOCAL_VIDEO;
// TODO：恢复
this.handler.sendMessage(message);
// 暂停
// surfaceViewRenderer.pauseVideo();
// 恢复
// surfaceViewRenderer.disableFpsReduction();
return surfaceViewRenderer;
}
public void remoteVideoRenderer(final MediaStream mediaStream) {
// 设置预览
final SurfaceViewRenderer surfaceViewRenderer = new SurfaceViewRenderer(this.context);
this.handler.post(() -> {
// 视频反转
surfaceViewRenderer.setMirror(false);
// 视频拉伸
surfaceViewRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL);
// 硬件拉伸
surfaceViewRenderer.setEnableHardwareScaler(true);
// 加载
surfaceViewRenderer.init(this.eglBase.getEglBaseContext(), null);
// 开始播放
final VideoTrack videoTrack = mediaStream.videoTracks.get(0);
videoTrack.setEnabled(true);
// 强制播放
if(!videoTrack.enabled()) {
videoTrack.setEnabled(true);
}
videoTrack.addSink(surfaceViewRenderer);
});
// 页面加载
final Message message = new Message();
message.obj = surfaceViewRenderer;
message.what = Config.WHAT_NEW_REMOTE_VIDEO;
message.what = flag;
this.handler.sendMessage(message);
// 暂停
// surfaceViewRenderer.pauseVideo();
// 恢复
// surfaceViewRenderer.disableFpsReduction();
return surfaceViewRenderer;
}
public void pauseAudio() {

20
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java
View File

@@ -1,5 +1,7 @@
package com.acgist.taoyao.media;
import org.webrtc.MediaStream;
/**
* 路由回调
*
@@ -9,15 +11,15 @@ public interface RouterCallback {
default void enterCallback(String rtpCapabilities, String sctpCapabilities) {};
default void sendTransportConnectCallback(String transportId, String dtlsParameters) {};
default String sendTransportProduceCallback(String kind, String transportId, String rtpParameters) {
return null;
};
default String sendTransportProduceCallback(String kind, String transportId, String rtpParameters) { return null; };
default void recvTransportConnectCallback(String transportId, String dtlsParameters) {};
default void newRemoteClientCallback() {};
default void closeRemoteClientCallback() {};
default void consumerPauseCallback() {};
default void consumerResumeCallback() {};
default void producerPauseCallback() {};
default void producerResumeCallback() {};
default void producerNewCallback(String kind, String producerId, long producerMediaTrackPointer) { };
default void producerPauseCallback(String producerId) {};
default void producerResumeCallback(String producerId) {};
default void producerCloseCallback(String producerId) {};
default void consumerNewCallback(String message, long consumerMediaTrackPointer) {};
default void consumerPauseCallback(String consumerId) {};
default void consumerResumeCallback(String consumerId) {};
default void consumerCloseCallback(String consumerId) {};
}

46
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Client.java
View File

@@ -1,11 +1,18 @@
package com.acgist.taoyao.media.client;
import android.os.Handler;
import android.os.Message;
import android.util.Log;
import com.acgist.taoyao.media.MediaManager;
import com.acgist.taoyao.media.config.Config;
import com.acgist.taoyao.media.signal.ITaoyao;
import org.webrtc.MediaStreamTrack;
import org.webrtc.RendererCommon;
import org.webrtc.SurfaceViewRenderer;
import org.webrtc.VideoTrack;
import java.io.Closeable;
/**
@@ -35,6 +42,10 @@ public abstract class Client implements Closeable {
* 媒体服务
*/
protected final MediaManager mediaManager;
/**
* 视频预览
*/
protected SurfaceViewRenderer surfaceViewRenderer;
public Client(String name, String clientId, Handler handler, ITaoyao taoyao) {
this.name = name;
@@ -44,9 +55,44 @@ public abstract class Client implements Closeable {
this.mediaManager = MediaManager.getInstance();
}
/**
* 播放音频
*/
public void playAudio() {
}
public void pauseAudio() {
}
public void resumeAudio() {
}
/**
* 播放视频
*/
public void playVideo() {
}
public void pauseVideo() {
if(this.surfaceViewRenderer != null) {
this.surfaceViewRenderer.pauseVideo();
}
}
public void resumeVideo() {
if(this.surfaceViewRenderer != null) {
// TODO：验证是否正确
this.surfaceViewRenderer.disableFpsReduction();
}
}
@Override
public void close() {
Log.i(this.getClass().getSimpleName(), "关闭终端：" + this.clientId);
if(this.surfaceViewRenderer != null) {
this.surfaceViewRenderer.release();
this.surfaceViewRenderer = null;
}
}
}

43
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java
View File

@@ -2,8 +2,12 @@ package com.acgist.taoyao.media.client;
import android.os.Handler;
import com.acgist.taoyao.media.config.Config;
import com.acgist.taoyao.media.signal.ITaoyao;
import org.webrtc.MediaStream;
import org.webrtc.VideoTrack;
/**
* 房间本地终端
*
@@ -11,13 +15,52 @@ import com.acgist.taoyao.media.signal.ITaoyao;
*/
public class LocalClient extends RoomClient {
/**
* 媒体流
*/
protected MediaStream mediaStream;
public LocalClient(String name, String clientId, Handler handler, ITaoyao taoyao) {
super(name, clientId, handler, taoyao);
}
public MediaStream getMediaStream() {
return this.mediaStream;
}
public void setMediaStream(MediaStream mediaStream) {
this.mediaStream = mediaStream;
}
@Override
public void playVideo() {
super.playVideo();
if(this.mediaStream == null) {
return;
}
final VideoTrack videoTrack = this.mediaStream.videoTracks.get(0);
if(this.surfaceViewRenderer == null) {
this.surfaceViewRenderer = this.mediaManager.buildSurfaceViewRenderer(Config.WHAT_NEW_LOCAL_VIDEO, videoTrack);
} else {
videoTrack.setEnabled(true);
}
}
@Override
public void playAudio() {
super.playAudio();
if(this.mediaStream == null) {
return;
}
this.mediaStream.audioTracks.forEach(v -> v.setEnabled(true));
}
@Override
public void close() {
super.close();
if(this.mediaStream != null) {
this.mediaStream.dispose();
}
}
}

42
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java
View File

@@ -1,9 +1,17 @@
package com.acgist.taoyao.media.client;
import android.os.Handler;
import android.provider.MediaStore;
import com.acgist.taoyao.media.config.Config;
import com.acgist.taoyao.media.signal.ITaoyao;
import org.webrtc.MediaStreamTrack;
import org.webrtc.VideoTrack;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
/**
* 房间远程终端
*
@@ -11,13 +19,47 @@ import com.acgist.taoyao.media.signal.ITaoyao;
*/
public class RemoteClient extends RoomClient {
/**
* 媒体流Track
* 消费者ID = 媒体流Track
*/
protected final Map<String, MediaStreamTrack> tracks;
public RemoteClient(String name, String clientId, Handler handler, ITaoyao taoyao) {
super(name, clientId, handler, taoyao);
this.tracks = new ConcurrentHashMap<>();
}
@Override
public void playVideo() {
super.playVideo();
final VideoTrack videoTrack = (VideoTrack) this.tracks.values().stream()
.filter(v -> MediaStreamTrack.VIDEO_TRACK_KIND.equals(v.kind()))
.findFirst()
.orElse(null);
if(videoTrack == null) {
return;
}
if(this.surfaceViewRenderer == null) {
this.surfaceViewRenderer = this.mediaManager.buildSurfaceViewRenderer(Config.WHAT_NEW_REMOTE_VIDEO, videoTrack);
} else {
videoTrack.setEnabled(true);
}
}
@Override
public void playAudio() {
super.playAudio();
this.tracks.values().stream()
.filter(v -> MediaStreamTrack.AUDIO_TRACK_KIND.equals(v.kind()))
.forEach(v -> v.setEnabled(true));
}
@Override
public void close() {
super.close();
this.tracks.values().forEach(MediaStreamTrack::dispose);
this.tracks.clear();
}
}

198
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java
View File

@@ -6,18 +6,24 @@ import android.util.Log;
import com.acgist.taoyao.boot.model.Message;
import com.acgist.taoyao.boot.utils.JSONUtils;
import com.acgist.taoyao.boot.utils.MapUtils;
import com.acgist.taoyao.boot.utils.PointerUtils;
import com.acgist.taoyao.media.MediaManager;
import com.acgist.taoyao.media.RouterCallback;
import com.acgist.taoyao.media.config.Config;
import com.acgist.taoyao.media.signal.ITaoyao;
import org.webrtc.AudioTrack;
import org.webrtc.MediaStream;
import org.webrtc.MediaStreamTrack;
import org.webrtc.PeerConnection;
import org.webrtc.PeerConnectionFactory;
import org.webrtc.VideoTrack;
import java.io.Closeable;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.ConcurrentHashMap;
/**
* 房间
@@ -42,7 +48,7 @@ public class Room implements Closeable, RouterCallback {
private final MediaManager mediaManager;
private volatile boolean enter;
private LocalClient localClient;
private List<RemoteClient> remoteClients;
private Map<String, RemoteClient> remoteClients;
private PeerConnection.RTCConfiguration rtcConfiguration;
private PeerConnectionFactory peerConnectionFactory;
private String sctpCapabilities;
@@ -54,39 +60,35 @@ public class Room implements Closeable, RouterCallback {
boolean dataConsume, boolean audioConsume, boolean videoConsume,
boolean dataProduce, boolean audioProduce, boolean videoProduce
) {
this.name = name;
this.name = name;
this.clientId = clientId;
this.roomId = roomId;
this.roomId = roomId;
this.password = password;
this.handler = handler;
this.taoyao = taoyao;
this.dataConsume = dataConsume;
this.handler = handler;
this.taoyao = taoyao;
this.dataConsume = dataConsume;
this.audioConsume = audioConsume;
this.videoConsume = videoConsume;
this.dataProduce = dataProduce;
this.dataProduce = dataProduce;
this.audioProduce = audioProduce;
this.videoProduce = videoProduce;
this.nativeRoomPointer = this.nativeNewRoom(roomId, this);
this.mediaManager = MediaManager.getInstance();
this.remoteClients = new CopyOnWriteArrayList<>();
this.remoteClients = new ConcurrentHashMap<>();
this.enter = false;
}
/**
* 远程终端列表
*/
private List<RemoteClient> remoteClientList;
public synchronized void enter() {
if(this.enter) {
if (this.enter) {
return;
}
final Message response = this.taoyao.request(this.taoyao.buildMessage("media::router::rtp::capabilities", "roomId", this.roomId));
if(response == null) {
if (response == null) {
Log.w(Room.class.getSimpleName(), "获取通道能力失败");
return;
}
this.localClient = new LocalClient(this.name, this.clientId, this.handler, this.taoyao);
this.localClient.setMediaStream(this.mediaManager.getMediaStream());
// STUN | TURN
final List<PeerConnection.IceServer> iceServers = new ArrayList<>();
// TODO：读取配置
@@ -98,14 +100,21 @@ public class Room implements Closeable, RouterCallback {
this.nativeEnter(this.nativeRoomPointer, JSONUtils.toJSON(rtpCapabilities), this.peerConnectionFactory.getNativePeerConnectionFactory(), this.rtcConfiguration);
}
public void produceMedia() {
if(this.audioProduce || this.videoProduce) {
public void mediaProduce() {
if (this.audioProduce || this.videoProduce) {
this.createSendTransport();
}
if(this.audioConsume || this.videoConsume) {
if (this.audioConsume || this.videoConsume) {
this.createRecvTransport();
}
final MediaStream mediaStream = this.localClient.getMediaStream();
final long mediaStreamPointer = PointerUtils.getNativePointer(mediaStream, "nativeStream");
if (this.audioProduce) {
this.nativeMediaProduceAudio(this.nativeRoomPointer, mediaStreamPointer);
}
if (this.videoProduce) {
this.nativeMediaProduceVideo(this.nativeRoomPointer, mediaStreamPointer);
}
}
private void createSendTransport() {
@@ -117,7 +126,7 @@ public class Room implements Closeable, RouterCallback {
"consuming", false,
"sctpCapabilities", this.dataProduce ? this.sctpCapabilities : null
));
if(response == null) {
if (response == null) {
Log.w(Room.class.getSimpleName(), "创建发送通道失败");
return;
}
@@ -132,42 +141,110 @@ public class Room implements Closeable, RouterCallback {
"consuming", true,
"sctpCapabilities", this.dataProduce ? this.sctpCapabilities : null
));
if(response == null) {
if (response == null) {
Log.w(Room.class.getSimpleName(), "创建接收通道失败");
return;
}
this.nativeCreateRecvTransport(this.nativeRoomPointer, JSONUtils.toJSON(response.body()));
}
public void mediaConsume(Message message, Map<String, Object> body) {
this.nativeMediaConsume(this.nativeRoomPointer, JSONUtils.toJSON(message));
}
/**
* 新增远程终端
*
* @param body 消息主体
*/
public void newRemoteClient(Map<String, Object> body) {
final String clientId = MapUtils.get(body, "clientId");
final Map<String, Object> status = MapUtils.get(body, "status");
final String name = MapUtils.get(status, "name");
final RemoteClient remoteClient = new RemoteClient(name, clientId, this.handler, this.taoyao);
this.remoteClients.add(remoteClient);
synchronized (this.remoteClients) {
final String clientId = MapUtils.get(body, "clientId");
final Map<String, Object> status = MapUtils.get(body, "status");
final String name = MapUtils.get(status, "name");
final RemoteClient remoteClient = new RemoteClient(name, clientId, this.handler, this.taoyao);
final RemoteClient old = this.remoteClients.put(clientId, remoteClient);
if(old != null) {
// 关闭旧的资源
old.close();
}
}
}
public void closeRemoteClient(String clientId) {
synchronized (this.remoteClients) {
final RemoteClient remoteClient = this.remoteClients.get(clientId);
if(remoteClient == null) {
return;
}
remoteClient.close();
}
}
@Override
public void close() {
Log.i(Room.class.getSimpleName(), "关闭房间：" + this.roomId);
this.localClient.close();
this.remoteClientList.forEach(RemoteClient::close);
this.remoteClients.values().forEach(v -> this.closeRemoteClient(v.clientId));
this.mediaManager.closeClient();
this.nativeCloseRoom(this.nativeRoomPointer);
}
public void mediaConsumerClose(String consumerId) {
this.taoyao.push(this.taoyao.buildMessage(
"media::consumer::close",
"roomId", this.roomId,
"consumerId", consumerId
));
}
public void mediaConsumerPause(String consumerId) {
this.taoyao.push(this.taoyao.buildMessage(
"media::consumer::pause",
"roomId", this.roomId,
"consumerId", consumerId
));
}
public void mediaConsumerResume(String consumerId) {
this.taoyao.push(this.taoyao.buildMessage(
"media::consumer::resume",
"roomId", this.roomId,
"consumerId", consumerId
));
}
public void mediaProducerClose(String producerId) {
this.taoyao.push(this.taoyao.buildMessage(
"media::producer::close",
"roomId", this.roomId,
"producerId", producerId
));
}
public void mediaProducerPause(String producerId) {
this.taoyao.push(this.taoyao.buildMessage(
"media::producer::pause",
"roomId", this.roomId,
"producerId", producerId
));
}
public void mediaProducerResume(String producerId) {
this.taoyao.push(this.taoyao.buildMessage(
"media::producer::resume",
"roomId", this.roomId,
"producerId", producerId
));
}
@Override
public void enterCallback(String rtpCapabilities, String sctpCapabilities) {
this.taoyao.request(this.taoyao.buildMessage(
"room::enter",
"roomId", this.roomId,
"password", this.password,
"rtpCapabilities", rtpCapabilities,
"roomId", this.roomId,
"password", this.password,
"rtpCapabilities", rtpCapabilities,
"sctpCapabilities", sctpCapabilities
));
this.enter = true;
@@ -177,8 +254,8 @@ public class Room implements Closeable, RouterCallback {
public void sendTransportConnectCallback(String transportId, String dtlsParameters) {
this.taoyao.request(this.taoyao.buildMessage(
"media::transport::webrtc::connect",
"roomId", this.roomId,
"transportId", transportId,
"roomId", this.roomId,
"transportId", transportId,
"dtlsParameters", JSONUtils.toMap(dtlsParameters)
));
}
@@ -187,9 +264,9 @@ public class Room implements Closeable, RouterCallback {
public String sendTransportProduceCallback(String kind, String transportId, String rtpParameters) {
final Message response = this.taoyao.request(this.taoyao.buildMessage(
"media::produce",
"kind", kind,
"roomId", this.roomId,
"transportId", transportId,
"kind", kind,
"roomId", this.roomId,
"transportId", transportId,
"rtpParameters", JSONUtils.toMap(rtpParameters)
));
final Map<String, Object> body = response.body();
@@ -200,22 +277,53 @@ public class Room implements Closeable, RouterCallback {
public void recvTransportConnectCallback(String transportId, String dtlsParameters) {
this.taoyao.request(this.taoyao.buildMessage(
"media::transport::webrtc::connect",
"roomId", this.roomId,
"transportId", transportId,
"roomId", this.roomId,
"transportId", transportId,
"dtlsParameters", JSONUtils.toMap(dtlsParameters)
));
}
private native void nativeEnter(
long nativePointer,
String rtpCapabilities,
long peerConnectionFactoryPointer,
PeerConnection.RTCConfiguration rtcConfiguration
);
@Override
public void consumerNewCallback(String message, long consumerMediaTrackPointer) {
final Message response = JSONUtils.toJava(message, Message.class);
final Map<String, Object> body = response.body();
final String kind = MapUtils.get(body, "kind");
final String clientId = MapUtils.get(body, "clientId");
final String consumerId = MapUtils.get(body, "consumerId");
final RemoteClient remoteClient = this.remoteClients.get(clientId);
if(remoteClient == null) {
// TODO：资源释放
return;
}
if(MediaStreamTrack.AUDIO_TRACK_KIND.equals(kind)) {
// WebRtcAudioTrack
final AudioTrack audioTrack = new AudioTrack(consumerMediaTrackPointer);
remoteClient.tracks.put(consumerId, audioTrack);
} else if(MediaStreamTrack.VIDEO_TRACK_KIND.equals(kind)) {
final VideoTrack videoTrack = new VideoTrack(consumerMediaTrackPointer);
remoteClient.tracks.put(consumerId, videoTrack);
remoteClient.playVideo();
} else {
Log.w(Room.class.getSimpleName(), "未知媒体类型：" + kind);
// TODO：资源释放
return;
}
this.taoyao.push(response);
}
private native void nativeEnter(long nativePointer, String rtpCapabilities, long peerConnectionFactoryPointer, PeerConnection.RTCConfiguration rtcConfiguration);
private native long nativeNewRoom(String roomId, RouterCallback routerCallback);
private native void nativeCloseRoom(long nativePointer);
private native void nativeCreateSendTransport(long nativeRoomPointer, String body);
private native void nativeCreateRecvTransport(long nativeRoomPointer, String body);
private native void nativeProduceMedia(long nativeRoomPointer, long mediaStreamPointer);
private native void nativeMediaProduceAudio(long nativeRoomPointer, long mediaStreamPointer);
private native void nativeMediaProduceVideo(long nativeRoomPointer, long mediaStreamPointer);
private native void nativeMediaConsume(long nativeRoomPointer, String message);
private native void nativeMediaProducerPause(long nativeRoomPointer, String producerId);
private native void nativeMediaProducerResume(long nativeRoomPointer, String producerId);
private native void nativeMediaProducerClose(long nativeRoomPointer, String producerId);
private native void nativeMediaConsumerPause(long nativeRoomPointer, String consumerId);
private native void nativeMediaConsumerResume(long nativeRoomPointer, String consumerId);
private native void nativeMediaConsumerClose(long nativeRoomPointer, String consumerId);
}

19
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RoomClient.java
View File

@@ -4,7 +4,7 @@ import android.os.Handler;
import com.acgist.taoyao.media.signal.ITaoyao;
import org.webrtc.MediaStream;
import org.webrtc.MediaStreamTrack;
/**
* 房间终端
@@ -14,30 +14,13 @@ import org.webrtc.MediaStream;
*/
public class RoomClient extends Client {
protected MediaStream mediaStream;
public RoomClient(String name, String clientId, Handler handler, ITaoyao taoyao) {
super(name, clientId, handler, taoyao);
}
/**
* 打开预览
*/
private void preview() {
}
@Override
public void close() {
super.close();
}
public MediaStream getMediaStream() {
return mediaStream;
}
public void setMediaStream(MediaStream mediaStream) {
this.mediaStream = mediaStream;
}
}

47
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java
View File

@@ -1,11 +1,13 @@
package com.acgist.taoyao.media.client;
import android.os.Handler;
import android.se.omapi.Session;
import android.util.Log;
import com.acgist.taoyao.boot.model.Message;
import com.acgist.taoyao.boot.utils.MapUtils;
import com.acgist.taoyao.media.MediaManager;
import com.acgist.taoyao.media.config.Config;
import com.acgist.taoyao.media.signal.ITaoyao;
import org.webrtc.DataChannel;
@@ -16,6 +18,7 @@ import org.webrtc.PeerConnection;
import org.webrtc.PeerConnectionFactory;
import org.webrtc.SdpObserver;
import org.webrtc.SessionDescription;
import org.webrtc.VideoTrack;
import java.util.ArrayList;
import java.util.List;
@@ -43,7 +46,7 @@ public class SessionClient extends Client {
/**
* 远程媒体
*/
private final List<MediaStream> remoteMediaStreams;
private MediaStream remoteMediaStream;
/**
* SDPObserver
*/
@@ -64,7 +67,6 @@ public class SessionClient extends Client {
public SessionClient(String sessionId, String name, String clientId, Handler handler, ITaoyao taoyao) {
super(name, clientId, handler, taoyao);
this.sessionId = sessionId;
this.remoteMediaStreams = new CopyOnWriteArrayList<>();
}
public void init() {
@@ -151,14 +153,36 @@ public class SessionClient extends Client {
}
}
@Override
public void playVideo() {
super.playVideo();
if(this.remoteMediaStream == null) {
return;
}
final VideoTrack videoTrack = this.remoteMediaStream.videoTracks.get(0);
if(this.surfaceViewRenderer == null) {
this.surfaceViewRenderer = this.mediaManager.buildSurfaceViewRenderer(Config.WHAT_NEW_REMOTE_VIDEO, videoTrack);
} else {
videoTrack.setEnabled(true);
}
}
@Override
public void playAudio() {
super.playAudio();
if(this.remoteMediaStream == null) {
return;
}
this.remoteMediaStream.audioTracks.forEach(v -> v.setEnabled(true));
}
@Override
public void close() {
super.close();
// 本地资源释放
// 本地资源释放：不要直接关闭MediaStream（共享资源）
this.mediaManager.closeClient();
// 远程资源释放
this.remoteMediaStreams.forEach(v -> v.dispose());
this.remoteMediaStreams.clear();
this.remoteMediaStream.dispose();
}
/**
@@ -201,14 +225,19 @@ public class SessionClient extends Client {
@Override
public void onAddStream(MediaStream mediaStream) {
Log.i(SessionClient.class.getSimpleName(), "添加远程媒体：" + SessionClient.this.clientId);
SessionClient.this.mediaStream = mediaStream;
SessionClient.this.mediaManager.remoteVideoRenderer(mediaStream);
SessionClient.this.remoteMediaStreams.add(mediaStream);
if(SessionClient.this.remoteMediaStream != null) {
// TODO：验证音频视频是否合在一起
}
SessionClient.this.remoteMediaStream = mediaStream;
SessionClient.this.playVideo();
SessionClient.this.playAudio();
}
@Override
public void onRemoveStream(MediaStream mediaStream) {
SessionClient.this.remoteMediaStreams.remove(mediaStream);
if(SessionClient.this.remoteMediaStream == mediaStream) {
}
}
@Override

2
taoyao-client-web/src/App.vue
View File

@@ -170,7 +170,7 @@ export default {
},
async roomEnter() {
await this.taoyao.roomEnter(this.room.roomId, this.room.password);
await this.taoyao.produceMedia();
await this.taoyao.mediaProduce();
this.roomVisible = false;
},
async roomInvite() {

56
taoyao-client-web/src/components/Taoyao.js
View File

@@ -1173,13 +1173,13 @@ class Taoyao extends RemoteClient {
kind,
type,
roomId,
appData,
clientId,
sourceId,
streamId,
producerId,
consumerId,
rtpParameters,
appData,
producerPaused,
} = message.body;
try {
@@ -1188,11 +1188,8 @@ class Taoyao extends RemoteClient {
kind,
producerId,
rtpParameters,
// NOTE: Force streamId to be same in mic and webcam and different
// in screen sharing so libwebrtc will just try to sync mic and
// webcam streams from the same remote peer.
//streamId: `${peerId}-${appData.share ? "share" : "mic-webcam"}`,
streamId: `${clientId}-${appData.share ? "share" : "mic-webcam"}`,
// 强制设置streamId，让libwebrtc同步麦克风和摄像头，屏幕共享不要求同步。
streamId: `${clientId}-${appData?.videoSource ? appData.videoSource : "unknown"}`,
appData, // Trick.
});
consumer.clientId = clientId;
@@ -1206,25 +1203,6 @@ class Taoyao extends RemoteClient {
mediasoupClient.parseScalabilityMode(
consumer.rtpParameters.encodings[0].scalabilityMode
);
// store.dispatch(
// stateActions.addConsumer(
// {
// id: consumer.id,
// type: type,
// locallyPaused: false,
// remotelyPaused: producerPaused,
// rtpParameters: consumer.rtpParameters,
// spatialLayers: spatialLayers,
// temporalLayers: temporalLayers,
// preferredSpatialLayer: spatialLayers - 1,
// preferredTemporalLayer: temporalLayers - 1,
// priority: 1,
// codec: consumer.rtpParameters.codecs[0].mimeType.split("/")[1],
// track: consumer.track,
// },
// peerId
// )
// );
self.push(message);
console.debug("远程媒体：", consumer);
const remoteClient = self.remoteClients.get(consumer.sourceId);
@@ -1244,7 +1222,7 @@ class Taoyao extends RemoteClient {
} else {
console.warn("远程终端没有实现服务代理：", remoteClient);
}
// If audio-only mode is enabled, pause it.
// 实现进入自动暂停视频，注：必须订阅所有类型媒体，不然媒体服务直接不会转发视频媒体
if (consumer.kind === "video" && !self.videoProduce) {
// this.pauseConsumer(consumer);
// TODO：实现
@@ -1509,7 +1487,7 @@ class Taoyao extends RemoteClient {
// return;
// }
await me.roomEnter(roomId, password);
await me.produceMedia();
await me.mediaProduce();
}
/**
* 离开房间
@@ -1583,7 +1561,7 @@ class Taoyao extends RemoteClient {
/**
* 生产媒体
*/
async produceMedia() {
async mediaProduce() {
const self = this;
if (!self.roomId) {
this.callbackError("无效房间");
@@ -1828,6 +1806,7 @@ class Taoyao extends RemoteClient {
self.callbackError("麦克风打开失败");
}
}
async closeAudioProducer() {
console.debug("closeAudioProducer()");
if (!this.audioProducer) {
@@ -1871,27 +1850,36 @@ class Taoyao extends RemoteClient {
let codec;
let encodings;
const codecOptions = {
videoGoogleStartBitrate: 1000,
videoGoogleStartBitrate: 400,
videoGoogleMaxBitrate : 1600,
videoGoogleMinBitrate : 800
};
// encodings :
// [
// { maxBitrate: 100000 },
// { maxBitrate: 300000 },
// { maxBitrate: 900000 }
// ],
// codecOptions :
// {
// videoGoogleStartBitrate : 1000
// }
if (self.forceH264) {
codec = self.mediasoupDevice.rtpCapabilities.codecs.find(
(c) => c.mimeType.toLowerCase() === "video/h264"
);
if (!codec) {
throw new Error(
"desired H264 codec+configuration is not supported"
);
self.callbackError("不支持H264视频编码");
}
} else if (self.forceVP9) {
codec = self.mediasoupDevice.rtpCapabilities.codecs.find(
(c) => c.mimeType.toLowerCase() === "video/vp9"
);
if (!codec) {
throw new Error("desired VP9 codec+configuration is not supported");
self.callbackError("不支持VP9视频编码");
}
}
if (this.useSimulcast) {
// If VP9 is the only available video codec then use SVC.
const firstVideoCodec =
this.mediasoupDevice.rtpCapabilities.codecs.find(
(c) => c.kind === "video"