qq27800734/taoyao
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[*] 去掉rtp功能

Browse Source
This commit is contained in:
acgist
2023-04-28 22:52:38 +08:00
parent 3dbf52eb1a
commit b8c9d42460
26 changed files with 21 additions and 364 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
README.md
View File

@@ -33,17 +33,16 @@
|功能|是否支持|是否实现|描述|
|:--|:--|:--|:--|
|P2P|支持|完成|P2P监控模式|
|WebRTC|支持|完成|Web终端不能同时进入多个房间|
|控制|支持|完成|实现所有控制信令|
|WebRTC|支持|完成|视频房间|
|控制|支持|完成|完整控制信令|
### 安卓终端功能
|功能|是否支持|是否实现|描述|
|:--|:--|:--|:--|
|P2P|支持|实现|P2P监控模式|
|WebRTC|支持|暂未实现|安卓终端支持同时进入多个房间|
|RTP|支持|暂未实现|支持房间RTP推流（不会拉流）|
|控制|支持|完成|实现部分控制信令|
|P2P|支持|完成|P2P监控模式|
|WebRTC|支持|完成|视频房间|
|控制|支持|完成|部分控制信令|
|拍照|支持|完成|拍照|
|录像|支持|完成|录制|
|变声|支持|暂未实现|变声器|
@@ -51,6 +50,8 @@
|美颜|支持|暂未实现|视频美颜|
|AI识别|支持|暂未实现|视频AI识别|
> 注意：Web不支持同时进入多个视频房间，安卓终端支持同时进入多个视频房间。
## 证书
本地开发测试安装`docs/certs`中的`ca.crt`证书到`受信任的根证书颁发机构`

2
taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java
View File

@@ -27,7 +27,6 @@ import androidx.core.app.ActivityCompat;
import com.acgist.taoyao.client.databinding.ActivityMainBinding;
import com.acgist.taoyao.client.signal.Taoyao;
import com.acgist.taoyao.media.MediaManager;
import com.acgist.taoyao.media.TransportType;
import com.acgist.taoyao.media.VideoSourceType;
import com.acgist.taoyao.media.config.Config;
@@ -128,7 +127,6 @@ public class MainActivity extends AppCompatActivity implements Serializable {
resources.getInteger(R.integer.iFrameInterval),
resources.getString(R.string.storagePathImage),
resources.getString(R.string.storagePathVideo),
TransportType.valueOf(resources.getString(R.string.transportType)),
VideoSourceType.valueOf(resources.getString(R.string.videoSourceType))
);
final Display display = this.getWindow().getContext().getDisplay();

10
taoyao-client-android/taoyao/client/src/main/res/values/settings.xml
View File

@@ -16,11 +16,10 @@
<string name="encrypt">DES</string>
<!-- 信令加密密钥 -->
<string name="encryptSecret">2SPWy+TF1zM=</string>
<!-- 文件存储目录 -->
<!-- 图片存储目录 -->
<string name="storagePathImage">/taoyao/image</string>
<!-- 视频存储目录 -->
<string name="storagePathVideo">/taoyao/video</string>
<!-- 传输类型：RTP|WEBRTC -->
<string name="transportType">WEBRTC</string>
<!-- 视频来源：FILE|BACK|FRONT|SCREEN -->
<string name="videoSourceType">BACK</string>
<!-- 媒体配置：是否消费数据 -->
@@ -35,9 +34,14 @@
<bool name="audioProduce">true</bool>
<!-- 媒体配置：是否生产视频 -->
<bool name="videoProduce">true</bool>
<!-- 图片质量 -->
<integer name="imageQuantity">100</integer>
<!-- 音频质量 -->
<string name="audioQuantity">fd-audio</string>
<!-- 视频质量 -->
<string name="videoQuantity">fd-video</string>
<!-- 音频通道数量 -->
<integer name="channelCount">1</integer>
<!-- 视频关键帧频率 -->
<integer name="iFrameInterval">1</integer>
</resources>

6
taoyao-client-android/taoyao/media/CMakeLists.txt
View File

@@ -29,12 +29,6 @@ set(
${SOURCE_DIR}/include/MediaManager.hpp
${SOURCE_DIR}/include/Room.hpp
${SOURCE_DIR}/include/RouterCallback.hpp
${SOURCE_DIR}/include/RtpAudioPublisher.hpp
${SOURCE_DIR}/include/RtpClient.hpp
${SOURCE_DIR}/include/RtpVideoPublisher.hpp
${SOURCE_DIR}/rtp/RtpAudioPublisher.cpp
${SOURCE_DIR}/rtp/RtpClient.cpp
${SOURCE_DIR}/rtp/RtpVideoPublisher.cpp
${SOURCE_DIR}/webrtc/MediaManager.cpp
${SOURCE_DIR}/webrtc/Room.cpp
${SOURCE_DIR}/webrtc/RouterCallback.cpp

5
taoyao-client-android/taoyao/media/src/main/cpp/include/RtpAudioPublisher.hpp
View File

@@ -1,5 +0,0 @@
#pragma once
namespace acgist {
}

5
taoyao-client-android/taoyao/media/src/main/cpp/include/RtpClient.hpp
View File

@@ -1,5 +0,0 @@
#pragma once
namespace acgist {
}

5
taoyao-client-android/taoyao/media/src/main/cpp/include/RtpVideoPublisher.hpp
View File

@@ -1,5 +0,0 @@
#pragma once
namespace acgist {
}

5
taoyao-client-android/taoyao/media/src/main/cpp/rtp/RtpAudioPublisher.cpp
View File

@@ -1,5 +0,0 @@
#include "RtpAudioPublisher.hpp"
namespace acgist {
}

5
taoyao-client-android/taoyao/media/src/main/cpp/rtp/RtpClient.cpp
View File

@@ -1,5 +0,0 @@
#include "RtpClient.hpp"
namespace acgist {
}

5
taoyao-client-android/taoyao/media/src/main/cpp/rtp/RtpVideoPublisher.cpp
View File

@@ -1,5 +0,0 @@
#include "RtpVideoPublisher.hpp"
namespace acgist {
}

2
taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp
View File

@@ -32,7 +32,6 @@ namespace acgist {
}
void OnConnectionStateChange(mediasoupclient::Transport* transport, const std::string& connectionState) override {
// TODO：restartIce？
}
std::future<std::string> OnProduce(mediasoupclient::SendTransport* transport, const std::string& kind, nlohmann::json rtpParameters, const nlohmann::json& appData) override {
@@ -401,7 +400,6 @@ namespace acgist {
// webrtc::PeerConnectionInterface::RTCConfiguration rtcConfiguration(webrtc::PeerConnectionInterface::RTCConfigurationType::kSafe);
webrtc::PeerConnectionInterface::RTCConfiguration rtcConfiguration(webrtc::PeerConnectionInterface::RTCConfigurationType::kAggressive);
webrtc::JavaParamRef<jobject> jRtcConfigurationRef(env, jRtcConfiguration);
// 注意
webrtc::jni::JavaToNativeRTCConfiguration(env, jRtcConfigurationRef, &rtcConfiguration);
const char* rtpCapabilities = env->GetStringUTFChars(jRtpCapabilities, nullptr);
room->enterRoom(

11
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java
View File

@@ -2,8 +2,6 @@ package com.acgist.taoyao.media;
import android.content.Context;
import android.content.Intent;
import android.media.MediaCodecInfo;
import android.media.MediaCodecList;
import android.media.projection.MediaProjection;
import android.os.Handler;
import android.os.Message;
@@ -43,8 +41,6 @@ import org.webrtc.VideoTrack;
import org.webrtc.audio.JavaAudioDeviceModule;
import java.util.Arrays;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
/**
* 媒体来源管理器
@@ -93,10 +89,6 @@ public final class MediaManager {
* 关键帧频率
*/
private int iFrameInterval;
/**
* 传输通道类型
*/
private TransportType transportType;
/**
* 视频来源类型
*/
@@ -214,7 +206,7 @@ public final class MediaManager {
int imageQuantity, String audioQuantity, String videoQuantity,
int channelCount, int iFrameInterval,
String imagePath, String videoPath,
TransportType transportType, VideoSourceType videoSourceType
VideoSourceType videoSourceType
) {
this.mainHandler = mainHandler;
this.context = context;
@@ -225,7 +217,6 @@ public final class MediaManager {
this.iFrameInterval = iFrameInterval;
this.imagePath = imagePath;
this.videoPath = videoPath;
this.transportType = transportType;
this.videoSourceType = videoSourceType;
}

20
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/TransportType.java
View File

@@ -1,20 +0,0 @@
package com.acgist.taoyao.media;
/**
* 传输类型
*
* @author acgist
*/
public enum TransportType {
/**
* RTP
* 注意：只能监控
*/
RTP,
/**
* WebRTC
*/
WEBRTC;
}

3
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/AudioMixer.java
View File

@@ -1,8 +1,5 @@
package com.acgist.taoyao.media.audio;
import org.webrtc.voiceengine.WebRtcAudioManager;
import org.webrtc.voiceengine.WebRtcAudioRecord;
/**
* 混音
*

1
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java
View File

@@ -11,7 +11,6 @@ import android.util.Log;
import com.acgist.taoyao.boot.utils.DateUtils;
import com.acgist.taoyao.media.MediaManager;
import com.acgist.taoyao.media.VideoSourceType;
import com.acgist.taoyao.media.signal.ITaoyao;
import org.webrtc.VideoFrame;

3
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java
View File

@@ -8,11 +8,8 @@ import com.acgist.taoyao.boot.utils.JSONUtils;
import com.acgist.taoyao.boot.utils.MapUtils;
import com.acgist.taoyao.boot.utils.PointerUtils;
import com.acgist.taoyao.media.RouterCallback;
import com.acgist.taoyao.media.VideoSourceType;
import com.acgist.taoyao.media.config.Config;
import com.acgist.taoyao.media.config.MediaAudioProperties;
import com.acgist.taoyao.media.config.MediaProperties;
import com.acgist.taoyao.media.config.MediaVideoProperties;
import com.acgist.taoyao.media.config.WebrtcProperties;
import com.acgist.taoyao.media.signal.ITaoyao;

6
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java
View File

@@ -4,17 +4,13 @@ import android.os.Handler;
import android.util.Log;
import com.acgist.taoyao.boot.model.Message;
import com.acgist.taoyao.boot.utils.JSONUtils;
import com.acgist.taoyao.boot.utils.ListUtils;
import com.acgist.taoyao.boot.utils.MapUtils;
import com.acgist.taoyao.media.VideoSourceType;
import com.acgist.taoyao.media.config.Config;
import com.acgist.taoyao.media.config.MediaProperties;
import com.acgist.taoyao.media.config.WebrtcProperties;
import com.acgist.taoyao.media.config.WebrtcStunProperties;
import com.acgist.taoyao.media.signal.ITaoyao;
import org.apache.commons.lang3.ArrayUtils;
import org.webrtc.DataChannel;
import org.webrtc.IceCandidate;
import org.webrtc.MediaConstraints;
@@ -24,11 +20,9 @@ import org.webrtc.PeerConnectionFactory;
import org.webrtc.SdpObserver;
import org.webrtc.SessionDescription;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.function.Consumer;
/**

2
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/WebrtcProperties.java
View File

@@ -6,9 +6,7 @@ import org.apache.commons.lang3.ArrayUtils;
import org.webrtc.PeerConnection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
/**
* WebRTC配置

2
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/signal/ITaoyao.java
View File

@@ -4,8 +4,6 @@ import android.util.Log;
import com.acgist.taoyao.boot.model.Message;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Future;
import java.util.function.Consumer;
import java.util.function.Function;

4
taoyao-client-media/README.md
View File

@@ -70,6 +70,10 @@ make -C worker
* [Medooze](https://github.com/medooze/media-server)
* [Mediasoup](https://github.com/versatica/mediasoup)
## RTP裸流
媒体服务主要使用`WebRTC`协议，同时支持接入`RTP`裸流，可以参考[RtpTest.java](../taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/rtp/RtpTest.java)配合`ffmpeg`使用`RTP`推拉流，具体代码需要自行实现。
## 协议
* https://www.ortc.org

13
taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/main/TaoyaoApplicationTests.java
View File

@@ -1,13 +0,0 @@
package com.acgist.taoyao.main;
import org.junit.jupiter.api.Test;
import org.springframework.boot.test.context.SpringBootTest;
@SpringBootTest
class TaoyaoApplicationTests {
@Test
void contextLoads() {
}
}

4
taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/signal/SocketSignalTest.java → taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/rtp/RtpTest.java
View File

@@ -1,4 +1,4 @@
package com.acgist.taoyao.signal;
package com.acgist.taoyao.rtp;
import java.io.InputStream;
import java.io.OutputStream;
@@ -19,7 +19,7 @@ import com.acgist.taoyao.signal.utils.CipherUtils;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class SocketSignalTest {
public class RtpTest {
@Test
void testSocket() throws Exception {

48
taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/service/Server.java
View File

@@ -1,48 +0,0 @@
package com.acgist.taoyao.service;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.ServerSocket;
import java.net.Socket;
import java.util.concurrent.Executor;
import java.util.concurrent.Executors;
import org.junit.jupiter.api.Test;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class Server {
public static final Executor EXECUTOR = Executors.newCachedThreadPool();
@Test
public void testServer() throws Exception {
final ServerSocket server = new ServerSocket(9999);
while(!server.isClosed()) {
final Socket accept = server.accept();
EXECUTOR.execute(() -> {
try {
this.execute(accept);
} catch (IOException e) {
log.error("异常", e);
}
});
}
server.close();
}
public void execute(Socket accept) throws IOException {
final InputStream inputStream = accept.getInputStream();
final OutputStream outputStream = accept.getOutputStream();
while(!accept.isClosed()) {
final byte[] bytes = new byte[1024];
final int length = inputStream.read(bytes);
log.info("收到消息：{}", new String(bytes, 0, length));
outputStream.write(bytes, 0, length);
outputStream.flush();
}
}
}

26
taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/signal/PlatformErrorProtocolTest.java
View File

@@ -1,26 +0,0 @@
package com.acgist.taoyao.signal;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import com.acgist.taoyao.annotation.TaoyaoTest;
import com.acgist.taoyao.boot.model.MessageCodeException;
import com.acgist.taoyao.main.TaoyaoApplication;
import com.acgist.taoyao.signal.protocol.platform.PlatformErrorProtocol;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@TaoyaoTest(classes = TaoyaoApplication.class)
public class PlatformErrorProtocolTest {
@Autowired
private PlatformErrorProtocol platformErrorProtocol;
@Test
public void testException() {
log.info("{}", this.platformErrorProtocol.build(MessageCodeException.of("自定义")));
log.info("{}", this.platformErrorProtocol.build(new NullPointerException("空指针")));
}
}

98
taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/signal/WebSocketClient.java
View File

@@ -1,98 +0,0 @@
package com.acgist.taoyao.signal;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.WebSocket;
import java.net.http.WebSocket.Listener;
import java.security.KeyManagementException;
import java.security.NoSuchAlgorithmException;
import java.security.SecureRandom;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.concurrent.CompletionStage;
import java.util.concurrent.CountDownLatch;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import lombok.extern.slf4j.Slf4j;
@Slf4j
public class WebSocketClient {
public static final WebSocket build(String uri, String clientId) throws InterruptedException {
return build(uri, clientId, null);
}
public static final WebSocket build(String uri, String clientId, CountDownLatch count) throws InterruptedException {
final Object lock = new Object();
try {
return HttpClient
.newBuilder()
.sslContext(newSSLContext())
.build()
.newWebSocketBuilder()
.buildAsync(URI.create(uri), new Listener() {
@Override
public void onOpen(WebSocket webSocket) {
webSocket.sendText(String.format("""
{"header":{"signal":"client::register","v":"1.0.0","id":"1"},"body":{"username":"taoyao","password":"taoyao","ip":"127.0.0.1","clientId":"%s"}}
""", clientId), true);
Listener.super.onOpen(webSocket);
}
@Override
public CompletionStage<?> onText(WebSocket webSocket, CharSequence data, boolean last) {
synchronized (lock) {
lock.notifyAll();
}
if(count == null) {
log.debug("收到WebSocket消息：{}", data);
} else {
count.countDown();
log.debug("收到WebSocket消息：{}-{}", count.getCount(), data);
}
return Listener.super.onText(webSocket, data, last);
}
})
.join();
} finally {
synchronized (lock) {
lock.wait(1000);
}
}
}
private static final SSLContext newSSLContext() {
SSLContext sslContext = null;
try {
// SSL协议：SSL、SSLv2、SSLv3、TLS、TLSv1、TLSv1.1、TLSv1.2、TLSv1.3
sslContext = SSLContext.getInstance("TLSv1.2");
sslContext.init(null, TRUST_ALL_CERT_MANAGER, new SecureRandom());
} catch (KeyManagementException | NoSuchAlgorithmException e) {
log.error("新建SSLContext异常", e);
try {
sslContext = SSLContext.getDefault();
} catch (NoSuchAlgorithmException ex) {
log.error("新建默认SSLContext异常", ex);
}
}
return sslContext;
}
private static final TrustManager[] TRUST_ALL_CERT_MANAGER = new TrustManager[] {
new X509TrustManager() {
@Override
public void checkClientTrusted(X509Certificate[] chain, String authType) throws CertificateException {
}
@Override
public void checkServerTrusted(X509Certificate[] chain, String authType) throws CertificateException {
}
@Override
public X509Certificate[] getAcceptedIssuers() {
return null;
}
}
};
}

81
taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/signal/WebSocketSignalTest.java
View File

@@ -1,81 +0,0 @@
package com.acgist.taoyao.signal;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.net.http.WebSocket;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import org.junit.jupiter.api.Test;
import com.acgist.taoyao.annotation.CostedTest;
import com.acgist.taoyao.annotation.TaoyaoTest;
import com.acgist.taoyao.main.TaoyaoApplication;
import lombok.extern.slf4j.Slf4j;
@Slf4j
@TaoyaoTest(classes = TaoyaoApplication.class)
class WebSocketSignalTest {
/**
* 防止GC
*/
private List<WebSocket> list = new ArrayList<>();
@Test
void testSignal() throws InterruptedException {
final WebSocket clientA = WebSocketClient.build("wss://localhost:8888/websocket.signal", "clientA");
final WebSocket clientB = WebSocketClient.build("wss://localhost:8888/websocket.signal", "clientB");
clientA.sendText("""
{"header":{"signal":"client::heartbeat","v":"1.0.0","id":"1"},"body":{}}
""", true).join();
assertNotNull(clientA);
assertNotNull(clientB);
}
@Test
@CostedTest(thread = 10, count = 100, waitRelease = 5000L)
void testThread() throws InterruptedException {
final int total = 100;
final CountDownLatch count = new CountDownLatch(total);
final WebSocket clientA = WebSocketClient.build("wss://localhost:8888/websocket.signal", "clientA", count);
final long aTime = System.currentTimeMillis();
for (int index = 0; index < total; index++) {
clientA.sendText("""
{"header":{"signal":"client::status","v":"1.0.0","id":"1"},"body":{}}
""", true).join();
}
this.list.add(clientA);
// final ExecutorService executor = Executors.newFixedThreadPool(10);
// for (int index = 0; index < total; index++) {
// executor.execute(() -> {
// synchronized (clientA) {
// clientA.sendText("""
// {"header":{"signal":"client::status","v":"1.0.0","id":"1"},"body":{}}
// """, true).join();
// }
// });
// }
count.await();
final long zTime = System.currentTimeMillis();
log.info("执行时间：{}", zTime - aTime);
log.info("当前连接数量：{}", this.list.size());
assertNotNull(clientA);
}
@Test
void testMax() throws InterruptedException {
final int size = 1024;
final CountDownLatch count = new CountDownLatch(size);
for (int index = 0; index < size; index++) {
final WebSocket clientA = WebSocketClient.build("wss://localhost:8888/websocket.signal", "clientA", count);
assertNotNull(clientA);
assertTrue(!(clientA.isInputClosed() || clientA.isOutputClosed()));
}
count.await();
}
}