diff --git a/README.md b/README.md index 9773670..40ddd38 100644 --- a/README.md +++ b/README.md @@ -30,14 +30,14 @@ |功能|是否支持|是否实现|描述| |:--|:--|:--|:--| -|P2P|支持|暂未实现|P2P监控模式| +|P2P|支持|实现|P2P监控模式| |WebRTC|支持|实现|Web终端不能同时进入多个房间| ### 安卓终端功能 |功能|是否支持|是否实现|描述| |:--|:--|:--|:--| -|P2P|支持|暂未实现|P2P监控模式| +|P2P|支持|实现|P2P监控模式| |WebRTC|支持|暂未实现|安卓终端支持同时进入多个房间| |RTP|支持|暂未实现|支持房间RTP推流（不会拉流）| diff --git a/taoyao-client-android/taoyao/boot/src/main/java/com/acgist/taoyao/boot/utils/MapUtils.java b/taoyao-client-android/taoyao/boot/src/main/java/com/acgist/taoyao/boot/utils/MapUtils.java new file mode 100644 index 0000000..cbe92c7 --- /dev/null +++ b/taoyao-client-android/taoyao/boot/src/main/java/com/acgist/taoyao/boot/utils/MapUtils.java @@ -0,0 +1,154 @@ +package com.acgist.taoyao.boot.utils; + +import java.math.BigDecimal; +import java.util.Map; + +/** + * Map工具 + * + * @author acgist + */ +public final class MapUtils { + + private MapUtils() { + } + + /** + * @param 参数泛型 + * + * @param body 消息主体 + * @param key 参数名称 + * + * @return 参数值 + */ + @SuppressWarnings("unchecked") + public static final T get(Map body, String key) { + if(body == null) { + return null; + } + return (T) body.get(key); + } + + /** + * @param 参数泛型 + * + * @param body 消息主体 + * @param key 参数名称 + * @param defaultValue 参数默认值 + * + * @return 参数值 + */ + @SuppressWarnings("unchecked") + public static final T get(Map body, String key, T defaultValue) { + if(body == null) { + return defaultValue; + } + final T t = (T) body.get(key); + return t == null ? defaultValue : t; + } + + /** + * @param body 消息主体 + * @param key 参数名称 + * + * @return 参数值 + */ + public static final Long getLong(Map body, String key) { + if(body == null) { + return null; + } + final Object object = body.get(key); + if(object == null) { + return null; + } else if(object instanceof Long value) { + return value; + } else if(object instanceof Integer value) { + return value.longValue(); + } else if(object instanceof Double value) { + return value.longValue(); + } + return new BigDecimal(object.toString()).longValue(); + } + + /** + * @param body 消息主体 + * @param key 参数名称 + * + * @return 参数值 + */ + public static final Double getDouble(Map body, String key) { + if(body == null) { + return null; + } + final Object object = body.get(key); + if(object == null) { + return null; + } else if(object instanceof Long value) { + return value.doubleValue(); + } else if(object instanceof Integer value) { + return value.doubleValue(); + } else if(object instanceof Double value) { + return value; + } + return new BigDecimal(object.toString()).doubleValue(); + } + + /** + * @param body 消息主体 + * @param key 参数名称 + * + * @return 参数值 + */ + public static final Integer getInteger(Map body, String key) { + if(body == null) { + return null; + } + final Object object = body.get(key); + if(object == null) { + return null; + } else if(object instanceof Long value) { + return value.intValue(); + } else if(object instanceof Integer value) { + return value; + } else if(object instanceof Double value) { + return value.intValue(); + } + return new BigDecimal(object.toString()).intValue(); + } + + /** + * @param body 消息主体 + * @param key 参数名称 + * + * @return 参数值 + */ + public static final Boolean getBoolean(Map body, String key) { + if(body == null) { + return null; + } + final Object object = body.get(key); + if(object == null) { + return null; + } else if(object instanceof Boolean value) { + return value; + } + return Boolean.valueOf(object.toString()); + } + + /** + * @param 参数泛型 + * + * @param body 消息主体 + * @param key 参数名称 + * + * @return 参数值 + */ + @SuppressWarnings("unchecked") + public static final T remove(Map body, String key) { + if(body == null) { + return null; + } + return (T) body.remove(key); + } + +} diff --git a/taoyao-client-android/taoyao/client/src/androidTest/java/com/acgist/taoyao/ExampleInstrumentedTest.java b/taoyao-client-android/taoyao/client/src/androidTest/java/com/acgist/taoyao/ExampleInstrumentedTest.java deleted file mode 100644 index 298364b..0000000 --- a/taoyao-client-android/taoyao/client/src/androidTest/java/com/acgist/taoyao/ExampleInstrumentedTest.java +++ /dev/null @@ -1,20 +0,0 @@ -package com.acgist.taoyao; - -import android.content.Context; - -import androidx.test.platform.app.InstrumentationRegistry; -import androidx.test.ext.junit.runners.AndroidJUnit4; - -import org.junit.Test; -import org.junit.runner.RunWith; - -import static org.junit.Assert.*; - -@RunWith(AndroidJUnit4.class) -public class ExampleInstrumentedTest { - @Test - public void useAppContext() { - Context appContext = InstrumentationRegistry.getInstrumentation().getTargetContext(); - assertEquals("com.acgist.taoyao", appContext.getPackageName()); - } -} \ No newline at end of file diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java index 3176ee6..7589617 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java @@ -2,12 +2,19 @@ package com.acgist.taoyao.client; import android.Manifest; import android.app.Activity; +import android.app.AlarmManager; +import android.app.Application; +import android.app.PendingIntent; +import android.content.Context; import android.content.Intent; import android.content.pm.PackageManager; +import android.content.res.Resources; import android.media.projection.MediaProjectionManager; import android.os.Bundle; import android.os.Handler; +import android.os.Looper; import android.os.Message; +import android.os.Process; import android.os.SystemClock; import android.util.Log; import android.view.Display; @@ -40,13 +47,15 @@ public class MainActivity extends AppCompatActivity implements Serializable { private MainHandler mainHandler; private ActivityMainBinding binding; - private ActivityResultLauncher activityResultLauncher; private MediaProjectionManager mediaProjectionManager; + private ActivityResultLauncher activityResultLauncher; @Override protected void onCreate(Bundle bundle) { Log.i(MainActivity.class.getSimpleName(), "onCreate"); super.onCreate(bundle); + // 全局异常 + this.catchAll(); // 请求权限 this.requestPermission(); // 启动点亮屏幕 @@ -64,6 +73,8 @@ public class MainActivity extends AppCompatActivity implements Serializable { this.registerMediaProjection(); this.binding.record.setOnClickListener(this::switchRecord); this.binding.settings.setOnClickListener(this::launchSettings); + // 加载媒体管理 + MediaManager.getInstance().initMedia(this.mainHandler, this.getApplicationContext()); } @Override @@ -95,6 +106,7 @@ public class MainActivity extends AppCompatActivity implements Serializable { Manifest.permission.ACCESS_WIFI_STATE, Manifest.permission.FOREGROUND_SERVICE, Manifest.permission.ACCESS_FINE_LOCATION, + Manifest.permission.ACCESS_NETWORK_STATE, Manifest.permission.READ_EXTERNAL_STORAGE, Manifest.permission.ACCESS_COARSE_LOCATION, Manifest.permission.RECEIVE_BOOT_COMPLETED, @@ -111,17 +123,18 @@ public class MainActivity extends AppCompatActivity implements Serializable { * 拉起媒体服务 */ private void launchMediaService() { - int times = 0; + if(this.mainHandler != null) { + return; + } + int waitCount = 0; + this.mainHandler = new MainHandler(this); final Display display = this.getWindow().getContext().getDisplay(); - while(Display.STATE_ON != display.getState() && times++ < 10) { + while(Display.STATE_ON != display.getState() && waitCount++ < 10) { SystemClock.sleep(100); } if(display.STATE_ON == display.getState()) { Log.i(MainActivity.class.getSimpleName(), "拉起媒体服务"); final Intent intent = new Intent(this, MediaService.class); - if(this.mainHandler == null) { - this.mainHandler = new MainHandler(this); - } intent.setAction(MediaService.Action.CONNECT.name()); intent.putExtra("mainHandler", this.mainHandler); this.startService(intent); @@ -152,15 +165,12 @@ public class MainActivity extends AppCompatActivity implements Serializable { ); } - private void switchRecord(View view) { + private synchronized void switchRecord(View view) { final MediaRecorder mediaRecorder = MediaRecorder.getInstance(); if(mediaRecorder.isActive()) { mediaRecorder.stop(); } else { - MediaManager.getInstance().init(this.mainHandler, this.getApplicationContext()); - MediaManager.getInstance().initAudio(); - MediaManager.getInstance().initVideo(); - MediaManager.getInstance().record(); + mediaRecorder.record(Resources.getSystem().getString(R.string.storagePathVideo)); } } @@ -192,8 +202,9 @@ public class MainActivity extends AppCompatActivity implements Serializable { super.handleMessage(message); Log.d(MainHandler.class.getSimpleName(), "Handler消息：" + message.what + " - " + message.obj); switch(message.what) { - case Config.WHAT_SCREEN_CAPTURE -> this.mainActivity.screenCapture(message); - case Config.WHAT_NEW_LOCAL_VIDEO -> this.mainActivity.newLocalVideo(message); + case Config.WHAT_SCREEN_CAPTURE -> this.mainActivity.screenCapture(message); + case Config.WHAT_NEW_LOCAL_VIDEO -> this.mainActivity.newLocalVideo(message); + case Config.WHAT_NEW_REMOTE_VIDEO -> this.mainActivity.newRemoteVideo(message); } } @@ -221,4 +232,31 @@ public class MainActivity extends AppCompatActivity implements Serializable { this.addContentView(surfaceView, layoutParams); } + private void newRemoteVideo(Message message) { + final SurfaceView surfaceView = (SurfaceView) message.obj; + final LinearLayout.LayoutParams layoutParams = new LinearLayout.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT); + layoutParams.weight = 1; + surfaceView.setZ(0F); + this.addContentView(surfaceView, layoutParams); + } + + private void catchAll() { + Log.i(MainActivity.class.getSimpleName(), "全局异常捕获"); + final Thread.UncaughtExceptionHandler old = Thread.getDefaultUncaughtExceptionHandler(); + Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { + @Override + public void uncaughtException(Thread t, Throwable e) { + Log.e(MediaService.class.getSimpleName(), "系统异常：" + t.getName(), e); + final Looper looper = Looper.myLooper(); + if(looper == Looper.getMainLooper()) { +// if(t.getId() == Looper.getMainLooper().getThread().getId()) { + // TODO：重启应用 + old.uncaughtException(t, e); + } else { + // 子线程 + } + } + }); + } + } \ No newline at end of file diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java index b046732..9944cb8 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java @@ -11,6 +11,7 @@ import android.net.wifi.WifiManager; import android.os.BatteryManager; import android.os.Handler; import android.os.HandlerThread; +import android.os.Process; import android.util.Log; import androidx.core.app.ActivityCompat; @@ -21,10 +22,13 @@ import com.acgist.taoyao.boot.model.MessageCode; import com.acgist.taoyao.boot.model.MessageCodeException; import com.acgist.taoyao.boot.utils.CloseableUtils; import com.acgist.taoyao.boot.utils.JSONUtils; +import com.acgist.taoyao.boot.utils.MapUtils; import com.acgist.taoyao.client.utils.IdUtils; +import com.acgist.taoyao.config.MediaProperties; import com.acgist.taoyao.media.MediaRecorder; -import com.acgist.taoyao.media.SessionClient; import com.acgist.taoyao.media.Room; +import com.acgist.taoyao.media.SessionClient; +import com.acgist.taoyao.signal.ITaoyao; import org.apache.commons.lang3.ArrayUtils; @@ -37,10 +41,8 @@ import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.util.Base64; import java.util.HashMap; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; import javax.crypto.BadPaddingException; import javax.crypto.Cipher; @@ -53,7 +55,7 @@ import javax.crypto.spec.SecretKeySpec; * * @author acgist */ -public final class Taoyao { +public final class Taoyao implements ITaoyao { /** * 端口 @@ -149,14 +151,18 @@ public final class Taoyao { private final HandlerThread heartbeatThread; private final Handler executeMessageHandler; private final HandlerThread executeMessageThread; + /** + * 媒体配置 + */ + private MediaProperties mediaProperties; /** * 房间列表 */ - private final List roomList; + private final Map rooms; /** - * P2P终端列表 + * 会话终端列表 */ - private final List p2pClientList; + private final Map sessionClients; public Taoyao( int port, String host, String version, @@ -196,8 +202,8 @@ public final class Taoyao { this.executeMessageThread = new HandlerThread("TaoyaoExecuteMessageThread"); this.executeMessageThread.start(); this.executeMessageHandler = new Handler(this.executeMessageThread.getLooper()); - this.roomList = new CopyOnWriteArrayList<>(); - this.p2pClientList = new CopyOnWriteArrayList<>(); + this.rooms = new ConcurrentHashMap<>(); + this.sessionClients = new ConcurrentHashMap<>(); } /** @@ -238,7 +244,7 @@ public final class Taoyao { if (this.socket.isConnected()) { this.input = this.socket.getInputStream(); this.output = this.socket.getOutputStream(); - this.register(); + this.clientRegister(); this.connect = true; synchronized (this) { this.notifyAll(); @@ -274,7 +280,7 @@ public final class Taoyao { this.connect(); } // 读取 - while (this.input != null && (length = this.input.read(bytes)) >= 0) { + while (!this.close && (length = this.input.read(bytes)) >= 0) { buffer.put(bytes, 0, length); while (buffer.position() > 0) { if (messageLength <= 0) { @@ -417,8 +423,8 @@ public final class Taoyao { this.heartbeatThread.quitSafely(); this.loopMessageThread.quitSafely(); this.executeMessageThread.quitSafely(); - this.roomList.forEach(Room::close); - this.p2pClientList.forEach(SessionClient::close); + this.rooms.values().forEach(Room::close); + this.sessionClients.values().forEach(SessionClient::close); } /** @@ -478,8 +484,12 @@ public final class Taoyao { } else { final Map body = message.body(); switch (header.getSignal()) { - case "client::register" -> this.register(message, body); - default -> Log.i(Taoyao.class.getSimpleName(), "没有适配信令：" + content); + case "client::config" -> this.clientConfig(message, body); + case "client::register" -> this.clientRegister(message, body); + case "session::call" -> this.sessionCall(message, body); + case "session::close" -> this.sessionClose(message, body); + case "session::exchange" -> this.sessionExchange(message, body); + default -> Log.d(Taoyao.class.getSimpleName(), "没有适配信令：" + content); } } } @@ -487,7 +497,7 @@ public final class Taoyao { /** * 注册 */ - private void register() { + private void clientRegister() { final Location location = this.location(); this.push(this.buildMessage( "client::register", @@ -509,7 +519,15 @@ public final class Taoyao { * @param message 消息 * @param body 消息主体 */ - private void register(Message message, Map body) { + private void clientConfig(Message message, Map body) { + this.mediaProperties = JSONUtils.toJava(JSONUtils.toJSON(body), MediaProperties.class); + } + + /** + * @param message 消息 + * @param body 消息主体 + */ + private void clientRegister(Message message, Map body) { final Integer index = (Integer) body.get("index"); if (index == null) { return; @@ -517,6 +535,36 @@ public final class Taoyao { IdUtils.setClientIndex(index); } + private void clientClose() { +// PowerManager manager = (PowerManager)this.getSystemService(Context.POWER_SERVICE); +// manager.reboot("重新启动系统") +// Process.killProcess(Process.myPid()); +// System.exit(0); + } + + private void sessionCall(Message message, Map body) { + final String name = MapUtils.get(body, "name"); + final String clientId = MapUtils.get(body, "clientId"); + final String sessionId = MapUtils.get(body, "sessionId"); + final SessionClient sessionClient = new SessionClient(sessionId, name, clientId, this); + this.sessionClients.put(sessionId, sessionClient); + sessionClient.init(); + sessionClient.offer(); + } + + private void sessionClose(Message message, Map body) { + } + + private void sessionExchange(Message message, Map body) { + final String sessionId = MapUtils.get(body, "sessionId"); + final SessionClient sessionClient = this.sessionClients.get(sessionId); + if(sessionClient == null) { + Log.w(Taoyao.class.getSimpleName(), "会话交换无效会话：" + sessionId); + return; + } + sessionClient.exchange(message, body); + } + /** * 心跳 */ diff --git a/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml b/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml index 9448baf..e8a4310 100644 --- a/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml +++ b/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml @@ -17,6 +17,4 @@ /taoyao/video WEBRTC - - \ No newline at end of file diff --git a/taoyao-client-android/taoyao/client/src/test/java/com/acgist/taoyao/ExampleUnitTest.java b/taoyao-client-android/taoyao/client/src/test/java/com/acgist/taoyao/ExampleUnitTest.java deleted file mode 100644 index 3de0e35..0000000 --- a/taoyao-client-android/taoyao/client/src/test/java/com/acgist/taoyao/ExampleUnitTest.java +++ /dev/null @@ -1,12 +0,0 @@ -package com.acgist.taoyao; - -import org.junit.Test; - -import static org.junit.Assert.*; - -public class ExampleUnitTest { - @Test - public void addition_isCorrect() { - assertEquals(4, 2 + 2); - } -} \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/CMakeLists.txt b/taoyao-client-android/taoyao/media/CMakeLists.txt index 8a4e274..eea198a 100644 --- a/taoyao-client-android/taoyao/media/CMakeLists.txt +++ b/taoyao-client-android/taoyao/media/CMakeLists.txt @@ -27,22 +27,20 @@ set( SOURCE_FILES ${SOURCE_DIR}/include/LocalClient.hpp ${SOURCE_DIR}/include/MediaRecorder.hpp - ${SOURCE_DIR}/include/MediasoupClient.hpp ${SOURCE_DIR}/include/RemoteClient.hpp ${SOURCE_DIR}/include/Room.hpp ${SOURCE_DIR}/include/RtpAudioPublisher.hpp ${SOURCE_DIR}/include/RtpClient.hpp ${SOURCE_DIR}/include/RtpVideoPublisher.hpp ${SOURCE_DIR}/include/SessionClient.hpp - ${SOURCE_DIR}/media/LocalClient.cpp - ${SOURCE_DIR}/media/MediaRecorder.cpp - ${SOURCE_DIR}/media/RemoteClient.cpp - ${SOURCE_DIR}/media/Room.cpp - ${SOURCE_DIR}/media/SessionClient.cpp ${SOURCE_DIR}/rtp/RtpAudioPublisher.cpp ${SOURCE_DIR}/rtp/RtpClient.cpp ${SOURCE_DIR}/rtp/RtpVideoPublisher.cpp - ${SOURCE_DIR}/webrtc/MediasoupClient.cpp + ${SOURCE_DIR}/webrtc/LocalClient.cpp + ${SOURCE_DIR}/webrtc/MediaRecorder.cpp + ${SOURCE_DIR}/webrtc/RemoteClient.cpp + ${SOURCE_DIR}/webrtc/Room.cpp + ${SOURCE_DIR}/webrtc/SessionClient.cpp ) set(LIBWEBRTC_BINARY_PATH ${LIBWEBRTC_BINARY_PATH}/${ANDROID_ABI} CACHE STRING "libwebrtc binary path" FORCE) diff --git a/taoyao-client-android/taoyao/media/build.gradle b/taoyao-client-android/taoyao/media/build.gradle index cc60b1f..5bb82e2 100644 --- a/taoyao-client-android/taoyao/media/build.gradle +++ b/taoyao-client-android/taoyao/media/build.gradle @@ -53,6 +53,7 @@ android { } dependencies { + implementation project(path: ':boot') implementation fileTree(dir: 'libs', include: ['*.jar']) implementation 'org.apache.commons:commons-lang3:3.12.0' implementation 'org.apache.commons:commons-collections4:4.4' diff --git a/taoyao-client-android/taoyao/media/src/main/AndroidManifest.xml b/taoyao-client-android/taoyao/media/src/main/AndroidManifest.xml index 992540c..cc9ff26 100644 --- a/taoyao-client-android/taoyao/media/src/main/AndroidManifest.xml +++ b/taoyao-client-android/taoyao/media/src/main/AndroidManifest.xml @@ -7,6 +7,7 @@ + diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/include/MediasoupClient.hpp b/taoyao-client-android/taoyao/media/src/main/cpp/include/MediasoupClient.hpp deleted file mode 100644 index f7883f8..0000000 --- a/taoyao-client-android/taoyao/media/src/main/cpp/include/MediasoupClient.hpp +++ /dev/null @@ -1,5 +0,0 @@ -#pragma once - -namespace acgist { - -} \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp b/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp index f7883f8..cf1e20d 100644 --- a/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp +++ b/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp @@ -1,5 +1,15 @@ #pragma once +#include "mediasoupclient.hpp" + namespace acgist { +class Room { +public: +// static mediasoupclient::Device * pDevice; +public: + Room(); + virtual ~Room(); +}; + } \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/media/Room.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/media/Room.cpp deleted file mode 100644 index 73f9bfb..0000000 --- a/taoyao-client-android/taoyao/media/src/main/cpp/media/Room.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "Room.hpp" - -namespace acgist { - -} \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/media/LocalClient.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/LocalClient.cpp similarity index 100% rename from taoyao-client-android/taoyao/media/src/main/cpp/media/LocalClient.cpp rename to taoyao-client-android/taoyao/media/src/main/cpp/webrtc/LocalClient.cpp diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/media/MediaRecorder.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediaRecorder.cpp similarity index 100% rename from taoyao-client-android/taoyao/media/src/main/cpp/media/MediaRecorder.cpp rename to taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediaRecorder.cpp diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediasoupClient.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediasoupClient.cpp deleted file mode 100644 index b428262..0000000 --- a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediasoupClient.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "MediasoupClient.hpp" - -namespace acgist { - -} \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/media/RemoteClient.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/RemoteClient.cpp similarity index 100% rename from taoyao-client-android/taoyao/media/src/main/cpp/media/RemoteClient.cpp rename to taoyao-client-android/taoyao/media/src/main/cpp/webrtc/RemoteClient.cpp diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp new file mode 100644 index 0000000..3786617 --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp @@ -0,0 +1,36 @@ +#include + +#include "Room.hpp" + +namespace acgist { + +Room::Room() { +} + +Room::~Room() { +} + +} + +void init() { + std::cout << "加载MediasoupClient：" << mediasoupclient::Version() << std::endl; + std::cout << "加载libwebrtc" << std::endl; + mediasoupclient::Initialize(); +// mediasoupclient::parseScalabilityMode("L2T3"); + // => { spatialLayers: 2, temporalLayers: 3 } +// mediasoupclient::parseScalabilityMode("L4T7_KEY_SHIFT"); + // => { spatialLayers: 4, temporalLayers: 7 } +} + +void load() { + // TODO：JNI信令交互 +// if(acgist::Room::pDevice == nullptr) { +// acgist::Room::pDevice = new mediasoupclient::Device(); +// acgist::Room::pDevice->Load(); +// } +} + +void stop() { + std::cout << "释放libwebrtc" << std::endl; + mediasoupclient::Cleanup(); +} \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/media/SessionClient.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/SessionClient.cpp similarity index 100% rename from taoyao-client-android/taoyao/media/src/main/cpp/media/SessionClient.cpp rename to taoyao-client-android/taoyao/media/src/main/cpp/webrtc/SessionClient.cpp diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/MediaAudioProperties.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/MediaAudioProperties.java new file mode 100644 index 0000000..026a4cf --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/MediaAudioProperties.java @@ -0,0 +1,60 @@ +package com.acgist.taoyao.config; + +/** + * 音频配置 + * + * @author acgist + */ +public class MediaAudioProperties { + + /** + * 音频格式 + * + * @author acgist + */ + public enum Format { + + G722, + PCMA, + PCMU, + OPUS; + + } + + /** + * 格式：G722|PCMA|PCMU|OPUS + */ + private Format format; + /** + * 采样数：8|16|32 + */ + private Integer sampleSize; + /** + * 采样率：8000|16000|32000|48000 + */ + private Integer sampleRate; + + public Format getFormat() { + return this.format; + } + + public void setFormat(Format format) { + this.format = format; + } + + public Integer getSampleSize() { + return this.sampleSize; + } + + public void setSampleSize(Integer sampleSize) { + this.sampleSize = sampleSize; + } + + public Integer getSampleRate() { + return this.sampleRate; + } + + public void setSampleRate(Integer sampleRate) { + this.sampleRate = sampleRate; + } +} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/MediaProperties.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/MediaProperties.java new file mode 100644 index 0000000..2f4fa05 --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/MediaProperties.java @@ -0,0 +1,204 @@ +package com.acgist.taoyao.config; + +import java.util.Map; + +/** + * 媒体配置 + * + * @author acgist + */ +public class MediaProperties { + + /** + * 最小视频宽度 + */ + private Integer minWidth; + /** + * 最大视频宽度 + */ + private Integer maxWidth; + /** + * 最小视频高度 + */ + private Integer minHeight; + /** + * 最大视频高度 + */ + private Integer maxHeight; + /** + * 最小视频码率 + */ + private Integer minBitrate; + /** + * 最大视频码率 + */ + private Integer maxBitrate; + /** + * 最小视频帧率 + */ + private Integer minFrameRate; + /** + * 最大视频帧率 + */ + private Integer maxFrameRate; + /** + * 最小音频采样数 + */ + private Integer minSampleSize; + /** + * 最大音频采样数 + */ + private Integer maxSampleSize; + /** + * 最小音频采样率 + */ + private Integer minSampleRate; + /** + * 最大音频采样率 + */ + private Integer maxSampleRate; + /** + * 音频默认配置 + */ + private MediaAudioProperties audio; + /** + * 视频默认配置 + */ + private MediaVideoProperties video; + /** + * 音频配置 + */ + private Map audios; + /** + * 视频配置 + */ + private Map videos; + + public Integer getMinWidth() { + return minWidth; + } + + public void setMinWidth(Integer minWidth) { + this.minWidth = minWidth; + } + + public Integer getMaxWidth() { + return maxWidth; + } + + public void setMaxWidth(Integer maxWidth) { + this.maxWidth = maxWidth; + } + + public Integer getMinHeight() { + return minHeight; + } + + public void setMinHeight(Integer minHeight) { + this.minHeight = minHeight; + } + + public Integer getMaxHeight() { + return maxHeight; + } + + public void setMaxHeight(Integer maxHeight) { + this.maxHeight = maxHeight; + } + + public Integer getMinBitrate() { + return minBitrate; + } + + public void setMinBitrate(Integer minBitrate) { + this.minBitrate = minBitrate; + } + + public Integer getMaxBitrate() { + return maxBitrate; + } + + public void setMaxBitrate(Integer maxBitrate) { + this.maxBitrate = maxBitrate; + } + + public Integer getMinFrameRate() { + return minFrameRate; + } + + public void setMinFrameRate(Integer minFrameRate) { + this.minFrameRate = minFrameRate; + } + + public Integer getMaxFrameRate() { + return maxFrameRate; + } + + public void setMaxFrameRate(Integer maxFrameRate) { + this.maxFrameRate = maxFrameRate; + } + + public Integer getMinSampleSize() { + return minSampleSize; + } + + public void setMinSampleSize(Integer minSampleSize) { + this.minSampleSize = minSampleSize; + } + + public Integer getMaxSampleSize() { + return maxSampleSize; + } + + public void setMaxSampleSize(Integer maxSampleSize) { + this.maxSampleSize = maxSampleSize; + } + + public Integer getMinSampleRate() { + return minSampleRate; + } + + public void setMinSampleRate(Integer minSampleRate) { + this.minSampleRate = minSampleRate; + } + + public Integer getMaxSampleRate() { + return maxSampleRate; + } + + public void setMaxSampleRate(Integer maxSampleRate) { + this.maxSampleRate = maxSampleRate; + } + + public MediaAudioProperties getAudio() { + return audio; + } + + public void setAudio(MediaAudioProperties audio) { + this.audio = audio; + } + + public MediaVideoProperties getVideo() { + return video; + } + + public void setVideo(MediaVideoProperties video) { + this.video = video; + } + + public Map getAudios() { + return audios; + } + + public void setAudios(Map audios) { + this.audios = audios; + } + + public Map getVideos() { + return videos; + } + + public void setVideos(Map videos) { + this.videos = videos; + } +} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/MediaVideoProperties.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/MediaVideoProperties.java new file mode 100644 index 0000000..28adcb2 --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/MediaVideoProperties.java @@ -0,0 +1,99 @@ +package com.acgist.taoyao.config; + +/** + * 视频配置 + * + * @author acgist + */ +public class MediaVideoProperties { + + /** + * 视频格式 + * + * @author acgist + */ + public enum Format { + + VP8, + VP9, + H264, + H265; + + } + + /** + * 格式：VP8|VP9|H264|H265 + */ + private Format format; + /** + * 码率：600|1200|1500|1800 + * 码率影响画质 + */ + private Integer bitrate; + /** + * 帧率：15|18|20|24|30|45 + * 帧率影响流畅 + */ + private Integer frameRate; + /** + * 分辨率：4096*2160|2560*1440|1920*1080|1280*720|720*480 + * 分辨率影响画面大小 + */ + private String resolution; + /** + * 宽度：4096|2560|1920|1280|720 + */ + private Integer width; + /** + * 高度：2160|1440|1080|720|480 + */ + private Integer height; + + public Format getFormat() { + return format; + } + + public void setFormat(Format format) { + this.format = format; + } + + public Integer getBitrate() { + return bitrate; + } + + public void setBitrate(Integer bitrate) { + this.bitrate = bitrate; + } + + public Integer getFrameRate() { + return frameRate; + } + + public void setFrameRate(Integer frameRate) { + this.frameRate = frameRate; + } + + public String getResolution() { + return resolution; + } + + public void setResolution(String resolution) { + this.resolution = resolution; + } + + public Integer getWidth() { + return width; + } + + public void setWidth(Integer width) { + this.width = width; + } + + public Integer getHeight() { + return height; + } + + public void setHeight(Integer height) { + this.height = height; + } +} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java index c3d267a..6cb2400 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java @@ -1,26 +1,17 @@ package com.acgist.taoyao.media; -import android.annotation.SuppressLint; import android.content.Context; import android.content.Intent; -import android.media.AudioFormat; -import android.media.AudioRecord; -import android.media.MediaFormat; import android.media.projection.MediaProjection; -import android.media.projection.MediaProjectionManager; -import android.os.Environment; import android.os.Handler; import android.os.Looper; import android.os.Message; import android.util.Log; -import android.view.Surface; import com.acgist.taoyao.config.Config; import org.webrtc.AudioSource; import org.webrtc.AudioTrack; -import org.webrtc.BuiltinAudioDecoderFactoryFactory; -import org.webrtc.BuiltinAudioEncoderFactoryFactory; import org.webrtc.Camera2Enumerator; import org.webrtc.CameraEnumerator; import org.webrtc.CameraVideoCapturer; @@ -37,18 +28,12 @@ import org.webrtc.SurfaceViewRenderer; import org.webrtc.VideoCapturer; import org.webrtc.VideoDecoderFactory; import org.webrtc.VideoEncoderFactory; -import org.webrtc.VideoFileRenderer; import org.webrtc.VideoFrame; import org.webrtc.VideoSink; import org.webrtc.VideoSource; import org.webrtc.VideoTrack; -import org.webrtc.audio.AudioDeviceModule; import org.webrtc.audio.JavaAudioDeviceModule; -import org.webrtc.voiceengine.WebRtcAudioManager; -import org.webrtc.voiceengine.WebRtcAudioRecord; -import org.webrtc.voiceengine.WebRtcAudioUtils; -import java.nio.file.Paths; import java.util.Arrays; import java.util.Iterator; import java.util.List; @@ -60,6 +45,7 @@ import java.util.List; * * https://zhuanlan.zhihu.com/p/82446482 * https://www.jianshu.com/p/97acd9a51909 + * https://juejin.cn/post/7036308428305727519 * https://blog.csdn.net/nanoage/article/details/127406494 * https://webrtc.org.cn/20190419_tutorial3_webrtc_android * https://blog.csdn.net/CSDN_Mew/article/details/103406781 @@ -99,13 +85,11 @@ public class MediaManager { private static final MediaManager INSTANCE = new MediaManager(); - private MediaManager() { - } - public static final MediaManager getInstance() { return INSTANCE; } + private volatile int clientCount; /** * 视频类型 */ @@ -155,15 +139,53 @@ public class MediaManager { // WebRtcAudioManager.setBlacklistDeviceForOpenSLESUsage(true); } + private MediaManager() { + this.clientCount = 0; + } + /** - * 加载媒体流 + * 新建终端 * - * @param context 上下文 + * @param type 视频类型：第一个终端进入有效 + * + * @return PeerConnectionFactory */ - public void init(Handler handler, Context context) { - this.type = Type.BACK; + public PeerConnectionFactory newClient(Type type) { + synchronized (this) { + if(this.clientCount <= 0) { + this.initMedia(type); + } + this.clientCount++; + } + return this.peerConnectionFactory; + } + + /** + * 关闭一个终端 + * + * @return 剩余终端数量 + */ + public int closeClient() { + synchronized (this) { + this.clientCount--; + if(this.clientCount <= 0) { + this.close(); + } + return this.clientCount; + } + } + + public void initMedia(Handler handler, Context context) { this.handler = handler; this.context = context; + } + + /** + * 加载媒体流 + */ + public void initMedia(Type type) { + Log.i(MediaManager.class.getSimpleName(), "加载媒体：" + type); + this.type = type; this.eglBase = EglBase.create(); PeerConnectionFactory.initialize( PeerConnectionFactory.InitializationOptions.builder(this.context) @@ -191,8 +213,10 @@ public class MediaManager { .createPeerConnectionFactory(); this.mediaStream = this.peerConnectionFactory.createLocalMediaStream("ARDAMS"); Arrays.stream(videoEncoderFactory.getSupportedCodecs()).forEach(v -> { - Log.i(MediaManager.class.getSimpleName(), "支持的视频解码器：" + v.name); + Log.d(MediaManager.class.getSimpleName(), "支持的视频解码器：" + v.name); }); + this.initAudio(); + this.initVideo(); } /** @@ -227,7 +251,7 @@ public class MediaManager { /** * 加载音频 */ - public void initAudio() { + private void initAudio() { // 关闭音频 this.closeAudioTrack(); // 加载音频 @@ -257,7 +281,7 @@ public class MediaManager { /** * 加载视频 */ - public void initVideo() { + private void initVideo() { this.closeVideoTrack(); if(this.videoCapturer != null) { this.videoCapturer.dispose(); @@ -291,8 +315,8 @@ public class MediaManager { this.videoCapturer = new ScreenCapturerAndroid(intent, new MediaProjection.Callback() { @Override public void onStop() { - super.onStop(); - Log.i(MediaManager.class.getSimpleName(), "停止屏幕捕获"); + super.onStop(); + Log.i(MediaManager.class.getSimpleName(), "停止屏幕捕获"); } }); this.initVideoTrack(); @@ -305,7 +329,11 @@ public class MediaManager { final SurfaceViewRenderer surfaceViewRenderer = this.preview(); // 加载视频 final SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("MediaVideoThread", this.eglBase.getEglBaseContext()); +// surfaceTextureHelper.setTextureSize(); +// surfaceTextureHelper.setFrameRotation(); final VideoSource videoSource = this.peerConnectionFactory.createVideoSource(this.videoCapturer.isScreencast()); + // 美颜水印 +// videoSource.setVideoProcessor(); this.videoCapturer.initialize(surfaceTextureHelper, this.context, videoSource.getCapturerObserver()); this.videoCapturer.startCapture(480, 640, 30); final VideoTrack videoTrack = this.peerConnectionFactory.createVideoTrack("ARDAMSv0", videoSource); @@ -314,20 +342,25 @@ public class MediaManager { videoTrack.setEnabled(true); this.mediaStream.addTrack(videoTrack); Log.i(MediaManager.class.getSimpleName(), "加载视频：" + videoTrack.id()); -// 二次处理：VideoProcessor + } + + public MediaStream getMediaStream() { + return this.mediaStream; } private SurfaceViewRenderer preview() { // 设置预览 final SurfaceViewRenderer surfaceViewRenderer = new SurfaceViewRenderer(this.context); - // 视频反转 - surfaceViewRenderer.setMirror(false); - // 视频拉伸 - surfaceViewRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL); - // 硬件拉伸 - surfaceViewRenderer.setEnableHardwareScaler(true); - // 加载 - surfaceViewRenderer.init(this.eglBase.getEglBaseContext(), null); + this.handler.post(() -> { + // 视频反转 + surfaceViewRenderer.setMirror(false); + // 视频拉伸 + surfaceViewRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL); + // 硬件拉伸 + surfaceViewRenderer.setEnableHardwareScaler(true); + // 加载 + surfaceViewRenderer.init(this.eglBase.getEglBaseContext(), null); + }); // 事件 // surfaceViewRenderer.setOnClickListener(); // TODO：迁移localvideo @@ -336,7 +369,8 @@ public class MediaManager { final Message message = new Message(); message.obj = surfaceViewRenderer; message.what = Config.WHAT_NEW_LOCAL_VIDEO; - this.handler.sendMessage(message); + // TODO：恢复 +// this.handler.sendMessage(message); // 暂停 // surfaceViewRenderer.pauseVideo(); // 恢复 @@ -344,6 +378,34 @@ public class MediaManager { return surfaceViewRenderer; } + public void remotePreview(final MediaStream mediaStream) { + // 设置预览 + final SurfaceViewRenderer surfaceViewRenderer = new SurfaceViewRenderer(this.context); + this.handler.post(() -> { + // 视频反转 + surfaceViewRenderer.setMirror(false); + // 视频拉伸 + surfaceViewRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL); + // 硬件拉伸 + surfaceViewRenderer.setEnableHardwareScaler(true); + // 加载 + surfaceViewRenderer.init(this.eglBase.getEglBaseContext(), null); + // 开始播放 + final VideoTrack videoTrack = mediaStream.videoTracks.get(0); + videoTrack.setEnabled(true); + videoTrack.addSink(surfaceViewRenderer); + }); + // 页面加载 + final Message message = new Message(); + message.obj = surfaceViewRenderer; + message.what = Config.WHAT_NEW_REMOTE_VIDEO; + this.handler.sendMessage(message); + // 暂停 +// surfaceViewRenderer.pauseVideo(); + // 恢复 +// surfaceViewRenderer.disableFpsReduction(); + } + public void pauseAudio() { synchronized (this.mediaStream.audioTracks) { this.mediaStream.audioTracks.forEach(a -> a.setEnabled(false)); @@ -374,10 +436,6 @@ public class MediaManager { } } - public void record() { - MediaRecorder.getInstance().init(System.currentTimeMillis() + ".mp4", null, null, 1, 1); - } - /** * 关闭声音 */ diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaRecorder.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaRecorder.java index 8721514..7c2b2f9 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaRecorder.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaRecorder.java @@ -1,6 +1,5 @@ package com.acgist.taoyao.media; -import android.graphics.YuvImage; import android.media.AudioFormat; import android.media.MediaCodec; import android.media.MediaCodecInfo; @@ -11,24 +10,17 @@ import android.os.Environment; import android.os.Handler; import android.os.HandlerThread; import android.util.Log; -import android.view.Surface; -import org.webrtc.EglBase; -import org.webrtc.GlRectDrawer; -import org.webrtc.HardwareVideoEncoderFactory; -import org.webrtc.VideoEncoderFactory; +import com.acgist.mediasoup.R; + import org.webrtc.VideoFrame; -import org.webrtc.VideoFrameDrawer; import org.webrtc.VideoSink; -import org.webrtc.YuvConverter; import org.webrtc.YuvHelper; import org.webrtc.audio.JavaAudioDeviceModule; import java.io.IOException; import java.nio.ByteBuffer; import java.nio.file.Paths; -import java.util.Optional; -import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.stream.Collectors; @@ -82,7 +74,7 @@ public final class MediaRecorder { */ public final VideoSink videoRecoder; - private MediaRecorder() { + static { final MediaCodecList mediaCodecList = new MediaCodecList(-1); for (MediaCodecInfo mediaCodecInfo : mediaCodecList.getCodecInfos()) { if (mediaCodecInfo.isEncoder()) { @@ -98,6 +90,9 @@ public final class MediaRecorder { } } } + } + + private MediaRecorder() { this.executorService = Executors.newFixedThreadPool(2); this.audioRecoder = audioSamples -> { }; @@ -135,7 +130,11 @@ public final class MediaRecorder { return this.active; } - public void init(String file, String audioFormat, String videoFormat, int width, int height) { + public void record(String path) { + this.record(path, System.currentTimeMillis() + ".mp4", null, null, 1, 1); + } + + public void record(String path, String file, String audioFormat, String videoFormat, int width, int height) { synchronized (MediaRecorder.INSTANCE) { this.file = file; this.active = true; @@ -143,7 +142,7 @@ public final class MediaRecorder { this.audioThread == null || !this.audioThread.isAlive() || this.videoThread == null || !this.videoThread.isAlive() ) { - this.initMediaMuxer(file); + this.initMediaMuxer(path, file); this.initAudioThread(MediaFormat.MIMETYPE_AUDIO_AAC, 96000, 44100, 1); this.initVideoThread(MediaFormat.MIMETYPE_VIDEO_AVC, 2500 * 1000, 30, 1, 1920, 1080); } @@ -324,10 +323,10 @@ public final class MediaRecorder { } } - private void initMediaMuxer(String file) { + private void initMediaMuxer(String path, String file) { try { this.mediaMuxer = new MediaMuxer( - Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_MOVIES).getAbsolutePath(), file).toAbsolutePath().toString(), + Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath(), path, file).toAbsolutePath().toString(), MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4 ); // 设置方向 diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/SessionClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/SessionClient.java index cec8f4b..9a33f9e 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/SessionClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/SessionClient.java @@ -1,39 +1,233 @@ package com.acgist.taoyao.media; +import android.se.omapi.Session; import android.util.Log; +import com.acgist.taoyao.boot.model.Message; +import com.acgist.taoyao.boot.utils.MapUtils; +import com.acgist.taoyao.signal.ITaoyao; + +import org.webrtc.DataChannel; +import org.webrtc.IceCandidate; +import org.webrtc.MediaConstraints; +import org.webrtc.MediaStream; +import org.webrtc.PeerConnection; +import org.webrtc.PeerConnectionFactory; +import org.webrtc.SdpObserver; +import org.webrtc.SessionDescription; + import java.io.Closeable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; /** * P2P终端 * 使用安卓SDK + WebRTC实现P2P会话 * + * https://zhuanlan.zhihu.com/p/82446482 + * * @author acgist */ public class SessionClient implements Closeable { + private final String id; + private final String name; private final String clientId; + private final ITaoyao taoyao; + private final MediaManager mediaManager; + private MediaStream mediaStream; + private SdpObserver sdpObserver; + private PeerConnection peerConnection; + private PeerConnection.Observer observer; + private PeerConnectionFactory peerConnectionFactory; - public SessionClient(String clientId) { + public SessionClient(String id, String name, String clientId, ITaoyao taoyao) { + this.id = id; + this.name = name; this.clientId = clientId; + this.taoyao = taoyao; + this.mediaManager = MediaManager.getInstance(); } - // 配置STUN穿透服务器 转发服务器 -// iceServers = new ArrayList<>(); -// PeerConnection.IceServer iceServer = PeerConnection.IceServer.builder(Constant.STUN).createIceServer(); -// iceServers.add(iceServer); -// streamList = new ArrayList<>(); -// PeerConnection.RTCConfiguration configuration = new PeerConnection.RTCConfiguration(iceServers); -// PeerConnectionObserver connectionObserver = getObserver(); -// peerConnection = peerConnectionFactory.createPeerConnection(configuration, connectionObserver); + public void init() { + this.peerConnectionFactory = this.mediaManager.newClient(MediaManager.Type.BACK); + // STUN | TURN + final List iceServers = new ArrayList<>(); + // TODO：读取配置 + final PeerConnection.IceServer iceServer = PeerConnection.IceServer.builder("stun:stun1.l.google.com:19302").createIceServer(); + iceServers.add(iceServer); + final PeerConnection.RTCConfiguration configuration = new PeerConnection.RTCConfiguration(iceServers); + this.observer = this.observer(); + this.sdpObserver = this.sdpObserver(); + this.peerConnection = this.peerConnectionFactory.createPeerConnection(configuration, this.observer); + this.peerConnection.addStream(this.mediaManager.getMediaStream()); + } -// pcConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")); -//pcConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")); -//pcConstraints.optional.add(new MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true")); + public void exchange(Message message, Map body) { + final String type = MapUtils.get(body, "type"); + switch(type) { + case "offer" -> this.offer(message, body); + case "answer" -> this.answer(message, body); + case "candidate" -> this.candidate(message, body); + default -> Log.d(SessionClient.class.getSimpleName(), "没有适配的会话指令：" + type); + } + } + + public void call(String clientId) { + final MediaConstraints mediaConstraints = new MediaConstraints(); + this.peerConnection.createOffer(this.sdpObserver, mediaConstraints); + } + + /** + * 提供媒体服务 + */ + public void offer() { + final MediaConstraints mediaConstraints = new MediaConstraints(); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("maxHeight", Integer.toString(1920))); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("maxWidth", Integer.toString(1080))); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("maxFrameRate", Integer.toString(15))); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("minFrameRate", Integer.toString(30))); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")); +// mediaConstraints.optional.add(new MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true")); + this.peerConnection.createOffer(this.sdpObserver, mediaConstraints); + } + + private void offer(Message message, Map body) { + final String sdp = MapUtils.get(body, "sdp"); + final String type = MapUtils.get(body, "type"); + final SessionDescription.Type sdpType = SessionDescription.Type.valueOf(type.toUpperCase()); + final SessionDescription sessionDescription = new SessionDescription(sdpType, sdp); + this.peerConnection.setRemoteDescription(this.sdpObserver, sessionDescription); + final MediaConstraints mediaConstraints = new MediaConstraints(); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("maxHeight", Integer.toString(1920))); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("maxWidth", Integer.toString(1080))); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("maxFrameRate", Integer.toString(15))); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("minFrameRate", Integer.toString(30))); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")); +// mediaConstraints.optional.add(new MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true")); + this.peerConnection.createAnswer(this.sdpObserver, mediaConstraints); + } + + private void answer(Message message, Map body) { + final String sdp = MapUtils.get(body, "sdp"); + final String type = MapUtils.get(body, "type"); + final SessionDescription.Type sdpType = SessionDescription.Type.valueOf(type.toUpperCase()); + final SessionDescription sessionDescription = new SessionDescription(sdpType, sdp); + this.peerConnection.setRemoteDescription(this.sdpObserver, sessionDescription); + } + + private void candidate(Message message, Map body) { + final Map candidate = MapUtils.get(body, "candidate"); + final String sdp = MapUtils.get(candidate, "candidate"); + final String sdpMid = MapUtils.get(candidate, "sdpMid"); + final Integer sdpMLineIndex = MapUtils.getInteger(candidate, "sdpMLineIndex"); + if(sdp == null || sdpMid == null || sdpMLineIndex == null) { + Log.w(SessionClient.class.getSimpleName(), "无效媒体协商：" + body); + } else { + final IceCandidate iceCandidate = new IceCandidate(sdpMid, sdpMLineIndex, sdp); + this.peerConnection.addIceCandidate(iceCandidate); + } + } @Override public void close() { Log.i(Room.class.getSimpleName(), "关闭终端：" + this.clientId); } + /** + * @return 监听 + */ + private PeerConnection.Observer observer() { + return new PeerConnection.Observer() { + @Override + public void onSignalingChange(PeerConnection.SignalingState signalingState) { + } + + @Override + public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) { + } + + @Override + public void onIceConnectionReceivingChange(boolean result) { + } + + @Override + public void onIceGatheringChange(PeerConnection.IceGatheringState iceGatheringState) { + } + + @Override + public void onIceCandidate(IceCandidate iceCandidate) { + Log.d(SessionClient.class.getSimpleName(), "发送媒体协商：" + SessionClient.this.id); + SessionClient.this.taoyao.push(SessionClient.this.taoyao.buildMessage( + "session::exchange", + "type", "candidate", + "candidate", iceCandidate, + "sessionId", SessionClient.this.id + )); + } + + @Override + public void onIceCandidatesRemoved(IceCandidate[] iceCandidates) { + } + + @Override + public void onAddStream(MediaStream mediaStream) { + Log.i(SessionClient.class.getSimpleName(), "添加远程媒体：" + SessionClient.this.clientId); + SessionClient.this.mediaStream = mediaStream; + SessionClient.this.mediaManager.remotePreview(mediaStream); + } + + @Override + public void onRemoveStream(MediaStream mediaStream) { + } + + @Override + public void onDataChannel(DataChannel dataChannel) { + } + + @Override + public void onRenegotiationNeeded() { + Log.d(SessionClient.class.getSimpleName(), "重新协商媒体：" + SessionClient.this.id); + if(peerConnection.connectionState() == PeerConnection.PeerConnectionState.CONNECTED) { + // TODO：重写协商 +// SessionClient.this.offer(); + } + } + }; + } + + private SdpObserver sdpObserver() { + return new SdpObserver() { + @Override + public void onCreateSuccess(SessionDescription sessionDescription) { + Log.d(SessionClient.class.getSimpleName(), "创建SDP成功：" + SessionClient.this.id); + SessionClient.this.peerConnection.setLocalDescription(this, sessionDescription); + SessionClient.this.taoyao.push(SessionClient.this.taoyao.buildMessage( + "session::exchange", + "sdp", sessionDescription.description, + "type", sessionDescription.type.toString().toLowerCase(), + "sessionId", SessionClient.this.id + )); + } + + @Override + public void onSetSuccess() { + Log.d(SessionClient.class.getSimpleName(), "设置SDP成功：" + SessionClient.this.id); + } + + @Override + public void onCreateFailure(String message) { + Log.w(SessionClient.class.getSimpleName(), "创建SDP失败：" + message); + } + + @Override + public void onSetFailure(String message) { + Log.w(SessionClient.class.getSimpleName(), "设置SDP失败：" + message); + } + }; + } + } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/signal/ITaoyao.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/signal/ITaoyao.java new file mode 100644 index 0000000..7a3359f --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/signal/ITaoyao.java @@ -0,0 +1,40 @@ +package com.acgist.taoyao.signal; + +import com.acgist.taoyao.boot.model.Message; + +/** + * 桃夭信令 + * + * @author acgist + */ +public interface ITaoyao { + + /** + * @param signal 信令 + * @param args 消息主体内容 + * + * @return 消息 + */ + Message buildMessage(String signal, Object ... args); + + /** + * @param signal 信令 + * @param body 消息主体 + * + * @return 消息 + */ + Message buildMessage(String signal, Object body); + + /** + * @param message 信令消息 + */ + void push(Message message); + + /** + * @param request 信令请求消息 + * + * @return 信令响应消息 + */ + Message request(Message request); + +} diff --git a/taoyao-client-web/src/components/Taoyao.js b/taoyao-client-web/src/components/Taoyao.js index 6c25fba..0fbdeba 100644 --- a/taoyao-client-web/src/components/Taoyao.js +++ b/taoyao-client-web/src/components/Taoyao.js @@ -2107,6 +2107,16 @@ class Taoyao extends RemoteClient { session.localVideoTrack = localStream.getVideoTracks()[0]; session.peerConnection.addTrack(session.localAudioTrack, localStream); session.peerConnection.addTrack(session.localVideoTrack, localStream); + session.peerConnection.createOffer().then(async description => { + await session.peerConnection.setLocalDescription(description); + me.push( + protocol.buildMessage("session::exchange", { + sdp : description.sdp, + type : description.type, + sessionId: sessionId + }) + ); + }); } async defaultSessionCall(message) { @@ -2118,6 +2128,7 @@ class Taoyao extends RemoteClient { const localStream = await me.getStream(); session.localAudioTrack = localStream.getAudioTracks()[0]; session.localVideoTrack = localStream.getVideoTracks()[0]; + // 相同Stream音视频同步 session.peerConnection.addTrack(session.localAudioTrack, localStream); session.peerConnection.addTrack(session.localVideoTrack, localStream); session.peerConnection.createOffer().then(async description => { @@ -2191,16 +2202,9 @@ class Taoyao extends RemoteClient { }; peerConnection.onnegotiationneeded = event => { console.debug("buildPeerConnection onnegotiationneeded", event); - session.peerConnection.createOffer().then(async description => { - await session.peerConnection.setLocalDescription(description); - me.push( - protocol.buildMessage("session::exchange", { - sdp : description.sdp, - type : description.type, - sessionId: sessionId - }) - ); - }); + if(peerConnection.connectionState === "connected") { + // TODO：重连 + } } return peerConnection; } diff --git a/taoyao-signal-server/README.md b/taoyao-signal-server/README.md index 6f9c815..2e5efa1 100644 --- a/taoyao-signal-server/README.md +++ b/taoyao-signal-server/README.md @@ -12,3 +12,16 @@ ## 信令格式 [信令格式](https://localhost:8888/protocol/list) + +## 特殊说明 + +### 消费者暂停恢复 + +* 消费者直接暂停消费：生产者生成数据、消费者接收数据 +* 媒体服务暂停消费者数据转发（默认）：生产者生成数据、消费者不会接收数据 + +### 生产者暂停恢复 + +* 消费者直接暂停接收：生产者生成数据、消费者接收数据 +* 媒体服务暂停生产者数据转发：生产者生成数据、消费者不会接收数据 +* 媒体服务暂停生产者数据转发同时暂停生产者生成（默认）：生产者不会生成数据、消费者不会接收数据 diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCallProtocol.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCallProtocol.java index 0f3b854..422219e 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCallProtocol.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCallProtocol.java @@ -2,8 +2,6 @@ package com.acgist.taoyao.signal.protocol.session; import java.util.Map; -import org.apache.tomcat.util.bcel.Const; - import com.acgist.taoyao.boot.annotation.Description; import com.acgist.taoyao.boot.annotation.Protocol; import com.acgist.taoyao.boot.config.Constant; @@ -14,14 +12,11 @@ import com.acgist.taoyao.signal.client.ClientType; import com.acgist.taoyao.signal.party.session.Session; import com.acgist.taoyao.signal.protocol.ProtocolSessionAdapter; -import lombok.extern.slf4j.Slf4j; - /** * 发起会话信令 * * @author acgist */ -@Slf4j @Protocol @Description( body = """ @@ -45,18 +40,18 @@ public class SessionCallProtocol extends ProtocolSessionAdapter { @Override public void execute(String clientId, ClientType clientType, Client client, Message message, Map body) { final String targetId = MapUtils.get(body, Constant.CLIENT_ID); - final Client target = this.clientManager.clients(targetId); + final Client target = this.clientManager.clients(targetId); final Session session = this.sessionManager.call(client, target); message.setBody(Map.of( - Constant.NAME, target.status().getName(), - Constant.CLIENT_ID, target.clientId(), + Constant.NAME, target.status().getName(), + Constant.CLIENT_ID, target.clientId(), Constant.SESSION_ID, session.getId() )); client.push(message); final Message callMessage = message.cloneWithoutBody(); callMessage.setBody(Map.of( - Constant.NAME, client.status().getName(), - Constant.CLIENT_ID, client.clientId(), + Constant.NAME, client.status().getName(), + Constant.CLIENT_ID, client.clientId(), Constant.SESSION_ID, session.getId() )); target.push(callMessage); diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCloseProtocol.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCloseProtocol.java index 58180c3..27bdcb5 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCloseProtocol.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionCloseProtocol.java @@ -10,14 +10,11 @@ import com.acgist.taoyao.signal.client.ClientType; import com.acgist.taoyao.signal.party.session.Session; import com.acgist.taoyao.signal.protocol.ProtocolSessionAdapter; -import lombok.extern.slf4j.Slf4j; - /** * 关闭媒体信令 * * @author acgist */ -@Slf4j @Protocol @Description( body = """ diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionExchangeProtocol.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionExchangeProtocol.java index 8e61005..5e2266b 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionExchangeProtocol.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/session/SessionExchangeProtocol.java @@ -10,16 +10,14 @@ import com.acgist.taoyao.signal.client.ClientType; import com.acgist.taoyao.signal.party.session.Session; import com.acgist.taoyao.signal.protocol.ProtocolSessionAdapter; -import lombok.extern.slf4j.Slf4j; - /** * 媒体交换信令 * * @author acgist */ -@Slf4j @Protocol @Description( + memo = "媒体交换协商：offer/answer/candidate", body = """ { }