diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java index 3d7bb2b..c77942a 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java @@ -4,6 +4,7 @@ import android.Manifest; import android.app.Activity; import android.content.Intent; import android.content.pm.PackageManager; +import android.content.res.Resources; import android.media.projection.MediaProjectionManager; import android.os.Bundle; import android.os.Handler; @@ -25,13 +26,15 @@ import androidx.annotation.NonNull; import androidx.appcompat.app.AppCompatActivity; import androidx.core.app.ActivityCompat; +import com.acgist.taoyao.boot.utils.DateUtils; import com.acgist.taoyao.client.databinding.ActivityMainBinding; import com.acgist.taoyao.client.signal.Taoyao; -import com.acgist.taoyao.config.Config; +import com.acgist.taoyao.media.TransportType; +import com.acgist.taoyao.media.config.Config; import com.acgist.taoyao.media.MediaManager; -import com.acgist.taoyao.media.MediaRecorder; import java.io.Serializable; +import java.time.LocalDateTime; import java.util.stream.Stream; /** @@ -141,7 +144,18 @@ public class MainActivity extends AppCompatActivity implements Serializable { int waitCount = 0; this.mainHandler = new MainHandler(this); // 不能写在service里面： Attempt to invoke virtual method 'android.view.View android.view.Window.getDecorView()' on a null object reference - MediaManager.getInstance().initContext(this.mainHandler, this.getApplicationContext()); + final Resources resources = this.getResources(); + MediaManager.getInstance().initContext( + this.mainHandler, this.getApplicationContext(), + resources.getBoolean(R.bool.preview), + resources.getBoolean(R.bool.playAudio), + resources.getBoolean(R.bool.playVideo), + resources.getBoolean(R.bool.audioConsume), + resources.getBoolean(R.bool.videoConsume), + resources.getBoolean(R.bool.audioProduce), + resources.getBoolean(R.bool.videoProduce), + TransportType.valueOf(resources.getString(R.string.transportType)) + ); final Display display = this.getWindow().getContext().getDisplay(); while (Display.STATE_ON != display.getState() && waitCount++ < 10) { SystemClock.sleep(100); @@ -188,19 +202,27 @@ public class MainActivity extends AppCompatActivity implements Serializable { ); } + /** + * 功能测试按钮根据实际情况设置功能 + * + * @param view View + */ private void action(View view) { this.threadHandler.post(() -> { // 进入房间 - Taoyao.taoyao.enterRoom("d8f1e91c-58d0-4e58-ad67-decc0fd61df2", null); + Taoyao.taoyao.roomEnter("d8f1e91c-58d0-4e58-ad67-decc0fd61df2", null); }); } - private synchronized void switchRecord(View view) { - final MediaRecorder mediaRecorder = MediaRecorder.getInstance(); - if (mediaRecorder.isActive()) { - mediaRecorder.stop(); + private void switchRecord(View view) { + final MediaManager mediaManager = MediaManager.getInstance(); + if (mediaManager.isRecording()) { + mediaManager.stopRecord(); } else { - mediaRecorder.record(this.getResources().getString(R.string.storagePathVideo)); + mediaManager.startRecord( + this.getResources().getString(R.string.storagePathVideo), + DateUtils.format(LocalDateTime.now(), DateUtils.DateTimeStyle.YYYYMMDDHH24MMSS) + ".mp4" + ); } } diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java index a710184..990c386 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java @@ -147,6 +147,7 @@ public class MediaService extends Service { resources.getInteger(R.integer.timeout), resources.getString(R.string.encrypt), resources.getString(R.string.encryptSecret), this.mainHandler, context ); + MediaManager.getInstance().initTaoyao(this.taoyao); Toast.makeText(this.getApplicationContext(), "连接信令", Toast.LENGTH_SHORT).show(); } diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java index b20f1a8..3e3e004 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java @@ -25,12 +25,11 @@ import com.acgist.taoyao.boot.utils.IdUtils; import com.acgist.taoyao.boot.utils.JSONUtils; import com.acgist.taoyao.boot.utils.MapUtils; import com.acgist.taoyao.client.R; -import com.acgist.taoyao.config.MediaProperties; +import com.acgist.taoyao.media.config.MediaProperties; import com.acgist.taoyao.media.MediaManager; -import com.acgist.taoyao.media.MediaRecorder; -import com.acgist.taoyao.media.Room; -import com.acgist.taoyao.media.SessionClient; -import com.acgist.taoyao.signal.ITaoyao; +import com.acgist.taoyao.media.client.Room; +import com.acgist.taoyao.media.client.SessionClient; +import com.acgist.taoyao.media.signal.ITaoyao; import org.apache.commons.lang3.ArrayUtils; @@ -158,7 +157,6 @@ public final class Taoyao implements ITaoyao { private final Handler executeMessageHandler; private final HandlerThread executeMessageThread; private final MediaManager mediaManager; - private final MediaRecorder mediaRecorder; /** * 媒体配置 */ @@ -212,7 +210,6 @@ public final class Taoyao implements ITaoyao { this.executeMessageThread.start(); this.executeMessageHandler = new Handler(this.executeMessageThread.getLooper()); this.mediaManager = MediaManager.getInstance(); - this.mediaRecorder = MediaRecorder.getInstance(); this.rooms = new ConcurrentHashMap<>(); this.sessionClients = new ConcurrentHashMap<>(); Taoyao.taoyao = this; @@ -447,6 +444,9 @@ public final class Taoyao implements ITaoyao { final Map map = new HashMap<>(); if (ArrayUtils.isNotEmpty(args)) { for (int index = 0; index < args.length; index += 2) { + if(args[index] == null || args[index + 1] == null) { + continue; + } map.put(args[index], args[index + 1]); } } @@ -503,11 +503,11 @@ public final class Taoyao implements ITaoyao { case "client::register" -> this.clientRegister(message, body); case "client::reboot" -> this.clientReboot(message, body); case "client::shutdown" -> this.clientShutdown(message, body); -// case "room::close" -> this.roomClose(message, body); -// case "room::enter" -> this.roomEnter(message, body); -// case "room::expel" -> this.roomExpel(message, body); + case "room::close" -> this.roomClose(message, body); + case "room::enter" -> this.roomEnter(message, body); +// case "room::expel" -> this.roomExpel(message, body); case "room::invite" -> this.roomInivte(message, body); -// case "room::leave" -> this.roomLeave(message, body); +// case "room::leave" -> this.roomLeave(message, body); case "session::call" -> this.sessionCall(message, body); case "session::close" -> this.sessionClose(message, body); case "session::exchange" -> this.sessionExchange(message, body); @@ -529,10 +529,10 @@ public final class Taoyao implements ITaoyao { "clientType", this.clientType, "latitude", location == null ? -1 : location.getLatitude(), "longitude", location == null ? -1 : location.getLongitude(), - "signal", this.wifiSignal(), + "signal", this.signal(), "battery", this.battery(), "charging", this.charging(), - "recording", MediaRecorder.getInstance().isActive() + "recording", this.mediaManager.isRecording() )); } @@ -569,26 +569,48 @@ public final class Taoyao implements ITaoyao { Process.killProcess(Process.myPid()); } + private void roomClose(Message message, Map body) { + final String roomId = MapUtils.get(body, "roomId"); + final Room room = this.rooms.remove(roomId); + if(room == null) { + return; + } + room.close(); + } + + private void roomEnter(Message message, Map body) { + final String roomId = MapUtils.get(body, "roomId"); + final Room room = this.rooms.get(roomId); + if(room == null) { + return; + } + room.newRemoteClient(body); + } + private void roomInivte(Message message, Map body) { final String roomId = MapUtils.get(body, "roomId"); final String password = MapUtils.get(body, "password"); - final Room room = this.enterRoom(roomId, password); - room.produceMedia(); + this.roomEnter(roomId, password); } - public Room enterRoom(String roomId, String password) { + public Room roomEnter(String roomId, String password) { final Resources resources = this.context.getResources(); final Room room = this.rooms.computeIfAbsent( roomId, key -> new Room( + this.name, this.clientId, key, password, + this.mainHandler, this, + resources.getBoolean(R.bool.dataConsume), resources.getBoolean(R.bool.audioConsume), resources.getBoolean(R.bool.videoConsume), resources.getBoolean(R.bool.audioProduce), - resources.getBoolean(R.bool.videoProduce), - this.mediaManager.nativeNewRoom(key), this) + resources.getBoolean(R.bool.dataProduce), + resources.getBoolean(R.bool.videoProduce) + ) ); room.enter(); + room.produceMedia(); return room; } @@ -596,7 +618,7 @@ public final class Taoyao implements ITaoyao { final String name = MapUtils.get(body, "name"); final String clientId = MapUtils.get(body, "clientId"); final String sessionId = MapUtils.get(body, "sessionId"); - final SessionClient sessionClient = new SessionClient(sessionId, name, clientId, this); + final SessionClient sessionClient = new SessionClient(sessionId, name, clientId, this.mainHandler, this); this.sessionClients.put(sessionId, sessionClient); sessionClient.init(); sessionClient.offer(); @@ -628,10 +650,10 @@ public final class Taoyao implements ITaoyao { "client::heartbeat", "latitude", location == null ? -1 : location.getLatitude(), "longitude", location == null ? -1 : location.getLongitude(), - "signal", this.wifiSignal(), + "signal", this.signal(), "battery", this.battery(), "charging", this.charging(), - "recording", MediaRecorder.getInstance().isActive() + "recording", this.mediaManager.isRecording() )); } @@ -639,7 +661,10 @@ public final class Taoyao implements ITaoyao { * @return 电量百分比 */ private int battery() { - return this.batteryManager == null ? -1 : this.batteryManager.getIntProperty(BatteryManager.BATTERY_PROPERTY_CAPACITY); + return + this.batteryManager == null ? + -1 : + this.batteryManager.getIntProperty(BatteryManager.BATTERY_PROPERTY_CAPACITY); } /** @@ -655,7 +680,7 @@ public final class Taoyao implements ITaoyao { /** * @return WIFI信号强度 */ - private int wifiSignal() { + private int signal() { if(this.wifiManager == null) { return -1; } @@ -681,17 +706,11 @@ public final class Taoyao implements ITaoyao { return null; } final Criteria criteria = new Criteria(); - // 功耗 criteria.setCostAllowed(false); - // 不要海拔 - criteria.setAltitudeRequired(false); - // 不要方位 criteria.setBearingRequired(false); - // 精度 + criteria.setAltitudeRequired(false); criteria.setAccuracy(Criteria.ACCURACY_FINE); - // 功耗 criteria.setPowerRequirement(Criteria.POWER_LOW); - // 最佳提供者 final String provider = this.locationManager.getBestProvider(criteria, true); if (provider == null) { return null; diff --git a/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml b/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml index 49ca46e..8ef90c5 100644 --- a/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml +++ b/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml @@ -1,7 +1,11 @@ - + true + + true + + true 1.0.0 @@ -17,9 +21,16 @@ /taoyao/video WEBRTC - + + false + true + true + + false + true + true \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/CMakeLists.txt b/taoyao-client-android/taoyao/media/CMakeLists.txt index c532cc2..8021f67 100644 --- a/taoyao-client-android/taoyao/media/CMakeLists.txt +++ b/taoyao-client-android/taoyao/media/CMakeLists.txt @@ -25,25 +25,17 @@ set( set( SOURCE_FILES - ${SOURCE_DIR}/include/LocalClient.hpp ${SOURCE_DIR}/include/Log.hpp ${SOURCE_DIR}/include/MediaManager.hpp - ${SOURCE_DIR}/include/MediaRecorder.hpp - ${SOURCE_DIR}/include/RemoteClient.hpp ${SOURCE_DIR}/include/Room.hpp ${SOURCE_DIR}/include/RtpAudioPublisher.hpp ${SOURCE_DIR}/include/RtpClient.hpp ${SOURCE_DIR}/include/RtpVideoPublisher.hpp - ${SOURCE_DIR}/include/SessionClient.hpp ${SOURCE_DIR}/rtp/RtpAudioPublisher.cpp ${SOURCE_DIR}/rtp/RtpClient.cpp ${SOURCE_DIR}/rtp/RtpVideoPublisher.cpp - ${SOURCE_DIR}/webrtc/LocalClient.cpp ${SOURCE_DIR}/webrtc/MediaManager.cpp - ${SOURCE_DIR}/webrtc/MediaRecorder.cpp - ${SOURCE_DIR}/webrtc/RemoteClient.cpp ${SOURCE_DIR}/webrtc/Room.cpp - ${SOURCE_DIR}/webrtc/SessionClient.cpp ) set(LIBWEBRTC_BINARY_PATH ${LIBWEBRTC_BINARY_PATH}/${ANDROID_ABI} CACHE STRING "libwebrtc binary path" FORCE) diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/include/LocalClient.hpp b/taoyao-client-android/taoyao/media/src/main/cpp/include/LocalClient.hpp deleted file mode 100644 index 3ef43c3..0000000 --- a/taoyao-client-android/taoyao/media/src/main/cpp/include/LocalClient.hpp +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#include - -#include "jni.h" -#include "mediasoupclient.hpp" - -namespace acgist { - - class LocalClient { - public: - std::string name; - std::string clientId; - mediasoupclient::Producer* producer; - public: - jmethodID newCallback; - jmethodID pauseCallback; - jmethodID resumeCallback; - jmethodID closeCallback; - public: - void pause(); - void resume(); - void close(); - }; - -} \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/include/MediaRecorder.hpp b/taoyao-client-android/taoyao/media/src/main/cpp/include/MediaRecorder.hpp deleted file mode 100644 index f7883f8..0000000 --- a/taoyao-client-android/taoyao/media/src/main/cpp/include/MediaRecorder.hpp +++ /dev/null @@ -1,5 +0,0 @@ -#pragma once - -namespace acgist { - -} \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/include/RemoteClient.hpp b/taoyao-client-android/taoyao/media/src/main/cpp/include/RemoteClient.hpp deleted file mode 100644 index dd314ea..0000000 --- a/taoyao-client-android/taoyao/media/src/main/cpp/include/RemoteClient.hpp +++ /dev/null @@ -1,26 +0,0 @@ -#pragma once - -#include - -#include "jni.h" -#include "mediasoupclient.hpp" - -namespace acgist { - - class RemoteClient { - public: - std::string name; - std::string clientId; - mediasoupclient::Consumer* consumer; - public: - jmethodID newCallback; - jmethodID pauseCallback; - jmethodID resumeCallback; - jmethodID closeCallback; - public: - void pause(); - void resume(); - void close(); - }; - -} \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp b/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp index 496fc90..d7a9f01 100644 --- a/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp +++ b/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp @@ -6,8 +6,6 @@ #include "jni.h" #include "Log.hpp" -#include "LocalClient.hpp" -#include "RemoteClient.hpp" #include "mediasoupclient.hpp" #include "sdk/android/src/jni/pc/peer_connection.h" @@ -17,10 +15,12 @@ namespace acgist { class Room { public: - mediasoupclient::Device* device; - mediasoupclient::PeerConnection* peerConnection; - acgist::LocalClient* localClient; - std::map remoteClients; + mediasoupclient::Device *device; + mediasoupclient::PeerConnection *peerConnection; + mediasoupclient::SendTransport *sendTransport; + mediasoupclient::RecvTransport *recvTransport; + mediasoupclient::SendTransport::Listener *sendListener; + mediasoupclient::RecvTransport::Listener *recvListener; jstring roomId; public: /** @@ -35,7 +35,14 @@ namespace acgist { Room(jstring roomId); virtual ~Room(); public: - void load(std::string rtpCapabilities, webrtc::PeerConnectionFactoryInterface* factory, webrtc::PeerConnectionInterface::RTCConfiguration& configuration); + void load( + std::string rtpCapabilities, + webrtc::PeerConnectionFactoryInterface *factory, + webrtc::PeerConnectionInterface::RTCConfiguration &rtcConfiguration + ); + void createSendTransport(std::string body); + void createRecvTransport(std::string body); + void produceMedia(webrtc::MediaStreamInterface mediaStream); void closeLocalClient(); void closeRemoteClient(); void close(); diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/include/SessionClient.hpp b/taoyao-client-android/taoyao/media/src/main/cpp/include/SessionClient.hpp deleted file mode 100644 index f7883f8..0000000 --- a/taoyao-client-android/taoyao/media/src/main/cpp/include/SessionClient.hpp +++ /dev/null @@ -1,5 +0,0 @@ -#pragma once - -namespace acgist { - -} \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/LocalClient.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/LocalClient.cpp deleted file mode 100644 index e63762a..0000000 --- a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/LocalClient.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "LocalClient.hpp" - -namespace acgist { - -} diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediaManager.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediaManager.cpp index fd64c51..31a2d73 100644 --- a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediaManager.cpp +++ b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediaManager.cpp @@ -3,23 +3,20 @@ namespace acgist { -extern "C" JNIEXPORT void JNICALL Java_com_acgist_taoyao_media_MediaManager_nativeInit(JNIEnv * env, jobject me) { - LOG_I("加载MediasoupClient：", mediasoupclient::Version().data()); - mediasoupclient::Initialize(); - // => { spatialLayers: 2, temporalLayers: 3 } -// mediasoupclient::parseScalabilityMode("L2T3"); - // => { spatialLayers: 4, temporalLayers: 7 } -// mediasoupclient::parseScalabilityMode("L4T7_KEY_SHIFT"); -} + extern "C" JNIEXPORT void JNICALL + Java_com_acgist_taoyao_media_MediaManager_nativeInit(JNIEnv *env, jobject me) { + LOG_I("加载MediasoupClient：", mediasoupclient::Version().data()); + mediasoupclient::Initialize(); + // => { spatialLayers: 2, temporalLayers: 3 } + // mediasoupclient::parseScalabilityMode("L2T3"); + // => { spatialLayers: 4, temporalLayers: 7 } + // mediasoupclient::parseScalabilityMode("L4T7_KEY_SHIFT"); + } -extern "C" JNIEXPORT void JNICALL Java_com_acgist_taoyao_media_MediaManager_nativeStop(JNIEnv * env, jobject me) { - std::cout << "释放mediasoupclient" << std::endl; - mediasoupclient::Cleanup(); -} - -extern "C" JNIEXPORT jlong JNICALL Java_com_acgist_taoyao_media_MediaManager_nativeNewRoom(JNIEnv * env, jobject me, jstring roomId) { - const Room* room = new Room(roomId); - return (jlong) room; -} + extern "C" JNIEXPORT void JNICALL + Java_com_acgist_taoyao_media_MediaManager_nativeStop(JNIEnv *env, jobject me) { + std::cout << "释放mediasoupclient" << std::endl; + mediasoupclient::Cleanup(); + } } diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediaRecorder.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediaRecorder.cpp deleted file mode 100644 index aaaa246..0000000 --- a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/MediaRecorder.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "MediaRecorder.hpp" - -namespace acgist { - -} \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/RemoteClient.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/RemoteClient.cpp deleted file mode 100644 index 6974b88..0000000 --- a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/RemoteClient.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "RemoteClient.hpp" - -namespace acgist { - -} \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp index bbcd2de..b35a6c8 100644 --- a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp +++ b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp @@ -2,61 +2,133 @@ namespace acgist { + class SendListener : public mediasoupclient::SendTransport::Listener { + + std::future OnConnect(mediasoupclient::Transport *transport, const nlohmann::json &dtlsParameters) override { + return std::future(); + } + + void OnConnectionStateChange(mediasoupclient::Transport *transport, const std::string &connectionState) override { + } + + std::future OnProduce(mediasoupclient::SendTransport *transport, const std::string &kind, nlohmann::json rtpParameters, const nlohmann::json &appData) override { + return std::future(); + } + + std::future OnProduceData(mediasoupclient::SendTransport *transport, const nlohmann::json &sctpStreamParameters, const std::string &label, const std::string &protocol, const nlohmann::json &appData) override { + return std::future(); + } + + }; + + class RecvListener : public mediasoupclient::RecvTransport::Listener { + + std::future OnConnect(mediasoupclient::Transport *transport, const nlohmann::json &dtlsParameters) override { + return std::future(); + } + + void OnConnectionStateChange(mediasoupclient::Transport *transport, const std::string &connectionState) override { + } + + }; + Room::Room(jstring roomId) { this->roomId = roomId; this->device = new mediasoupclient::Device(); + this->sendListener = new SendListener(); + this->recvListener = new RecvListener(); } Room::~Room() { + delete this->device; + delete this->sendListener; + delete this->sendTransport; + delete this->recvListener; + delete this->recvTransport; } - void Room::load(std::string rtpCapabilities, webrtc::PeerConnectionFactoryInterface* factory, webrtc::PeerConnectionInterface::RTCConfiguration& configuration) { - // TODO：PeerConnectionFactory复用 + void Room::load( + std::string rtpCapabilities, + webrtc::PeerConnectionFactoryInterface *factory, + webrtc::PeerConnectionInterface::RTCConfiguration &rtcConfiguration + ) { nlohmann::json json; + // TODO：全局 mediasoupclient::PeerConnection::Options options; - options.config = configuration; + options.config = rtcConfiguration; options.factory = factory; json["routerRtpCapabilities"] = nlohmann::json::parse(rtpCapabilities); this->device->Load(json, &options); } + void Room::createSendTransport(std::string body) { + nlohmann::json json = nlohmann::json::parse(body); + this->sendTransport = this->device->CreateSendTransport( + this->sendListener, + json["transportId"], + json["iceCandidates"], + json["iceParameters"], + json["dtlsParameters"], + json["sctpParameters"] + // TODO：全局options + ); + } + + void Room::createRecvTransport(std::string body) { + nlohmann::json json = nlohmann::json::parse(body); + this->recvTransport = this->device->CreateRecvTransport( + this->recvListener, + json["transportId"], + json["iceCandidates"], + json["iceParameters"], + json["dtlsParameters"], + json["sctpParameters"] + // TODO：全局options + ); + } + + void Room::produceMedia(webrtc::MediaStreamInterface mediaStream) { +// this->device->CanProduce(); + } + void Room::close() { delete this->device; this->device = nullptr; } - extern "C" JNIEXPORT void JNICALL Java_com_acgist_taoyao_media_Room_nativeLoad( + extern "C" JNIEXPORT void JNICALL + Java_com_acgist_taoyao_media_client_Room_nativeLoad( JNIEnv* env, jobject me, - jlong nativePointer, jstring rtpCapabilities, - jlong factoryPointer, jobject configuration + jlong nativeRoomPointer, jstring jRtpCapabilities, + jlong factoryPointer, jobject jRtcConfiguration ) { - Room *room = (Room *) nativePointer; + Room* room = (Room*) nativeRoomPointer; webrtc::PeerConnectionInterface::RTCConfiguration rtcConfiguration(webrtc::PeerConnectionInterface::RTCConfigurationType::kAggressive); - // TODO：为什么不能转换 - jobject configurationGlobal = webrtc::jni::NewGlobalRef(env, configuration); - webrtc::JavaParamRef configurationRef(configurationGlobal); + // TODO：为什么不能转换？测试是否因为stun配置问题 + jobject jRtcConfigurationGlobal = webrtc::jni::NewGlobalRef(env, jRtcConfiguration); + webrtc::JavaParamRef jRtcConfigurationRef(jRtcConfigurationGlobal); // webrtc::jni::JavaToNativeMediaConstraints() - webrtc::jni::JavaToNativeRTCConfiguration(env, webrtc::JavaParamRef(env, configurationGlobal), &rtcConfiguration); - webrtc::jni::DeleteGlobalRef(env, configurationGlobal); - const char* routerRtpCapabilities = env->GetStringUTFChars(rtpCapabilities, 0); + webrtc::jni::JavaToNativeRTCConfiguration(env, jRtcConfigurationRef, &rtcConfiguration); + webrtc::jni::DeleteGlobalRef(env, jRtcConfigurationGlobal); + const char* rtpCapabilities = env->GetStringUTFChars(jRtpCapabilities, 0); room->load( - routerRtpCapabilities, + rtpCapabilities, reinterpret_cast(factoryPointer), // (webrtc::PeerConnectionFactoryInterface*) factoryPointer, rtcConfiguration ); -// env->ReleaseStringUTFChars(rtpCapabilities, routerRtpCapabilities); -// delete routerRtpCapabilities; + env->ReleaseStringUTFChars(jRtpCapabilities, rtpCapabilities); +// delete rtpCapabilities; } - extern "C" JNIEXPORT void JNICALL Java_com_acgist_taoyao_media_Room_nativeNewClient(JNIEnv * env, jobject me) { - + extern "C" JNIEXPORT jlong JNICALL + Java_com_acgist_taoyao_media_client_Room_nativeNewRoom(JNIEnv *env, jobject me, jstring roomId) { + const Room* room = new Room(roomId); + return (jlong) room; } - extern "C" JNIEXPORT void JNICALL Java_com_acgist_taoyao_media_Room_nativeCloseClient(JNIEnv * env, jobject me) { - } - - extern "C" JNIEXPORT void JNICALL Java_com_acgist_taoyao_media_Room_nativeCloseRoom(JNIEnv * env, jobject me, jlong nativePointer) { + extern "C" JNIEXPORT void JNICALL + Java_com_acgist_taoyao_media_client_Room_nativeCloseRoom(JNIEnv *env, jobject me, jlong nativeRoomPointer) { // JNIEXPORT void JNICALL // Java_nativeMethod // (JNIEnv *env, jobject thiz) { @@ -75,9 +147,28 @@ namespace acgist { // jobject gThiz = (jobject)env->NewGlobalRef(thiz);//thiz为JAVA对象 // (*obj).javaObj = (jint)gThiz; - Room* room = (Room*) nativePointer; + Room* room = (Room*) nativeRoomPointer; room->close(); delete room; } + extern "C" JNIEXPORT void JNICALL + Java_com_acgist_taoyao_media_client_Room_nativeCreateSendTransport(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jBody) { + Room* room = (Room*) nativeRoomPointer; + const char* body = env->GetStringUTFChars(jBody, 0); + room->createSendTransport(body); + env->ReleaseStringUTFChars(jBody, body); + } + + extern "C" JNIEXPORT void JNICALL + Java_com_acgist_taoyao_media_client_Room_nativeCreateRecvTransport(JNIEnv *env, jobject me, jlong nativeRoomPointer, jstring jBody) { + Room* room = (Room*) nativeRoomPointer; + } + + extern "C" JNIEXPORT void JNICALL + Java_com_acgist_taoyao_media_client_Room_nativeProduceMedia(JNIEnv *env, jobject me, jlong nativeRoomPointer, jlong mediaStreamPointer) { + Room* room = (Room*) nativeRoomPointer; + webrtc::MediaStreamInterface* mediaStream = reinterpret_cast(mediaStreamPointer); + } + } diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/SessionClient.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/SessionClient.cpp deleted file mode 100644 index 4883440..0000000 --- a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/SessionClient.cpp +++ /dev/null @@ -1,5 +0,0 @@ -#include "SessionClient.hpp" - -namespace acgist { - -} \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/RouterCallback.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/RouterCallback.java deleted file mode 100644 index bb94691..0000000 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/RouterCallback.java +++ /dev/null @@ -1,16 +0,0 @@ -package com.acgist.taoyao; - -/** - * 路由回调 - */ -public interface RouterCallback { - - void enterCallback(); - void newRemoteClientCallback(); - void closeRemoteClientCallback(); - void consumerPauseCallback(); - void consumerResumeCallback(); - void producerPauseCallback(); - void producerResumeCallback(); - -} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/LocalClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/LocalClient.java deleted file mode 100644 index f90e3df..0000000 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/LocalClient.java +++ /dev/null @@ -1,39 +0,0 @@ -package com.acgist.taoyao.media; - -import java.util.logging.Handler; - -/** - * 房间本地终端 - * - * @author acgist - */ -public class LocalClient extends RoomClient { - - /** - * 传输类型 - * - * @author acgist - */ - public enum TransportType { - - /** - * RTP - */ - RTP, - /** - * WebRTC - */ - WEBRTC; - - } - - public LocalClient(String name, String clientId, Handler handler) { - super(name, clientId, handler); - } - - @Override - public void close() { - super.close(); - } - -} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java index cff1568..3f2efb6 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java @@ -6,11 +6,12 @@ import android.media.MediaCodecInfo; import android.media.MediaCodecList; import android.media.projection.MediaProjection; import android.os.Handler; -import android.os.Looper; import android.os.Message; import android.util.Log; -import com.acgist.taoyao.config.Config; +import com.acgist.taoyao.media.client.RecordClient; +import com.acgist.taoyao.media.config.Config; +import com.acgist.taoyao.media.signal.ITaoyao; import org.webrtc.AudioSource; import org.webrtc.AudioTrack; @@ -58,7 +59,7 @@ import java.util.stream.IntStream; *

* TODO：动态码率（BITRATE_MODE_VBR、BITRATE_MODE） */ -public class MediaManager { +public final class MediaManager { /** * 视频来源类型 @@ -103,10 +104,46 @@ public class MediaManager { * 当前终端数量 */ private volatile int clientCount; + /** + * 是否预览视频 + */ + private boolean preview; + /** + * 是否打开音频播放 + */ + private boolean playAudio; + /** + * 是否打开视频播放 + */ + private boolean playVideo; + /** + * 是否消费音频 + */ + private boolean audioConsume; + /** + * 是否消费视频 + */ + private boolean videoConsume; + /** + * 是否生产音频 + */ + private boolean audioProduce; + /** + * 是否生产视频 + */ + private boolean videoProduce; /** * 视频来源类型 */ private Type type; + /** + * 传输通道类型 + */ + private TransportType transportType; + /** + * 信令 + */ + private ITaoyao taoyao; /** * Handler */ @@ -119,6 +156,10 @@ public class MediaManager { * EGL */ private EglBase eglBase; + /** + * 录制终端 + */ + private RecordClient recordClient; /** * 媒体流：声音、视频 */ @@ -127,14 +168,14 @@ public class MediaManager { * 视频捕获 */ private VideoCapturer videoCapturer; + /** + * 本地视频预览 + */ + private SurfaceViewRenderer localVideoRenderer; /** * PeerConnectionFactory */ private PeerConnectionFactory peerConnectionFactory; - /** - * 媒体录制 - */ - private final MediaRecorder mediaRecorder; static { // 采样 @@ -151,14 +192,14 @@ public class MediaManager { for (MediaCodecInfo mediaCodecInfo : mediaCodecList.getCodecInfos()) { // if (mediaCodecInfo.isEncoder()) { final String[] supportedTypes = mediaCodecInfo.getSupportedTypes(); - Log.d(MediaRecorder.class.getSimpleName(), "编码器名称：" + mediaCodecInfo.getName()); - Log.d(MediaRecorder.class.getSimpleName(), "编码器类型：" + String.join(" , ", supportedTypes)); + Log.d(RecordClient.class.getSimpleName(), "编码器名称：" + mediaCodecInfo.getName()); + Log.d(RecordClient.class.getSimpleName(), "编码器类型：" + String.join(" , ", supportedTypes)); for (String supportType : supportedTypes) { final MediaCodecInfo.CodecCapabilities codecCapabilities = mediaCodecInfo.getCapabilitiesForType(supportType); final int[] colorFormats = codecCapabilities.colorFormats; - Log.d(MediaRecorder.class.getSimpleName(), "编码器格式：" + codecCapabilities.getMimeType()); + Log.d(RecordClient.class.getSimpleName(), "编码器格式：" + codecCapabilities.getMimeType()); // MediaCodecInfo.CodecCapabilities.COLOR_* - Log.d(MediaRecorder.class.getSimpleName(), "编码器支持格式：" + IntStream.of(colorFormats).boxed().map(String::valueOf).collect(Collectors.joining(" , "))); + Log.d(RecordClient.class.getSimpleName(), "编码器支持格式：" + IntStream.of(colorFormats).boxed().map(String::valueOf).collect(Collectors.joining(" , "))); } // } } @@ -166,25 +207,57 @@ public class MediaManager { private MediaManager() { this.clientCount = 0; - this.mediaRecorder = MediaRecorder.getInstance(); } /** - * 初始化上下文 - * - * @param handler Handler - * @param context 上下文 + * @param handler Handler + * @param context 上下文 + * @param preview 是否预览视频 + * @param playAudio 是否播放音频 + * @param playVideo 是否播放视频 + * @param audioConsume 是否消费音频 + * @param videoConsume 是否消费视频 + * @param audioProduce 是否生产音频 + * @param videoProduce 是否生产视频 */ - public void initContext(Handler handler, Context context) { + public void initContext( + Handler handler, Context context, + boolean preview, boolean playAudio, boolean playVideo, + boolean audioConsume, boolean videoConsume, + boolean audioProduce, boolean videoProduce, + TransportType transportType + ) { this.handler = handler; this.context = context; + this.preview = preview; + this.playAudio = playAudio; + this.playVideo = playVideo; + this.audioConsume = audioConsume; + this.videoConsume = videoConsume; + this.audioProduce = audioProduce; + this.videoProduce = videoProduce; + this.transportType = transportType; PeerConnectionFactory.initialize( PeerConnectionFactory.InitializationOptions.builder(this.context) -// .setEnableInternalTracer(true) - .createInitializationOptions() +// .setEnableInternalTracer(true) + .createInitializationOptions() ); } + /** + * @param taoyao 信令 + */ + public void initTaoyao(ITaoyao taoyao) { + this.taoyao = taoyao; + } + + /** + * @return 是否可用 + */ + public boolean available() { + return this.handler != null && this.context != null && this.taoyao != null; + } + /** * 新建终端 * 第一个终端进入时没有初始化时，初始化所有资源。 @@ -195,7 +268,7 @@ public class MediaManager { */ public PeerConnectionFactory newClient(Type type) { synchronized (this) { - if (this.clientCount <= 0 && !MediaRecorder.getInstance().isActive()) { + if (this.clientCount <= 0) { this.initMedia(type); this.nativeInit(); } @@ -221,24 +294,32 @@ public class MediaManager { } } - public void initRecord(Type type) { + public boolean isRecording() { + return this.recordClient != null; + } + + public RecordClient startRecord(String path, String filename) { synchronized (this) { - if(this.clientCount <= 0 && !MediaRecorder.getInstance().isActive()) { - this.initMedia(type); - this.nativeInit(); - } + this.recordClient = new RecordClient(path, filename, this.handler, this.taoyao); + this.recordClient.start(); + return this.recordClient; } } public void stopRecord() { synchronized (this) { - if(this.clientCount <= 0) { - this.close(); - this.nativeStop(); - } + this.recordClient.close(); + this.recordClient = null; } } + /** + * @return 照片路径 + */ + public String photograph() { + return null; + } + /** * 加载媒体 * @@ -253,7 +334,11 @@ public class MediaManager { final JavaAudioDeviceModule javaAudioDeviceModule = JavaAudioDeviceModule.builder(this.context) // .setAudioSource(android.media.MediaRecorder.AudioSource.MIC) // 本地音频 - .setSamplesReadyCallback(MediaRecorder.getInstance().audioRecoder) + .setSamplesReadyCallback(audioSamples -> { + if(this.recordClient != null) { + this.recordClient.putAudio(audioSamples); + } + }) // 远程音频 // .setAudioTrackStateCallback() // .setUseHardwareNoiseSuppressor(true) @@ -293,7 +378,6 @@ public class MediaManager { @Override public void onCameraSwitchDone(boolean success) { } - @Override public void onCameraSwitchError(String message) { } @@ -385,7 +469,6 @@ public class MediaManager { * 加载视频 */ private void initVideoTrack() { - final SurfaceViewRenderer surfaceViewRenderer = this.localPreview(); // 加载视频 final SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("MediaVideoThread", this.eglBase.getEglBaseContext()); // surfaceTextureHelper.setTextureSize(); @@ -396,8 +479,15 @@ public class MediaManager { this.videoCapturer.initialize(surfaceTextureHelper, this.context, videoSource.getCapturerObserver()); this.videoCapturer.startCapture(480, 640, 30); final VideoTrack videoTrack = this.peerConnectionFactory.createVideoTrack("ARDAMSv0", videoSource); - videoTrack.addSink(surfaceViewRenderer); -// videoTrack.addSink(MediaRecorder.getInstance().videoRecoder); + if(preview) { + this.localVideoRenderer = this.localVideoRenderer(); + videoTrack.addSink(this.localVideoRenderer); + } + videoTrack.addSink(videoFrame -> { + if(this.recordClient != null) { + this.recordClient.putVideo(videoFrame); + } + }); videoTrack.setEnabled(true); this.mediaStream.addTrack(videoTrack); Log.i(MediaManager.class.getSimpleName(), "加载视频：" + videoTrack.id()); @@ -407,7 +497,7 @@ public class MediaManager { return this.mediaStream; } - private SurfaceViewRenderer localPreview() { + private SurfaceViewRenderer localVideoRenderer() { // 设置预览 final SurfaceViewRenderer surfaceViewRenderer = new SurfaceViewRenderer(this.context); this.handler.post(() -> { @@ -437,7 +527,7 @@ public class MediaManager { return surfaceViewRenderer; } - public void remotePreview(final MediaStream mediaStream) { + public void remoteVideoRenderer(final MediaStream mediaStream) { // 设置预览 final SurfaceViewRenderer surfaceViewRenderer = new SurfaceViewRenderer(this.context); this.handler.post(() -> { @@ -600,6 +690,5 @@ public class MediaManager { private native void nativeInit(); private native void nativeStop(); - public native long nativeNewRoom(String roomId); } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/Room.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/Room.java deleted file mode 100644 index 6a02fbb..0000000 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/Room.java +++ /dev/null @@ -1,102 +0,0 @@ -package com.acgist.taoyao.media; - -import android.provider.MediaStore; -import android.util.Log; - -import com.acgist.taoyao.boot.model.Message; -import com.acgist.taoyao.boot.utils.JSONUtils; -import com.acgist.taoyao.boot.utils.MapUtils; -import com.acgist.taoyao.signal.ITaoyao; - -import org.webrtc.PeerConnection; -import org.webrtc.PeerConnectionFactory; - -import java.io.Closeable; -import java.util.ArrayList; -import java.util.List; - -/** - * 房间 - * - * @author acgist - */ -public class Room implements Closeable { - - private final String roomId; - private final String password; - private final long nativePointer; - private final ITaoyao taoyao; - private final MediaManager mediaManager; - private volatile boolean enter; - private PeerConnection.RTCConfiguration rtcConfiguration; - private PeerConnectionFactory peerConnectionFactory; - - public Room( - String roomId, String password, - boolean audioConsume, boolean videoConsume, - boolean audioProduce, boolean videoProduce, - long nativePointer, ITaoyao taoyao - ) { - this.roomId = roomId; - this.password = password; - this.nativePointer = nativePointer; - this.taoyao = taoyao; - this.mediaManager = MediaManager.getInstance(); - this.enter = false; - } - - /** - * 远程终端列表 - */ - private List remoteClientList; - - @Override - public void close() { - Log.i(Room.class.getSimpleName(), "关闭房间：" + this.roomId); - this.mediaManager.closeClient(); - this.remoteClientList.forEach(RemoteClient::close); - } - - public synchronized void enter() { - if(this.enter) { - return; - } - final Message response = this.taoyao.request(this.taoyao.buildMessage("media::router::rtp::capabilities", "roomId", this.roomId)); - if(response == null) { - return; - } - // STUN | TURN - final List iceServers = new ArrayList<>(); - // TODO：读取配置 - final PeerConnection.IceServer iceServer = PeerConnection.IceServer.builder("stun:stun1.l.google.com:19302").createIceServer(); - iceServers.add(iceServer); - this.rtcConfiguration = new PeerConnection.RTCConfiguration(iceServers); - - this.rtcConfiguration.screencastMinBitrate = 100; - this.rtcConfiguration.enableDtlsSrtp = true; - - this.peerConnectionFactory = this.mediaManager.newClient(MediaManager.Type.BACK); - final Object rtpCapabilities = MapUtils.get(response.body(), "rtpCapabilities"); - this.nativeLoad(this.nativePointer, JSONUtils.toJSON(rtpCapabilities), this.peerConnectionFactory.getNativePeerConnectionFactory(), this.rtcConfiguration); - } - - public void enterCallback(String rtpCapabilities, String sctpCapabilities) { - this.taoyao.request(this.taoyao.buildMessage( - "room::enter", - "roomId", this.roomId, - "password", this.password, - "rtpCapabilities", rtpCapabilities, - "sctpCapabilities", sctpCapabilities - )); - this.enter = true; - } - - public void produceMedia() { - } - - private native void nativeLoad(long nativePointer, String rtpCapabilities, long peerConnectionFactory, PeerConnection.RTCConfiguration rtcConfiguration); - private native void nativeNewClient(); - private native void nativeCloseClient(); - private native void nativeCloseRoom(long nativePointer); - -} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RoomClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RoomClient.java deleted file mode 100644 index bf2cfe6..0000000 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RoomClient.java +++ /dev/null @@ -1,46 +0,0 @@ -package com.acgist.taoyao.media; - -import android.util.Log; - -import org.webrtc.AudioTrack; -import org.webrtc.MediaStream; -import org.webrtc.VideoTrack; - -import java.io.Closeable; -import java.io.IOException; -import java.util.logging.Handler; - -/** - * 房间终端 - * 使用SDK + NDK + Mediasoup实现多人会话 - * - * @author acgist - */ -public class RoomClient implements Closeable { - - protected final String name; - protected final String clientId; - protected final Handler handler; - protected AudioTrack audioTrack; - protected VideoTrack videoTrack; - protected MediaStream mediaStream; - - public RoomClient(String name, String clientId, Handler handler) { - this.name = name; - this.clientId = clientId; - this.handler = handler; - } - - /** - * 打开预览 - */ - private void preview() { - - } - - @Override - public void close() { - Log.i(Room.class.getSimpleName(), "关闭终端：" + this.clientId); - } - -} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java new file mode 100644 index 0000000..5a31ee2 --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java @@ -0,0 +1,18 @@ +package com.acgist.taoyao.media; + +/** + * 路由回调 + * + * @author acgist + */ +public interface RouterCallback { + + default void enterCallback() {}; + default void newRemoteClientCallback() {}; + default void closeRemoteClientCallback() {}; + default void consumerPauseCallback() {}; + default void consumerResumeCallback() {}; + default void producerPauseCallback() {}; + default void producerResumeCallback() {}; + +} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/TransportType.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/TransportType.java new file mode 100644 index 0000000..e298321 --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/TransportType.java @@ -0,0 +1,20 @@ +package com.acgist.taoyao.media; + +/** + * 传输类型 + * + * @author acgist + */ +public enum TransportType { + + /** + * RTP + * 注意：只能监控 + */ + RTP, + /** + * WebRTC + */ + WEBRTC; + +} \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/VoiceChangerProcesser.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/VoiceChangerProcesser.java new file mode 100644 index 0000000..b47b1e1 --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/VoiceChangerProcesser.java @@ -0,0 +1,9 @@ +package com.acgist.taoyao.media.audio; + +/** + * 变声处理器 + * + * @author acgist + */ +public class VoiceChangerProcesser { +} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Client.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Client.java new file mode 100644 index 0000000..7eaf428 --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Client.java @@ -0,0 +1,52 @@ +package com.acgist.taoyao.media.client; + +import android.os.Handler; +import android.util.Log; + +import com.acgist.taoyao.media.MediaManager; +import com.acgist.taoyao.media.signal.ITaoyao; + +import java.io.Closeable; + +/** + * 终端 + * + * @author acgist + */ +public abstract class Client implements Closeable { + + /** + * 终端名称 + */ + protected final String name; + /** + * 终端ID + */ + protected final String clientId; + /** + * Handler + */ + protected final Handler handler; + /** + * 信令通道 + */ + protected final ITaoyao taoyao; + /** + * 媒体服务 + */ + protected final MediaManager mediaManager; + + public Client(String name, String clientId, Handler handler, ITaoyao taoyao) { + this.name = name; + this.clientId = clientId; + this.taoyao = taoyao; + this.handler = handler; + this.mediaManager = MediaManager.getInstance(); + } + + @Override + public void close() { + Log.i(this.getClass().getSimpleName(), "关闭终端：" + this.clientId); + } + +} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java new file mode 100644 index 0000000..4742dbf --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java @@ -0,0 +1,23 @@ +package com.acgist.taoyao.media.client; + +import android.os.Handler; + +import com.acgist.taoyao.media.signal.ITaoyao; + +/** + * 房间本地终端 + * + * @author acgist + */ +public class LocalClient extends RoomClient { + + public LocalClient(String name, String clientId, Handler handler, ITaoyao taoyao) { + super(name, clientId, handler, taoyao); + } + + @Override + public void close() { + super.close(); + } + +} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaRecorder.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java similarity index 61% rename from taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaRecorder.java rename to taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java index 95f3453..3b809c9 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaRecorder.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java @@ -1,9 +1,8 @@ -package com.acgist.taoyao.media; +package com.acgist.taoyao.media.client; import android.media.AudioFormat; import android.media.MediaCodec; import android.media.MediaCodecInfo; -import android.media.MediaCodecList; import android.media.MediaFormat; import android.media.MediaMuxer; import android.os.Environment; @@ -11,19 +10,13 @@ import android.os.Handler; import android.os.HandlerThread; import android.util.Log; -import com.acgist.mediasoup.R; +import com.acgist.taoyao.media.MediaManager; +import com.acgist.taoyao.media.signal.ITaoyao; -import org.webrtc.JavaI420Buffer; -import org.webrtc.RtpSender; -import org.webrtc.RtpTransceiver; import org.webrtc.TextureBufferImpl; -import org.webrtc.ThreadUtils; import org.webrtc.VideoFrame; -import org.webrtc.VideoSink; -import org.webrtc.YuvConverter; import org.webrtc.YuvHelper; import org.webrtc.audio.JavaAudioDeviceModule; -import org.webrtc.voiceengine.WebRtcAudioRecord; import java.io.File; import java.io.IOException; @@ -32,8 +25,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import java.util.stream.Collectors; -import java.util.stream.IntStream; /** * 录像机 @@ -42,78 +33,76 @@ import java.util.stream.IntStream; * * @author acgist */ -public final class MediaRecorder { - - private static final MediaRecorder INSTANCE = new MediaRecorder(); - - public static final MediaRecorder getInstance() { - return INSTANCE; - } +public class RecordClient extends Client { /** * 是否正在录像 */ private volatile boolean active; + /** + * 音频准备录制 + */ private volatile boolean audioActive; + /** + * 视频准备录制 + */ private volatile boolean videoActive; + /** + * 录制时间戳 + */ private volatile long pts; - private String file; + /** + * 录制文件名称 + */ + private final String filename; + /** + * 录制文件路径 + */ + private final String filepath; /** * 音频编码 */ private MediaCodec audioCodec; + /** + * 音频处理线程 + */ private HandlerThread audioThread; + /** + * 音频Handler + */ + private Handler audioHandler; /** * 视频编码 */ private MediaCodec videoCodec; + /** + * 视频处理线程 + */ private HandlerThread videoThread; + /** + * 视频Handler + */ private Handler videoHandler; - private ExecutorService executorService; /** * 媒体合成器 */ private MediaMuxer mediaMuxer; /** - * 音频录制 + * 线程池 */ - public final JavaAudioDeviceModule.SamplesReadyCallback audioRecoder; - /** - * 视频录制 - */ - public final VideoSink videoRecoder; + private final ExecutorService executorService; - private MediaRecorder() { + public RecordClient(String path, String filename, Handler handler, ITaoyao taoyao) { + super("本地录像", "LocalRecordClient", handler, taoyao); + this.filename = filename; + final Path filePath = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath(), path, filename); + final File parentFile = filePath.getParent().toFile(); + if(!parentFile.exists()) { + parentFile.mkdirs(); + } + this.filepath = filePath.toString(); + Log.i(RecordClient.class.getSimpleName(), "录制视频文件：" + this.filepath); this.executorService = Executors.newFixedThreadPool(2); - this.audioRecoder = audioSamples -> { - }; - this.videoRecoder = videoFrame -> { - if (this.active && this.videoActive) { - this.executorService.submit(() -> { -// TextureBufferImpl -// videoFrame.retain(); - final TextureBufferImpl buffer = (TextureBufferImpl) videoFrame.getBuffer(); - final int outputFrameSize = videoFrame.getRotatedWidth() * videoFrame.getRotatedHeight() * 3 / 2; - final ByteBuffer outputFrameBuffer = ByteBuffer.allocateDirect(outputFrameSize); - final int index = this.videoCodec.dequeueInputBuffer(1000L * 1000); -// YuvImage：截图 - // YV12 - VideoFrame.I420Buffer i420 = videoFrame.getBuffer().toI420(); -// i420.retain(); - Log.i(MediaRecorder.class.getSimpleName(), "视频信息：" + videoFrame.getRotatedWidth() + " - " + videoFrame.getRotatedHeight()); -// YuvHelper.I420Copy(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight()); - // NV12 - YuvHelper.I420ToNV12(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight()); -// YuvHelper.I420Rotate(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight(), videoFrame.getRotation()); - final ByteBuffer x = this.videoCodec.getInputBuffer(index); -// i420.release(); - x.put(outputFrameBuffer.array()); - this.videoCodec.queueInputBuffer(index, 0, outputFrameSize, System.currentTimeMillis(), 0); -// this.putVideo(outputFrameBuffer, System.currentTimeMillis()); -// videoFrame.release(); - }); - } - }; } /** @@ -123,26 +112,24 @@ public final class MediaRecorder { return this.active; } - public void record(String path) { - if(this.active) { - return; + public void start() { + synchronized (this) { + if(this.active) { + return; + } + this.active = true; + this.record(null, null, 1, 1); } - this.record(path, System.currentTimeMillis() + ".mp4", null, null, 1, 1); } - public void record(String path, String file, String audioFormat, String videoFormat, int width, int height) { - synchronized (MediaRecorder.INSTANCE) { - MediaManager.getInstance().initRecord(MediaManager.Type.BACK); - this.file = file; - this.active = true; - if ( - this.audioThread == null || !this.audioThread.isAlive() || - this.videoThread == null || !this.videoThread.isAlive() - ) { - this.initMediaMuxer(path, file); - this.initAudioThread(MediaFormat.MIMETYPE_AUDIO_AAC, 96000, 44100, 1); - this.initVideoThread(MediaFormat.MIMETYPE_VIDEO_AVC, 2500 * 1000, 30, 1, 1920, 1080); - } + private void record(String audioFormat, String videoFormat, int width, int height) { + if ( + this.audioThread == null || !this.audioThread.isAlive() || + this.videoThread == null || !this.videoThread.isAlive() + ) { + this.initMediaMuxer(); + this.initAudioThread(MediaFormat.MIMETYPE_AUDIO_AAC, 96000, 44100, 1); + this.initVideoThread(MediaFormat.MIMETYPE_VIDEO_AVC, 2500 * 1000, 30, 1, 1920, 1080); } } @@ -163,7 +150,7 @@ public final class MediaRecorder { audioFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 8 * 1024); this.audioCodec.configure(audioFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); } catch (Exception e) { - Log.e(MediaRecorder.class.getSimpleName(), "加载音频录制线程异常", e); + Log.e(RecordClient.class.getSimpleName(), "加载音频录制线程异常", e); } final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); this.audioThread = new HandlerThread("AudioRecoderThread"); @@ -178,17 +165,17 @@ public final class MediaRecorder { outputIndex = this.audioCodec.dequeueOutputBuffer(info, 1000L * 1000); if (outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { } else if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { - synchronized (MediaRecorder.INSTANCE) { + synchronized (this) { trackIndex = this.mediaMuxer.addTrack(this.audioCodec.getOutputFormat()); - Log.i(MediaRecorder.class.getSimpleName(), "开始录制音频：" + trackIndex); + Log.i(RecordClient.class.getSimpleName(), "开始录制音频：" + trackIndex); if (this.videoActive) { - Log.i(MediaRecorder.class.getSimpleName(), "开始录制文件：" + this.file); + Log.i(RecordClient.class.getSimpleName(), "开始录制文件：" + this.filename); this.pts = System.currentTimeMillis(); this.mediaMuxer.start(); - MediaRecorder.INSTANCE.notifyAll(); + this.notifyAll(); } else if (this.active) { try { - MediaRecorder.INSTANCE.wait(); + this.wait(); } catch (InterruptedException e) { } } @@ -202,16 +189,16 @@ public final class MediaRecorder { this.audioCodec.releaseOutputBuffer(outputIndex, false); } } - synchronized (MediaRecorder.INSTANCE) { + synchronized (this) { if (this.audioCodec != null) { - Log.i(MediaRecorder.class.getSimpleName(), "结束录制音频"); + Log.i(RecordClient.class.getSimpleName(), "结束录制音频"); this.audioCodec.stop(); this.audioCodec.release(); this.audioCodec = null; } this.audioActive = false; if (this.mediaMuxer != null && !this.videoActive) { - Log.i(MediaRecorder.class.getSimpleName(), "结束录制文件：" + this.file); + Log.i(RecordClient.class.getSimpleName(), "结束录制文件：" + this.filename); this.mediaMuxer.stop(); this.mediaMuxer.release(); this.mediaMuxer = null; @@ -220,8 +207,46 @@ public final class MediaRecorder { }); } - public void putAudio(byte[] bytes) { + public void putAudio(JavaAudioDeviceModule.AudioSamples audioSamples) { + if(this.active && this.audioActive) { + } + } + public void putVideo(VideoFrame videoFrame) { + if (this.active && this.videoActive) { + this.executorService.submit(() -> { +// TextureBufferImpl +// videoFrame.retain(); + final TextureBufferImpl buffer = (TextureBufferImpl) videoFrame.getBuffer(); + final int outputFrameSize = videoFrame.getRotatedWidth() * videoFrame.getRotatedHeight() * 3 / 2; + final ByteBuffer outputFrameBuffer = ByteBuffer.allocateDirect(outputFrameSize); + final int index = this.videoCodec.dequeueInputBuffer(1000L * 1000); +// YuvImage：截图 + // YV12 + VideoFrame.I420Buffer i420 = videoFrame.getBuffer().toI420(); +// i420.retain(); + Log.i(RecordClient.class.getSimpleName(), "视频信息：" + videoFrame.getRotatedWidth() + " - " + videoFrame.getRotatedHeight()); +// YuvHelper.I420Copy(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight()); + // NV12 + YuvHelper.I420ToNV12(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight()); +// YuvHelper.I420Rotate(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight(), videoFrame.getRotation()); + final ByteBuffer x = this.videoCodec.getInputBuffer(index); +// i420.release(); + x.put(outputFrameBuffer.array()); + this.videoCodec.queueInputBuffer(index, 0, outputFrameSize, System.currentTimeMillis(), 0); +// this.putVideo(outputFrameBuffer, System.currentTimeMillis()); +// videoFrame.release(); +// while (this.active && this.videoActive) { +// final int index = this.videoCodec.dequeueInputBuffer(1000L * 1000); +// if (index < 0) { +// continue; +// } +// final ByteBuffer byteBuffer = this.videoCodec.getInputBuffer(index); +// byteBuffer.put(buffer); +// this.videoCodec.queueInputBuffer(index, 0, buffer.capacity(), pts, 0); +// } + }); + } } /** @@ -245,7 +270,7 @@ public final class MediaRecorder { videoFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1); this.videoCodec.configure(videoFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); } catch (Exception e) { - Log.e(MediaRecorder.class.getSimpleName(), "加载视频录制线程异常", e); + Log.e(RecordClient.class.getSimpleName(), "加载视频录制线程异常", e); } final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); this.videoThread = new HandlerThread("VideoRecoderThread"); @@ -260,17 +285,17 @@ public final class MediaRecorder { outputIndex = this.videoCodec.dequeueOutputBuffer(info, 1000L * 1000); if (outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { } else if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { - synchronized (MediaRecorder.INSTANCE) { + synchronized (this) { trackIndex = this.mediaMuxer.addTrack(this.videoCodec.getOutputFormat()); - Log.i(MediaRecorder.class.getSimpleName(), "开始录制视频：" + trackIndex); + Log.i(RecordClient.class.getSimpleName(), "开始录制视频：" + trackIndex); if (this.audioActive) { - Log.i(MediaRecorder.class.getSimpleName(), "开始录制文件：" + this.file); + Log.i(RecordClient.class.getSimpleName(), "开始录制文件：" + this.filename); this.pts = System.currentTimeMillis(); this.mediaMuxer.start(); - MediaRecorder.INSTANCE.notifyAll(); + this.notifyAll(); } else if (this.active) { try { - MediaRecorder.INSTANCE.wait(); + this.wait(); } catch (InterruptedException e) { } } @@ -282,7 +307,7 @@ public final class MediaRecorder { info.presentationTimeUs = (info.presentationTimeUs - this.pts) * 1000; this.mediaMuxer.writeSampleData(trackIndex, outputBuffer, info); this.videoCodec.releaseOutputBuffer(outputIndex, false); - Log.d(MediaRecorder.class.getSimpleName(), "录制视频帧（时间戳）：" + (info.presentationTimeUs / 1000000F)); + Log.d(RecordClient.class.getSimpleName(), "录制视频帧（时间戳）：" + (info.presentationTimeUs / 1000000F)); // if(info.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME) { // } else if(info.flags == MediaCodec.BUFFER_FLAG_CODEC_CONFIG) { // } else if(info.flags == MediaCodec.BUFFER_FLAG_END_OF_STREAM) { @@ -290,16 +315,16 @@ public final class MediaRecorder { // } } } - synchronized (MediaRecorder.INSTANCE) { + synchronized (this) { if (this.videoCodec != null) { - Log.i(MediaRecorder.class.getSimpleName(), "结束录制视频"); + Log.i(RecordClient.class.getSimpleName(), "结束录制视频"); this.videoCodec.stop(); this.videoCodec.release(); this.videoCodec = null; } this.videoActive = false; if (this.mediaMuxer != null && !this.audioActive) { - Log.i(MediaRecorder.class.getSimpleName(), "结束录制文件：" + this.file); + Log.i(RecordClient.class.getSimpleName(), "结束录制文件：" + this.filename); this.mediaMuxer.stop(); this.mediaMuxer.release(); this.mediaMuxer = null; @@ -308,27 +333,10 @@ public final class MediaRecorder { }); } - public void putVideo(ByteBuffer buffer, long pts) { - while (this.active && this.videoActive) { - final int index = this.videoCodec.dequeueInputBuffer(1000L * 1000); - if (index < 0) { - continue; - } - final ByteBuffer byteBuffer = this.videoCodec.getInputBuffer(index); - byteBuffer.put(buffer); - this.videoCodec.queueInputBuffer(index, 0, buffer.capacity(), pts, 0); - } - } - - private void initMediaMuxer(String path, String file) { + private void initMediaMuxer() { try { - final Path filePath = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath(), path, file); - final File parentFile = filePath.getParent().toFile(); - if(!parentFile.exists()) { - parentFile.mkdirs(); - } this.mediaMuxer = new MediaMuxer( - filePath.toAbsolutePath().toString(), + this.filepath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4 ); // 设置方向 @@ -338,23 +346,28 @@ public final class MediaRecorder { } } - public void stop() { - synchronized (MediaRecorder.INSTANCE) { - Log.i(MediaRecorder.class.getSimpleName(), "结束录制：" + this.file); - this.active = false; - if (audioThread != null) { - this.audioThread.quitSafely(); - this.audioThread = null; - } - if (this.videoThread != null) { - this.videoThread.quitSafely(); - this.videoThread = null; - } - if(this.executorService != null) { - this.executorService.shutdown(); - this.executorService = null; - } - MediaRecorder.INSTANCE.notifyAll(); + private void stop() { + Log.i(RecordClient.class.getSimpleName(), "结束录制：" + this.filepath); + this.active = false; + if (audioThread != null) { + this.audioThread.quitSafely(); + } + if (this.videoThread != null) { + this.videoThread.quitSafely(); + } + if(this.executorService != null) { + this.executorService.shutdown(); + } + synchronized (this) { + this.notifyAll(); + } + } + + @Override + public void close() { + synchronized (this) { + super.close(); + this.stop(); } } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RemoteClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java similarity index 56% rename from taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RemoteClient.java rename to taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java index 9d9cd72..7d5ce9f 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RemoteClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java @@ -1,8 +1,8 @@ -package com.acgist.taoyao.media; +package com.acgist.taoyao.media.client; -import java.io.Closeable; -import java.io.IOException; -import java.util.logging.Handler; +import android.os.Handler; + +import com.acgist.taoyao.media.signal.ITaoyao; /** * 房间远程终端 @@ -11,8 +11,8 @@ import java.util.logging.Handler; */ public class RemoteClient extends RoomClient { - public RemoteClient(String name, String clientId, Handler handler) { - super(name, clientId, handler); + public RemoteClient(String name, String clientId, Handler handler, ITaoyao taoyao) { + super(name, clientId, handler, taoyao); } @Override diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java new file mode 100644 index 0000000..e7b5b84 --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java @@ -0,0 +1,176 @@ +package com.acgist.taoyao.media.client; + +import android.os.Handler; +import android.util.Log; + +import com.acgist.taoyao.boot.model.Message; +import com.acgist.taoyao.boot.utils.JSONUtils; +import com.acgist.taoyao.boot.utils.MapUtils; +import com.acgist.taoyao.media.MediaManager; +import com.acgist.taoyao.media.RouterCallback; +import com.acgist.taoyao.media.signal.ITaoyao; + +import org.webrtc.PeerConnection; +import org.webrtc.PeerConnectionFactory; + +import java.io.Closeable; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; + +/** + * 房间 + * + * @author acgist + */ +public class Room implements Closeable, RouterCallback { + + private final String name; + private final String clientId; + private final String roomId; + private final String password; + private final Handler handler; + private final ITaoyao taoyao; + private final boolean dataConsume; + private final boolean audioConsume; + private final boolean videoConsume; + private final boolean dataProduce; + private final boolean audioProduce; + private final boolean videoProduce; + private final long nativeRoomPointer; + private final MediaManager mediaManager; + private volatile boolean enter; + private LocalClient localClient; + private List remoteClients; + private PeerConnection.RTCConfiguration rtcConfiguration; + private PeerConnectionFactory peerConnectionFactory; + private String sctpCapabilities; + + public Room( + String name, String clientId, + String roomId, String password, + Handler handler, ITaoyao taoyao, + boolean dataConsume, boolean audioConsume, boolean videoConsume, + boolean dataProduce, boolean audioProduce, boolean videoProduce + ) { + this.name = name; + this.clientId = clientId; + this.roomId = roomId; + this.password = password; + this.handler = handler; + this.taoyao = taoyao; + this.dataConsume = dataConsume; + this.audioConsume = audioConsume; + this.videoConsume = videoConsume; + this.dataProduce = dataProduce; + this.audioProduce = audioProduce; + this.videoProduce = videoProduce; + this.nativeRoomPointer = this.nativeNewRoom(roomId); + this.mediaManager = MediaManager.getInstance(); + this.remoteClients = new CopyOnWriteArrayList<>(); + this.enter = false; + } + + /** + * 远程终端列表 + */ + private List remoteClientList; + + public synchronized void enter() { + if(this.enter) { + return; + } + final Message response = this.taoyao.request(this.taoyao.buildMessage("media::router::rtp::capabilities", "roomId", this.roomId)); + if(response == null) { + Log.w(Room.class.getSimpleName(), "获取通道能力失败"); + return; + } + this.localClient = new LocalClient(this.name, this.clientId, this.handler, this.taoyao); + // STUN | TURN + final List iceServers = new ArrayList<>(); + // TODO：读取配置 + final PeerConnection.IceServer iceServer = PeerConnection.IceServer.builder("stun:stun1.l.google.com:19302").createIceServer(); + iceServers.add(iceServer); + this.rtcConfiguration = new PeerConnection.RTCConfiguration(iceServers); + this.peerConnectionFactory = this.mediaManager.newClient(MediaManager.Type.BACK); + final Object rtpCapabilities = MapUtils.get(response.body(), "rtpCapabilities"); + this.nativeLoad(this.nativeRoomPointer, JSONUtils.toJSON(rtpCapabilities), this.peerConnectionFactory.getNativePeerConnectionFactory(), this.rtcConfiguration); + } + + public void produceMedia() { + if(this.audioProduce || this.videoProduce) { + this.createSendTransport(); + } + if(this.audioConsume || this.videoConsume) { + this.createRecvTransport(); + } + + } + + private void createSendTransport() { + final Message response = this.taoyao.request(this.taoyao.buildMessage( + "media::transport::webrtc::create", + "roomId", this.roomId, + "forceTcp", false, + "producing", true, + "consuming", false, + "sctpCapabilities", this.dataProduce ? this.sctpCapabilities : null + )); + if(response == null) { + Log.w(Room.class.getSimpleName(), "创建发送通道失败"); + return; + } + final Map body = response.body(); + this.nativeCreateSendTransport(this.nativeRoomPointer, JSONUtils.toJSON(body)); + } + + private void createRecvTransport() { + } + + /** + * 新增远程终端 + * + * @param body 消息主体 + */ + public void newRemoteClient(Map body) { + final String clientId = MapUtils.get(body, "clientId"); + final Map status = MapUtils.get(body, "status"); + final String name = MapUtils.get(status, "name"); + final RemoteClient remoteClient = new RemoteClient(name, clientId, this.handler, this.taoyao); + this.remoteClients.add(remoteClient); + } + + @Override + public void close() { + Log.i(Room.class.getSimpleName(), "关闭房间：" + this.roomId); + this.localClient.close(); + this.remoteClientList.forEach(RemoteClient::close); + this.mediaManager.closeClient(); + this.nativeCloseRoom(this.nativeRoomPointer); + } + + public void enterCallback(String rtpCapabilities, String sctpCapabilities) { + this.taoyao.request(this.taoyao.buildMessage( + "room::enter", + "roomId", this.roomId, + "password", this.password, + "rtpCapabilities", rtpCapabilities, + "sctpCapabilities", sctpCapabilities + )); + this.enter = true; + } + + private native void nativeLoad( + long nativePointer, + String rtpCapabilities, + long peerConnectionFactoryPointer, + PeerConnection.RTCConfiguration rtcConfiguration + ); + private native long nativeNewRoom(String roomId); + private native void nativeCloseRoom(long nativePointer); + private native void nativeCreateSendTransport(long nativeRoomPointer, String body); + private native void nativeCreateRecvTransport(long nativeRoomPointer, String body); + private native void nativeProduceMedia(long nativeRoomPointer, long mediaStreamPointer); + +} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RoomClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RoomClient.java new file mode 100644 index 0000000..70bcd86 --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RoomClient.java @@ -0,0 +1,43 @@ +package com.acgist.taoyao.media.client; + +import android.os.Handler; + +import com.acgist.taoyao.media.signal.ITaoyao; + +import org.webrtc.MediaStream; + +/** + * 房间终端 + * 使用SDK + NDK + Mediasoup实现多人会话 + * + * @author acgist + */ +public class RoomClient extends Client { + + protected MediaStream mediaStream; + + public RoomClient(String name, String clientId, Handler handler, ITaoyao taoyao) { + super(name, clientId, handler, taoyao); + } + + /** + * 打开预览 + */ + private void preview() { + + } + + @Override + public void close() { + super.close(); + } + + public MediaStream getMediaStream() { + return mediaStream; + } + + public void setMediaStream(MediaStream mediaStream) { + this.mediaStream = mediaStream; + } + +} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/SessionClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java similarity index 69% rename from taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/SessionClient.java rename to taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java index 47ad065..8be4898 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/SessionClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java @@ -1,15 +1,15 @@ -package com.acgist.taoyao.media; +package com.acgist.taoyao.media.client; -import android.se.omapi.Session; +import android.os.Handler; import android.util.Log; import com.acgist.taoyao.boot.model.Message; import com.acgist.taoyao.boot.utils.MapUtils; -import com.acgist.taoyao.signal.ITaoyao; +import com.acgist.taoyao.media.MediaManager; +import com.acgist.taoyao.media.signal.ITaoyao; import org.webrtc.DataChannel; import org.webrtc.IceCandidate; -import org.webrtc.JniCommon; import org.webrtc.MediaConstraints; import org.webrtc.MediaStream; import org.webrtc.PeerConnection; @@ -17,10 +17,10 @@ import org.webrtc.PeerConnectionFactory; import org.webrtc.SdpObserver; import org.webrtc.SessionDescription; -import java.io.Closeable; import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; /** * P2P终端 @@ -30,25 +30,41 @@ import java.util.Map; * * @author acgist */ -public class SessionClient implements Closeable { +public class SessionClient extends Client { - private final String id; - private final String name; - private final String clientId; - private final ITaoyao taoyao; - private final MediaManager mediaManager; + /** + * 会话ID + */ + private final String sessionId; + /** + * 本地媒体 + */ private MediaStream mediaStream; + /** + * 远程媒体 + */ + private final List remoteMediaStreams; + /** + * SDPObserver + */ private SdpObserver sdpObserver; + /** + * Peer连接 + */ private PeerConnection peerConnection; + /** + * Peer连接Observer + */ private PeerConnection.Observer observer; + /** + * Peer连接工厂 + */ private PeerConnectionFactory peerConnectionFactory; - public SessionClient(String id, String name, String clientId, ITaoyao taoyao) { - this.id = id; - this.name = name; - this.clientId = clientId; - this.taoyao = taoyao; - this.mediaManager = MediaManager.getInstance(); + public SessionClient(String sessionId, String name, String clientId, Handler handler, ITaoyao taoyao) { + super(name, clientId, handler, taoyao); + this.sessionId = sessionId; + this.remoteMediaStreams = new CopyOnWriteArrayList<>(); } public void init() { @@ -60,9 +76,10 @@ public class SessionClient implements Closeable { iceServers.add(iceServer); final PeerConnection.RTCConfiguration configuration = new PeerConnection.RTCConfiguration(iceServers); this.observer = this.observer(); + this.mediaStream = this.mediaManager.getMediaStream(); this.sdpObserver = this.sdpObserver(); this.peerConnection = this.peerConnectionFactory.createPeerConnection(configuration, this.observer); - this.peerConnection.addStream(this.mediaManager.getMediaStream()); + this.peerConnection.addStream(this.mediaStream); } public void exchange(Message message, Map body) { @@ -78,6 +95,7 @@ public class SessionClient implements Closeable { public void call(String clientId) { final MediaConstraints mediaConstraints = new MediaConstraints(); this.peerConnection.createOffer(this.sdpObserver, mediaConstraints); + // TODO：实现主动拉取别人 } /** @@ -85,13 +103,13 @@ public class SessionClient implements Closeable { */ public void offer() { final MediaConstraints mediaConstraints = new MediaConstraints(); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("maxHeight", Integer.toString(1920))); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("maxWidth", Integer.toString(1080))); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("maxFrameRate", Integer.toString(15))); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("minFrameRate", Integer.toString(30))); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")); -// mediaConstraints.optional.add(new MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("maxHeight", Integer.toString(1920))); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("maxWidth", Integer.toString(1080))); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("maxFrameRate", Integer.toString(15))); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("minFrameRate", Integer.toString(30))); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")); +// mediaConstraints.optional.add(new MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true")); this.peerConnection.createOffer(this.sdpObserver, mediaConstraints); } @@ -102,13 +120,13 @@ public class SessionClient implements Closeable { final SessionDescription sessionDescription = new SessionDescription(sdpType, sdp); this.peerConnection.setRemoteDescription(this.sdpObserver, sessionDescription); final MediaConstraints mediaConstraints = new MediaConstraints(); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("maxHeight", Integer.toString(1920))); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("maxWidth", Integer.toString(1080))); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("maxFrameRate", Integer.toString(15))); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("minFrameRate", Integer.toString(30))); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")); -// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")); -// mediaConstraints.optional.add(new MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("maxHeight", Integer.toString(1920))); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("maxWidth", Integer.toString(1080))); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("maxFrameRate", Integer.toString(15))); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("minFrameRate", Integer.toString(30))); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveAudio", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("OfferToReceiveVideo", "true")); +// mediaConstraints.optional.add(new MediaConstraints.KeyValuePair("DtlsSrtpKeyAgreement", "true")); this.peerConnection.createAnswer(this.sdpObserver, mediaConstraints); } @@ -135,7 +153,12 @@ public class SessionClient implements Closeable { @Override public void close() { - Log.i(Room.class.getSimpleName(), "关闭终端：" + this.clientId); + super.close(); + // 本地资源释放 + this.mediaManager.closeClient(); + // 远程资源释放 + this.remoteMediaStreams.forEach(v -> v.dispose()); + this.remoteMediaStreams.clear(); } /** @@ -143,6 +166,7 @@ public class SessionClient implements Closeable { */ private PeerConnection.Observer observer() { return new PeerConnection.Observer() { + @Override public void onSignalingChange(PeerConnection.SignalingState signalingState) { } @@ -161,12 +185,12 @@ public class SessionClient implements Closeable { @Override public void onIceCandidate(IceCandidate iceCandidate) { - Log.d(SessionClient.class.getSimpleName(), "发送媒体协商：" + SessionClient.this.id); + Log.d(SessionClient.class.getSimpleName(), "发送媒体协商：" + SessionClient.this.sessionId); SessionClient.this.taoyao.push(SessionClient.this.taoyao.buildMessage( "session::exchange", "type", "candidate", "candidate", iceCandidate, - "sessionId", SessionClient.this.id + "sessionId", SessionClient.this.sessionId )); } @@ -178,11 +202,13 @@ public class SessionClient implements Closeable { public void onAddStream(MediaStream mediaStream) { Log.i(SessionClient.class.getSimpleName(), "添加远程媒体：" + SessionClient.this.clientId); SessionClient.this.mediaStream = mediaStream; - SessionClient.this.mediaManager.remotePreview(mediaStream); + SessionClient.this.mediaManager.remoteVideoRenderer(mediaStream); + SessionClient.this.remoteMediaStreams.add(mediaStream); } @Override public void onRemoveStream(MediaStream mediaStream) { + SessionClient.this.remoteMediaStreams.remove(mediaStream); } @Override @@ -191,32 +217,34 @@ public class SessionClient implements Closeable { @Override public void onRenegotiationNeeded() { - Log.d(SessionClient.class.getSimpleName(), "重新协商媒体：" + SessionClient.this.id); + Log.d(SessionClient.class.getSimpleName(), "重新协商媒体：" + SessionClient.this.sessionId); if(peerConnection.connectionState() == PeerConnection.PeerConnectionState.CONNECTED) { // TODO：重写协商 // SessionClient.this.offer(); } } + }; } private SdpObserver sdpObserver() { return new SdpObserver() { + @Override public void onCreateSuccess(SessionDescription sessionDescription) { - Log.d(SessionClient.class.getSimpleName(), "创建SDP成功：" + SessionClient.this.id); + Log.d(SessionClient.class.getSimpleName(), "创建SDP成功：" + SessionClient.this.sessionId); SessionClient.this.peerConnection.setLocalDescription(this, sessionDescription); SessionClient.this.taoyao.push(SessionClient.this.taoyao.buildMessage( "session::exchange", "sdp", sessionDescription.description, "type", sessionDescription.type.toString().toLowerCase(), - "sessionId", SessionClient.this.id + "sessionId", SessionClient.this.sessionId )); } @Override public void onSetSuccess() { - Log.d(SessionClient.class.getSimpleName(), "设置SDP成功：" + SessionClient.this.id); + Log.d(SessionClient.class.getSimpleName(), "设置SDP成功：" + SessionClient.this.sessionId); } @Override @@ -228,6 +256,7 @@ public class SessionClient implements Closeable { public void onSetFailure(String message) { Log.w(SessionClient.class.getSimpleName(), "设置SDP失败：" + message); } + }; } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/Config.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/Config.java similarity index 93% rename from taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/Config.java rename to taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/Config.java index 273dfe3..6232f23 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/Config.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/Config.java @@ -1,4 +1,4 @@ -package com.acgist.taoyao.config; +package com.acgist.taoyao.media.config; /** * 配置 diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/MediaAudioProperties.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/MediaAudioProperties.java similarity index 95% rename from taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/MediaAudioProperties.java rename to taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/MediaAudioProperties.java index 026a4cf..3beb8b1 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/MediaAudioProperties.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/MediaAudioProperties.java @@ -1,4 +1,4 @@ -package com.acgist.taoyao.config; +package com.acgist.taoyao.media.config; /** * 音频配置 diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/MediaProperties.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/MediaProperties.java similarity index 99% rename from taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/MediaProperties.java rename to taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/MediaProperties.java index 2f4fa05..37fb72a 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/MediaProperties.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/MediaProperties.java @@ -1,4 +1,4 @@ -package com.acgist.taoyao.config; +package com.acgist.taoyao.media.config; import java.util.Map; diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/MediaVideoProperties.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/MediaVideoProperties.java similarity index 97% rename from taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/MediaVideoProperties.java rename to taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/MediaVideoProperties.java index 28adcb2..ae4f54b 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/MediaVideoProperties.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/MediaVideoProperties.java @@ -1,4 +1,4 @@ -package com.acgist.taoyao.config; +package com.acgist.taoyao.media.config; /** * 视频配置 diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/signal/ITaoyao.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/signal/ITaoyao.java similarity index 94% rename from taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/signal/ITaoyao.java rename to taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/signal/ITaoyao.java index 7a3359f..2a0ce12 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/signal/ITaoyao.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/signal/ITaoyao.java @@ -1,4 +1,4 @@ -package com.acgist.taoyao.signal; +package com.acgist.taoyao.media.signal; import com.acgist.taoyao.boot.model.Message; diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/AiProcesser.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/AiProcesser.java new file mode 100644 index 0000000..5c7c898 --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/AiProcesser.java @@ -0,0 +1,9 @@ +package com.acgist.taoyao.media.video; + +/** + * AI处理器 + * + * @author acgist + */ +public class AiProcesser { +} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/BeautyProcesser.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/BeautyProcesser.java new file mode 100644 index 0000000..4997753 --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/BeautyProcesser.java @@ -0,0 +1,9 @@ +package com.acgist.taoyao.media.video; + +/** + * 美颜处理器 + * + * @author acgist + */ +public class BeautyProcesser { +} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/WatermarkProcesser.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/WatermarkProcesser.java new file mode 100644 index 0000000..a5c6499 --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/WatermarkProcesser.java @@ -0,0 +1,9 @@ +package com.acgist.taoyao.media.video; + +/** + * 水印处理器 + * + * @author acgist + */ +public class WatermarkProcesser { +} diff --git a/taoyao-client-web/src/components/Taoyao.js b/taoyao-client-web/src/components/Taoyao.js index 9b319c6..a9a33b3 100644 --- a/taoyao-client-web/src/components/Taoyao.js +++ b/taoyao-client-web/src/components/Taoyao.js @@ -1603,6 +1603,7 @@ class Taoyao extends RemoteClient { setTimeout(() => audioTrack.stop(), 30000); }); } + // TODO：代码提取 if (self.dataProduce || self.audioProduce || self.videoProduce) { const response = await self.request( protocol.buildMessage("media::transport::webrtc::create", { @@ -1635,9 +1636,10 @@ class Taoyao extends RemoteClient { proprietaryConstraints: { optional: [{ googDscp: true }], }, - additionalSettings: + additionalSettings: { // TODO：加密解密 - { encodedInsertableStreams: false }, + encodedInsertableStreams: false + }, }); self.sendTransport.on( "connect", @@ -1726,8 +1728,8 @@ class Taoyao extends RemoteClient { // indicating 'server' here and vice-versa. role: "auto", }, - iceServers: [], sctpParameters, + iceServers: [], additionalSettings: { // TODO：加密解密 encodedInsertableStreams: false,