From ce0afef889b12ee87080c4d981eb796aa4b3c751 Mon Sep 17 00:00:00 2001 From: acgist <289547414@qq.com> Date: Fri, 14 Apr 2023 08:35:54 +0800 Subject: [PATCH] =?UTF-8?q?[+]=20=E5=BD=95=E5=83=8F=E6=8B=8D=E7=85=A7?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 12 +- taoyao-client-android/README.md | 24 + .../com/acgist/taoyao/boot/model/Message.java | 4 - .../acgist/taoyao/boot/utils/JSONUtils.java | 7 +- .../acgist/taoyao/client/MainActivity.java | 85 +--- .../acgist/taoyao/client/MediaService.java | 18 + .../acgist/taoyao/client/TaoyaoReceiver.java | 1 - .../acgist/taoyao/client/signal/Taoyao.java | 45 +- .../src/main/res/layout/activity_main.xml | 47 +- .../client/src/main/res/values/settings.xml | 2 + .../media/src/main/cpp/include/Room.hpp | 4 +- .../src/main/cpp/include/RouterCallback.hpp | 1 + .../taoyao/media/src/main/cpp/webrtc/Room.cpp | 15 +- .../src/main/cpp/webrtc/RouterCallback.cpp | 16 +- .../com/acgist/taoyao/media/Broadcaster.java | 53 +++ .../com/acgist/taoyao/media/MediaManager.java | 194 +++++--- .../acgist/taoyao/media/RouterCallback.java | 1 + .../acgist/taoyao/media/client/Client.java | 12 +- .../taoyao/media/client/CloseableClient.java | 8 +- .../taoyao/media/client/LocalClient.java | 6 +- .../taoyao/media/client/PhotographClient.java | 113 ++++- .../taoyao/media/client/RecordClient.java | 435 +++++++++++------- .../taoyao/media/client/RemoteClient.java | 5 +- .../com/acgist/taoyao/media/client/Room.java | 14 +- .../taoyao/media/client/RoomClient.java | 6 +- .../taoyao/media/client/SessionClient.java | 6 +- .../taoyao/media/signal/ITaoyaoListener.java | 2 - taoyao-signal-server/README.md | 14 +- .../signal/protocol/ProtocolAdapter.java | 1 - .../protocol/ProtocolSessionAdapter.java | 1 - 30 files changed, 735 insertions(+), 417 deletions(-) create mode 100644 taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/Broadcaster.java diff --git a/README.md b/README.md index f9721f0..4839c4a 100644 --- a/README.md +++ b/README.md @@ -32,9 +32,9 @@ |功能|是否支持|是否实现|描述| |:--|:--|:--|:--| -|P2P|支持|实现|P2P监控模式| -|WebRTC|支持|实现|Web终端不能同时进入多个房间| -|控制|支持|实现|实现所有控制信令| +|P2P|支持|完成|P2P监控模式| +|WebRTC|支持|完成|Web终端不能同时进入多个房间| +|控制|支持|完成|实现所有控制信令| ### 安卓终端功能 @@ -43,9 +43,9 @@ |P2P|支持|实现|P2P监控模式| |WebRTC|支持|暂未实现|安卓终端支持同时进入多个房间| |RTP|支持|暂未实现|支持房间RTP推流（不会拉流）| -|控制|支持|实现|实现部分控制信令| -|拍照|支持|暂未实现|拍照| -|录像|支持|暂未实现|本地高清录制| +|控制|支持|完成|实现部分控制信令| +|拍照|支持|完成|拍照| +|录像|支持|完成|录制| |变声|支持|暂未实现|变声器| |水印|支持|暂未实现|视频水印| |美颜|支持|暂未实现|视频美颜| diff --git a/taoyao-client-android/README.md b/taoyao-client-android/README.md index 0810594..9cf0122 100644 --- a/taoyao-client-android/README.md +++ b/taoyao-client-android/README.md @@ -44,3 +44,27 @@ * `org.webrtc:google-webrtc` * `io.github.haiyangwu:mediasoup-client` + + +``` +/** + * YUV终端 + * + * Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y + * Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y + * Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y + * Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y + * U U U U U U V V V V V V U V U V U V V U V U V U + * V V V V V V U U U U U U U V U V U V V U V U V U + * - I420 - - YV12 - - NV12 - - NV21 - + * + * I420 = YUV420P = YU12 + * NV12 = YUV420SP + * + * RGB和YUV转换算法：BT.601（标清）、BT.709（高清）、BT.2020（超高清） + * + * YuvHelper + * + * @author acgist + */ +``` diff --git a/taoyao-client-android/taoyao/boot/src/main/java/com/acgist/taoyao/boot/model/Message.java b/taoyao-client-android/taoyao/boot/src/main/java/com/acgist/taoyao/boot/model/Message.java index 4905e1c..69c70b1 100644 --- a/taoyao-client-android/taoyao/boot/src/main/java/com/acgist/taoyao/boot/model/Message.java +++ b/taoyao-client-android/taoyao/boot/src/main/java/com/acgist/taoyao/boot/model/Message.java @@ -1,15 +1,11 @@ package com.acgist.taoyao.boot.model; -import android.util.Log; - import com.acgist.taoyao.boot.utils.JSONUtils; import com.fasterxml.jackson.annotation.JsonIncludeProperties; import org.apache.commons.lang3.StringUtils; import java.io.Serializable; -import java.util.HashMap; -import java.util.Map; /** diff --git a/taoyao-client-android/taoyao/boot/src/main/java/com/acgist/taoyao/boot/utils/JSONUtils.java b/taoyao-client-android/taoyao/boot/src/main/java/com/acgist/taoyao/boot/utils/JSONUtils.java index 5dd910f..38a9dcc 100644 --- a/taoyao-client-android/taoyao/boot/src/main/java/com/acgist/taoyao/boot/utils/JSONUtils.java +++ b/taoyao-client-android/taoyao/boot/src/main/java/com/acgist/taoyao/boot/utils/JSONUtils.java @@ -21,7 +21,12 @@ import java.text.SimpleDateFormat; import java.time.LocalDate; import java.time.LocalDateTime; import java.time.LocalTime; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TimeZone; /** * JSON工具 diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java index 05d74df..1b10d0a 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java @@ -16,9 +16,8 @@ import android.util.Log; import android.view.Display; import android.view.SurfaceView; import android.view.View; -import android.view.ViewGroup; import android.view.WindowManager; -import android.widget.LinearLayout; +import android.widget.GridLayout; import androidx.activity.result.ActivityResultLauncher; import androidx.activity.result.contract.ActivityResultContracts; @@ -26,15 +25,13 @@ import androidx.annotation.NonNull; import androidx.appcompat.app.AppCompatActivity; import androidx.core.app.ActivityCompat; -import com.acgist.taoyao.boot.utils.DateUtils; import com.acgist.taoyao.client.databinding.ActivityMainBinding; import com.acgist.taoyao.client.signal.Taoyao; +import com.acgist.taoyao.media.MediaManager; import com.acgist.taoyao.media.TransportType; import com.acgist.taoyao.media.config.Config; -import com.acgist.taoyao.media.MediaManager; import java.io.Serializable; -import java.time.LocalDateTime; import java.util.stream.Stream; /** @@ -46,7 +43,6 @@ public class MainActivity extends AppCompatActivity implements Serializable { private Handler threadHandler; private MainHandler mainHandler; - private HandlerThread handlerThread; private ActivityMainBinding binding; private MediaProjectionManager mediaProjectionManager; private ActivityResultLauncher activityResultLauncher; @@ -55,7 +51,6 @@ public class MainActivity extends AppCompatActivity implements Serializable { protected void onCreate(Bundle bundle) { Log.i(MainActivity.class.getSimpleName(), "onCreate"); super.onCreate(bundle); - this.catchAllException(); this.requestPermission(); this.launchMediaService(); this.setTurnScreenOn(true); @@ -64,11 +59,10 @@ public class MainActivity extends AppCompatActivity implements Serializable { this.binding = ActivityMainBinding.inflate(this.getLayoutInflater()); this.setContentView(this.binding.getRoot()); this.binding.getRoot().setZ(100F); - this.buildHandlerThread(); this.registerMediaProjection(); this.binding.action.setOnClickListener(this::action); - this.binding.record.setOnClickListener(this::switchRecord); - this.binding.settings.setOnClickListener(this::launchSettings); + this.binding.record.setOnClickListener(this::record); + this.binding.settings.setOnClickListener(this::settings); this.binding.photograph.setOnClickListener(this::photograph); } @@ -90,25 +84,6 @@ public class MainActivity extends AppCompatActivity implements Serializable { super.onDestroy(); } - private void catchAllException() { - Log.i(MainActivity.class.getSimpleName(), "全局异常捕获"); - final Thread.UncaughtExceptionHandler old = Thread.getDefaultUncaughtExceptionHandler(); - Thread.setDefaultUncaughtExceptionHandler(new Thread.UncaughtExceptionHandler() { - @Override - public void uncaughtException(Thread t, Throwable e) { - Log.e(MediaService.class.getSimpleName(), "系统异常：" + t.getName(), e); - final Looper looper = Looper.myLooper(); - if (looper == Looper.getMainLooper()) { -// if(t.getId() == Looper.getMainLooper().getThread().getId()) { - // TODO：重启应用 - old.uncaughtException(t, e); - } else { - // 子线程 - } - } - }); - } - /** * 请求权限 */ @@ -143,7 +118,6 @@ public class MainActivity extends AppCompatActivity implements Serializable { } int waitCount = 0; this.mainHandler = new MainHandler(this); - // 不能写在service里面： Attempt to invoke virtual method 'android.view.View android.view.Window.getDecorView()' on a null object reference final Resources resources = this.getResources(); MediaManager.getInstance().initContext( this.mainHandler, this.getApplicationContext(), @@ -172,15 +146,6 @@ public class MainActivity extends AppCompatActivity implements Serializable { } } - private void buildHandlerThread() { - if (this.handlerThread != null) { - return; - } - this.handlerThread = new HandlerThread("MainHandleThread"); - this.handlerThread.start(); - this.threadHandler = new Handler(this.handlerThread.getLooper()); - } - private void registerMediaProjection() { if (this.activityResultLauncher != null && this.mediaProjectionManager != null) { return; @@ -209,13 +174,18 @@ public class MainActivity extends AppCompatActivity implements Serializable { * @param view View */ private void action(View view) { + if (this.threadHandler == null) { + final HandlerThread handlerThread = new HandlerThread("ActionThread"); + handlerThread.start(); + this.threadHandler = new Handler(handlerThread.getLooper()); + } this.threadHandler.post(() -> { // 进入房间 Taoyao.taoyao.roomEnter("1e6707a5-6846-405e-95de-632aa01569aa", null); }); } - private void switchRecord(View view) { + private void record(View view) { final MediaManager mediaManager = MediaManager.getInstance(); if (mediaManager.isRecording()) { mediaManager.stopRecordVideoCapture(); @@ -224,17 +194,13 @@ public class MainActivity extends AppCompatActivity implements Serializable { } } - /** - * 拉起设置页面 - * - * @param view View - */ - private void launchSettings(View view) { + private void settings(View view) { final Intent intent = new Intent(this, SettingsActivity.class); this.startActivity(intent); } private void photograph(View view) { + MediaManager.getInstance().photograph(); } /** @@ -256,8 +222,8 @@ public class MainActivity extends AppCompatActivity implements Serializable { Log.d(MainHandler.class.getSimpleName(), "Handler消息：" + message.what + " - " + message.obj); switch (message.what) { case Config.WHAT_SCREEN_CAPTURE -> this.mainActivity.screenCapture(message); - case Config.WHAT_NEW_LOCAL_VIDEO -> this.mainActivity.newLocalVideo(message); - case Config.WHAT_NEW_REMOTE_VIDEO -> this.mainActivity.newRemoteVideo(message); + case Config.WHAT_NEW_LOCAL_VIDEO, + Config.WHAT_NEW_REMOTE_VIDEO -> this.mainActivity.previewVideo(message); } } @@ -273,24 +239,21 @@ public class MainActivity extends AppCompatActivity implements Serializable { } /** - * 新建用户视频 + * 预览用户视频 * * @param message 消息 */ - private void newLocalVideo(Message message) { + private void previewVideo(Message message) { + final GridLayout video = this.binding.video; + final int count = video.getChildCount(); + final GridLayout.Spec rowSpec = GridLayout.spec(count / 2); + final GridLayout.Spec columnSpec = GridLayout.spec(count % 2); + GridLayout.LayoutParams layoutParams = new GridLayout.LayoutParams(rowSpec, columnSpec); + layoutParams.width = 0; + layoutParams.height = 0; final SurfaceView surfaceView = (SurfaceView) message.obj; - final LinearLayout.LayoutParams layoutParams = new LinearLayout.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT); - layoutParams.weight = 1; surfaceView.setZ(0F); - this.addContentView(surfaceView, layoutParams); - } - - private void newRemoteVideo(Message message) { - final SurfaceView surfaceView = (SurfaceView) message.obj; - final LinearLayout.LayoutParams layoutParams = new LinearLayout.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT); - layoutParams.weight = 1; - surfaceView.setZ(0F); - this.addContentView(surfaceView, layoutParams); + video.addView(surfaceView, layoutParams); } } \ No newline at end of file diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java index 3bb38b0..303ae70 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java @@ -11,6 +11,7 @@ import android.content.SharedPreferences; import android.content.res.Resources; import android.graphics.BitmapFactory; import android.os.Binder; +import android.os.Environment; import android.os.Handler; import android.os.IBinder; import android.util.Log; @@ -22,6 +23,10 @@ import com.acgist.taoyao.client.signal.Taoyao; import com.acgist.taoyao.media.MediaManager; import com.acgist.taoyao.media.signal.ITaoyaoListener; +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; + /** * 媒体服务 * @@ -70,6 +75,8 @@ public class MediaService extends Service { :: https://gitee.com/acgist/taoyao """); super.onCreate(); + this.mkdir(R.string.storagePathImage); + this.mkdir(R.string.storagePathVideo); } @Override @@ -191,4 +198,15 @@ public class MediaService extends Service { MediaManager.getInstance().initScreen(intent.getParcelableExtra("data")); } + private void mkdir(int id) { + final Path imagePath = Paths.get( + Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath(), + this.getResources().getString(id) + ); + final File file = imagePath.toFile(); + if(!file.exists()) { + file.mkdirs(); + } + } + } \ No newline at end of file diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/TaoyaoReceiver.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/TaoyaoReceiver.java index c5557ac..e45f3f2 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/TaoyaoReceiver.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/TaoyaoReceiver.java @@ -18,7 +18,6 @@ public class TaoyaoReceiver extends BroadcastReceiver { if (Intent.ACTION_BOOT_COMPLETED.equals(intent.getAction())) { this.launchPreview(context); } else { - // TODO：重启关机释放资源（录像） } } diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java index 116bae3..bd7e167 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java @@ -25,11 +25,11 @@ import com.acgist.taoyao.boot.utils.IdUtils; import com.acgist.taoyao.boot.utils.JSONUtils; import com.acgist.taoyao.boot.utils.MapUtils; import com.acgist.taoyao.client.R; -import com.acgist.taoyao.media.config.MediaAudioProperties; -import com.acgist.taoyao.media.config.MediaProperties; import com.acgist.taoyao.media.MediaManager; import com.acgist.taoyao.media.client.Room; import com.acgist.taoyao.media.client.SessionClient; +import com.acgist.taoyao.media.config.MediaAudioProperties; +import com.acgist.taoyao.media.config.MediaProperties; import com.acgist.taoyao.media.config.MediaVideoProperties; import com.acgist.taoyao.media.config.WebrtcProperties; import com.acgist.taoyao.media.signal.ITaoyao; @@ -158,12 +158,33 @@ public final class Taoyao implements ITaoyao { * 请求消息：同步消息 */ private final Map requestMessage; + /** + * 循环消息Handler + */ private final Handler loopMessageHandler; + /** + * 循环消息线程 + */ private final HandlerThread loopMessageThread; - private final Handler heartbeatHandler; - private final HandlerThread heartbeatThread; + /** + * 消息处理Handler + */ private final Handler executeMessageHandler; + /** + * 消息处理线程 + */ private final HandlerThread executeMessageThread; + /** + * 心跳Handler + */ + private final Handler heartbeatHandler; + /** + * 心跳线程 + */ + private final HandlerThread heartbeatThread; + /** + * 媒体来源管理器 + */ private final MediaManager mediaManager; /** * 房间列表 @@ -173,6 +194,9 @@ public final class Taoyao implements ITaoyao { * 会话终端列表 */ private final Map sessionClients; + /** + * 全局静态变量 + */ public static Taoyao taoyao; public Taoyao( @@ -203,17 +227,20 @@ public final class Taoyao implements ITaoyao { this.batteryManager = context.getSystemService(BatteryManager.class); this.locationManager = context.getSystemService(LocationManager.class); this.requestMessage = new ConcurrentHashMap<>(); - this.loopMessageThread = new HandlerThread("TaoyaoLoopMessageThread"); + this.loopMessageThread = new HandlerThread("LoopMessageThread"); + this.loopMessageThread.setDaemon(true); this.loopMessageThread.start(); this.loopMessageHandler = new Handler(this.loopMessageThread.getLooper()); this.loopMessageHandler.post(this::loopMessage); - this.heartbeatThread = new HandlerThread("TaoyaoHeartbeatThread"); + this.executeMessageThread = new HandlerThread("ExecuteMessageThread"); + this.executeMessageThread.setDaemon(true); + this.executeMessageThread.start(); + this.executeMessageHandler = new Handler(this.executeMessageThread.getLooper()); + this.heartbeatThread = new HandlerThread("HeartbeatThread"); + this.heartbeatThread.setDaemon(true); this.heartbeatThread.start(); this.heartbeatHandler = new Handler(this.heartbeatThread.getLooper()); this.heartbeatHandler.postDelayed(this::heartbeat, 30L * 1000); - this.executeMessageThread = new HandlerThread("TaoyaoExecuteMessageThread"); - this.executeMessageThread.start(); - this.executeMessageHandler = new Handler(this.executeMessageThread.getLooper()); this.mediaManager = MediaManager.getInstance(); this.rooms = new ConcurrentHashMap<>(); this.sessionClients = new ConcurrentHashMap<>(); diff --git a/taoyao-client-android/taoyao/client/src/main/res/layout/activity_main.xml b/taoyao-client-android/taoyao/client/src/main/res/layout/activity_main.xml index 4be9000..6f2c12d 100644 --- a/taoyao-client-android/taoyao/client/src/main/res/layout/activity_main.xml +++ b/taoyao-client-android/taoyao/client/src/main/res/layout/activity_main.xml @@ -7,6 +7,27 @@ android:layout_height="match_parent" tools:context="com.acgist.taoyao.client.MainActivity"> + + + + - - + app:layout_constraintBottom_toTopOf="@+id/action" + app:layout_constraintStart_toEndOf="@+id/record" /> \ No newline at end of file diff --git a/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml b/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml index 8ef90c5..2dc7863 100644 --- a/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml +++ b/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml @@ -33,4 +33,6 @@ true true + 100 + fd-video \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp b/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp index 4d55837..c2430e2 100644 --- a/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp +++ b/taoyao-client-android/taoyao/media/src/main/cpp/include/Room.hpp @@ -29,7 +29,7 @@ namespace acgist { Room(std::string roomId, JavaVM* javaVM, jobject routerCallback); virtual ~Room(); public: - void enter(JNIEnv* env, std::string rtpCapabilities, webrtc::PeerConnectionFactoryInterface* factory, webrtc::PeerConnectionInterface::RTCConfiguration& rtcConfiguration); + void enterRoom(JNIEnv* env, std::string rtpCapabilities, webrtc::PeerConnectionFactoryInterface* factory, webrtc::PeerConnectionInterface::RTCConfiguration& rtcConfiguration); void createSendTransport(JNIEnv* env, std::string body); void createRecvTransport(JNIEnv* env, std::string body); void mediaProduceAudio(JNIEnv* env, webrtc::MediaStreamInterface* mediaStream); @@ -41,7 +41,7 @@ namespace acgist { void mediaConsumerPause(JNIEnv* env, std::string consumerId); void mediaConsumerResume(JNIEnv* env, std::string consumerId); void mediaConsumerClose(JNIEnv* env, std::string consumerId); - void close(); + void closeRoom(); }; } \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/include/RouterCallback.hpp b/taoyao-client-android/taoyao/media/src/main/cpp/include/RouterCallback.hpp index c6c1f75..2f2c165 100644 --- a/taoyao-client-android/taoyao/media/src/main/cpp/include/RouterCallback.hpp +++ b/taoyao-client-android/taoyao/media/src/main/cpp/include/RouterCallback.hpp @@ -13,6 +13,7 @@ namespace acgist { jobject routerCallback; public: void enterRoomCallback(JNIEnv* env, std::string rtpCapabilities, std::string sctpCapabilities); + void closeRoomCallback(JNIEnv* env); void sendTransportConnectCallback(JNIEnv* env, std::string transportId, std::string dtlsParameters); void recvTransportConnectCallback(JNIEnv* env, std::string transportId, std::string dtlsParameters); std::string sendTransportProduceCallback(JNIEnv* env, std::string kind, std::string transportId, std::string rtpParameters); diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp index a22ff62..be0ce56 100644 --- a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp +++ b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/Room.cpp @@ -154,7 +154,7 @@ namespace acgist { this->javaVM->DetachCurrentThread(); } - void Room::enter( + void Room::enterRoom( JNIEnv* env, std::string rtpCapabilities, webrtc::PeerConnectionFactoryInterface* factory, @@ -325,16 +325,23 @@ namespace acgist { this->consumerCloseCallback(env, consumerId); } - void Room::close() { + void Room::closeRoom() { this->audioProducer->Close(); this->videoProducer->Close(); std::map::iterator iterator; for (iterator = this->consumers.begin(); iterator != this->consumers.end(); iterator++) { iterator->second->Close(); } +// std::for_each(this->consumers.begin(), this->consumers.end(), [](mediasoupclient::Consumer* consumer) { +// consumer->Close(); +// }); this->consumers.clear(); this->sendTransport->Close(); this->recvTransport->Close(); + JNIEnv* env; + this->javaVM->AttachCurrentThread(&env, nullptr); + this->closeRoomCallback(env); + this->javaVM->DetachCurrentThread(); } extern "C" JNIEXPORT jlong JNICALL @@ -365,7 +372,7 @@ namespace acgist { // webrtc::jni::JavaToNativeMediaConstraints() // webrtc::jni::JavaToNativeRTCConfiguration(env, jRtcConfigurationRef, &rtcConfiguration); const char* rtpCapabilities = env->GetStringUTFChars(jRtpCapabilities, nullptr); - room->enter( + room->enterRoom( env, rtpCapabilities, reinterpret_cast(factoryPointer), @@ -380,7 +387,7 @@ namespace acgist { extern "C" JNIEXPORT void JNICALL Java_com_acgist_taoyao_media_client_Room_nativeCloseRoom(JNIEnv* env, jobject me, jlong nativeRoomPointer) { Room* room = (Room*) nativeRoomPointer; - room->close(); + room->closeRoom(); delete room; } diff --git a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/RouterCallback.cpp b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/RouterCallback.cpp index 5c0bff4..b6ac323 100644 --- a/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/RouterCallback.cpp +++ b/taoyao-client-android/taoyao/media/src/main/cpp/webrtc/RouterCallback.cpp @@ -20,6 +20,16 @@ namespace acgist { env->DeleteLocalRef(jCallbackClazz); } + void RouterCallback::closeRoomCallback(JNIEnv* env) { + jclass jCallbackClazz = env->GetObjectClass(this->routerCallback); + jmethodID closeRoomCallback = env->GetMethodID(jCallbackClazz, "closeRoomCallback", "()V"); + env->CallVoidMethod( + this->routerCallback, + closeRoomCallback + ); + env->DeleteLocalRef(jCallbackClazz); + } + void RouterCallback::sendTransportConnectCallback(JNIEnv* env, std::string transportId, std::string dtlsParameters) { jclass jCallbackClazz = env->GetObjectClass(this->routerCallback); jmethodID sendTransportConnectCallback = env->GetMethodID(jCallbackClazz, "sendTransportConnectCallback", "(Ljava/lang/String;Ljava/lang/String;)V"); @@ -86,9 +96,9 @@ namespace acgist { jclass jCallbackClazz = env->GetObjectClass(this->routerCallback); jmethodID producerNewCallback = env->GetMethodID(jCallbackClazz, "producerNewCallback", "(Ljava/lang/String;java/lang/String;J;J;)V"); const char* cKind = kind.data(); - jstring jKind = env->NewStringUTF(cKind); + jstring jKind = env->NewStringUTF(cKind); const char* cProducerId = producerId.data(); - jstring jProducerId = env->NewStringUTF(cProducerId); + jstring jProducerId = env->NewStringUTF(cProducerId); env->CallVoidMethod( this->routerCallback, producerNewCallback, @@ -112,7 +122,7 @@ namespace acgist { jclass jCallbackClazz = env->GetObjectClass(this->routerCallback); jmethodID consumerNewCallback = env->GetMethodID(jCallbackClazz, "consumerNewCallback", "(Ljava/lang/String;J;J;)V"); const char* cMessage = message.data(); - jstring jMessage = env->NewStringUTF(cMessage); + jstring jMessage = env->NewStringUTF(cMessage); env->CallVoidMethod( this->routerCallback, consumerNewCallback, diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/Broadcaster.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/Broadcaster.java new file mode 100644 index 0000000..dc1948a --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/Broadcaster.java @@ -0,0 +1,53 @@ +package com.acgist.taoyao.media; + +import android.content.Context; +import android.speech.tts.TextToSpeech; +import android.util.Log; + +import java.util.Locale; +import java.util.UUID; + +/** + * 声音播报 + * + * @author acgist + */ +public final class Broadcaster { + + private static final Broadcaster INSTANCE = new Broadcaster(); + + public static final Broadcaster getInstance() { + return INSTANCE; + } + + private TextToSpeech textToSpeech; + + private Broadcaster() { + } + + public void init(Context context) { + this.textToSpeech = new TextToSpeech(context, new TextToSpeechInitListener()); + } + + public void broadcast(String text) { +// this.textToSpeech.stop(); + this.textToSpeech.speak(text, TextToSpeech.QUEUE_FLUSH, null, UUID.randomUUID().toString()); + } + + public void shutdown() { + this.textToSpeech.shutdown(); + } + + private class TextToSpeechInitListener implements TextToSpeech.OnInitListener { + @Override + public void onInit(int status) { + Log.i(Broadcaster.class.getSimpleName(), "加载TTS：" + status); + if(status == TextToSpeech.SUCCESS) { + Broadcaster.this.textToSpeech.setLanguage(Locale.CANADA); + Broadcaster.this.textToSpeech.setPitch(1.0F); + Broadcaster.this.textToSpeech.setSpeechRate(1.0F); + } + } + } + +} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java index e0d3b57..5d38e4b 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java @@ -2,15 +2,16 @@ package com.acgist.taoyao.media; import android.content.Context; import android.content.Intent; +import android.media.AudioAttributes; +import android.media.ImageReader; import android.media.MediaCodecInfo; import android.media.MediaCodecList; +import android.media.MediaRecorder; import android.media.projection.MediaProjection; import android.os.Handler; import android.os.Message; -import android.provider.MediaStore; import android.util.Log; -import com.acgist.taoyao.boot.utils.DateUtils; import com.acgist.taoyao.media.client.PhotographClient; import com.acgist.taoyao.media.client.RecordClient; import com.acgist.taoyao.media.config.Config; @@ -38,18 +39,18 @@ import org.webrtc.SurfaceViewRenderer; import org.webrtc.VideoCapturer; import org.webrtc.VideoDecoderFactory; import org.webrtc.VideoEncoderFactory; -import org.webrtc.VideoFrame; +import org.webrtc.VideoFileRenderer; import org.webrtc.VideoSource; import org.webrtc.VideoTrack; import org.webrtc.audio.JavaAudioDeviceModule; -import java.time.LocalDateTime; import java.util.Arrays; -import java.util.function.Consumer; /** * 媒体来源管理器 * + * 注意：镜头选择可以使用代码实现，如果可以经理直接进行物理旋转。 + * * @author acgist * * TODO：动态码率（BITRATE_MODE_VBR、BITRATE_MODE） @@ -117,7 +118,7 @@ public final class MediaManager { /** * Handler */ - private Handler handler; + private Handler mainHandler; /** * 上下文 */ @@ -142,6 +143,10 @@ public final class MediaManager { * EGL */ private EglBase eglBase; + /** + * EGL共享上下文 + */ + private EglBase.Context shareEglContext; /** * 媒体流：声音、视频 */ @@ -232,11 +237,11 @@ public final class MediaManager { * @return 是否可用 */ public boolean available() { - return this.handler != null && this.context != null && this.taoyao != null; + return this.mainHandler != null && this.context != null && this.taoyao != null; } /** - * @param handler Handler + * @param mainHandler Handler * @param context 上下文 * @param playAudio 是否播放音频 * @param playVideo 是否播放视频 @@ -246,14 +251,14 @@ public final class MediaManager { * @param videoProduce 是否生产视频 */ public void initContext( - Handler handler, Context context, + Handler mainHandler, Context context, boolean playAudio, boolean playVideo, boolean audioConsume, boolean videoConsume, boolean audioProduce, boolean videoProduce, String imagePath, String videoPath, TransportType transportType ) { - this.handler = handler; + this.mainHandler = mainHandler; this.context = context; this.playAudio = playAudio; this.playVideo = playVideo; @@ -266,9 +271,10 @@ public final class MediaManager { this.transportType = transportType; PeerConnectionFactory.initialize( PeerConnectionFactory.InitializationOptions.builder(this.context) -// .setFieldTrials("WebRTC-H264HighProfile/Enabled/") -// .setEnableInternalTracer(true) - .createInitializationOptions() +// .setFieldTrials("WebRTC-H264HighProfile/Enabled/") +// .setNativeLibraryName("jingle_peerconnection_so") +// .setEnableInternalTracer(true) + .createInitializationOptions() ); } @@ -333,34 +339,19 @@ public final class MediaManager { */ private void initMedia(VideoSourceType videoSourceType) { Log.i(MediaManager.class.getSimpleName(), "加载媒体：" + videoSourceType); - this.videoSourceType = videoSourceType; this.eglBase = EglBase.create(); - final VideoDecoderFactory videoDecoderFactory = new DefaultVideoDecoderFactory(this.eglBase.getEglBaseContext()); - final VideoEncoderFactory videoEncoderFactory = new DefaultVideoEncoderFactory(this.eglBase.getEglBaseContext(), true, true); - final JavaAudioDeviceModule javaAudioDeviceModule = JavaAudioDeviceModule.builder(this.context) -// .setAudioSource(android.media.MediaRecorder.AudioSource.MIC) - // 本地音频 - .setSamplesReadyCallback(audioSamples -> { - if(this.recordClient != null) { - this.recordClient.putAudio(audioSamples); - } - }) - // 超低延迟 -// .setUseLowLatency() - // 远程音频 -// .setAudioTrackStateCallback() -// .setAudioFormat(AudioFormat.ENCODING_PCM_32BIT) -// .setUseHardwareNoiseSuppressor(true) -// .setUseHardwareAcousticEchoCanceler(true) - .createAudioDeviceModule(); + this.shareEglContext = this.eglBase.getEglBaseContext(); + this.videoSourceType = videoSourceType; + final VideoDecoderFactory videoDecoderFactory = new DefaultVideoDecoderFactory(this.shareEglContext); + final VideoEncoderFactory videoEncoderFactory = new DefaultVideoEncoderFactory(this.shareEglContext, true, true); + final JavaAudioDeviceModule javaAudioDeviceModule = this.javaAudioDeviceModule(); this.peerConnectionFactory = PeerConnectionFactory.builder() + .setVideoDecoderFactory(videoDecoderFactory) + .setVideoEncoderFactory(videoEncoderFactory) .setAudioDeviceModule(javaAudioDeviceModule) - // 变声 // .setAudioProcessingFactory() // .setAudioEncoderFactoryFactory(new BuiltinAudioEncoderFactoryFactory()) // .setAudioDecoderFactoryFactory(new BuiltinAudioDecoderFactoryFactory()) - .setVideoDecoderFactory(videoDecoderFactory) - .setVideoEncoderFactory(videoEncoderFactory) .createPeerConnectionFactory(); this.mediaStream = this.peerConnectionFactory.createLocalMediaStream("Taoyao"); Arrays.stream(videoEncoderFactory.getSupportedCodecs()).forEach(v -> { @@ -370,12 +361,58 @@ public final class MediaManager { this.initVideo(); } + private JavaAudioDeviceModule javaAudioDeviceModule() { +// final AudioAttributes audioAttributes = new AudioAttributes.Builder() +// .setFlags(AudioAttributes.FLAG_AUDIBILITY_ENFORCED) +// .setUsage(AudioAttributes.USAGE_VOICE_COMMUNICATION) +// .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) +// .build(); + final JavaAudioDeviceModule javaAudioDeviceModule = JavaAudioDeviceModule.builder(this.context) +// .setSampleRate() +// .setAudioSource(MediaRecorder.AudioSource.MIC) +// .setAudioFormat(AudioFormat.ENCODING_PCM_16BIT) +// .setAudioAttributes(audioAttributes) +// .setUseStereoInput() +// .setUseStereoOutput() + // 超低延迟 +// .setUseLowLatency() + .setSamplesReadyCallback(audioSamples -> { + if(this.recordClient != null) { + this.recordClient.putAudio(audioSamples); + } + }) + .setAudioTrackStateCallback(new JavaAudioDeviceModule.AudioTrackStateCallback() { + @Override + public void onWebRtcAudioTrackStart() { + Log.i(MediaManager.class.getSimpleName(), "WebRTC声音Track开始"); + } + @Override + public void onWebRtcAudioTrackStop() { + Log.i(MediaManager.class.getSimpleName(), "WebRTC声音Track结束"); + } + }) + .setAudioRecordStateCallback(new JavaAudioDeviceModule.AudioRecordStateCallback() { + @Override + public void onWebRtcAudioRecordStart() { + Log.i(MediaManager.class.getSimpleName(), "WebRTC声音录制开始"); + } + @Override + public void onWebRtcAudioRecordStop() { + Log.i(MediaManager.class.getSimpleName(), "WebRTC声音录制结束"); + } + }) +// .setUseHardwareNoiseSuppressor(true) +// .setUseHardwareAcousticEchoCanceler(true) + .createAudioDeviceModule(); +// javaAudioDeviceModule.setSpeakerMute(false); +// javaAudioDeviceModule.setMicrophoneMute(false); + return javaAudioDeviceModule; + } + /** * 加载音频 */ private void initAudio() { - // 关闭音频 - this.closeAudio(); // 加载音频 final MediaConstraints mediaConstraints = new MediaConstraints(); // 高音过滤 @@ -404,22 +441,22 @@ public final class MediaManager { * 加载视频 */ private void initVideo() { - // 关闭视频 - this.closeVideo(); // 加载视频 Log.i(MediaManager.class.getSimpleName(), "加载视频：" + this.videoSourceType); - if (this.videoSourceType.isCamera()) { + if (this.videoSourceType == VideoSourceType.FILE) { + this.initFile(); + } else if (this.videoSourceType.isCamera()) { this.initCamera(); - } else if (this.videoSourceType == VideoSourceType.FILE) { } else if (this.videoSourceType == VideoSourceType.SCREEN) { - final Message message = new Message(); - message.what = Config.WHAT_SCREEN_CAPTURE; - this.handler.sendMessage(message); + this.initSharePromise(); } else { // 其他来源 } } + private void initFile() { + } + /** * 加载摄像头 */ @@ -440,6 +477,12 @@ public final class MediaManager { this.initVideoTrack(); } + private void initSharePromise() { + final Message message = new Message(); + message.what = Config.WHAT_SCREEN_CAPTURE; + this.mainHandler.sendMessage(message); + } + /** * 加载屏幕 * @@ -456,7 +499,7 @@ public final class MediaManager { */ private void initVideoTrack() { // 加载视频 - this.surfaceTextureHelper = SurfaceTextureHelper.create("MediaVideoThread", this.eglBase.getEglBaseContext()); + this.surfaceTextureHelper = SurfaceTextureHelper.create("MediaVideoThread", this.shareEglContext); // this.surfaceTextureHelper.setTextureSize(); // this.surfaceTextureHelper.setFrameRotation(); // 次码流 @@ -471,14 +514,15 @@ public final class MediaManager { this.recordVideoCapturer.initialize(this.surfaceTextureHelper, this.context, this.recordVideoSource.getCapturerObserver()); this.recordVideoTrack = this.peerConnectionFactory.createVideoTrack("TaoyaoV1", this.recordVideoSource); this.recordVideoTrack.addSink(videoFrame -> { - if(this.recordClient != null) { + // 录制 + if (this.recordClient != null) { + videoFrame.retain(); this.recordClient.putVideo(videoFrame); } - if(this.photographClient != null) { - synchronized (this.photographClient) { - this.photographClient.photograph(videoFrame); - this.photographClient = null; - } + // 拍照 + if (this.photographClient != null) { + videoFrame.retain(); + this.photographClient.photograph(videoFrame); } }); this.recordVideoTrack.setEnabled(true); @@ -515,7 +559,7 @@ public final class MediaManager { } catch (InterruptedException e) { Log.e(MediaManager.class.getSimpleName(), "暂停视频捕获异常", e); } - this.videoCapturer.startCapture(this.mediaVideoProperties.getHeight(), this.mediaVideoProperties.getWidth(), this.mediaVideoProperties.getFrameRate()); + this.videoCapturer.startCapture(this.mediaVideoProperties.getWidth(), this.mediaVideoProperties.getHeight(), this.mediaVideoProperties.getFrameRate()); } } @@ -532,9 +576,10 @@ public final class MediaManager { if (this.videoSourceType == videoSourceType) { return; } - this.videoSourceType = videoSourceType; Log.i(MediaManager.class.getSimpleName(), "设置视频来源：" + videoSourceType); - if (this.videoSourceType.isCamera() && videoSourceType.isCamera()) { + final VideoSourceType old = this.videoSourceType; + this.videoSourceType = videoSourceType; + if (old.isCamera() && videoSourceType.isCamera()) { // TODO：测试是否需要完全重置 final CameraVideoCapturer cameraVideoCapturer = (CameraVideoCapturer) this.videoCapturer; cameraVideoCapturer.switchCamera(new CameraVideoCapturer.CameraSwitchHandler() { @@ -564,7 +609,7 @@ public final class MediaManager { return; } else { this.shareClientCount++; - this.videoCapturer.startCapture(this.mediaVideoProperties.getHeight(), this.mediaVideoProperties.getWidth(), this.mediaVideoProperties.getFrameRate()); + this.videoCapturer.startCapture(this.mediaVideoProperties.getWidth(), this.mediaVideoProperties.getHeight(), this.mediaVideoProperties.getFrameRate()); } } } @@ -595,10 +640,12 @@ public final class MediaManager { return null; } String filepath; + // TODO：质量读取 + this.photographClient = new PhotographClient(100, this.imagePath); if(this.recordClient == null) { + // TODO：质量读取 final MediaVideoProperties mediaVideoProperties = this.mediaProperties.getVideos().get("fd-video"); - this.recordVideoCapturer.startCapture(mediaVideoProperties.getHeight(), mediaVideoProperties.getWidth(), mediaVideoProperties.getFrameRate()); - this.photographClient = new PhotographClient(this.imagePath); + this.recordVideoCapturer.startCapture(mediaVideoProperties.getWidth(), mediaVideoProperties.getHeight(), mediaVideoProperties.getFrameRate()); filepath = this.photographClient.waitForPhotograph(); try { this.recordVideoCapturer.stopCapture(); @@ -606,9 +653,9 @@ public final class MediaManager { Log.e(MediaManager.class.getSimpleName(), "关闭视频捕获（主码流）异常", e); } } else { - this.photographClient = new PhotographClient(this.imagePath); filepath = this.photographClient.waitForPhotograph(); } + this.photographClient = null; return filepath; } } @@ -621,11 +668,12 @@ public final class MediaManager { this.recordClient = new RecordClient( this.videoPath, this.taoyao, - this.handler + this.mainHandler ); this.recordClient.start(); + // TODO：质量读取 final MediaVideoProperties mediaVideoProperties = this.mediaProperties.getVideos().get("fd-video"); - this.recordVideoCapturer.startCapture(mediaVideoProperties.getHeight(), mediaVideoProperties.getWidth(), mediaVideoProperties.getFrameRate()); + this.recordVideoCapturer.startCapture(mediaVideoProperties.getWidth(), mediaVideoProperties.getHeight(), mediaVideoProperties.getFrameRate()); return this.recordClient; } } @@ -635,13 +683,13 @@ public final class MediaManager { if(this.recordClient == null) { return; } else { - this.recordClient.close(); - this.recordClient = null; try { this.recordVideoCapturer.stopCapture(); } catch (InterruptedException e) { Log.e(MediaManager.class.getSimpleName(), "关闭视频捕获（主码流）异常", e); } + this.recordClient.close(); + this.recordClient = null; } } } @@ -658,7 +706,7 @@ public final class MediaManager { ) { // 预览控件 final SurfaceViewRenderer surfaceViewRenderer = new SurfaceViewRenderer(this.context); - this.handler.post(() -> { + this.mainHandler.post(() -> { // 视频反转 surfaceViewRenderer.setMirror(false); // 视频拉伸 @@ -666,7 +714,7 @@ public final class MediaManager { // 硬件拉伸 surfaceViewRenderer.setEnableHardwareScaler(true); // 加载OpenSL ES - surfaceViewRenderer.init(this.eglBase.getEglBaseContext(), null); + surfaceViewRenderer.init(this.shareEglContext, null); // 强制播放 if(!videoTrack.enabled()) { videoTrack.setEnabled(true); @@ -677,7 +725,7 @@ public final class MediaManager { final Message message = new Message(); message.obj = surfaceViewRenderer; message.what = flag; - this.handler.sendMessage(message); + this.mainHandler.sendMessage(message); return surfaceViewRenderer; } @@ -729,8 +777,6 @@ public final class MediaManager { * 关闭视频 */ private void closeVideo() { - this.stopVideoCapture(); - this.stopRecordVideoCapture(); if(this.videoTrack != null) { this.videoTrack.dispose(); this.videoTrack = null; @@ -765,11 +811,12 @@ public final class MediaManager { if (this.eglBase != null) { this.eglBase.release(); this.eglBase = null; + this.shareEglContext = null; } - if (this.mediaStream != null) { - this.mediaStream.dispose(); - this.mediaStream = null; - } +// if (this.mediaStream != null) { +// this.mediaStream.dispose(); +// this.mediaStream = null; +// } if (this.peerConnectionFactory != null) { this.peerConnectionFactory.dispose(); this.peerConnectionFactory = null; @@ -792,7 +839,7 @@ public final class MediaManager { * * @author acgist */ - private static class MediaCameraEventsHandler implements CameraVideoCapturer.CameraEventsHandler { + private class MediaCameraEventsHandler implements CameraVideoCapturer.CameraEventsHandler { @Override public void onCameraError(String message) { @@ -825,13 +872,14 @@ public final class MediaManager { /** * 屏幕录制回调 */ - private static class ScreenCallback extends MediaProjection.Callback { + private class ScreenCallback extends MediaProjection.Callback { @Override public void onStop() { super.onStop(); Log.i(MediaManager.class.getSimpleName(), "停止屏幕捕获"); } + } private native void nativeInit(); diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java index 03c3d47..9315dbb 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RouterCallback.java @@ -8,6 +8,7 @@ package com.acgist.taoyao.media; public interface RouterCallback { default void enterRoomCallback(String rtpCapabilities, String sctpCapabilities) {}; + default void closeRoomCallback() {}; default void sendTransportConnectCallback(String transportId, String dtlsParameters) {}; default void recvTransportConnectCallback(String transportId, String dtlsParameters) {}; default String sendTransportProduceCallback(String kind, String transportId, String rtpParameters) { return null; }; diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Client.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Client.java index b261956..ab3fdb9 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Client.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Client.java @@ -1,19 +1,11 @@ package com.acgist.taoyao.media.client; import android.os.Handler; -import android.os.Message; import android.util.Log; -import com.acgist.taoyao.media.MediaManager; -import com.acgist.taoyao.media.config.Config; import com.acgist.taoyao.media.signal.ITaoyao; -import org.webrtc.MediaStreamTrack; -import org.webrtc.RendererCommon; import org.webrtc.SurfaceViewRenderer; -import org.webrtc.VideoTrack; - -import java.io.Closeable; /** * 终端 @@ -35,8 +27,8 @@ public abstract class Client extends CloseableClient { */ protected SurfaceViewRenderer surfaceViewRenderer; - public Client(String name, String clientId, ITaoyao taoyao, Handler handler) { - super(taoyao, handler); + public Client(String name, String clientId, ITaoyao taoyao, Handler mainHandler) { + super(taoyao, mainHandler); this.name = name; this.clientId = clientId; } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/CloseableClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/CloseableClient.java index 733411a..49b723a 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/CloseableClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/CloseableClient.java @@ -5,8 +5,6 @@ import android.os.Handler; import com.acgist.taoyao.media.MediaManager; import com.acgist.taoyao.media.signal.ITaoyao; -import org.webrtc.PeerConnectionFactory; - import java.io.Closeable; /** @@ -32,17 +30,17 @@ public abstract class CloseableClient implements Closeable { /** * Handler */ - protected final Handler handler; + protected final Handler mainHandler; /** * 媒体服务 */ protected final MediaManager mediaManager; - public CloseableClient(ITaoyao taoyao, Handler handler) { + public CloseableClient(ITaoyao taoyao, Handler mainHandler) { this.init = false; this.close = false; this.taoyao = taoyao; - this.handler = handler; + this.mainHandler = mainHandler; this.mediaManager = MediaManager.getInstance(); } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java index c659cf0..11e7e85 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/LocalClient.java @@ -6,9 +6,7 @@ import com.acgist.taoyao.boot.utils.ListUtils; import com.acgist.taoyao.media.config.Config; import com.acgist.taoyao.media.signal.ITaoyao; -import org.webrtc.AudioTrack; import org.webrtc.MediaStream; -import org.webrtc.VideoTrack; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -32,8 +30,8 @@ public class LocalClient extends RoomClient { protected long audioProducerPointer; protected long videoProducerPointer; - public LocalClient(String name, String clientId, ITaoyao taoyao, Handler handler) { - super(name, clientId, taoyao, handler); + public LocalClient(String name, String clientId, ITaoyao taoyao, Handler mainHandler) { + super(name, clientId, taoyao, mainHandler); this.tracks = new ConcurrentHashMap<>(); } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java index 5df4b51..6f21924 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java @@ -1,15 +1,22 @@ package com.acgist.taoyao.media.client; +import android.graphics.Bitmap; +import android.graphics.BitmapFactory; +import android.graphics.ImageFormat; +import android.graphics.Rect; +import android.graphics.YuvImage; import android.os.Environment; -import android.provider.MediaStore; import android.util.Log; import com.acgist.taoyao.boot.utils.DateUtils; import org.webrtc.VideoFrame; +import java.io.ByteArrayOutputStream; import java.io.File; -import java.nio.file.Path; +import java.io.FileOutputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; import java.nio.file.Paths; import java.time.LocalDateTime; @@ -20,26 +27,51 @@ import java.time.LocalDateTime; */ public class PhotographClient { + private final int quantity; private final String filename; private final String filepath; - private VideoFrame videoFrame; - public PhotographClient(String path) { - this.filename = DateUtils.format(LocalDateTime.now(), DateUtils.DateTimeStyle.YYYYMMDDHH24MMSS) + ".mp4"; - final Path filePath = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath(), path, filename); - final File parentFile = filePath.getParent().toFile(); - if(!parentFile.exists()) { - parentFile.mkdirs(); - } - this.filepath = filePath.toString(); + public PhotographClient(int quantity, String path) { + this.quantity = quantity; + this.filename = DateUtils.format(LocalDateTime.now(), DateUtils.DateTimeStyle.YYYYMMDDHH24MMSS) + ".jpg"; + this.filepath = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath(), path, this.filename).toString(); Log.i(RecordClient.class.getSimpleName(), "拍摄照片文件：" + this.filepath); } public String photograph(VideoFrame videoFrame) { + final Thread thread = new Thread(() -> this.photographBackground(videoFrame)); + thread.setName("PhotographThread"); + thread.setDaemon(true); + thread.start(); + return this.filepath; + } + + private void photographBackground(VideoFrame videoFrame) { + final File file = new File(this.filepath); + try ( + final OutputStream output = new FileOutputStream(file); + final ByteArrayOutputStream byteArray = new ByteArrayOutputStream(); + ) { + final VideoFrame.I420Buffer i420 = videoFrame.getBuffer().toI420(); + final int width = i420.getWidth(); + final int height = i420.getHeight(); + // YuvHelper转换颜色溢出 + final YuvImage image = this.i420ToYuvImage(i420, width, height); + i420.release(); + videoFrame.release(); + image.compressToJpeg(new Rect(0, 0, width, height), this.quantity, byteArray); + final byte[] array = byteArray.toByteArray(); + final Bitmap bitmap = BitmapFactory.decodeByteArray(array, 0, array.length); +// final Matrix matrix = new Matrix(); +// matrix.setRotate(90); +// final Bitmap matrixBitmap = Bitmap.createBitmap(bitmap, 0, 0, width, height, matrix, false); + bitmap.compress(Bitmap.CompressFormat.JPEG, this.quantity, output); + } catch (Exception e) { + Log.e(PhotographClient.class.getSimpleName(), "拍照异常", e); + } synchronized (this) { this.notifyAll(); } - return this.filepath; } public String waitForPhotograph() { @@ -53,4 +85,61 @@ public class PhotographClient { return this.filepath; } + private YuvImage i420ToYuvImage(VideoFrame.I420Buffer i420, int width, int height) { + final ByteBuffer[] yuvPlanes = new ByteBuffer[] { + i420.getDataY(), i420.getDataU(), i420.getDataV() + }; + final int[] yuvStrides = new int[] { + i420.getStrideY(), i420.getStrideU(), i420.getStrideV() + }; + if ( + yuvStrides[0] != width || + yuvStrides[1] != width / 2 || + yuvStrides[2] != width / 2 + ) { + return i420ToYuvImage(yuvPlanes, yuvStrides, width, height); + } + final byte[] bytes = new byte[yuvStrides[0] * height + yuvStrides[1] * height / 2 + yuvStrides[2] * height / 2]; + final ByteBuffer yBuffer = ByteBuffer.wrap(bytes, 0, width * height); + this.copyPlane(yuvPlanes[0], yBuffer); + final byte[] uvBytes = new byte[width / 2 * height / 2]; + final ByteBuffer uvBuffer = ByteBuffer.wrap(uvBytes, 0, uvBytes.length); + this.copyPlane(yuvPlanes[2], uvBuffer); + for (int row = 0; row < height / 2; row++) { + for (int col = 0; col < width / 2; col++) { + bytes[width * height + row * width + col * 2] = uvBytes[row * width / 2 + col]; + } + } + this.copyPlane(yuvPlanes[1], uvBuffer); + for (int row = 0; row < height / 2; row++) { + for (int col = 0; col < width / 2; col++) { + bytes[width * height + row * width + col * 2 + 1] = uvBytes[row * width / 2 + col]; + } + } + return new YuvImage(bytes, ImageFormat.NV21, width, height, null); + } + + private YuvImage i420ToYuvImage(ByteBuffer[] yuvPlanes, int[] yuvStrides, int width, int height) { + int i = 0; + final byte[] bytes = new byte[width * height * 3 / 2]; + for (int row = 0; row < height; row++) { + for (int col = 0; col < width; col++) { + bytes[i++] = yuvPlanes[0].get(col + row * yuvStrides[0]); + } + } + for (int row = 0; row < height / 2; row++) { + for (int col = 0; col < width / 2; col++) { + bytes[i++] = yuvPlanes[2].get(col + row * yuvStrides[2]); + bytes[i++] = yuvPlanes[1].get(col + row * yuvStrides[1]); + } + } + return new YuvImage(bytes, ImageFormat.NV21, width, height, null); + } + + private void copyPlane(ByteBuffer src, ByteBuffer dst) { + src.position(0).limit(src.capacity()); + dst.put(src); + dst.position(0).limit(dst.capacity()); + } + } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java index 64cc373..40db0af 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java @@ -1,5 +1,6 @@ package com.acgist.taoyao.media.client; +import android.graphics.YuvImage; import android.media.AudioFormat; import android.media.MediaCodec; import android.media.MediaCodecInfo; @@ -9,14 +10,19 @@ import android.os.Environment; import android.os.Handler; import android.os.HandlerThread; import android.util.Log; +import android.view.Surface; import com.acgist.taoyao.boot.utils.DateUtils; import com.acgist.taoyao.media.MediaManager; import com.acgist.taoyao.media.VideoSourceType; import com.acgist.taoyao.media.signal.ITaoyao; +import org.webrtc.EglBase; +import org.webrtc.GlRectDrawer; import org.webrtc.TextureBufferImpl; import org.webrtc.VideoFrame; +import org.webrtc.VideoFrameDrawer; +import org.webrtc.VideoSink; import org.webrtc.YuvHelper; import org.webrtc.audio.JavaAudioDeviceModule; @@ -26,24 +32,30 @@ import java.nio.ByteBuffer; import java.nio.file.Path; import java.nio.file.Paths; import java.time.LocalDateTime; +import java.util.Queue; +import java.util.concurrent.BlockingQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; /** * 录像机 * + * https://www.freesion.com/article/448330501/ * https://blog.csdn.net/nanoage/article/details/127406494 * https://webrtc.org.cn/20190419_tutorial3_webrtc_android * https://blog.csdn.net/CSDN_Mew/article/details/103406781 * https://blog.csdn.net/Tong_Hou/article/details/112116349 * https://blog.csdn.net/u011418943/article/details/127108642 * https://blog.csdn.net/m0_60259116/article/details/126875532 - * https://blog.csdn.net/csdn_shen0221/article/details/120331004 * https://blog.csdn.net/csdn_shen0221/article/details/119982257 + * https://blog.csdn.net/csdn_shen0221/article/details/120331004 + * https://github.com/flutter-webrtc/flutter-webrtc/blob/main/android/src/main/java/com/cloudwebrtc/webrtc/record/VideoFileRenderer.java * * @author acgist */ -public class RecordClient extends Client { +public class RecordClient extends Client implements VideoSink, JavaAudioDeviceModule.SamplesReadyCallback { /** * 音频准备录制 @@ -53,10 +65,6 @@ public class RecordClient extends Client { * 视频准备录制 */ private volatile boolean videoActive; - /** - * 录制时间戳 - */ - private volatile long pts; /** * 录制文件名称 */ @@ -93,22 +101,15 @@ public class RecordClient extends Client { * 媒体合成器 */ private MediaMuxer mediaMuxer; - /** - * 线程池 - */ - private final ExecutorService executorService; + private final BlockingQueue audioSamplesQueue; + private final BlockingQueue videoFrameQueue; - public RecordClient(String path, ITaoyao taoyao, Handler handler) { - super("本地录像", "LocalRecordClient", taoyao, handler); + public RecordClient(String path, ITaoyao taoyao, Handler mainHandler) { + super("本地录像", "LocalRecordClient", taoyao, mainHandler); this.filename = DateUtils.format(LocalDateTime.now(), DateUtils.DateTimeStyle.YYYYMMDDHH24MMSS) + ".mp4"; - final Path filePath = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath(), path, filename); - final File parentFile = filePath.getParent().toFile(); - if(!parentFile.exists()) { - parentFile.mkdirs(); - } - this.filepath = filePath.toString(); - Log.i(RecordClient.class.getSimpleName(), "录制视频文件：" + this.filepath); - this.executorService = Executors.newFixedThreadPool(2); + this.filepath = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath(), path, this.filename).toString(); + this.audioSamplesQueue = new LinkedBlockingQueue<>(); + this.videoFrameQueue = new LinkedBlockingQueue<>(); } public void start() { @@ -116,6 +117,7 @@ public class RecordClient extends Client { if(this.init) { return; } + Log.i(RecordClient.class.getSimpleName(), "录制视频文件：" + this.filepath); super.init(); this.mediaManager.newClient(VideoSourceType.BACK); this.record(null, null, 1, 1); @@ -141,111 +143,131 @@ public class RecordClient extends Client { */ private void initAudioThread(String audioType, int bitRate, int sampleRate, int channelCount) { try { - this.audioCodec = MediaCodec.createEncoderByType(audioType); final MediaFormat audioFormat = MediaFormat.createAudioFormat(audioType, sampleRate, channelCount); // audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitRate); audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, AudioFormat.ENCODING_PCM_16BIT); audioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC); - audioFormat.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR); +// audioFormat.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR); audioFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 8 * 1024); + this.audioCodec = MediaCodec.createEncoderByType(audioType); this.audioCodec.configure(audioFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); } catch (Exception e) { Log.e(RecordClient.class.getSimpleName(), "加载音频录制线程异常", e); } - final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); this.audioThread = new HandlerThread("AudioRecoderThread"); this.audioThread.start(); - final Handler audioHandler = new Handler(this.audioThread.getLooper()); - audioHandler.post(() -> { - int trackIndex = -1; - int outputIndex; - this.audioCodec.start(); - this.audioActive = true; - while (!this.close) { - outputIndex = this.audioCodec.dequeueOutputBuffer(info, 1000L * 1000); - if (outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { - } else if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { - synchronized (this) { - trackIndex = this.mediaMuxer.addTrack(this.audioCodec.getOutputFormat()); - Log.i(RecordClient.class.getSimpleName(), "开始录制音频：" + trackIndex); - if (this.videoActive) { - Log.i(RecordClient.class.getSimpleName(), "开始录制文件：" + this.filename); - this.pts = System.currentTimeMillis(); - this.mediaMuxer.start(); - this.notifyAll(); - } else if (!this.close) { - try { - this.wait(); - } catch (InterruptedException e) { - } - } - } - } else if (outputIndex >= 0) { - final ByteBuffer outputBuffer = this.audioCodec.getOutputBuffer(outputIndex); - outputBuffer.position(info.offset); - outputBuffer.limit(info.offset + info.size); - info.presentationTimeUs = (info.presentationTimeUs - this.pts) * 1000; -// this.mediaMuxer.writeSampleData(trackIndex, outputBuffer, info); - this.audioCodec.releaseOutputBuffer(outputIndex, false); - } - } - synchronized (this) { - if (this.audioCodec != null) { - Log.i(RecordClient.class.getSimpleName(), "结束录制音频"); - this.audioCodec.stop(); - this.audioCodec.release(); - this.audioCodec = null; - } - this.audioActive = false; - if (this.mediaMuxer != null && !this.videoActive) { - Log.i(RecordClient.class.getSimpleName(), "结束录制文件：" + this.filename); - this.mediaMuxer.stop(); - this.mediaMuxer.release(); - this.mediaMuxer = null; - } - } - }); + this.audioHandler = new Handler(this.audioThread.getLooper()); + this.audioHandler.post(this::audioCodec); } - public void putAudio(JavaAudioDeviceModule.AudioSamples audioSamples) { - if(!this.close && this.audioActive) { + private void audioCodec() { + int trackIndex = -1; + int outputIndex; + long pts = 0L; + this.audioCodec.start(); + this.audioActive = true; + JavaAudioDeviceModule.AudioSamples audioSamples = null; + final MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); + while (!this.close) { + + + try { + audioSamples = this.audioSamplesQueue.poll(1000, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Log.e(RecordClient.class.getSimpleName(), "录制线程等待异常", e); + } + if(audioSamples == null) { + continue; + } + int index = this.audioCodec.dequeueInputBuffer(1000L * 1000); + if (index >= 0) { + final byte[] data = audioSamples.getData(); + final ByteBuffer buffer = this.audioCodec.getInputBuffer(index); + buffer.put(data); + this.audioCodec.queueInputBuffer(index, 0, data.length, this.audioPts, 0); + this.audioPts += data.length * 125 / 12; // 1000000 microseconds / 48000hz / 2 bytes +// presTime += data.length * 125 / 12; // 1000000 microseconds / 48000hz / 2 bytes +// presTime += data.length * (1_000_000 / audioSamples.getSampleRate() / 2); //16位最后那个数字是2，8位是1 + } else { + // WARN + } + audioSamples = null; + + + outputIndex = this.audioCodec.dequeueOutputBuffer(bufferInfo, 1000L * 1000); + if (outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { +// } else if (outputIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { + } else if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + synchronized (this) { + trackIndex = this.mediaMuxer.addTrack(this.audioCodec.getOutputFormat()); + Log.i(RecordClient.class.getSimpleName(), "开始录制音频：" + trackIndex); + if (this.videoActive) { + Log.i(RecordClient.class.getSimpleName(), "开始录制文件：" + this.filename); + this.mediaMuxer.start(); + this.notifyAll(); + } else if (!this.close) { + try { + this.wait(); + } catch (InterruptedException e) { + Log.e(RecordClient.class.getSimpleName(), "录制线程等待异常", e); + } + } + } + } else if (outputIndex >= 0) { + if(pts == 0L) { + pts = bufferInfo.presentationTimeUs; + } + final ByteBuffer outputBuffer = this.audioCodec.getOutputBuffer(outputIndex); + outputBuffer.position(bufferInfo.offset); + outputBuffer.limit(bufferInfo.offset + bufferInfo.size); + bufferInfo.presentationTimeUs -= pts; +// this.mediaMuxer.writeSampleData(trackIndex, outputBuffer, info); + this.audioCodec.releaseOutputBuffer(outputIndex, false); + Log.d(RecordClient.class.getSimpleName(), "录制音频帧（时间戳）：" + bufferInfo.flags + " - " + (bufferInfo.presentationTimeUs / 1_000_000F)); +// if (bufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME == MediaCodec.BUFFER_FLAG_KEY_FRAME) { +// } else if (bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG == MediaCodec.BUFFER_FLAG_CODEC_CONFIG) { +// } else if (bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM == MediaCodec.BUFFER_FLAG_END_OF_STREAM) { +// } else if (bufferInfo.flags & MediaCodec.BUFFER_FLAG_PARTIAL_FRAME == MediaCodec.BUFFER_FLAG_PARTIAL_FRAME) { +// } + if((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == MediaCodec.BUFFER_FLAG_END_OF_STREAM) { +// this.close(); + break; + } + } else { + + } + } + synchronized (this) { + if (this.audioCodec != null) { + Log.i(RecordClient.class.getSimpleName(), "结束录制音频"); + this.audioCodec.stop(); + this.audioCodec.release(); + this.audioCodec = null; + } + this.audioActive = false; + if (this.mediaMuxer != null && !this.videoActive) { + Log.i(RecordClient.class.getSimpleName(), "结束录制文件：" + this.filename); + this.mediaMuxer.stop(); + this.mediaMuxer.release(); + this.mediaMuxer = null; + } } } - public void putVideo(VideoFrame videoFrame) { - if (!this.close && this.videoActive) { - this.executorService.submit(() -> { -// TextureBufferImpl -// videoFrame.retain(); - final TextureBufferImpl buffer = (TextureBufferImpl) videoFrame.getBuffer(); - final int outputFrameSize = videoFrame.getRotatedWidth() * videoFrame.getRotatedHeight() * 3 / 2; - final ByteBuffer outputFrameBuffer = ByteBuffer.allocateDirect(outputFrameSize); - final int index = this.videoCodec.dequeueInputBuffer(1000L * 1000); -// YuvImage：截图 - // YV12 - VideoFrame.I420Buffer i420 = videoFrame.getBuffer().toI420(); -// i420.retain(); - Log.i(RecordClient.class.getSimpleName(), "视频信息：" + videoFrame.getRotatedWidth() + " - " + videoFrame.getRotatedHeight()); -// YuvHelper.I420Copy(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight()); - // NV12 - YuvHelper.I420ToNV12(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight()); -// YuvHelper.I420Rotate(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight(), videoFrame.getRotation()); - final ByteBuffer x = this.videoCodec.getInputBuffer(index); -// i420.release(); - x.put(outputFrameBuffer.array()); - this.videoCodec.queueInputBuffer(index, 0, outputFrameSize, System.currentTimeMillis(), 0); -// this.putVideo(outputFrameBuffer, System.currentTimeMillis()); -// videoFrame.release(); -// while (this.active && this.videoActive) { -// final int index = this.videoCodec.dequeueInputBuffer(1000L * 1000); -// if (index < 0) { -// continue; -// } -// final ByteBuffer byteBuffer = this.videoCodec.getInputBuffer(index); -// byteBuffer.put(buffer); -// this.videoCodec.queueInputBuffer(index, 0, buffer.capacity(), pts, 0); -// } - }); + private volatile long audioPts = 0; + + /** + * @param audioSamples PCM数据 + */ + public void putAudio(JavaAudioDeviceModule.AudioSamples audioSamples) { + if(this.close || !this.audioActive) { + return; + } + Log.i(RecordClient.class.getSimpleName(), "音频信息：" + audioSamples.getAudioFormat()); + try { + this.audioSamplesQueue.put(audioSamples); + } catch (InterruptedException e) { + Log.e(RecordClient.class.getSimpleName(), "录制线程等待异常", e); } } @@ -253,84 +275,142 @@ public class RecordClient extends Client { * @param videoType 视频格式 * @param bitRate 比特率：800 * 1000 | 1600 * 1000 | 2500 * 1000 * @param frameRate 帧率：30 - * @param iFrameInterval 关键帧频率：1 + * @param iFrameInterval 关键帧频率：1 ~ 5 * @param width 宽度：1920 * @param height 高度：1080 */ private void initVideoThread(String videoType, int bitRate, int frameRate, int iFrameInterval, int width, int height) { try { - this.videoCodec = MediaCodec.createEncoderByType(videoType); - final MediaFormat videoFormat = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, 360, 480); -// videoFormat.setInteger(MediaFormat.KEY_LEVEL, MediaCodecInfo.CodecProfileLevel.AVCLevel31); -// videoFormat.setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileHigh); - videoFormat.setInteger(MediaFormat.KEY_BIT_RATE, 800 * 1000); + final MediaFormat videoFormat = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, 1920, 1080); +// videoFormat.setInteger(MediaFormat.KEY_LEVEL, MediaCodecInfo.CodecProfileLevel.AVCLevel31); +// videoFormat.setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileHigh); +// videoFormat.setInteger(MediaFormat.KEY_BIT_RATE, 1920 * 1080 * 5); + videoFormat.setInteger(MediaFormat.KEY_BIT_RATE, 6_000_000); +// videoFormat.setInteger(MediaFormat.KEY_BIT_RATE, 800 * 1000); videoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30); -// videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible); - videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible); +// videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); + videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible); videoFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1); + this.videoCodec = MediaCodec.createEncoderByType(videoType); this.videoCodec.configure(videoFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); } catch (Exception e) { Log.e(RecordClient.class.getSimpleName(), "加载视频录制线程异常", e); } - final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); this.videoThread = new HandlerThread("VideoRecoderThread"); this.videoThread.start(); this.videoHandler = new Handler(this.videoThread.getLooper()); - this.videoHandler.post(() -> { - int trackIndex = -1; - int outputIndex; - this.videoCodec.start(); - this.videoActive = true; - while (!this.close) { - outputIndex = this.videoCodec.dequeueOutputBuffer(info, 1000L * 1000); - if (outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { - } else if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { - synchronized (this) { - trackIndex = this.mediaMuxer.addTrack(this.videoCodec.getOutputFormat()); - Log.i(RecordClient.class.getSimpleName(), "开始录制视频：" + trackIndex); - if (this.audioActive) { - Log.i(RecordClient.class.getSimpleName(), "开始录制文件：" + this.filename); - this.pts = System.currentTimeMillis(); - this.mediaMuxer.start(); - this.notifyAll(); - } else if (!this.close) { - try { - this.wait(); - } catch (InterruptedException e) { - } + this.videoHandler.post(this::videoCodec); + } + + private void videoCodec() { + int trackIndex = -1; + int outputIndex; + long pts = 0L; + + this.videoCodec.start(); + this.videoActive = true; + final MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); + VideoFrame videoFrame = null; + while (!this.close) { + try { + videoFrame = this.videoFrameQueue.poll(1000, TimeUnit.MILLISECONDS); + } catch (InterruptedException e) { + Log.e(RecordClient.class.getSimpleName(), "录制线程等待异常", e); + } + if(videoFrame == null) { + continue; + } + + + final TextureBufferImpl buffer = (TextureBufferImpl) videoFrame.getBuffer(); + final int outputFrameSize = videoFrame.getRotatedWidth() * videoFrame.getRotatedHeight() * 3 / 2; +// final ByteBuffer outputFrameBuffer = ByteBuffer.allocateDirect(outputFrameSize); + final int index = this.videoCodec.dequeueInputBuffer(1000L * 1000); + VideoFrame.I420Buffer i420 = buffer.toI420(); + final ByteBuffer bufferx = this.videoCodec.getInputBuffer(index); +// YuvHelper.I420Copy(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight()); +// YuvHelper.I420Rotate(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight(), videoFrame.getRotation()); + YuvHelper.I420ToNV12(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), bufferx, i420.getWidth(), i420.getHeight()); + i420.release(); + videoFrame.release(); +// bufferx.put(outputFrameBuffer.array()); + this.videoCodec.queueInputBuffer(index, 0, outputFrameSize, videoFrame.getTimestampNs(), 0); + + +// this.videoFrameDrawer.drawFrame(videoFrame, this.glRectDrawer, null, 0, 0, videoFrame.getRotatedWidth(), videoFrame.getRotatedHeight()); +// videoFrame.release(); +// videoFrame = null; +// this.eglBase.swapBuffers(); + outputIndex = this.videoCodec.dequeueOutputBuffer(bufferInfo, 1000L * 1000); + if (outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { +// } else if (outputIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { + } else if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + synchronized (this) { + trackIndex = this.mediaMuxer.addTrack(this.videoCodec.getOutputFormat()); + Log.i(RecordClient.class.getSimpleName(), "开始录制视频：" + trackIndex); + if (this.audioActive) { + Log.i(RecordClient.class.getSimpleName(), "开始录制文件：" + this.filename); + this.mediaMuxer.start(); + this.notifyAll(); + } else if (!this.close) { + try { + this.wait(); + } catch (InterruptedException e) { + Log.e(RecordClient.class.getSimpleName(), "录制线程等待异常", e); } } - } else if (outputIndex >= 0) { - final ByteBuffer outputBuffer = this.videoCodec.getOutputBuffer(outputIndex); - outputBuffer.position(info.offset); - outputBuffer.limit(info.offset + info.size); - info.presentationTimeUs = (info.presentationTimeUs - this.pts) * 1000; - this.mediaMuxer.writeSampleData(trackIndex, outputBuffer, info); - this.videoCodec.releaseOutputBuffer(outputIndex, false); - Log.d(RecordClient.class.getSimpleName(), "录制视频帧（时间戳）：" + (info.presentationTimeUs / 1000000F)); -// if(info.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME) { -// } else if(info.flags == MediaCodec.BUFFER_FLAG_CODEC_CONFIG) { -// } else if(info.flags == MediaCodec.BUFFER_FLAG_END_OF_STREAM) { -// } else if(info.flags == MediaCodec.BUFFER_FLAG_PARTIAL_FRAME) { -// } } + } else if (outputIndex >= 0) { + if(pts == 0L) { + pts = bufferInfo.presentationTimeUs / 1000; + } + final ByteBuffer outputBuffer = this.videoCodec.getOutputBuffer(outputIndex); + outputBuffer.position(bufferInfo.offset); + outputBuffer.limit(bufferInfo.offset + bufferInfo.size); + bufferInfo.presentationTimeUs /= 1000; + bufferInfo.presentationTimeUs -= pts; + this.mediaMuxer.writeSampleData(trackIndex, outputBuffer, bufferInfo); + this.videoCodec.releaseOutputBuffer(outputIndex, false); + Log.d(RecordClient.class.getSimpleName(), "录制视频帧（时间戳）：" + bufferInfo.flags + " - " + (bufferInfo.presentationTimeUs / 1_000_000F)); +// if (bufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME == MediaCodec.BUFFER_FLAG_KEY_FRAME) { +// } else if (bufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG == MediaCodec.BUFFER_FLAG_CODEC_CONFIG) { +// } else if (bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM == MediaCodec.BUFFER_FLAG_END_OF_STREAM) { +// } else if (bufferInfo.flags & MediaCodec.BUFFER_FLAG_PARTIAL_FRAME == MediaCodec.BUFFER_FLAG_PARTIAL_FRAME) { +// } + if((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == MediaCodec.BUFFER_FLAG_END_OF_STREAM) { +// this.close(); + break; + } + } else { } - synchronized (this) { - if (this.videoCodec != null) { - Log.i(RecordClient.class.getSimpleName(), "结束录制视频"); - this.videoCodec.stop(); - this.videoCodec.release(); - this.videoCodec = null; - } - this.videoActive = false; - if (this.mediaMuxer != null && !this.audioActive) { - Log.i(RecordClient.class.getSimpleName(), "结束录制文件：" + this.filename); - this.mediaMuxer.stop(); - this.mediaMuxer.release(); - this.mediaMuxer = null; - } + } + synchronized (this) { + if (this.videoCodec != null) { + Log.i(RecordClient.class.getSimpleName(), "结束录制视频"); + this.videoCodec.stop(); + this.videoCodec.release(); + this.videoCodec = null; } - }); + this.videoActive = false; + if (this.mediaMuxer != null && !this.audioActive) { + Log.i(RecordClient.class.getSimpleName(), "结束录制文件：" + this.filename); + this.mediaMuxer.stop(); + this.mediaMuxer.release(); + this.mediaMuxer = null; + } + } + } + + public void putVideo(VideoFrame videoFrame) { + if (this.close || !this.videoActive) { + return; + } + Log.i(RecordClient.class.getSimpleName(), "视频信息：" + videoFrame.getRotatedWidth() + " - " + videoFrame.getRotatedHeight()); + try { + this.videoFrameQueue.put(videoFrame); + } catch (InterruptedException e) { + Log.e(RecordClient.class.getSimpleName(), "录制线程等待异常", e); + } } private void initMediaMuxer() { @@ -360,12 +440,25 @@ public class RecordClient extends Client { if (this.videoThread != null) { this.videoThread.quitSafely(); } - if(this.executorService != null) { - this.executorService.shutdown(); - } this.notifyAll(); this.mediaManager.closeClient(); } } + public String getFilename() { + return this.filename; + } + + public String getFilepath() { + return this.filepath; + } + + @Override + public void onFrame(VideoFrame videoFrame) { + } + + @Override + public void onWebRtcAudioRecordSamplesReady(JavaAudioDeviceModule.AudioSamples audioSamples) { + } + } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java index 3fb7df3..ad14561 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RemoteClient.java @@ -9,7 +9,6 @@ import com.acgist.taoyao.media.signal.ITaoyao; import org.webrtc.MediaStreamTrack; import org.webrtc.VideoTrack; -import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import java.util.stream.Collectors; @@ -29,8 +28,8 @@ public class RemoteClient extends RoomClient { protected long audioConsumerPointer; protected long videoConsumerPointer; - public RemoteClient(String name, String clientId, ITaoyao taoyao, Handler handler) { - super(name, clientId, taoyao, handler); + public RemoteClient(String name, String clientId, ITaoyao taoyao, Handler mainHandler) { + super(name, clientId, taoyao, mainHandler); this.tracks = new ConcurrentHashMap<>(); } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java index 2c463cb..5bc46d4 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java @@ -7,7 +7,6 @@ import com.acgist.taoyao.boot.model.Message; import com.acgist.taoyao.boot.utils.JSONUtils; import com.acgist.taoyao.boot.utils.MapUtils; import com.acgist.taoyao.boot.utils.PointerUtils; -import com.acgist.taoyao.media.MediaManager; import com.acgist.taoyao.media.RouterCallback; import com.acgist.taoyao.media.VideoSourceType; import com.acgist.taoyao.media.config.Config; @@ -54,12 +53,12 @@ public class Room extends CloseableClient implements RouterCallback { public Room( String name, String clientId, String roomId, String password, - ITaoyao taoyao, Handler handler, + ITaoyao taoyao, Handler mainHandler, boolean preview, boolean dataConsume, boolean audioConsume, boolean videoConsume, boolean dataProduce, boolean audioProduce, boolean videoProduce ) { - super(taoyao, handler); + super(taoyao, mainHandler); this.name = name; this.clientId = clientId; this.roomId = roomId; @@ -83,7 +82,7 @@ public class Room extends CloseableClient implements RouterCallback { super.init(); this.peerConnectionFactory = this.mediaManager.newClient(VideoSourceType.BACK); this.mediaManager.startVideoCapture(); - this.localClient = new LocalClient(this.name, this.clientId, this.taoyao, this.handler); + this.localClient = new LocalClient(this.name, this.clientId, this.taoyao, this.mainHandler); this.localClient.setMediaStream(this.mediaManager.getMediaStream()); // STUN | TURN final List iceServers = new ArrayList<>(); @@ -164,7 +163,7 @@ public class Room extends CloseableClient implements RouterCallback { final String clientId = MapUtils.get(body, "clientId"); final Map status = MapUtils.get(body, "status"); final String name = MapUtils.get(status, "name"); - final RemoteClient remoteClient = new RemoteClient(name, clientId, this.taoyao, this.handler); + final RemoteClient remoteClient = new RemoteClient(name, clientId, this.taoyao, this.mainHandler); final RemoteClient old = this.remoteClients.put(clientId, remoteClient); if(old != null) { // 关闭旧的资源 @@ -285,6 +284,11 @@ public class Room extends CloseableClient implements RouterCallback { )); } + @Override + public void closeRoomCallback() { + + } + @Override public void sendTransportConnectCallback(String transportId, String dtlsParameters) { this.taoyao.request(this.taoyao.buildMessage( diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RoomClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RoomClient.java index 6b4a97d..9f9b096 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RoomClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RoomClient.java @@ -4,8 +4,6 @@ import android.os.Handler; import com.acgist.taoyao.media.signal.ITaoyao; -import org.webrtc.MediaStreamTrack; - /** * 房间终端 * 使用SDK + NDK + Mediasoup实现多人会话 @@ -14,8 +12,8 @@ import org.webrtc.MediaStreamTrack; */ public class RoomClient extends Client { - public RoomClient(String name, String clientId, ITaoyao taoyao, Handler handler) { - super(name, clientId, taoyao, handler); + public RoomClient(String name, String clientId, ITaoyao taoyao, Handler mainHandler) { + super(name, clientId, taoyao, mainHandler); } @Override diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java index 7d70f93..f320a29 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java @@ -6,7 +6,6 @@ import android.util.Log; import com.acgist.taoyao.boot.model.Message; import com.acgist.taoyao.boot.utils.ListUtils; import com.acgist.taoyao.boot.utils.MapUtils; -import com.acgist.taoyao.media.MediaManager; import com.acgist.taoyao.media.VideoSourceType; import com.acgist.taoyao.media.config.Config; import com.acgist.taoyao.media.signal.ITaoyao; @@ -19,7 +18,6 @@ import org.webrtc.PeerConnection; import org.webrtc.PeerConnectionFactory; import org.webrtc.SdpObserver; import org.webrtc.SessionDescription; -import org.webrtc.VideoTrack; import java.util.ArrayList; import java.util.List; @@ -64,8 +62,8 @@ public class SessionClient extends Client { */ private PeerConnectionFactory peerConnectionFactory; - public SessionClient(String sessionId, String name, String clientId, ITaoyao taoyao, Handler handler) { - super(name, clientId, taoyao, handler); + public SessionClient(String sessionId, String name, String clientId, ITaoyao taoyao, Handler mainHandler) { + super(name, clientId, taoyao, mainHandler); this.sessionId = sessionId; } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/signal/ITaoyaoListener.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/signal/ITaoyaoListener.java index 67057d9..cddfdfc 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/signal/ITaoyaoListener.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/signal/ITaoyaoListener.java @@ -2,8 +2,6 @@ package com.acgist.taoyao.media.signal; import com.acgist.taoyao.boot.model.Message; -import java.util.Map; - /** * 信令监听 * diff --git a/taoyao-signal-server/README.md b/taoyao-signal-server/README.md index 2e5efa1..d6857fa 100644 --- a/taoyao-signal-server/README.md +++ b/taoyao-signal-server/README.md @@ -13,15 +13,7 @@ [信令格式](https://localhost:8888/protocol/list) -## 特殊说明 +## 协议 -### 消费者暂停恢复 - -* 消费者直接暂停消费：生产者生成数据、消费者接收数据 -* 媒体服务暂停消费者数据转发（默认）：生产者生成数据、消费者不会接收数据 - -### 生产者暂停恢复 - -* 消费者直接暂停接收：生产者生成数据、消费者接收数据 -* 媒体服务暂停生产者数据转发：生产者生成数据、消费者不会接收数据 -* 媒体服务暂停生产者数据转发同时暂停生产者生成（默认）：生产者不会生成数据、消费者不会接收数据 +* https://www.ortc.org +* https://www.webrtc.org diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolAdapter.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolAdapter.java index e070d94..ae60b42 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolAdapter.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolAdapter.java @@ -14,7 +14,6 @@ import com.acgist.taoyao.boot.service.IdService; import com.acgist.taoyao.signal.client.ClientManager; import com.acgist.taoyao.signal.event.ApplicationEventAdapter; import com.acgist.taoyao.signal.party.media.RoomManager; -import com.acgist.taoyao.signal.party.session.SessionManager; import lombok.extern.slf4j.Slf4j; diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolSessionAdapter.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolSessionAdapter.java index bff01b3..34b2d44 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolSessionAdapter.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/ProtocolSessionAdapter.java @@ -10,7 +10,6 @@ import com.acgist.taoyao.boot.model.MessageCodeException; import com.acgist.taoyao.boot.utils.MapUtils; import com.acgist.taoyao.signal.client.Client; import com.acgist.taoyao.signal.client.ClientType; -import com.acgist.taoyao.signal.party.media.Room; import com.acgist.taoyao.signal.party.session.Session; import com.acgist.taoyao.signal.party.session.SessionManager;