From ed66875a47ae133c559d7a8e27c3e01bb9a56142 Mon Sep 17 00:00:00 2001 From: acgist <289547414@qq.com> Date: Sat, 1 Apr 2023 02:28:33 +0800 Subject: [PATCH] =?UTF-8?q?[+]=20=E8=A7=86=E9=A2=91=E6=8D=95=E8=8E=B7?= =?UTF-8?q?=EF=BC=9A=E5=89=8D=E7=BD=AE=E3=80=81=E5=90=8E=E7=BD=AE=E3=80=81?= =?UTF-8?q?=E5=B1=8F=E5=B9=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 18 +- docs/Deploy.md | 64 ++-- docs/WebRTC.md | 38 +++ .../client/src/main/AndroidManifest.xml | 6 +- .../acgist/taoyao/client/MainActivity.java | 85 +++-- .../acgist/taoyao/client/MediaService.java | 61 +++- .../client/media/RoomClientManager.java | 4 - .../taoyao/client/p2p/P2PClientManager.java | 4 - .../acgist/taoyao/client/signal/Taoyao.java | 16 + .../client/src/main/res/drawable/record.xml | 5 + .../client/src/main/res/drawable/settings.png | Bin 6468 -> 0 bytes .../client/src/main/res/drawable/settings.xml | 5 + .../src/main/res/layout/activity_main.xml | 17 + .../client/src/main/res/values/settings.xml | 4 + .../client/src/main/res/values/strings.xml | 1 + .../taoyao/media/src/main/AndroidManifest.xml | 4 + .../java/com/acgist/taoyao/config/Config.java | 16 +- .../com/acgist/taoyao/media/LocalClient.java | 17 + .../com/acgist/taoyao/media/MediaManager.java | 139 ++++++-- .../acgist/taoyao/media/MediaRecorder.java | 297 +++++++++++++++++- .../com/acgist/taoyao/media/P2PClient.java | 28 +- .../com/acgist/taoyao/media/RemoteClient.java | 16 +- .../java/com/acgist/taoyao/media/Room.java | 31 +- .../com/acgist/taoyao/media/RoomClient.java | 13 +- .../acgist/taoyao/boot/utils/NetUtils.java | 1 + .../signal/config/camera/AiProperties.java | 2 +- 26 files changed, 756 insertions(+), 136 deletions(-) delete mode 100644 taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/media/RoomClientManager.java delete mode 100644 taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/p2p/P2PClientManager.java create mode 100644 taoyao-client-android/taoyao/client/src/main/res/drawable/record.xml delete mode 100644 taoyao-client-android/taoyao/client/src/main/res/drawable/settings.png create mode 100644 taoyao-client-android/taoyao/client/src/main/res/drawable/settings.xml diff --git a/README.md b/README.md index 5a4f363..6e183ff 100644 --- a/README.md +++ b/README.md @@ -23,14 +23,22 @@ |:--|:--|:--| |taoyao-client-web|Web终端|Web终端| |taoyao-client-media|媒体终端|媒体服务| -|taoyao-client-android|安卓终端|安卓终端| |taoyao-signal-server|信令服务|终端信令控制| +|taoyao-client-android|安卓终端|安卓终端| -## 功能 +### Web终端功能 -Web、信令已经完成大部分音视频功能，还有部分视频质量调整功能没有完成。 +|功能|是否支持|是否实现|描述| +|P2P|支持|暂未实现|P2P监控模式| +|WebRTC|支持|实现|Web终端不能同时进入多个房间| -Android还在学习之中... +### 安卓终端功能 + +|功能|是否支持|是否实现|描述| +|P2P|支持|暂未实现|P2P监控模式| +|WebRTC|支持|暂未实现|安卓终端支持同时进入多个房间| +|RTP|支持|暂未实现|支持房间RTP推流（不会拉流）| +|||| ## 证书 @@ -54,8 +62,6 @@ Android还在学习之中... ## TODO -* P2P -* RTP * 标识 -> ID * 所有字段获取 -> get * 优化JS错误回调 -> platform::error diff --git a/docs/Deploy.md b/docs/Deploy.md index b4e6aa5..54ff12f 100644 --- a/docs/Deploy.md +++ b/docs/Deploy.md @@ -403,6 +403,17 @@ firewall-cmd --list-ports ``` mkdir /data/certs cd /data/certs + +# CA证书 + +openssl genrsa -out ca.key 2048 +openssl req -x509 -new -key ca.key -out ca.crt -days 3650 +openssl x509 -in ca.crt -subject -issuer -noout +# subject= /C=cn/ST=gd/L=gz/O=acgist/OU=acgist/CN=acgist.com +# issuer= /C=cn/ST=gd/L=gz/O=acgist/OU=acgist/CN=acgist.com + +# Server证书信息 + vim server.ext --- @@ -422,58 +433,19 @@ DNS.3=www.acgist.com DNS.4=taoyao.acgist.com --- -# CA -openssl genrsa -out ca.key 2048 -openssl req -x509 -new -key ca.key -out ca.crt -days 3650 -openssl x509 -in ca.crt -subject -issuer -noout -# subject= /C=cn/ST=gd/L=gz/O=acgist/OU=acgist/CN=acgist.com -# issuer= /C=cn/ST=gd/L=gz/O=acgist/OU=acgist/CN=acgist.com - -# Server +# Server证书 openssl genrsa -out server.key 2048 openssl req -new -key server.key -out server.csr +# 设置信息：-subj "/C=cn/ST=gd/L=gz/O=acgist/OU=taoyao/CN=taoyao.acgist.com" openssl x509 -req -in server.csr -out server.crt -CA ca.crt -CAkey ca.key -CAcreateserial -days 3650 -extfile server.ext openssl x509 -in server.crt -subject -issuer -noout # subject= /C=cn/ST=gd/L=gz/O=acgist/OU=taoyao/CN=taoyao.acgist.com # issuer= /C=cn/ST=gd/L=gz/O=acgist/OU=acgist/CN=acgist.com openssl pkcs12 -export -clcerts -in server.crt -inkey server.key -out server.p12 -name taoyao -``` - -## licenses - -``` -List of licenses: -webrtc, -abseil-cpp, -android_deps, -android_deps:com_android_support_support_annotations.*, -android_ndk, -android_sdk, -androidx, -base64, -boringssl, -crc32c, -fft, -fiat, -g711, -g722, -ijar, -jdk, -libaom, -libevent, -libjpeg_turbo, -libsrtp, -libvpx, -libyuv, -nasm, -ooura, -opus, -pffft, -protobuf, -rnnoise, -sigslot, -spl_sqrt_floor, -usrsctp, -zlib +# 不要导出ca证书：-clcerts +# 设置密码：-passout pass:123456 +# keytool -importkeystore -v -srckeystore server.p12 -srcstoretype pkcs12 -destkeystore server.jks -deststoretype jks +# 原始密码：-srcstorepass 123456 +# 设置密码：-deststorepass 123456 ``` diff --git a/docs/WebRTC.md b/docs/WebRTC.md index c9c66f4..4b52251 100644 --- a/docs/WebRTC.md +++ b/docs/WebRTC.md @@ -87,3 +87,41 @@ cmake . -B build \ make -C build make install -C build ``` + +## licenses + +``` +List of licenses: +webrtc, +abseil-cpp, +android_deps, +android_deps:com_android_support_support_annotations.*, +android_ndk, +android_sdk, +androidx, +base64, +boringssl, +crc32c, +fft, +fiat, +g711, +g722, +ijar, +jdk, +libaom, +libevent, +libjpeg_turbo, +libsrtp, +libvpx, +libyuv, +nasm, +ooura, +opus, +pffft, +protobuf, +rnnoise, +sigslot, +spl_sqrt_floor, +usrsctp, +zlib +``` diff --git a/taoyao-client-android/taoyao/client/src/main/AndroidManifest.xml b/taoyao-client-android/taoyao/client/src/main/AndroidManifest.xml index 93be3b9..f2613a3 100644 --- a/taoyao-client-android/taoyao/client/src/main/AndroidManifest.xml +++ b/taoyao-client-android/taoyao/client/src/main/AndroidManifest.xml @@ -11,6 +11,7 @@ android:supportsRtl="true"> @@ -29,7 +31,9 @@ + android:exported="false" + android:foregroundServiceType="mediaProjection" /> + activityResultLauncher; + private MediaProjectionManager mediaProjectionManager; @Override protected void onCreate(Bundle bundle) { @@ -52,7 +62,8 @@ public class MainActivity extends AppCompatActivity implements Serializable { // 布局 this.binding = ActivityMainBinding.inflate(this.getLayoutInflater()); this.setContentView(this.binding.getRoot()); - // 设置按钮 + this.registerMediaProjection(); + this.binding.record.setOnClickListener(this::switchRecord); this.binding.settings.setOnClickListener(this::launchSettings); } @@ -83,27 +94,18 @@ public class MainActivity extends AppCompatActivity implements Serializable { Manifest.permission.INTERNET, Manifest.permission.RECORD_AUDIO, Manifest.permission.ACCESS_WIFI_STATE, + Manifest.permission.FOREGROUND_SERVICE, Manifest.permission.ACCESS_FINE_LOCATION, Manifest.permission.READ_EXTERNAL_STORAGE, Manifest.permission.ACCESS_COARSE_LOCATION, Manifest.permission.RECEIVE_BOOT_COMPLETED, Manifest.permission.WRITE_EXTERNAL_STORAGE }; - boolean allGranted = true; - for (String permission : permissions) { - if(this.getApplicationContext().checkCallingOrSelfPermission(permission) == PackageManager.PERMISSION_GRANTED) { - Log.i(MediaService.class.getSimpleName(), "授权成功：" + permission); - } else { - allGranted = false; - Log.w(MediaService.class.getSimpleName(), "授权失败：" + permission); - } + if(Stream.of(permissions).map(this.getApplicationContext()::checkSelfPermission).allMatch(v -> v == PackageManager.PERMISSION_GRANTED)) { + Log.i(MediaService.class.getSimpleName(), "授权成功"); + } else { + ActivityCompat.requestPermissions(this, permissions, 0); } - if(!allGranted) { - Toast.makeText(this, "授权失败", Toast.LENGTH_SHORT).show(); - } - MediaManager.getInstance().init(this.getApplicationContext()); - MediaManager.getInstance().initAudio(); - MediaManager.getInstance().initVideo(); } /** @@ -129,6 +131,40 @@ public class MainActivity extends AppCompatActivity implements Serializable { } } + private void registerMediaProjection() { + if(this.activityResultLauncher != null && this.mediaProjectionManager != null) { + return; + } + this.mediaProjectionManager = this.getApplicationContext().getSystemService(MediaProjectionManager.class); + this.activityResultLauncher = this.registerForActivityResult( + new ActivityResultContracts.StartActivityForResult(), + result -> { + if(result.getResultCode() == Activity.RESULT_OK) { + Log.i(MediaManager.class.getSimpleName(), "屏幕捕获成功"); + final Intent intent = new Intent(this, MediaService.class); + intent.setAction(MediaService.Action.SCREEN_RECORD.name()); + intent.putExtra("data", result.getData()); + intent.putExtra("code", result.getResultCode()); + this.startService(intent); + } else { + Log.w(MainActivity.class.getSimpleName(), "屏幕捕获失败：" + result.getResultCode()); + } + } + ); + } + + private void switchRecord(View view) { + final MediaRecorder mediaRecorder = MediaRecorder.getInstance(); + if(mediaRecorder.isActive()) { + mediaRecorder.stop(); + } else { + MediaManager.getInstance().init(this.mainHandler, this.getApplicationContext()); + MediaManager.getInstance().initAudio(); + MediaManager.getInstance().initVideo(); + mediaRecorder.init(System.currentTimeMillis() + ".mp4", null, null, 1, 1); + } + } + /** * 拉起设置页面 * @@ -158,7 +194,7 @@ public class MainActivity extends AppCompatActivity implements Serializable { Log.d(MainHandler.class.getSimpleName(), "Handler消息：" + message.what + " - " + message.obj); switch(message.what) { case Config.WHAT_SCREEN_CAPTURE -> this.mainActivity.screenCapture(message); - case Config.WHAT_NEW_CLIENT_VIDEO -> this.mainActivity.newClientVideo(message); + case Config.WHAT_NEW_LOCAL_VIDEO -> this.mainActivity.newLocalVideo(message); } } @@ -170,18 +206,7 @@ public class MainActivity extends AppCompatActivity implements Serializable { * @param message 消息 */ private void screenCapture(Message message) { - final ActivityResultLauncher activityResultLauncher = this.registerForActivityResult( - new ActivityResultContracts.StartActivityForResult(), - result -> { - if(result.getResultCode() == RESULT_OK) { - Log.i(MediaManager.class.getSimpleName(), "开始屏幕捕获"); - message.getCallback().run(); - } else { - Log.w(MainActivity.class.getSimpleName(), "屏幕捕获失败：" + result.getResultCode()); - } - } - ); - activityResultLauncher.launch((Intent) message.obj); + this.activityResultLauncher.launch(this.mediaProjectionManager.createScreenCaptureIntent()); } /** @@ -189,7 +214,9 @@ public class MainActivity extends AppCompatActivity implements Serializable { * * @param message 消息 */ - private void newClientVideo(Message message) { + private void newLocalVideo(Message message) { + final SurfaceView surfaceView = (SurfaceView) message.obj; + this.addContentView(surfaceView, new ViewGroup.LayoutParams(ViewGroup.LayoutParams.WRAP_CONTENT, ViewGroup.LayoutParams.WRAP_CONTENT)); } } \ No newline at end of file diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java index 0867b8d..87ac9ba 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MediaService.java @@ -1,12 +1,15 @@ package com.acgist.taoyao.client; -import android.Manifest; +import android.app.Notification; +import android.app.NotificationChannel; +import android.app.NotificationManager; +import android.app.PendingIntent; import android.app.Service; import android.content.Context; import android.content.Intent; import android.content.SharedPreferences; -import android.content.pm.PackageManager; import android.content.res.Resources; +import android.graphics.BitmapFactory; import android.location.LocationManager; import android.net.wifi.WifiManager; import android.os.BatteryManager; @@ -16,7 +19,10 @@ import android.os.IBinder; import android.util.Log; import android.widget.Toast; +import androidx.core.app.NotificationCompat; + import com.acgist.taoyao.client.signal.Taoyao; +import com.acgist.taoyao.media.MediaManager; /** * 媒体服务 @@ -40,7 +46,9 @@ public class MediaService extends Service { // 连接 CONNECT, // 重连 - RECONNECT; + RECONNECT, + // 屏幕录制 + SCREEN_RECORD; } @@ -63,16 +71,11 @@ public class MediaService extends Service { public int onStartCommand(Intent intent, int flags, int startId) { Log.i(MediaService.class.getSimpleName(), "onStartCommand"); if (Action.CONNECT.name().equals(intent.getAction())) { - if (this.taoyao == null) { - Log.d(MediaService.class.getSimpleName(), "打开信令连接"); - this.mainHandler = (Handler) intent.getSerializableExtra("mainHandler"); - this.connect(); - } else { - Log.d(MediaService.class.getSimpleName(), "信令已经连接"); - } + this.openConnect(intent); } else if (Action.RECONNECT.name().equals(intent.getAction())) { - Log.d(MediaService.class.getSimpleName(), "重新连接信令"); - this.connect(); + this.reconnect(); + } else if (Action.SCREEN_RECORD.name().equals(intent.getAction())) { + this.screenRecord(intent); } else { Log.w(MediaService.class.getSimpleName(), "未知动作：" + intent.getAction()); } @@ -86,6 +89,21 @@ public class MediaService extends Service { this.close(); } + private void openConnect(Intent intent) { + if (this.taoyao == null) { + Log.d(MediaService.class.getSimpleName(), "打开信令连接"); + this.mainHandler = (Handler) intent.getSerializableExtra("mainHandler"); + this.connect(); + } else { + Log.d(MediaService.class.getSimpleName(), "信令已经连接"); + } + } + + private void reconnect() { + Log.d(MediaService.class.getSimpleName(), "重新连接信令"); + this.connect(); + } + /** * 连接信令 */ @@ -125,4 +143,23 @@ public class MediaService extends Service { this.taoyao = null; } + public void screenRecord(Intent intent) { + final Intent notificationIntent = new Intent(this, MediaService.class); + final PendingIntent pendingIntent = PendingIntent.getActivity(this, 0, notificationIntent, 0); + final NotificationCompat.Builder notificationBuilder = new NotificationCompat.Builder(this, "NOTIFICATION_CHANNEL_ID") + .setSmallIcon(R.mipmap.ic_launcher_foreground) + .setLargeIcon(BitmapFactory.decodeResource(this.getResources(), R.mipmap.ic_launcher_foreground)) + .setTicker("NOTIFICATION_TICKER") + .setContentTitle("屏幕录制") + .setContentText("屏幕录制共享") + .setContentIntent(pendingIntent); + final Notification notification = notificationBuilder.build(); + final NotificationChannel channel = new NotificationChannel("NOTIFICATION_CHANNEL_ID", "NOTIFICATION_CHANNEL_NAME", NotificationManager.IMPORTANCE_DEFAULT); + channel.setDescription("NOTIFICATION_CHANNEL_DESC"); + final NotificationManager notificationManager = this.getSystemService(NotificationManager.class); + notificationManager.createNotificationChannel(channel); + this.startForeground((int) System.currentTimeMillis(), notification); + MediaManager.getInstance().screenRecord(intent.getParcelableExtra("data")); + } + } \ No newline at end of file diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/media/RoomClientManager.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/media/RoomClientManager.java deleted file mode 100644 index 6cb60f5..0000000 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/media/RoomClientManager.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.acgist.taoyao.client.media; - -public class RoomClientManager { -} diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/p2p/P2PClientManager.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/p2p/P2PClientManager.java deleted file mode 100644 index 93f3226..0000000 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/p2p/P2PClientManager.java +++ /dev/null @@ -1,4 +0,0 @@ -package com.acgist.taoyao.client.p2p; - -public class P2PClientManager { -} diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java index 2c58c74..c750906 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java @@ -22,6 +22,8 @@ import com.acgist.taoyao.boot.utils.CloseableUtils; import com.acgist.taoyao.boot.utils.JSONUtils; import com.acgist.taoyao.media.MediaRecorder; import com.acgist.taoyao.client.utils.IdUtils; +import com.acgist.taoyao.media.P2PClient; +import com.acgist.taoyao.media.Room; import org.apache.commons.lang3.ArrayUtils; @@ -34,8 +36,10 @@ import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.util.Base64; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -156,6 +160,14 @@ public final class Taoyao { * 定时任务线程池 */ private final ScheduledExecutorService scheduled; + /** + * 房间列表 + */ + private final List roomList; + /** + * P2P终端列表 + */ + private final List p2pClientList; public Taoyao( int port, String host, String version, @@ -191,6 +203,8 @@ public final class Taoyao { this.scheduled = Executors.newScheduledThreadPool(1); this.executor.submit(this::loopMessage); this.scheduled.scheduleWithFixedDelay(this::heartbeat, 30, 30, TimeUnit.SECONDS); + this.roomList = new CopyOnWriteArrayList<>(); + this.p2pClientList = new CopyOnWriteArrayList<>(); } /** @@ -416,6 +430,8 @@ public final class Taoyao { this.disconnect(); this.executor.shutdown(); this.scheduled.shutdown(); + this.roomList.forEach(Room::close); + this.p2pClientList.forEach(P2PClient::close); } /** diff --git a/taoyao-client-android/taoyao/client/src/main/res/drawable/record.xml b/taoyao-client-android/taoyao/client/src/main/res/drawable/record.xml new file mode 100644 index 0000000..b3d8455 --- /dev/null +++ b/taoyao-client-android/taoyao/client/src/main/res/drawable/record.xml @@ -0,0 +1,5 @@ + + + diff --git a/taoyao-client-android/taoyao/client/src/main/res/drawable/settings.png b/taoyao-client-android/taoyao/client/src/main/res/drawable/settings.png deleted file mode 100644 index 523e42e3bbdb3c3cf7f70cc5fe66bd1a2b3eab93..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6468 zcmaiZcQoA37xriMZnY>8R$oC_ElRM;vTE1_u}ZY)5=)fGV)c?>l_()vln}inTGU0< z=&K|M`RYOTI8uO1Pb&&B}h#rB)UU z(ydk&jFAlFm~OigGDPQ5Q!vIdL!6apK5h_M&HF;>$BmGU=e}S-9PsjM&_H03r!pE` z4EHng2;qeAPrRn_x8up<$F&gi<{%j;`j>US-~{oibyjvgHy zb(&G%#wCm4cC(A;y^d9e>o*L_Gj(f^jWnyRQ6lR8)Sw#)dhO>)>o z1^#I#hZ+8`%7Xt5liP~4A<)h0iPL1S^k!+6OjtE{^Cpus6+emxcK3dfZ@MDPutsQ+ zVO>VBQ@(~9Lr=o_+(M8w@#sb8zpLw_FHc<6^E;=}|GsC%jASM44=|*gkx}rhU0q#8 zqUgE~jwnEbSal7>l>@}UyKq?*oQ&7gRN!XGl4RyEJ!&J#M4)SeuZK%v$PI8TLX|Oj zJx!?WeHM0itGVW`n~zuGS|c>!a6B}_@aiap*iBw6IlN(&Ay6Or55}*w|HNkQ@R8LO zlb(tw480R3mNvNfpKnW?GSl+1`U70nXLer8U8lHb=_uD{dA+}ymgHdP*9*iI=N2z) z*b%>GDG&OLP@WKKTJS@&8?kIb@0;w5MAQ0wTQ0sHEuCVotZj+X&%MwApN5G0g2y;4 zIM5VGsZdZ6W$KCcInX|%LusPFqumlj5rlz@wotJ~+7YDZ-exeQdMQu_;Kt4RD7a28 zll!z+R8)Z4|NK%{>F6eRE-*l-~IQ zYIs6M9a<|&vpg?-*}lKOEB-h~Yx1988tF6v|oz^?ks;OD??gYv06UpU*0qUt3< zre5%bu*TUZmhWc*ufK@NM<(i-EF|aAatkA=$nm(VesY2s_M!bJhGsBNdb4Y6sr7cg^t+8@bToyUrpT@~tkq)yytp~sUlHu?QjSYe|s@kiE>ka3mN;wvu1HvXuMhB3jr)o6ucfiRea zwIz18{w8l*?>Y`l0bpzgnNqOl3!NVszp0!(-tc@mKRLa87EEK{d1-nHmWC#*E=2c! zQ|20`62(Ove4xkwS$JQ^Ta>mg2wEi_6s$aVWC(7zDg(ioo4+c)m-^5ZE)3`W%m0wH zZy6OIs0IIhirs?T-uqBd!5M^VFi6695U17h|yCL4jWQ z?u+B>ddYv8S*F3HNZtyUkf#aqbw(cKV<8VT&d1arbVB`WR{=Ts{GtbnUlm~imAdJA zUPf=UMoiCqv<_1_6;gO8w9GO@#$@g>?Z??B#t81FV|*|Ai=d&YuV*BY4{{hP)w4Y;oR4B2E4&EMVe7o9Xlc?) zrTLi2gbl4h3z?O43WN@)(&0Jl8q)ZM>?)~XC&73C#oouvDrY~Bg8rpszzH*IWm}5E z3AzYqJC{~I1-`3+FfVB&YOykSCM|hap)QrG(1=s)n=mM<2V=*--C>^K$Z|Rs_uiEj zqEjl@3R^eawE}OMNqVeWFX+3{s5E&l=$H8>jVIsxO+BNYF)VU3u2>90tt5)cc;t8q zPI0gsYatsPVS?6lYxKpdtlH_OC5!wlx`6>;+3irSlPIO(qT6|K@{C2-g;HPXOF_h# zAi3;pNHgE@HHUaB!){wT{Tc zJE=xeEdRSLVJj+RIq`FkB|`MBttp&%-^no&)kAw;eDD7{kigN!*Qv{mqA{^0T28+B zjzLAC{StC850MX`MdSoZB?c!%`MOK}#^LD`f7>-UIKiUr-B-;2gV__S(!TckzIS{ItM-Ye^|Co%V6;)$)Cmf=ihOYgUz4sfe8}5K$qb?tko?iO*OT8voFNope71xP^rcG|#F{+yq^30-=(F}gB@xxqT9L!1-`aHVh;H3 zvU04WCqL>iLT-_Pxyz%i=;snK$D|^IoO@aitUAfLQTKL|IQ349c2b>^(we=5cHx35 zOcNTZu)a-sRAiC!u z$VG$eC>m&Novn`PslAeq{>x<(Bq!YC=}ts#KW(vza4_R%E}bFw4uRJt`>;t73T1~l z=Fjb}k4}3qe|BRVUw0zDsT3as3hlD$T{8#(`I?`PVrY(0S!of7u6ih-$jou0L19D| zOO3=&Z$~K}K9>Eb3>03IxVm`<`a9zhH`5jV4V2%bztH=lqBTUY<5TEyQMnqmQF>Y73Ehxfl7%cK_)+wFsSMB@vCqc04wvbRz5X^GKIE#crh& zi&UUzUOCh586o<(D?#CnNwT&SaBC79V_N@e?F6+`k^=Ow2#O#7R5xCQ_;Tp@c_rry z-5G6Dn#j8~y1N?avJ&;6@16zQoVc%u}W@RqB^q#!W|tOV?tbbL$= zmdv35_v`4W_D)bT3E9UnTrQRERVz;G7lb}(%O8BvTl%)?&Eus$lTjbUaP3DUvQb@y(r%- za>rCCG1v478S>%Q{6rE)H5GbTA&}TG*>4gV=Eb9Yjo}gV7&=mFzK}QW?FJ+5$ z|F#DPIUZ{v7a7ux!izPp3@0FEGBngQ0W;!aM+6Tr6C7$8C5Dp$3C?)QN9cUJ&xRTD z$sb}U=VpUv0fjocmEIzb)ViB!5=-4sH1Id{Y-J@)usn85@m+_O(y;UvKh}@L&*|kN=2w&CP>}viAXpNw~**cwO68%0zv+mXL@HSv3Ax2cF)_d8v6k3wA zvEquZk7>*#eUbNnTSeAO>I;~a4~FFaFdRH2fM^hMC?iQ(c$8}6G~P`VmfBZtEvo!< z6m~=WcP8Mt3p1fJ&UjRG%&27US})0#jh)XLEhN7=3w($C#igwj5u3fwZAT=;b*OGo zk%N;MgSx(J*CL|qVU`cZ1-@wx*#2owTRJPuU96jh7~$e%5g(LCl%3NbSvBsA@_Hn< zRa9V?*2+0}53di}tKt+}IC;tjuTVGbm(K{KmFd8c^D^)HN;a4Hb4|fpR)3zxlR9G*1m9 z*M~*fUSctXjuyh}pY &^h89?j!w~Bi5ngOm>p+G4?6Se5PdpLpGAnYtt=d_nb?L z7A@}nGm8xMG>HP-Fo}{wEdKT-aa@;jWd+?>X54`0yGG-IB51Bimyk ztLGyw4Mc!ZlZ~Z z-eHUFa_dLRs$nbbS57Z4FI7v}AE)}B;yFRGaHXW+yF(7I+rL*gt93RtaLZ}zOx(?h zUO|Z;)sXZCVO;hhf0aV+ei_>HFZcbnwij5~^^g6|PfKARf=&aP&<>g(Lq7xEnM&q_ z=Sl1mKXbA&v8bt)=aZZbR_QslA7~Xp`5Jd_ku^_miu)Hn0ulQ1H1EeEDFK zquiX{zZGQj$*8BaU>M#jgcXPm%sHt8SW!WZvz+f|wi6{lbXDtHPJ0qYd2Ed}69!8SMO`B#SZvCN0FbgAkPu)CShK*(Nbn##eXaE`<6xwFun>Udw4^35|{ z;=V4;E$le)XzxbnbGB=Xby+??AM#B)y)MMfB1i(O;~H5FFz*7T5Ff73M0*MswFDQv zMpyN%43gwb1;tgaG=*B0zbmWY7c8w|G7LLK$aIf(<1NY-knM^jE0!`M)eD>k<;z~# z9c3Yn?`DsKUe0r{LRWF1yddOp`xH9gq-^R1o*vGQlh^uhT%M&65#Tnm)>`f0*ovlE z$Zyti@yXkGfhU^^)bz`Zd1}Yr=gy$=mW*OVaRRbi=rhU+Lpx_$%zG))V0zjKVFo{T zbj2yvZP%ZLZk+)Z54-oL#tM&{2kUp0qQsk$t2Y4*4QmS8nSZTL=Et+7r^lblcahP0 zXsi)vi#%>mF$v3DcFxVNGOkAfM?<=2&_DEgVz{0iP)}nNbIc46Xgu<7Zrcz#6yMcqTuMDq#V$eRSQ?{O0?5g`mcFZHU)5)En&6&ce{3k z>XNr1NU=Pi`Ux3=jG(mpqt$wl`InSp+wvFMeS<`J0ws-~LPCIraFW~5-%h78Lt}7+&K6L3CNOq4 zBRcD4VOS&`So}nlNpyrnNe)v9FXI?)0Lh8@&r~xXB!<1?5e5m%@G#G~8UeJD$k>Y( zXGhAsZ}pNX-weyW^04?WzdqUuqzQ9swI+&PBBJ1)Mz%c7E5E4F+^1}V60G4Iky|~@ zOxl%Spcaunlh@vHDl+M^Ml+@J_!G(pXFQJ5bn1o4qqf7GQB&81>2Sj^Gf)H!`9_Yk z9sSdN58ZcBP?1kB`~>eWXqLy+2!eWkX$bSRrU(<%2YRw^m$z-j(%>!CJsQv0E4829 z);ly*+spubp_AfKNu+9hS*_$}=4a$=H;95LPV~PN3>XX(_|5Z_Ykeu5*(yU#0ILKl zmY&M`u-ofeoM*Z-99h_u^JRm-arT|QQ3xyCOKD*4m3P>BaYcn+&VK2K0A88U0Ws>2 zdhsKh9PW+|3C%}!uicZLZMkN?(`;7E8I>-**u2&7!L@}`!q`e*=zgr6RSdB^vs}l`ktzMO0aEjl{B#N7oVPZJ67jEk8L}=(?I+BKl!Md|InWD*c4t<7DjNx znPBzUC)FRm(vuCf8O>0{+_q73gtR5R~Rr_twB+uyIJ_~u48ci8E^-Rynt(EGg zcy5UiX0ZTevqKx6FnW6WbMfKdZ(0H;IE{-uv*KqVqr0;AmfNBzM2D=**f;Js47;fS z+6KnW%5+Y-m4?&?c#67`+kKC1oE1PE@a^aCH)RA6Q=Ps2lip^24H zM;$K&EhcA`evUD4D=JK{+cedHnjGvihuI9wpR@I>`f@h+TQ}?Kr5DP-d!rSaz-hYx zvo{5;anvYxP=Q;UCH4Z-_R0PFcgXelK7PK2_=O?n^}@R*4mMk>cW$>uu!e_YVpkBG zrmA|TR3(}`T8zjs8@-RmFxFB!qlk4`W47|M;f9d0Y@%&Qr~WDf1H;P=LtEm2x?pTI zw*r#iIfs_1Ci%nneW`%@S16zU5QM*1ztiHiwSgFsW0&?bu4S5R6#wBJjV;+juXx&t zP&`ku!^?QS!q9zbfvi^@4qw6xEwi}g;WD+_=MQF*+m2qngQ(J$6;zVb<$p~&~WjjZ~FqyO+denmP+3=pTY#E0SyNXW!|$kVcrb z?@n@sh<+KTVAL6Y>VQ#&Aa?>0&#j#GvRU&Es@}GTL5VR@MX8X#DE@{M@O8{er!E z?RnaxdCbcj&VEG`eJ)#@;VD49-p1Psp`Hv9!3i(8aTHtRnwv*&q8ekjTocCzHebIh z);#ukw?TCyAgRu;6CcpuO|=6es0+IbY$=nwr2Q1=`&7NNSeWZ6^WDC< zQhsai?>}&~toikt0dV!6^60XHAyDXh>*zWqH7i^fHX4t-Npjxh8eZylhg0CqjQj*s zhr%hqH#bn7WC(>T?o* diff --git a/taoyao-client-android/taoyao/client/src/main/res/drawable/settings.xml b/taoyao-client-android/taoyao/client/src/main/res/drawable/settings.xml new file mode 100644 index 0000000..298a5a1 --- /dev/null +++ b/taoyao-client-android/taoyao/client/src/main/res/drawable/settings.xml @@ -0,0 +1,5 @@ + + + diff --git a/taoyao-client-android/taoyao/client/src/main/res/layout/activity_main.xml b/taoyao-client-android/taoyao/client/src/main/res/layout/activity_main.xml index 608ed46..460b74f 100644 --- a/taoyao-client-android/taoyao/client/src/main/res/layout/activity_main.xml +++ b/taoyao-client-android/taoyao/client/src/main/res/layout/activity_main.xml @@ -7,6 +7,23 @@ android:layout_height="match_parent" tools:context="com.acgist.taoyao.client.MainActivity"> + + /taoyao/image /taoyao/video + + WEBRTC + + \ No newline at end of file diff --git a/taoyao-client-android/taoyao/client/src/main/res/values/strings.xml b/taoyao-client-android/taoyao/client/src/main/res/values/strings.xml index 0c851a1..e86f118 100644 --- a/taoyao-client-android/taoyao/client/src/main/res/values/strings.xml +++ b/taoyao-client-android/taoyao/client/src/main/res/values/strings.xml @@ -3,6 +3,7 @@ 桃夭 桃夭终端预览 桃夭终端设置 + 录制 设置 连接 信令端口 diff --git a/taoyao-client-android/taoyao/media/src/main/AndroidManifest.xml b/taoyao-client-android/taoyao/media/src/main/AndroidManifest.xml index 800a77f..992540c 100644 --- a/taoyao-client-android/taoyao/media/src/main/AndroidManifest.xml +++ b/taoyao-client-android/taoyao/media/src/main/AndroidManifest.xml @@ -5,10 +5,14 @@ + + + + \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/Config.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/Config.java index a40debc..273dfe3 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/Config.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/config/Config.java @@ -12,8 +12,20 @@ public class Config { */ public static final int WHAT_SCREEN_CAPTURE = 1000; /** - * 新建用户视频 + * 新建本地音频 */ - public static final int WHAT_NEW_CLIENT_VIDEO = 1001; + public static final int WHAT_NEW_LOCAL_AUDIO = 2000; + /** + * 新建本地视频 + */ + public static final int WHAT_NEW_LOCAL_VIDEO = 2001; + /** + * 新建远程音频 + */ + public static final int WHAT_NEW_REMOTE_AUDIO = 2002; + /** + * 新建远程视频 + */ + public static final int WHAT_NEW_REMOTE_VIDEO = 2003; } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/LocalClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/LocalClient.java index ad1f1a9..17410f7 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/LocalClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/LocalClient.java @@ -9,8 +9,25 @@ import java.util.logging.Handler; */ public class LocalClient extends RoomClient { + /** + * 传输类型 + * + * @author acgist + */ + public enum TransportType { + + RTP, + WEBRTC; + + } + public LocalClient(String name, String clientId, Handler handler) { super(name, clientId, handler); } + @Override + public void close() { + super.close(); + } + } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java index cfec6de..85f4ca5 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java @@ -2,9 +2,11 @@ package com.acgist.taoyao.media; import android.content.Context; import android.content.Intent; +import android.media.AudioRecord; import android.media.projection.MediaProjection; import android.media.projection.MediaProjectionManager; import android.os.Handler; +import android.os.Looper; import android.os.Message; import android.util.Log; @@ -15,18 +17,29 @@ import org.webrtc.AudioTrack; import org.webrtc.Camera2Enumerator; import org.webrtc.CameraEnumerator; import org.webrtc.CameraVideoCapturer; +import org.webrtc.DefaultVideoDecoderFactory; +import org.webrtc.DefaultVideoEncoderFactory; import org.webrtc.EglBase; import org.webrtc.MediaConstraints; import org.webrtc.MediaStream; import org.webrtc.PeerConnectionFactory; +import org.webrtc.RendererCommon; import org.webrtc.ScreenCapturerAndroid; import org.webrtc.SurfaceTextureHelper; import org.webrtc.SurfaceViewRenderer; import org.webrtc.VideoCapturer; +import org.webrtc.VideoDecoderFactory; +import org.webrtc.VideoEncoderFactory; +import org.webrtc.VideoFrame; +import org.webrtc.VideoSink; import org.webrtc.VideoSource; import org.webrtc.VideoTrack; +import org.webrtc.audio.JavaAudioDeviceModule; +import org.webrtc.voiceengine.WebRtcAudioManager; +import org.webrtc.voiceengine.WebRtcAudioRecord; import org.webrtc.voiceengine.WebRtcAudioUtils; +import java.util.Arrays; import java.util.Iterator; import java.util.List; @@ -34,6 +47,18 @@ import java.util.List; * 媒体来源管理器 * * @author acgist + * + * https://zhuanlan.zhihu.com/p/82446482 + * https://www.jianshu.com/p/97acd9a51909 + * https://blog.csdn.net/nanoage/article/details/127406494 + * https://webrtc.org.cn/20190419_tutorial3_webrtc_android + * https://blog.csdn.net/CSDN_Mew/article/details/103406781 + * https://blog.csdn.net/Tong_Hou/article/details/112116349 + * https://blog.csdn.net/u011418943/article/details/127108642 + * https://blog.csdn.net/csdn_shen0221/article/details/120331004 + * https://blog.csdn.net/csdn_shen0221/article/details/119982257 + * + * TODO：动态码率（BITRATE_MODE_VBR、BITRATE_MODE） */ public class MediaManager { @@ -84,6 +109,14 @@ public class MediaManager { * 上下文 */ private Context context; + /** + * 本地终端 + */ + private LocalClient localClient; + /** + * + */ + private EglBase eglBase; /** * 媒体流：声音、主码流（预览流）、次码流 */ @@ -107,6 +140,8 @@ public class MediaManager { WebRtcAudioUtils.setWebRtcBasedNoiseSuppressor(true); // 回声小丑 WebRtcAudioUtils.setWebRtcBasedAcousticEchoCanceler(true); + // 使用OpenSL ES +// WebRtcAudioManager.setBlacklistDeviceForOpenSLESUsage(true); } /** @@ -114,16 +149,38 @@ public class MediaManager { * * @param context 上下文 */ - public void init(Context context) { + public void init(Handler handler, Context context) { this.type = Type.BACK; + this.handler = handler; this.context = context; + this.eglBase = EglBase.create(); PeerConnectionFactory.initialize( PeerConnectionFactory.InitializationOptions.builder(this.context) .setEnableInternalTracer(true) .createInitializationOptions() ); - this.peerConnectionFactory = PeerConnectionFactory.builder().createPeerConnectionFactory(); + final VideoDecoderFactory videoDecoderFactory = new DefaultVideoDecoderFactory(this.eglBase.getEglBaseContext()); + final VideoEncoderFactory videoEncoderFactory = new DefaultVideoEncoderFactory(this.eglBase.getEglBaseContext(), true, true); + final JavaAudioDeviceModule javaAudioDeviceModule = JavaAudioDeviceModule.builder(this.context) + // 本地音频 + .setSamplesReadyCallback(MediaRecorder.getInstance().audioRecoder) + // 远程音频 +// .setAudioTrackStateCallback() +// .setUseHardwareNoiseSuppressor(true) +// .setUseHardwareAcousticEchoCanceler(true) + .createAudioDeviceModule(); + this.peerConnectionFactory = PeerConnectionFactory.builder() +// .setAudioProcessingFactory() +// .setAudioDecoderFactoryFactory() +// .setAudioEncoderFactoryFactory() + .setAudioDeviceModule(javaAudioDeviceModule) + .setVideoDecoderFactory(videoDecoderFactory) + .setVideoEncoderFactory(videoEncoderFactory) + .createPeerConnectionFactory(); this.mediaStream = this.peerConnectionFactory.createLocalMediaStream("ARDAMS"); + Arrays.stream(videoEncoderFactory.getSupportedCodecs()).forEach(v -> { + Log.i(MediaManager.class.getSimpleName(), "支持的视频解码器：" + v.name); + }); } /** @@ -163,6 +220,25 @@ public class MediaManager { this.closeAudioTrack(); // 加载音频 final MediaConstraints mediaConstraints = new MediaConstraints(); + // 高音过滤 +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googHighpassFilter", "true")); +// // 自动增益 +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googAutoGainControl", "true")); +// // 回声消除 +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googEchoCancellation", "true")); +// // 噪音处理 +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googNoiseSuppression", "true")); + // 更多 +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googEchoCancellation", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googEchoCancellation2", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googDAEchoCancellation", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googTypingNoiseDetection", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googAutoGainControl", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googAutoGainControl2", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googNoiseSuppression", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googNoiseSuppression2", "true")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googAudioMirroring", "false")); +// mediaConstraints.mandatory.add(new MediaConstraints.KeyValuePair("googHighpassFilter", "true")); final AudioSource audioSource = this.peerConnectionFactory.createAudioSource(mediaConstraints); final AudioTrack audioTrack = this.peerConnectionFactory.createAudioTrack("ARDAMSa0", audioSource); audioTrack.setEnabled(true); @@ -192,26 +268,25 @@ public class MediaManager { Log.d(MediaManager.class.getSimpleName(), "忽略摄像头：" + name); } } + this.initVideoTrack(); } else if(this.type == Type.FILE) { Log.i(MediaManager.class.getSimpleName(), "加载视频（文件）"); } else if(this.type == Type.SCREEN) { Log.i(MediaManager.class.getSimpleName(), "加载视频（录屏）"); - final MediaProjectionManager mediaProjectionManager = this.context.getSystemService(MediaProjectionManager.class); - final Intent intent = mediaProjectionManager.createScreenCaptureIntent(); - final Message message = Message.obtain(this.handler, () -> { - this.videoCapturer = new ScreenCapturerAndroid(intent, new MediaProjection.Callback() { - @Override - public void onStop() { - super.onStop(); - Log.i(MediaManager.class.getSimpleName(), "停止屏幕捕获"); - } - }); - }); - message.obj = intent; + final Message message = new Message(); message.what = Config.WHAT_SCREEN_CAPTURE; this.handler.sendMessage(message); -// this.handler.dispatchMessage(message); } + } + + public void screenRecord(Intent intent) { + this.videoCapturer = new ScreenCapturerAndroid(intent, new MediaProjection.Callback() { + @Override + public void onStop() { + super.onStop(); + Log.i(MediaManager.class.getSimpleName(), "停止屏幕捕获"); + } + }); this.initVideoTrack(); } @@ -219,20 +294,42 @@ public class MediaManager { * 加载视频 */ private void initVideoTrack() { - // 设置预览 - final SurfaceViewRenderer surfaceViewRenderer = new SurfaceViewRenderer(this.context); - surfaceViewRenderer.setMirror(true); - surfaceViewRenderer.setEnableHardwareScaler(true); + final SurfaceViewRenderer surfaceViewRenderer = this.preview(); // 加载视频 - final EglBase eglBase = EglBase.create(); - final SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("MediaVideoThread", eglBase.getEglBaseContext()); + final SurfaceTextureHelper surfaceTextureHelper = SurfaceTextureHelper.create("MediaVideoThread", this.eglBase.getEglBaseContext()); final VideoSource videoSource = this.peerConnectionFactory.createVideoSource(this.videoCapturer.isScreencast()); this.videoCapturer.initialize(surfaceTextureHelper, this.context, videoSource.getCapturerObserver()); this.videoCapturer.startCapture(640, 480, 30); final VideoTrack videoTrack = this.peerConnectionFactory.createVideoTrack("ARDAMSv0", videoSource); + videoTrack.addSink(surfaceViewRenderer); + videoTrack.addSink(MediaRecorder.getInstance().videoRecoder); videoTrack.setEnabled(true); this.mediaStream.addTrack(videoTrack); Log.i(MediaManager.class.getSimpleName(), "加载视频：" + videoTrack.id()); +// 二次处理：VideoProcessor + } + + private SurfaceViewRenderer preview() { + // 设置预览 + final SurfaceViewRenderer surfaceViewRenderer = new SurfaceViewRenderer(this.context); + // 视频反转 + surfaceViewRenderer.setMirror(false); + // 视频拉伸 + surfaceViewRenderer.setScalingType(RendererCommon.ScalingType.SCALE_ASPECT_FILL); + // 硬件拉伸 + surfaceViewRenderer.setEnableHardwareScaler(true); + // 加载 + surfaceViewRenderer.init(this.eglBase.getEglBaseContext(), null); + // 事件 +// surfaceViewRenderer.setOnClickListener(); + // TODO：迁移localvideo +// surfaceViewRenderer.release(); + // 页面加载 + final Message message = new Message(); + message.obj = surfaceViewRenderer; + message.what = Config.WHAT_NEW_LOCAL_VIDEO; + this.handler.sendMessage(message); + return surfaceViewRenderer; } public void pauseAudio() { diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaRecorder.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaRecorder.java index 2f5cb9c..a4e379f 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaRecorder.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaRecorder.java @@ -1,23 +1,102 @@ package com.acgist.taoyao.media; +import android.media.AudioFormat; +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaCodecList; +import android.media.MediaFormat; +import android.media.MediaMuxer; +import android.os.Environment; +import android.util.Log; + +import org.webrtc.VideoFrame; +import org.webrtc.VideoSink; +import org.webrtc.audio.JavaAudioDeviceModule; +import org.webrtc.voiceengine.WebRtcAudioRecord; + +import java.io.File; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.file.Paths; +import java.util.stream.Collectors; +import java.util.stream.IntStream; +import java.util.stream.Stream; + /** * 录像机 * + * https://blog.csdn.net/m0_60259116/article/details/126875532 + * * @author acgist */ public final class MediaRecorder { - /** - * 是否正在录像 - */ - private boolean active; - private static final MediaRecorder INSTANCE = new MediaRecorder(); + private MediaRecorder() { + final MediaCodecList mediaCodecList = new MediaCodecList(-1); + for (MediaCodecInfo mediaCodecInfo : mediaCodecList.getCodecInfos()) { + if (mediaCodecInfo.isEncoder()) { + final String[] supportedTypes = mediaCodecInfo.getSupportedTypes(); + Log.d(MediaRecorder.class.getSimpleName(), "编码器名称：" + mediaCodecInfo.getName()); + Log.d(MediaRecorder.class.getSimpleName(), "编码器类型：" + String.join(" , ", supportedTypes)); + for (String supportType : supportedTypes) { + final MediaCodecInfo.CodecCapabilities codecCapabilities = mediaCodecInfo.getCapabilitiesForType(supportType); + final int[] colorFormats = codecCapabilities.colorFormats; + Log.d(MediaRecorder.class.getSimpleName(), "编码器格式：" + codecCapabilities.getMimeType()); +// MediaCodecInfo.CodecCapabilities.COLOR_* + Log.d(MediaRecorder.class.getSimpleName(), "编码器支持格式：" + IntStream.of(colorFormats).boxed().map(String::valueOf).collect(Collectors.joining(" , "))); + } + } + } + this.audioRecoder = audioSamples -> { + Log.d(MediaRecorder.class.getSimpleName(), audioSamples + " - 音频"); + }; + this.videoRecoder = videoFrame -> { +// Log.d(MediaRecorder.class.getSimpleName(), videoFrame + " - 视频"); + if(this.active && this.videoActive) { + final VideoFrame.Buffer buffer = videoFrame.getBuffer(); + final VideoFrame.I420Buffer i420Buffer = buffer.toI420(); + i420Buffer.getDataU(); +// this.putVideo(videoFrame.getBuffer(), videoFrame.getTimestampNs()); + } + }; + } + public static final MediaRecorder getInstance() { return INSTANCE; } + /** + * 是否正在录像 + */ + private volatile boolean active; + private volatile boolean audioActive; + private volatile boolean videoActive; + private volatile long pts; + /** + * 音频编码 + */ + private MediaCodec audioCodec; + private Thread audioThread; + /** + * 视频编码 + */ + private MediaCodec videoCodec; + private Thread videoThread; + /** + * 媒体合成器 + */ + private MediaMuxer mediaMuxer; + /** + * 音频录制 + */ + public final JavaAudioDeviceModule.SamplesReadyCallback audioRecoder; + /** + * 视频录制 + */ + public final VideoSink videoRecoder; + /** * @return 是否正在录像 */ @@ -25,4 +104,212 @@ public final class MediaRecorder { return this.active; } + public void init(String file, String audioFormat, String videoFormat, int width, int height) { + synchronized (MediaRecorder.INSTANCE) { + this.active = true; + if( + this.audioThread == null || !this.audioThread.isAlive() || + this.videoThread == null || !this.videoThread.isAlive() + ) { + this.initMediaMuxer(file); + this.initAudioThread(MediaFormat.MIMETYPE_AUDIO_AAC, 96000, 44100, 1); + this.initVideoThread(MediaFormat.MIMETYPE_VIDEO_AVC, 2500 * 1000, 30, 1, 1920, 1080); + } +// this.audioCodec = MediaCodec.createByCodecName(); + } + } + + /** + * @param audioType 类型 + * @param bitRate 比特率：96 * 1000 | 128 * 1000 | 256 * 1000 + * @param sampleRate 采样率：32000 | 44100 | 48000 + * @param channelCount 通道数量 + */ + private void initAudioThread(String audioType, int bitRate, int sampleRate, int channelCount) { + try { + this.audioCodec = MediaCodec.createEncoderByType(audioType); + final MediaFormat audioFormat = MediaFormat.createAudioFormat(audioType, sampleRate, channelCount); +// audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitRate); + audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, AudioFormat.ENCODING_PCM_16BIT); + audioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC); + audioFormat.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR); + audioFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 8 * 1024); + this.audioCodec.configure(audioFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + } catch (Exception e) { + Log.e(MediaRecorder.class.getSimpleName(), "加载音频录制线程异常", e); + } + final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); + this.audioThread = new Thread(() -> { + int trackIndex; + int outputIndex; + synchronized (MediaRecorder.INSTANCE) { + Log.i(MediaRecorder.class.getSimpleName(), "开始录制音频"); + this.audioCodec.start(); + this.audioActive = true; + trackIndex = this.mediaMuxer.addTrack(this.audioCodec.getOutputFormat()); + if(this.videoActive) { + Log.i(MediaRecorder.class.getSimpleName(), "开始录制文件"); + this.pts = System.currentTimeMillis(); + this.mediaMuxer.start(); + MediaRecorder.INSTANCE.notifyAll(); + } else { + try { + MediaRecorder.INSTANCE.wait(); + } catch (InterruptedException e) { + } + } + } + while(this.active) { + outputIndex = this.audioCodec.dequeueOutputBuffer(info, 1000L * 1000); + if(outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { + } else if(outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + } else { + final ByteBuffer outputBuffer = this.audioCodec.getOutputBuffer(outputIndex); + outputBuffer.position(info.offset); + outputBuffer.limit(info.offset + info.size); + info.presentationTimeUs = info.presentationTimeUs - this.pts; + this.mediaMuxer.writeSampleData(trackIndex, outputBuffer, info); + this.audioCodec.releaseOutputBuffer(outputIndex, false); + } + } + synchronized (MediaRecorder.INSTANCE) { + if(this.audioCodec != null) { + Log.i(MediaRecorder.class.getSimpleName(), "结束录制音频"); + this.audioCodec.stop(); + this.audioCodec.release(); + this.audioCodec = null; + } + this.audioActive = false; + if(this.mediaMuxer != null && !this.videoActive) { + Log.i(MediaRecorder.class.getSimpleName(), "结束录制文件"); + this.mediaMuxer.stop(); + this.mediaMuxer.release(); + this.mediaMuxer = null; + } + } + }); + this.audioThread.setName("AudioRecoder"); + this.audioThread.start(); + } + + public void putAudio(byte[] bytes) { + + } + + /** + * @param videoType 视频格式 + * @param bitRate 比特率：800 * 1000 | 1600 * 1000 | 2500 * 1000 + * @param frameRate 帧率：30 + * @param iFrameInterval 关键帧频率：1 + * @param width 宽度：1920 + * @param height 高度：1080 + */ + private void initVideoThread(String videoType, int bitRate, int frameRate, int iFrameInterval, int width, int height) { + try { + this.videoCodec = MediaCodec.createEncoderByType(videoType); + final MediaFormat videoFormat = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, width, height); + videoFormat.setInteger(MediaFormat.KEY_BIT_RATE, 2500000); +// videoFormat.setInteger(MediaFormat.KEY_LEVEL, MediaCodecInfo.CodecProfileLevel.AVCLevel32); +// videoFormat.setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileHigh); + videoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30); +// videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); + videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible); + videoFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1); + this.videoCodec.configure(videoFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + } catch (Exception e) { + Log.e(MediaRecorder.class.getSimpleName(), "加载视频录制线程异常", e); + } + final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); + this.videoThread = new Thread(() -> { + int trackIndex; + int outputIndex; + synchronized (MediaRecorder.INSTANCE) { + Log.i(MediaRecorder.class.getSimpleName(), "开始录制视频"); + this.videoCodec.start(); + this.videoActive = true; + trackIndex = this.mediaMuxer.addTrack(this.videoCodec.getOutputFormat()); + if(this.audioActive) { + Log.i(MediaRecorder.class.getSimpleName(), "开始录制文件"); + this.pts = System.currentTimeMillis(); + this.mediaMuxer.start(); + MediaRecorder.INSTANCE.notifyAll(); + } else { + try { + MediaRecorder.INSTANCE.wait(); + } catch (InterruptedException e) { + } + } + } + while(this.active) { + outputIndex = this.videoCodec.dequeueOutputBuffer(info, 1000L * 1000); + if(outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { + } else if(outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + } else { + Log.i(MediaRecorder.class.getSimpleName(), "======" + info.size); + final ByteBuffer outputBuffer = this.audioCodec.getOutputBuffer(outputIndex); + outputBuffer.position(info.offset); + outputBuffer.limit(info.offset + info.size); + info.presentationTimeUs = info.presentationTimeUs - this.pts; + this.mediaMuxer.writeSampleData(trackIndex, outputBuffer, info); + this.audioCodec.releaseOutputBuffer(outputIndex, false); +// if(info.flags == MediaCodec.BUFFER_FLAG_KEY_FRAME) { +// } else if(info.flags == MediaCodec.BUFFER_FLAG_CODEC_CONFIG) { +// } else if(info.flags == MediaCodec.BUFFER_FLAG_END_OF_STREAM) { +// } else if(info.flags == MediaCodec.BUFFER_FLAG_PARTIAL_FRAME) { +// } + } + } + synchronized (MediaRecorder.INSTANCE) { + if(this.videoCodec != null) { + Log.i(MediaRecorder.class.getSimpleName(), "结束录制视频"); + this.videoCodec.stop(); + this.videoCodec.release(); + this.videoCodec = null; + } + this.videoActive = false; + if(this.mediaMuxer != null && !this.audioActive) { + Log.i(MediaRecorder.class.getSimpleName(), "结束录制文件"); + this.mediaMuxer.stop(); + this.mediaMuxer.release(); + this.mediaMuxer = null; + } + } + }); + this.videoThread.setName("VideoRecoder"); + this.videoThread.start(); + } + + public void putVideo(byte[] bytes, long pts) { + while(this.active && this.videoActive) { + final int index = this.videoCodec.dequeueInputBuffer(1000L * 1000); + if(index < 0) { + continue; + } + final ByteBuffer byteBuffer = this.videoCodec.getInputBuffer(index); + byteBuffer.put(bytes); + this.videoCodec.queueInputBuffer(index, 0, bytes.length, pts, 0); + } + } + + private void initMediaMuxer(String file) { + try { + this.mediaMuxer = new MediaMuxer( + Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_MOVIES).getAbsolutePath(), file).toAbsolutePath().toString(), + MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4 + ); + // 设置方向 +// this.mediaMuxer.setOrientationHint(); + } catch (IOException e) { + Log.e(MediaManager.class.getSimpleName(), "加载媒体合成器异常", e); + } + } + + public void stop() { + synchronized(MediaRecorder.INSTANCE) { + this.active = false; + this.audioThread = null; + this.videoThread = null; + } + } + } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/P2PClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/P2PClient.java index 9027cb4..7a42958 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/P2PClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/P2PClient.java @@ -1,10 +1,36 @@ package com.acgist.taoyao.media; +import android.util.Log; + +import java.io.Closeable; +import java.io.IOException; + /** * P2P终端 * 使用安卓SDK + WebRTC实现P2P会话 * * @author acgist */ -public class P2PClient { +public class P2PClient implements Closeable { + + private final String clientId; + + public P2PClient(String clientId) { + this.clientId = clientId; + } + + // 配置STUN穿透服务器 转发服务器 +// iceServers = new ArrayList<>(); +// PeerConnection.IceServer iceServer = PeerConnection.IceServer.builder(Constant.STUN).createIceServer(); +// iceServers.add(iceServer); +// streamList = new ArrayList<>(); +// PeerConnection.RTCConfiguration configuration = new PeerConnection.RTCConfiguration(iceServers); +// PeerConnectionObserver connectionObserver = getObserver(); +// peerConnection = peerConnectionFactory.createPeerConnection(configuration, connectionObserver); + + @Override + public void close() { + Log.i(Room.class.getSimpleName(), "关闭终端：" + this.clientId); + } + } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RemoteClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RemoteClient.java index 34ecc9f..9d9cd72 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RemoteClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RemoteClient.java @@ -1,9 +1,23 @@ package com.acgist.taoyao.media; +import java.io.Closeable; +import java.io.IOException; +import java.util.logging.Handler; + /** * 房间远程终端 * * @author acgist */ -public class RemoteClient { +public class RemoteClient extends RoomClient { + + public RemoteClient(String name, String clientId, Handler handler) { + super(name, clientId, handler); + } + + @Override + public void close() { + super.close(); + } + } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/Room.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/Room.java index bbf8ab4..610a12b 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/Room.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/Room.java @@ -1,4 +1,33 @@ package com.acgist.taoyao.media; -public class Room { +import android.util.Log; + +import java.io.Closeable; +import java.io.IOException; +import java.util.List; + +/** + * 房间 + * + * @author acgist + */ +public class Room implements Closeable { + + private final String id; + + public Room(String id) { + this.id = id; + } + + /** + * 远程终端列表 + */ + private List remoteClientList; + + @Override + public void close() { + Log.i(Room.class.getSimpleName(), "关闭房间：" + this.id); + this.remoteClientList.forEach(RemoteClient::close); + } + } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RoomClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RoomClient.java index 9216fe4..bf2cfe6 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RoomClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/RoomClient.java @@ -1,18 +1,22 @@ package com.acgist.taoyao.media; +import android.util.Log; + import org.webrtc.AudioTrack; import org.webrtc.MediaStream; import org.webrtc.VideoTrack; +import java.io.Closeable; +import java.io.IOException; import java.util.logging.Handler; /** * 房间终端 - * 使用NDK + Mediasoup实现多人会话 + * 使用SDK + NDK + Mediasoup实现多人会话 * * @author acgist */ -public class RoomClient { +public class RoomClient implements Closeable { protected final String name; protected final String clientId; @@ -34,4 +38,9 @@ public class RoomClient { } + @Override + public void close() { + Log.i(Room.class.getSimpleName(), "关闭终端：" + this.clientId); + } + } diff --git a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/utils/NetUtils.java b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/utils/NetUtils.java index c11e210..a1fd0ba 100644 --- a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/utils/NetUtils.java +++ b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/utils/NetUtils.java @@ -109,6 +109,7 @@ public final class NetUtils { if(Boolean.FALSE.equals(NetUtils.ipRewriteProperties.getEnabled())) { return sourceIp; } + log.debug("重写地址：{} - {}", sourceIp, clientIp); try { final InetAddress sourceAddress = NetUtils.realAddress(sourceIp); final InetAddress clientAddress = NetUtils.realAddress(clientIp); diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/config/camera/AiProperties.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/config/camera/AiProperties.java index 5234cb9..76b635a 100644 --- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/config/camera/AiProperties.java +++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/config/camera/AiProperties.java @@ -14,7 +14,7 @@ import lombok.Setter; @Setter @Schema(title = "AI识别配置", description = "AI识别配置") public class AiProperties { - + /** * 识别类型 *