diff --git a/taoyao-client-android/README.md b/taoyao-client-android/README.md
index 9cf0122..7832e12 100644
--- a/taoyao-client-android/README.md
+++ b/taoyao-client-android/README.md
@@ -45,26 +45,19 @@
* `org.webrtc:google-webrtc`
* `io.github.haiyangwu:mediasoup-client`
+## YUV
```
-/**
- * YUV终端
- *
- * Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y
- * Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y
- * Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y
- * Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y
- * U U U U U U V V V V V V U V U V U V V U V U V U
- * V V V V V V U U U U U U U V U V U V V U V U V U
- * - I420 - - YV12 - - NV12 - - NV21 -
- *
- * I420 = YUV420P = YU12
- * NV12 = YUV420SP
- *
- * RGB和YUV转换算法：BT.601（标清）、BT.709（高清）、BT.2020（超高清）
- *
- * YuvHelper
- *
- * @author acgist
- */
+Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y
+Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y
+Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y
+Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y Y
+U U U U U U V V V V V V U V U V U V V U V U V U
+V V V V V V U U U U U U U V U V U V V U V U V U
+- I420 - - YV12 - - NV12 - - NV21 -
+
+I420 = YUV420P = YU12
+NV12 = YUV420SP
+
+RGB和YUV转换算法：BT.601（标清）、BT.709（高清）、BT.2020（超高清）
```
diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java
index 1b10d0a..c4a5c3c 100644
--- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java
+++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java
@@ -121,12 +121,11 @@ public class MainActivity extends AppCompatActivity implements Serializable {
final Resources resources = this.getResources();
MediaManager.getInstance().initContext(
this.mainHandler, this.getApplicationContext(),
- resources.getBoolean(R.bool.playAudio),
- resources.getBoolean(R.bool.playVideo),
- resources.getBoolean(R.bool.audioConsume),
- resources.getBoolean(R.bool.videoConsume),
- resources.getBoolean(R.bool.audioProduce),
- resources.getBoolean(R.bool.videoProduce),
+ resources.getInteger(R.integer.imageQuantity),
+ resources.getString(R.string.audioQuantity),
+ resources.getString(R.string.videoQuantity),
+ resources.getInteger(R.integer.channelCount),
+ resources.getInteger(R.integer.iFrameInterval),
resources.getString(R.string.storagePathImage),
resources.getString(R.string.storagePathVideo),
TransportType.valueOf(resources.getString(R.string.transportType))
diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java
index bd7e167..e1ed1f6 100644
--- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java
+++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java
@@ -1,8 +1,7 @@
package com.acgist.taoyao.client.signal;
-import android.Manifest;
+import android.annotation.SuppressLint;
import android.content.Context;
-import android.content.pm.PackageManager;
import android.content.res.Resources;
import android.location.Criteria;
import android.location.Location;
@@ -16,8 +15,6 @@ import android.os.PowerManager;
import android.os.Process;
import android.util.Log;
-import androidx.core.app.ActivityCompat;
-
import com.acgist.taoyao.boot.model.Header;
import com.acgist.taoyao.boot.model.Message;
import com.acgist.taoyao.boot.utils.CloseableUtils;
@@ -728,12 +725,15 @@ public final class Taoyao implements ITaoyao {
key, password,
this, this.mainHandler,
resources.getBoolean(R.bool.preview),
+ resources.getBoolean(R.bool.playAudio),
+ resources.getBoolean(R.bool.playVideo),
resources.getBoolean(R.bool.dataConsume),
resources.getBoolean(R.bool.audioConsume),
resources.getBoolean(R.bool.videoConsume),
resources.getBoolean(R.bool.audioProduce),
resources.getBoolean(R.bool.dataProduce),
- resources.getBoolean(R.bool.videoProduce)
+ resources.getBoolean(R.bool.videoProduce),
+ this.mediaManager.getMediaProperties()
)
);
final boolean success = room.enter();
@@ -783,7 +783,20 @@ public final class Taoyao implements ITaoyao {
final String name = MapUtils.get(body, "name");
final String clientId = MapUtils.get(body, "clientId");
final String sessionId = MapUtils.get(body, "sessionId");
- final SessionClient sessionClient = new SessionClient(sessionId, name, clientId, this, this.mainHandler);
+ final Resources resources = this.context.getResources();
+ final SessionClient sessionClient = new SessionClient(
+ sessionId, name, clientId, this, this.mainHandler,
+ resources.getBoolean(R.bool.preview),
+ resources.getBoolean(R.bool.playAudio),
+ resources.getBoolean(R.bool.playVideo),
+ resources.getBoolean(R.bool.dataConsume),
+ resources.getBoolean(R.bool.audioConsume),
+ resources.getBoolean(R.bool.videoConsume),
+ resources.getBoolean(R.bool.audioProduce),
+ resources.getBoolean(R.bool.dataProduce),
+ resources.getBoolean(R.bool.videoProduce),
+ this.mediaManager.getMediaProperties()
+ );
this.sessionClients.put(sessionId, sessionClient);
sessionClient.init();
sessionClient.offer();
@@ -884,16 +897,11 @@ public final class Taoyao implements ITaoyao {
/**
* @return 位置
*/
+ @SuppressLint("MissingPermission")
private Location location() {
if (this.locationManager == null) {
return null;
}
- if (
- ActivityCompat.checkSelfPermission(this.context, Manifest.permission.ACCESS_FINE_LOCATION) != PackageManager.PERMISSION_GRANTED &&
- ActivityCompat.checkSelfPermission(this.context, Manifest.permission.ACCESS_COARSE_LOCATION) != PackageManager.PERMISSION_GRANTED
- ) {
- return null;
- }
final Criteria criteria = new Criteria();
criteria.setCostAllowed(false);
criteria.setBearingRequired(false);
diff --git a/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml b/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml
index 2dc7863..6775e9b 100644
--- a/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml
+++ b/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml
@@ -34,5 +34,8 @@
true
100
+ fd-audio
fd-video
+ 1
+ 1
\ No newline at end of file
diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java
index 5d38e4b..610b2e0 100644
--- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java
+++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java
@@ -2,11 +2,8 @@ package com.acgist.taoyao.media;
import android.content.Context;
import android.content.Intent;
-import android.media.AudioAttributes;
-import android.media.ImageReader;
import android.media.MediaCodecInfo;
import android.media.MediaCodecList;
-import android.media.MediaRecorder;
import android.media.projection.MediaProjection;
import android.os.Handler;
import android.os.Message;
@@ -39,7 +36,6 @@ import org.webrtc.SurfaceViewRenderer;
import org.webrtc.VideoCapturer;
import org.webrtc.VideoDecoderFactory;
import org.webrtc.VideoEncoderFactory;
-import org.webrtc.VideoFileRenderer;
import org.webrtc.VideoSource;
import org.webrtc.VideoTrack;
import org.webrtc.audio.JavaAudioDeviceModule;
@@ -71,30 +67,6 @@ public final class MediaManager {
* 当前媒体共享数量
*/
private volatile int shareClientCount;
- /**
- * 是否打开音频播放
- */
- private boolean playAudio;
- /**
- * 是否打开视频播放
- */
- private boolean playVideo;
- /**
- * 是否消费音频
- */
- private boolean audioConsume;
- /**
- * 是否消费视频
- */
- private boolean videoConsume;
- /**
- * 是否生产音频
- */
- private boolean audioProduce;
- /**
- * 是否生产视频
- */
- private boolean videoProduce;
/**
* 视频路径
*/
@@ -103,6 +75,20 @@ public final class MediaManager {
* 图片路径
*/
private String videoPath;
+ /**
+ * 图片质量
+ */
+ private int imageQuantity;
+ /**
+ * 音频质量
+ */
+ private String audioQuantity;
+ /**
+ * 视频质量
+ */
+ private String videoQuantity;
+ private int channelCount;
+ private int iFrameInterval;
/**
* 视频来源类型
*/
@@ -243,39 +229,24 @@ public final class MediaManager {
/**
* @param mainHandler Handler
* @param context 上下文
- * @param playAudio 是否播放音频
- * @param playVideo 是否播放视频
- * @param audioConsume 是否消费音频
- * @param videoConsume 是否消费视频
- * @param audioProduce 是否生产音频
- * @param videoProduce 是否生产视频
*/
public void initContext(
Handler mainHandler, Context context,
- boolean playAudio, boolean playVideo,
- boolean audioConsume, boolean videoConsume,
- boolean audioProduce, boolean videoProduce,
+ int imageQuantity, String audioQuantity, String videoQuantity,
+ int channelCount, int iFrameInterval,
String imagePath, String videoPath,
TransportType transportType
) {
this.mainHandler = mainHandler;
- this.context = context;
- this.playAudio = playAudio;
- this.playVideo = playVideo;
- this.audioConsume = audioConsume;
- this.videoConsume = videoConsume;
- this.audioProduce = audioProduce;
- this.videoProduce = videoProduce;
+ this.context = context;
+ this.imageQuantity = imageQuantity;
+ this.audioQuantity = audioQuantity;
+ this.videoQuantity = videoQuantity;
+ this.channelCount = channelCount;
+ this.iFrameInterval = iFrameInterval;
this.imagePath = imagePath;
this.videoPath = videoPath;
this.transportType = transportType;
- PeerConnectionFactory.initialize(
- PeerConnectionFactory.InitializationOptions.builder(this.context)
-// .setFieldTrials("WebRTC-H264HighProfile/Enabled/")
-// .setNativeLibraryName("jingle_peerconnection_so")
-// .setEnableInternalTracer(true)
- .createInitializationOptions()
- );
}
/**
@@ -302,6 +273,7 @@ public final class MediaManager {
}
}
if (this.clientCount <= 0) {
+ this.initPeerConnectionFactory();
this.initMedia(videoSourceType);
this.nativeInit();
}
@@ -323,6 +295,7 @@ public final class MediaManager {
if (this.clientCount <= 0) {
this.close();
this.nativeStop();
+ this.stopPeerConnectionFactory();
}
return this.clientCount;
}
@@ -332,6 +305,24 @@ public final class MediaManager {
return this.recordClient != null;
}
+ public MediaProperties getMediaProperties() {
+ return this.mediaProperties;
+ }
+
+ private void initPeerConnectionFactory() {
+ PeerConnectionFactory.initialize(
+ PeerConnectionFactory.InitializationOptions.builder(this.context)
+// .setFieldTrials("WebRTC-H264HighProfile/Enabled/")
+// .setNativeLibraryName("jingle_peerconnection_so")
+// .setEnableInternalTracer(true)
+ .createInitializationOptions()
+ );
+ }
+
+ private void stopPeerConnectionFactory() {
+ PeerConnectionFactory.shutdownInternalTracer();
+ }
+
/**
* 加载媒体
*
@@ -635,16 +626,16 @@ public final class MediaManager {
public String photograph() {
synchronized (this) {
- if(this.recordVideoCapturer == null) {
- // 如果没有拉流不能拍照
- return null;
- }
String filepath;
- // TODO：质量读取
- this.photographClient = new PhotographClient(100, this.imagePath);
+ if(this.clientCount <= 0) {
+ final MediaVideoProperties mediaVideoProperties = this.mediaProperties.getVideos().get(this.videoQuantity);
+ final PhotographClient photographClient = new PhotographClient(this.imageQuantity, this.imagePath);
+ filepath = photographClient.photograph(mediaVideoProperties.getWidth(), mediaVideoProperties.getHeight(), VideoSourceType.BACK, this.context);
+ return filepath;
+ }
+ this.photographClient = new PhotographClient(this.imageQuantity, this.imagePath);
if(this.recordClient == null) {
- // TODO：质量读取
- final MediaVideoProperties mediaVideoProperties = this.mediaProperties.getVideos().get("fd-video");
+ final MediaVideoProperties mediaVideoProperties = this.mediaProperties.getVideos().get(this.videoQuantity);
this.recordVideoCapturer.startCapture(mediaVideoProperties.getWidth(), mediaVideoProperties.getHeight(), mediaVideoProperties.getFrameRate());
filepath = this.photographClient.waitForPhotograph();
try {
@@ -665,15 +656,16 @@ public final class MediaManager {
if(this.recordClient != null) {
return this.recordClient;
}
+ final MediaAudioProperties mediaAudioProperties = this.mediaProperties.getAudios().get(this.audioQuantity);
+ final MediaVideoProperties mediaVideoProperties = this.mediaProperties.getVideos().get(this.videoQuantity);
this.recordClient = new RecordClient(
- this.videoPath,
- this.taoyao,
- this.mainHandler
+ mediaAudioProperties.getBitrate(), mediaAudioProperties.getSampleRate(), this.channelCount,
+ mediaVideoProperties.getBitrate(), mediaVideoProperties.getFrameRate(), this.iFrameInterval, mediaVideoProperties.getWidth(), mediaVideoProperties.getHeight(),
+ this.videoPath, this.taoyao, this.mainHandler
);
- this.recordClient.start();
- // TODO：质量读取
- final MediaVideoProperties mediaVideoProperties = this.mediaProperties.getVideos().get("fd-video");
+ this.newClient(VideoSourceType.BACK);
this.recordVideoCapturer.startCapture(mediaVideoProperties.getWidth(), mediaVideoProperties.getHeight(), mediaVideoProperties.getFrameRate());
+ this.recordClient.start();
return this.recordClient;
}
}
@@ -683,27 +675,25 @@ public final class MediaManager {
if(this.recordClient == null) {
return;
} else {
+ this.recordClient.close();
+ this.recordClient = null;
try {
this.recordVideoCapturer.stopCapture();
} catch (InterruptedException e) {
Log.e(MediaManager.class.getSimpleName(), "关闭视频捕获（主码流）异常", e);
}
- this.recordClient.close();
- this.recordClient = null;
+ this.closeClient();
}
}
}
/**
- * @param flag Config.WHAT_*
- * @param videoTrack 视频媒体流Track
+ * @param flag Config.WHAT_*
+ * @param videoTrack 视频媒体流Track
*
* @return 播放控件
*/
- public SurfaceViewRenderer buildSurfaceViewRenderer(
- final int flag,
- final VideoTrack videoTrack
- ) {
+ public SurfaceViewRenderer buildSurfaceViewRenderer(final int flag, final VideoTrack videoTrack) {
// 预览控件
final SurfaceViewRenderer surfaceViewRenderer = new SurfaceViewRenderer(this.context);
this.mainHandler.post(() -> {
@@ -729,44 +719,14 @@ public final class MediaManager {
return surfaceViewRenderer;
}
- public void pauseAudio() {
- synchronized (this.mediaStream.audioTracks) {
- this.mediaStream.audioTracks.forEach(a -> a.setEnabled(false));
- }
- }
-
- public void resumeAudio() {
- synchronized (this.mediaStream.audioTracks) {
- this.mediaStream.audioTracks.forEach(a -> a.setEnabled(true));
- }
- }
-
- public void pauseVideo() {
- synchronized (this.mediaStream.videoTracks) {
- this.mediaStream.videoTracks.forEach(v -> v.setEnabled(false));
- }
- synchronized (this.mediaStream.preservedVideoTracks) {
- this.mediaStream.preservedVideoTracks.forEach(v -> v.setEnabled(false));
- }
- }
-
- public void resumeVideo() {
- synchronized (this.mediaStream.videoTracks) {
- this.mediaStream.videoTracks.forEach(v -> v.setEnabled(true));
- }
- synchronized (this.mediaStream.preservedVideoTracks) {
- this.mediaStream.preservedVideoTracks.forEach(v -> v.setEnabled(true));
- }
- }
-
/**
* 关闭声音
*/
private void closeAudio() {
- if(this.audioTrack != null) {
- this.audioTrack.dispose();
- this.audioTrack = null;
- }
+// if(this.audioTrack != null) {
+// this.audioTrack.dispose();
+// this.audioTrack = null;
+// }
if(this.audioSource != null) {
this.audioSource.dispose();
this.audioSource = null;
@@ -777,10 +737,10 @@ public final class MediaManager {
* 关闭视频
*/
private void closeVideo() {
- if(this.videoTrack != null) {
- this.videoTrack.dispose();
- this.videoTrack = null;
- }
+// if(this.videoTrack != null) {
+// this.videoTrack.dispose();
+// this.videoTrack = null;
+// }
if(this.videoSource != null) {
this.videoSource.dispose();
this.videoSource = null;
@@ -789,6 +749,9 @@ public final class MediaManager {
this.videoCapturer.dispose();
this.videoCapturer = null;
}
+ }
+
+ private void closeRecord() {
if(this.recordVideoTrack != null) {
this.recordVideoTrack.dispose();
this.recordVideoTrack = null;
@@ -801,10 +764,6 @@ public final class MediaManager {
this.recordVideoCapturer.dispose();
this.recordVideoCapturer = null;
}
- if(this.surfaceTextureHelper != null) {
- this.surfaceTextureHelper.dispose();
- this.surfaceTextureHelper = null;
- }
}
private void closeMedia() {
@@ -813,10 +772,14 @@ public final class MediaManager {
this.eglBase = null;
this.shareEglContext = null;
}
-// if (this.mediaStream != null) {
-// this.mediaStream.dispose();
-// this.mediaStream = null;
-// }
+ if (this.mediaStream != null) {
+ this.mediaStream.dispose();
+ this.mediaStream = null;
+ }
+ if(this.surfaceTextureHelper != null) {
+ this.surfaceTextureHelper.dispose();
+ this.surfaceTextureHelper = null;
+ }
if (this.peerConnectionFactory != null) {
this.peerConnectionFactory.dispose();
this.peerConnectionFactory = null;
@@ -829,9 +792,8 @@ public final class MediaManager {
private void close() {
this.closeAudio();
this.closeVideo();
+ this.closeRecord();
this.closeMedia();
-// PeerConnectionFactory.shutdownInternalTracer();
-// PeerConnectionFactory.stopInternalTracingCapture();
}
/**
diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java
index 6f21924..2c9dcb2 100644
--- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java
+++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java
@@ -1,24 +1,42 @@
package com.acgist.taoyao.media.client;
+import android.annotation.SuppressLint;
+import android.content.Context;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.ImageFormat;
import android.graphics.Rect;
import android.graphics.YuvImage;
+import android.hardware.camera2.CameraAccessException;
+import android.hardware.camera2.CameraCaptureSession;
+import android.hardware.camera2.CameraCharacteristics;
+import android.hardware.camera2.CameraDevice;
+import android.hardware.camera2.CameraManager;
+import android.hardware.camera2.CaptureRequest;
+import android.hardware.camera2.TotalCaptureResult;
+import android.hardware.camera2.params.OutputConfiguration;
+import android.hardware.camera2.params.SessionConfiguration;
+import android.media.Image;
+import android.media.ImageReader;
import android.os.Environment;
import android.util.Log;
+import android.view.Surface;
import com.acgist.taoyao.boot.utils.DateUtils;
+import com.acgist.taoyao.media.VideoSourceType;
import org.webrtc.VideoFrame;
import java.io.ByteArrayOutputStream;
import java.io.File;
import java.io.FileOutputStream;
+import java.io.IOException;
import java.io.OutputStream;
import java.nio.ByteBuffer;
import java.nio.file.Paths;
import java.time.LocalDateTime;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
/**
* 拍照终端
@@ -30,19 +48,35 @@ public class PhotographClient {
private final int quantity;
private final String filename;
private final String filepath;
+ private volatile boolean wait;
+ private volatile boolean finish;
+ private Surface surface;
+ private ImageReader imageReader;
+ private CameraDevice cameraDevice;
+ private CameraManager cameraManager;
+ private CameraCaptureSession cameraCaptureSession;
public PhotographClient(int quantity, String path) {
this.quantity = quantity;
this.filename = DateUtils.format(LocalDateTime.now(), DateUtils.DateTimeStyle.YYYYMMDDHH24MMSS) + ".jpg";
this.filepath = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath(), path, this.filename).toString();
+ this.wait = true;
+ this.finish = false;
Log.i(RecordClient.class.getSimpleName(), "拍摄照片文件：" + this.filepath);
}
+ // ================ 拉流拍照 ================ //
+
public String photograph(VideoFrame videoFrame) {
- final Thread thread = new Thread(() -> this.photographBackground(videoFrame));
- thread.setName("PhotographThread");
- thread.setDaemon(true);
- thread.start();
+ if(this.wait) {
+ this.wait = false;
+ final Thread thread = new Thread(() -> this.photographBackground(videoFrame));
+ thread.setName("PhotographThread");
+ thread.setDaemon(true);
+ thread.start();
+ } else {
+ videoFrame.release();
+ }
return this.filepath;
}
@@ -59,7 +93,8 @@ public class PhotographClient {
final YuvImage image = this.i420ToYuvImage(i420, width, height);
i420.release();
videoFrame.release();
- image.compressToJpeg(new Rect(0, 0, width, height), this.quantity, byteArray);
+ final Rect rect = new Rect(0, 0, width, height);
+ image.compressToJpeg(rect, this.quantity, byteArray);
final byte[] array = byteArray.toByteArray();
final Bitmap bitmap = BitmapFactory.decodeByteArray(array, 0, array.length);
// final Matrix matrix = new Matrix();
@@ -69,13 +104,21 @@ public class PhotographClient {
} catch (Exception e) {
Log.e(PhotographClient.class.getSimpleName(), "拍照异常", e);
}
+ this.notifyWait();
+ }
+
+ private void notifyWait() {
synchronized (this) {
+ this.finish = true;
this.notifyAll();
}
}
public String waitForPhotograph() {
synchronized (this) {
+ if(this.finish) {
+ return this.filepath;
+ }
try {
this.wait(5000);
} catch (InterruptedException e) {
@@ -86,60 +129,133 @@ public class PhotographClient {
}
private YuvImage i420ToYuvImage(VideoFrame.I420Buffer i420, int width, int height) {
- final ByteBuffer[] yuvPlanes = new ByteBuffer[] {
- i420.getDataY(), i420.getDataU(), i420.getDataV()
- };
- final int[] yuvStrides = new int[] {
- i420.getStrideY(), i420.getStrideU(), i420.getStrideV()
- };
- if (
- yuvStrides[0] != width ||
- yuvStrides[1] != width / 2 ||
- yuvStrides[2] != width / 2
- ) {
- return i420ToYuvImage(yuvPlanes, yuvStrides, width, height);
- }
- final byte[] bytes = new byte[yuvStrides[0] * height + yuvStrides[1] * height / 2 + yuvStrides[2] * height / 2];
- final ByteBuffer yBuffer = ByteBuffer.wrap(bytes, 0, width * height);
- this.copyPlane(yuvPlanes[0], yBuffer);
- final byte[] uvBytes = new byte[width / 2 * height / 2];
- final ByteBuffer uvBuffer = ByteBuffer.wrap(uvBytes, 0, uvBytes.length);
- this.copyPlane(yuvPlanes[2], uvBuffer);
- for (int row = 0; row < height / 2; row++) {
- for (int col = 0; col < width / 2; col++) {
- bytes[width * height + row * width + col * 2] = uvBytes[row * width / 2 + col];
- }
- }
- this.copyPlane(yuvPlanes[1], uvBuffer);
- for (int row = 0; row < height / 2; row++) {
- for (int col = 0; col < width / 2; col++) {
- bytes[width * height + row * width + col * 2 + 1] = uvBytes[row * width / 2 + col];
- }
- }
- return new YuvImage(bytes, ImageFormat.NV21, width, height, null);
- }
-
- private YuvImage i420ToYuvImage(ByteBuffer[] yuvPlanes, int[] yuvStrides, int width, int height) {
- int i = 0;
- final byte[] bytes = new byte[width * height * 3 / 2];
+ int index = 0;
+ final int yy = i420.getStrideY();
+ final int uu = i420.getStrideU();
+ final int vv = i420.getStrideV();
+ final ByteBuffer y = i420.getDataY();
+ final ByteBuffer u = i420.getDataU();
+ final ByteBuffer v = i420.getDataV();
+ final byte[] nv21 = new byte[width * height * 3 / 2];
for (int row = 0; row < height; row++) {
for (int col = 0; col < width; col++) {
- bytes[i++] = yuvPlanes[0].get(col + row * yuvStrides[0]);
+ nv21[index++] = y.get(col + row * yy);
}
}
- for (int row = 0; row < height / 2; row++) {
- for (int col = 0; col < width / 2; col++) {
- bytes[i++] = yuvPlanes[2].get(col + row * yuvStrides[2]);
- bytes[i++] = yuvPlanes[1].get(col + row * yuvStrides[1]);
+ final int halfWidth = width / 2;
+ final int halfHeight = height / 2;
+ for (int row = 0; row < halfHeight; row++) {
+ for (int col = 0; col < halfWidth; col++) {
+ nv21[index++] = v.get(col + row * vv);
+ nv21[index++] = u.get(col + row * uu);
}
}
- return new YuvImage(bytes, ImageFormat.NV21, width, height, null);
+ return new YuvImage(nv21, ImageFormat.NV21, width, height, null);
}
- private void copyPlane(ByteBuffer src, ByteBuffer dst) {
- src.position(0).limit(src.capacity());
- dst.put(src);
- dst.position(0).limit(dst.capacity());
+ // ================ Camera2拍照 ================ //
+
+ @SuppressLint("MissingPermission")
+ public String photograph(int width, int height, VideoSourceType type, Context context) {
+ this.cameraManager = context.getSystemService(CameraManager.class);
+ this.imageReader = ImageReader.newInstance(width, height, ImageFormat.JPEG, 1);
+ this.surface = this.imageReader.getSurface();
+ this.imageReader.setOnImageAvailableListener(this.imageAvailableListener, null);
+ try {
+ final String cameraId = String.valueOf(type == VideoSourceType.BACK ? CameraCharacteristics.LENS_FACING_BACK : CameraCharacteristics.LENS_FACING_FRONT);
+ this.cameraManager.openCamera(cameraId, this.cameraDeviceStateCallback, null);
+ } catch (CameraAccessException e) {
+ Log.e(PhotographClient.class.getSimpleName(), "拍照异常", e);
+ PhotographClient.this.closeCamera();
+ }
+ return this.filepath;
+ }
+
+ private ImageReader.OnImageAvailableListener imageAvailableListener = new ImageReader.OnImageAvailableListener() {
+ @Override
+ public void onImageAvailable(ImageReader imageReader) {
+ final Image image = imageReader.acquireLatestImage();
+ final ByteBuffer byteBuffer = image.getPlanes()[0].getBuffer();
+ final byte[] bytes = new byte[byteBuffer.remaining()];
+ byteBuffer.get(bytes);
+ final File file = new File(PhotographClient.this.filepath);
+ try (
+ final OutputStream output = new FileOutputStream(file);
+ ) {
+ output.write(bytes,0,bytes.length);
+ } catch (IOException e) {
+ Log.e(PhotographClient.class.getSimpleName(), "拍照异常", e);
+ PhotographClient.this.closeCamera();
+ } finally {
+ image.close();
+ PhotographClient.this.closeCamera();
+ }
+ }
+ };
+
+ private CameraDevice.StateCallback cameraDeviceStateCallback = new CameraDevice.StateCallback() {
+ @Override
+ public void onOpened(CameraDevice cameraDevice) {
+ PhotographClient.this.cameraDevice = cameraDevice;
+ try {
+ PhotographClient.this.cameraDevice.createCaptureSession(new SessionConfiguration(
+ SessionConfiguration.SESSION_REGULAR,
+ List.of(new OutputConfiguration(PhotographClient.this.surface)),
+ Runnable::run,
+ PhotographClient.this.cameraCaptureSessionStateCallback
+ ));
+ } catch (CameraAccessException e) {
+ Log.e(PhotographClient.class.getSimpleName(), "拍照异常", e);
+ PhotographClient.this.closeCamera();
+ }
+ }
+ @Override
+ public void onDisconnected(CameraDevice cameraDevice) {
+ PhotographClient.this.closeCamera();
+ }
+ @Override
+ public void onError(CameraDevice cameraDevice, int error) {
+ PhotographClient.this.closeCamera();
+ }
+ };
+
+ private CameraCaptureSession.StateCallback cameraCaptureSessionStateCallback = new CameraCaptureSession.StateCallback() {
+ @Override
+ public void onConfigured(CameraCaptureSession cameraCaptureSession) {
+ try {
+ PhotographClient.this.cameraCaptureSession = cameraCaptureSession;
+ final CaptureRequest.Builder builder = PhotographClient.this.cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);
+ builder.addTarget(PhotographClient.this.surface);
+ cameraCaptureSession.setRepeatingRequest(builder.build(), null, null);
+ } catch (CameraAccessException e) {
+ Log.e(PhotographClient.class.getSimpleName(), "拍照异常", e);
+ PhotographClient.this.closeCamera();
+ }
+ }
+ @Override
+ public void onConfigureFailed(CameraCaptureSession session) {
+ PhotographClient.this.closeCamera();
+ }
+ };
+
+ private void closeCamera() {
+ if(this.cameraCaptureSession != null) {
+ this.cameraCaptureSession.close();
+ this.cameraCaptureSession = null;
+ }
+ if(this.cameraDevice != null) {
+ this.cameraDevice.close();
+ this.cameraDevice = null;
+ }
+ // 最后释放ImageReader
+ if(this.surface != null) {
+ this.surface.release();
+ this.surface = null;
+ }
+ if(this.imageReader != null) {
+ this.imageReader.close();
+ this.imageReader = null;
+ }
}
}
diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java
index 40db0af..11b4290 100644
--- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java
+++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java
@@ -1,7 +1,5 @@
package com.acgist.taoyao.media.client;
-import android.graphics.YuvImage;
-import android.media.AudioFormat;
import android.media.MediaCodec;
import android.media.MediaCodecInfo;
import android.media.MediaFormat;
@@ -10,18 +8,13 @@ import android.os.Environment;
import android.os.Handler;
import android.os.HandlerThread;
import android.util.Log;
-import android.view.Surface;
import com.acgist.taoyao.boot.utils.DateUtils;
import com.acgist.taoyao.media.MediaManager;
import com.acgist.taoyao.media.VideoSourceType;
import com.acgist.taoyao.media.signal.ITaoyao;
-import org.webrtc.EglBase;
-import org.webrtc.GlRectDrawer;
-import org.webrtc.TextureBufferImpl;
import org.webrtc.VideoFrame;
-import org.webrtc.VideoFrameDrawer;
import org.webrtc.VideoSink;
import org.webrtc.YuvHelper;
import org.webrtc.audio.JavaAudioDeviceModule;
@@ -29,34 +22,22 @@ import org.webrtc.audio.JavaAudioDeviceModule;
import java.io.File;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.LocalDateTime;
-import java.util.Queue;
import java.util.concurrent.BlockingQueue;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
/**
* 录像机
*
- * https://www.freesion.com/article/448330501/
- * https://blog.csdn.net/nanoage/article/details/127406494
- * https://webrtc.org.cn/20190419_tutorial3_webrtc_android
- * https://blog.csdn.net/CSDN_Mew/article/details/103406781
- * https://blog.csdn.net/Tong_Hou/article/details/112116349
- * https://blog.csdn.net/u011418943/article/details/127108642
- * https://blog.csdn.net/m0_60259116/article/details/126875532
- * https://blog.csdn.net/csdn_shen0221/article/details/119982257
- * https://blog.csdn.net/csdn_shen0221/article/details/120331004
- * https://github.com/flutter-webrtc/flutter-webrtc/blob/main/android/src/main/java/com/cloudwebrtc/webrtc/record/VideoFileRenderer.java
- *
* @author acgist
*/
public class RecordClient extends Client implements VideoSink, JavaAudioDeviceModule.SamplesReadyCallback {
+ private static final long WAIT_TIME_MS = 50;
+ private static final long WAIT_TIME_US = WAIT_TIME_MS * 1000;
+
/**
* 音频准备录制
*/
@@ -73,6 +54,40 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
* 录制文件路径
*/
private final String filepath;
+ /**
+ * 比特率：96 * 1000 | 128 * 1000 | 256 * 1000
+ * 比特率：96 * 1024 | 128 * 1024 | 256 * 1024
+ */
+ private final int audioBitRate;
+ /**
+ * 采样率：32000 | 44100 | 48000
+ */
+ private final int sampleRate;
+ /**
+ * 通道数量：1 | 2
+ */
+ private final int channelCount;
+ /**
+ * 比特率：800 * 1000 | 1600 * 1000 | 2500 * 1000
+ * 比特率：800 * 1024 | 1600 * 1024 | 2500 * 1024
+ */
+ private final int videoBitRate;
+ /**
+ * 帧率：15 | 20 | 25 | 30
+ */
+ private final int frameRate;
+ /**
+ * 关键帧频率：1 ~ 5
+ */
+ private final int iFrameInterval;
+ /**
+ * 宽度：1920
+ */
+ private final int width;
+ /**
+ * 高度：1080
+ */
+ private final int height;
/**
* 音频编码
*/
@@ -101,11 +116,29 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
* 媒体合成器
*/
private MediaMuxer mediaMuxer;
+ /**
+ * 音频队列
+ */
private final BlockingQueue audioSamplesQueue;
+ /**
+ * 视频队列
+ */
private final BlockingQueue videoFrameQueue;
- public RecordClient(String path, ITaoyao taoyao, Handler mainHandler) {
+ public RecordClient(
+ int audioBitRate, int sampleRate, int channelCount,
+ int videoBitRate, int frameRate, int iFrameInterval, int width, int height,
+ String path, ITaoyao taoyao, Handler mainHandler
+ ) {
super("本地录像", "LocalRecordClient", taoyao, mainHandler);
+ this.audioBitRate = audioBitRate * 1024;
+ this.sampleRate = sampleRate;
+ this.channelCount = channelCount;
+ this.videoBitRate = videoBitRate * 1024;
+ this.frameRate = frameRate;
+ this.iFrameInterval = iFrameInterval;
+ this.width = width;
+ this.height = height;
this.filename = DateUtils.format(LocalDateTime.now(), DateUtils.DateTimeStyle.YYYYMMDDHH24MMSS) + ".mp4";
this.filepath = Paths.get(Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_DOCUMENTS).getAbsolutePath(), path, this.filename).toString();
this.audioSamplesQueue = new LinkedBlockingQueue<>();
@@ -119,36 +152,29 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
}
Log.i(RecordClient.class.getSimpleName(), "录制视频文件：" + this.filepath);
super.init();
- this.mediaManager.newClient(VideoSourceType.BACK);
- this.record(null, null, 1, 1);
- }
- }
-
- private void record(String audioFormat, String videoFormat, int width, int height) {
- if (
- this.audioThread == null || !this.audioThread.isAlive() ||
- this.videoThread == null || !this.videoThread.isAlive()
- ) {
- this.initMediaMuxer();
- this.initAudioThread(MediaFormat.MIMETYPE_AUDIO_AAC, 96000, 44100, 1);
- this.initVideoThread(MediaFormat.MIMETYPE_VIDEO_AVC, 2500 * 1000, 30, 1, 1920, 1080);
+ if (
+ this.audioThread == null || !this.audioThread.isAlive() ||
+ this.videoThread == null || !this.videoThread.isAlive()
+ ) {
+ this.initMediaMuxer();
+ this.initAudioThread(MediaFormat.MIMETYPE_AUDIO_AAC);
+ this.initVideoThread(MediaFormat.MIMETYPE_VIDEO_AVC);
+ }
}
}
/**
- * @param audioType 类型
- * @param bitRate 比特率：96 * 1000 | 128 * 1000 | 256 * 1000
- * @param sampleRate 采样率：32000 | 44100 | 48000
- * @param channelCount 通道数量
+ * @param audioType 类型
*/
- private void initAudioThread(String audioType, int bitRate, int sampleRate, int channelCount) {
+ private void initAudioThread(String audioType) {
try {
- final MediaFormat audioFormat = MediaFormat.createAudioFormat(audioType, sampleRate, channelCount);
-// audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, bitRate);
- audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, AudioFormat.ENCODING_PCM_16BIT);
+ final MediaFormat audioFormat = MediaFormat.createAudioFormat(audioType, this.sampleRate, this.channelCount);
+// audioFormat.setString(MediaFormat.KEY_MIME, audioType);
+// audioFormat.setInteger(MediaFormat.KEY_SAMPLE_RATE, this.sampleRate);
+// audioFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, this.channelCount);
+ audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, this.audioBitRate);
audioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
-// audioFormat.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR);
- audioFormat.setInteger(MediaFormat.KEY_MAX_INPUT_SIZE, 8 * 1024);
+// audioFormat.setInteger(MediaFormat.KEY_BITRATE_MODE, MediaCodecInfo.EncoderCapabilities.BITRATE_MODE_VBR);
this.audioCodec = MediaCodec.createEncoderByType(audioType);
this.audioCodec.configure(audioFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
} catch (Exception e) {
@@ -160,48 +186,45 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
this.audioHandler.post(this::audioCodec);
}
+ private volatile long audioPts = 0;
+
private void audioCodec() {
+ long pts = 0L;
int trackIndex = -1;
int outputIndex;
- long pts = 0L;
this.audioCodec.start();
this.audioActive = true;
JavaAudioDeviceModule.AudioSamples audioSamples = null;
final MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
while (!this.close) {
-
-
try {
- audioSamples = this.audioSamplesQueue.poll(1000, TimeUnit.MILLISECONDS);
+ audioSamples = this.audioSamplesQueue.poll(WAIT_TIME_MS, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Log.e(RecordClient.class.getSimpleName(), "录制线程等待异常", e);
}
if(audioSamples == null) {
continue;
}
- int index = this.audioCodec.dequeueInputBuffer(1000L * 1000);
+ int index = this.audioCodec.dequeueInputBuffer(WAIT_TIME_US);
if (index >= 0) {
final byte[] data = audioSamples.getData();
final ByteBuffer buffer = this.audioCodec.getInputBuffer(index);
buffer.put(data);
this.audioCodec.queueInputBuffer(index, 0, data.length, this.audioPts, 0);
- this.audioPts += data.length * 125 / 12; // 1000000 microseconds / 48000hz / 2 bytes
-// presTime += data.length * 125 / 12; // 1000000 microseconds / 48000hz / 2 bytes
-// presTime += data.length * (1_000_000 / audioSamples.getSampleRate() / 2); //16位最后那个数字是2，8位是1
+ // 1000000 microseconds / 48000 hz / 2 bytes
+ this.audioPts += data.length * (1_000_000 / audioSamples.getSampleRate() / 2);
} else {
// WARN
}
audioSamples = null;
-
-
- outputIndex = this.audioCodec.dequeueOutputBuffer(bufferInfo, 1000L * 1000);
+ outputIndex = this.audioCodec.dequeueOutputBuffer(bufferInfo, WAIT_TIME_US);
if (outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
// } else if (outputIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
} else if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
synchronized (this) {
trackIndex = this.mediaMuxer.addTrack(this.audioCodec.getOutputFormat());
Log.i(RecordClient.class.getSimpleName(), "开始录制音频：" + trackIndex);
- if (this.videoActive) {
+ if (!this.close && this.videoActive) {
Log.i(RecordClient.class.getSimpleName(), "开始录制文件：" + this.filename);
this.mediaMuxer.start();
this.notifyAll();
@@ -230,15 +253,15 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
// } else if (bufferInfo.flags & MediaCodec.BUFFER_FLAG_PARTIAL_FRAME == MediaCodec.BUFFER_FLAG_PARTIAL_FRAME) {
// }
if((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == MediaCodec.BUFFER_FLAG_END_OF_STREAM) {
-// this.close();
+ this.close();
break;
}
} else {
-
+ // WARN
}
}
synchronized (this) {
- if (this.audioCodec != null) {
+ if (this.audioCodec != null && this.audioActive) {
Log.i(RecordClient.class.getSimpleName(), "结束录制音频");
this.audioCodec.stop();
this.audioCodec.release();
@@ -247,15 +270,13 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
this.audioActive = false;
if (this.mediaMuxer != null && !this.videoActive) {
Log.i(RecordClient.class.getSimpleName(), "结束录制文件：" + this.filename);
- this.mediaMuxer.stop();
+// this.mediaMuxer.stop();
this.mediaMuxer.release();
this.mediaMuxer = null;
}
}
}
- private volatile long audioPts = 0;
-
/**
* @param audioSamples PCM数据
*/
@@ -272,25 +293,18 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
}
/**
- * @param videoType 视频格式
- * @param bitRate 比特率：800 * 1000 | 1600 * 1000 | 2500 * 1000
- * @param frameRate 帧率：30
- * @param iFrameInterval 关键帧频率：1 ~ 5
- * @param width 宽度：1920
- * @param height 高度：1080
+ * @param videoType 视频格式
*/
- private void initVideoThread(String videoType, int bitRate, int frameRate, int iFrameInterval, int width, int height) {
+ private void initVideoThread(String videoType) {
try {
- final MediaFormat videoFormat = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, 1920, 1080);
+ final MediaFormat videoFormat = MediaFormat.createVideoFormat(MediaFormat.MIMETYPE_VIDEO_AVC, this.width, this.height);
// videoFormat.setInteger(MediaFormat.KEY_LEVEL, MediaCodecInfo.CodecProfileLevel.AVCLevel31);
// videoFormat.setInteger(MediaFormat.KEY_PROFILE, MediaCodecInfo.CodecProfileLevel.AVCProfileHigh);
-// videoFormat.setInteger(MediaFormat.KEY_BIT_RATE, 1920 * 1080 * 5);
- videoFormat.setInteger(MediaFormat.KEY_BIT_RATE, 6_000_000);
-// videoFormat.setInteger(MediaFormat.KEY_BIT_RATE, 800 * 1000);
- videoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30);
-// videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
- videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible);
- videoFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1);
+ videoFormat.setInteger(MediaFormat.KEY_BIT_RATE, this.videoBitRate);
+ videoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, this.frameRate);
+// videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface);
+ videoFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatYUV420Flexible);
+ videoFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, this.iFrameInterval);
this.videoCodec = MediaCodec.createEncoderByType(videoType);
this.videoCodec.configure(videoFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
} catch (Exception e) {
@@ -303,52 +317,39 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
}
private void videoCodec() {
- int trackIndex = -1;
+ long pts = 0L;
+ int trackIndex = -1;
int outputIndex;
- long pts = 0L;
-
+ VideoFrame videoFrame = null;
this.videoCodec.start();
this.videoActive = true;
final MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
- VideoFrame videoFrame = null;
while (!this.close) {
try {
- videoFrame = this.videoFrameQueue.poll(1000, TimeUnit.MILLISECONDS);
+ videoFrame = this.videoFrameQueue.poll(WAIT_TIME_MS, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Log.e(RecordClient.class.getSimpleName(), "录制线程等待异常", e);
}
if(videoFrame == null) {
continue;
}
-
-
- final TextureBufferImpl buffer = (TextureBufferImpl) videoFrame.getBuffer();
- final int outputFrameSize = videoFrame.getRotatedWidth() * videoFrame.getRotatedHeight() * 3 / 2;
-// final ByteBuffer outputFrameBuffer = ByteBuffer.allocateDirect(outputFrameSize);
- final int index = this.videoCodec.dequeueInputBuffer(1000L * 1000);
- VideoFrame.I420Buffer i420 = buffer.toI420();
- final ByteBuffer bufferx = this.videoCodec.getInputBuffer(index);
-// YuvHelper.I420Copy(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight());
-// YuvHelper.I420Rotate(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), outputFrameBuffer, i420.getWidth(), i420.getHeight(), videoFrame.getRotation());
- YuvHelper.I420ToNV12(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), bufferx, i420.getWidth(), i420.getHeight());
+ final int videoFrameSize = videoFrame.getRotatedWidth() * videoFrame.getRotatedHeight() * 3 / 2;
+ final int index = this.videoCodec.dequeueInputBuffer(WAIT_TIME_US);
+ final VideoFrame.I420Buffer i420 = videoFrame.getBuffer().toI420();
+ final ByteBuffer inputByteBuffer = this.videoCodec.getInputBuffer(index);
+ YuvHelper.I420ToNV12(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), inputByteBuffer, i420.getWidth(), i420.getHeight());
i420.release();
videoFrame.release();
-// bufferx.put(outputFrameBuffer.array());
- this.videoCodec.queueInputBuffer(index, 0, outputFrameSize, videoFrame.getTimestampNs(), 0);
-
-
-// this.videoFrameDrawer.drawFrame(videoFrame, this.glRectDrawer, null, 0, 0, videoFrame.getRotatedWidth(), videoFrame.getRotatedHeight());
-// videoFrame.release();
-// videoFrame = null;
-// this.eglBase.swapBuffers();
- outputIndex = this.videoCodec.dequeueOutputBuffer(bufferInfo, 1000L * 1000);
+ this.videoCodec.queueInputBuffer(index, 0, videoFrameSize, videoFrame.getTimestampNs(), 0);
+ videoFrame = null;
+ outputIndex = this.videoCodec.dequeueOutputBuffer(bufferInfo, WAIT_TIME_US);
if (outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
// } else if (outputIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
} else if (outputIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
synchronized (this) {
trackIndex = this.mediaMuxer.addTrack(this.videoCodec.getOutputFormat());
Log.i(RecordClient.class.getSimpleName(), "开始录制视频：" + trackIndex);
- if (this.audioActive) {
+ if (!this.close && this.audioActive) {
Log.i(RecordClient.class.getSimpleName(), "开始录制文件：" + this.filename);
this.mediaMuxer.start();
this.notifyAll();
@@ -358,6 +359,7 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
} catch (InterruptedException e) {
Log.e(RecordClient.class.getSimpleName(), "录制线程等待异常", e);
}
+ } else {
}
}
} else if (outputIndex >= 0) {
@@ -378,14 +380,15 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
// } else if (bufferInfo.flags & MediaCodec.BUFFER_FLAG_PARTIAL_FRAME == MediaCodec.BUFFER_FLAG_PARTIAL_FRAME) {
// }
if((bufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) == MediaCodec.BUFFER_FLAG_END_OF_STREAM) {
-// this.close();
+ this.close();
break;
}
} else {
+ // WARN
}
}
synchronized (this) {
- if (this.videoCodec != null) {
+ if (this.videoCodec != null && this.videoActive) {
Log.i(RecordClient.class.getSimpleName(), "结束录制视频");
this.videoCodec.stop();
this.videoCodec.release();
@@ -394,7 +397,7 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
this.videoActive = false;
if (this.mediaMuxer != null && !this.audioActive) {
Log.i(RecordClient.class.getSimpleName(), "结束录制文件：" + this.filename);
- this.mediaMuxer.stop();
+// this.mediaMuxer.stop();
this.mediaMuxer.release();
this.mediaMuxer = null;
}
@@ -440,8 +443,12 @@ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceMo
if (this.videoThread != null) {
this.videoThread.quitSafely();
}
+ final File file = new File(this.filepath);
+ if(file.length() <= 0) {
+ Log.i(RecordClient.class.getSimpleName(), "删除没有录制数据文件：" + this.filepath);
+ file.delete();
+ }
this.notifyAll();
- this.mediaManager.closeClient();
}
}
diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java
index 5bc46d4..ccc3d04 100644
--- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java
+++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/Room.java
@@ -10,6 +10,9 @@ import com.acgist.taoyao.boot.utils.PointerUtils;
import com.acgist.taoyao.media.RouterCallback;
import com.acgist.taoyao.media.VideoSourceType;
import com.acgist.taoyao.media.config.Config;
+import com.acgist.taoyao.media.config.MediaAudioProperties;
+import com.acgist.taoyao.media.config.MediaProperties;
+import com.acgist.taoyao.media.config.MediaVideoProperties;
import com.acgist.taoyao.media.signal.ITaoyao;
import org.webrtc.AudioTrack;
@@ -36,27 +39,49 @@ public class Room extends CloseableClient implements RouterCallback {
private final String roomId;
private final String password;
private final boolean preview;
+ private final boolean playAudio;
+ private final boolean playVideo;
private final boolean dataConsume;
private final boolean audioConsume;
private final boolean videoConsume;
private final boolean dataProduce;
private final boolean audioProduce;
private final boolean videoProduce;
+ private final MediaProperties mediaProperties;
+ private final Map remoteClients;
private final long nativeRoomPointer;
private LocalClient localClient;
- private Map remoteClients;
private PeerConnection.RTCConfiguration rtcConfiguration;
private PeerConnectionFactory peerConnectionFactory;
private String rtpCapabilities;
private String sctpCapabilities;
+ /**
+ *
+ * @param name
+ * @param clientId
+ * @param roomId
+ * @param password
+ * @param taoyao
+ * @param mainHandler
+ * @param preview 是否预览视频
+ * @param playAudio 是否播放音频
+ * @param playVideo 是否播放视频
+ * @param dataConsume 是否消费数据
+ * @param audioConsume 是否消费音频
+ * @param videoConsume 是否消费视频
+ * @param dataProduce 是否生产数据
+ * @param audioProduce 是否生产音频
+ * @param videoProduce 是否生产视频
+ */
public Room(
String name, String clientId,
String roomId, String password,
ITaoyao taoyao, Handler mainHandler,
- boolean preview,
+ boolean preview, boolean playAudio, boolean playVideo,
boolean dataConsume, boolean audioConsume, boolean videoConsume,
- boolean dataProduce, boolean audioProduce, boolean videoProduce
+ boolean dataProduce, boolean audioProduce, boolean videoProduce,
+ MediaProperties mediaProperties
) {
super(taoyao, mainHandler);
this.name = name;
@@ -64,14 +89,17 @@ public class Room extends CloseableClient implements RouterCallback {
this.roomId = roomId;
this.password = password;
this.preview = preview;
+ this.playAudio = playAudio;
+ this.playVideo = playVideo;
this.dataConsume = dataConsume;
this.audioConsume = audioConsume;
this.videoConsume = videoConsume;
this.dataProduce = dataProduce;
this.audioProduce = audioProduce;
this.videoProduce = videoProduce;
- this.nativeRoomPointer = this.nativeNewRoom(roomId, this);
+ this.mediaProperties = mediaProperties;
this.remoteClients = new ConcurrentHashMap<>();
+ this.nativeRoomPointer = this.nativeNewRoom(roomId, this);
}
public boolean enter() {
diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java
index f320a29..1443aed 100644
--- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java
+++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java
@@ -8,6 +8,7 @@ import com.acgist.taoyao.boot.utils.ListUtils;
import com.acgist.taoyao.boot.utils.MapUtils;
import com.acgist.taoyao.media.VideoSourceType;
import com.acgist.taoyao.media.config.Config;
+import com.acgist.taoyao.media.config.MediaProperties;
import com.acgist.taoyao.media.signal.ITaoyao;
import org.webrtc.DataChannel;
@@ -37,6 +38,16 @@ public class SessionClient extends Client {
* 会话ID
*/
private final String sessionId;
+ private final boolean preview;
+ private final boolean playAudio;
+ private final boolean playVideo;
+ private final boolean dataConsume;
+ private final boolean audioConsume;
+ private final boolean videoConsume;
+ private final boolean dataProduce;
+ private final boolean audioProduce;
+ private final boolean videoProduce;
+ private final MediaProperties mediaProperties;
/**
* 本地媒体
*/
@@ -62,9 +73,25 @@ public class SessionClient extends Client {
*/
private PeerConnectionFactory peerConnectionFactory;
- public SessionClient(String sessionId, String name, String clientId, ITaoyao taoyao, Handler mainHandler) {
+ public SessionClient(
+ String sessionId, String name, String clientId, ITaoyao taoyao, Handler mainHandler,
+ boolean preview, boolean playAudio, boolean playVideo,
+ boolean dataConsume, boolean audioConsume, boolean videoConsume,
+ boolean dataProduce, boolean audioProduce, boolean videoProduce,
+ MediaProperties mediaProperties
+ ) {
super(name, clientId, taoyao, mainHandler);
this.sessionId = sessionId;
+ this.preview = preview;
+ this.playAudio = playAudio;
+ this.playVideo = playVideo;
+ this.dataConsume = dataConsume;
+ this.audioConsume = audioConsume;
+ this.videoConsume = videoConsume;
+ this.dataProduce = dataProduce;
+ this.audioProduce = audioProduce;
+ this.videoProduce = videoProduce;
+ this.mediaProperties = mediaProperties;
}
@Override
diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/MediaAudioProperties.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/MediaAudioProperties.java
index 3beb8b1..6cfea01 100644
--- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/MediaAudioProperties.java
+++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/MediaAudioProperties.java
@@ -26,7 +26,11 @@ public class MediaAudioProperties {
*/
private Format format;
/**
- * 采样数：8|16|32
+ * 比特率：96|128|256
+ */
+ private Integer bitrate;
+ /**
+ * 采样位数：8|16|32
*/
private Integer sampleSize;
/**
@@ -42,6 +46,14 @@ public class MediaAudioProperties {
this.format = format;
}
+ public Integer getBitrate() {
+ return bitrate;
+ }
+
+ public void setBitrate(Integer bitrate) {
+ this.bitrate = bitrate;
+ }
+
public Integer getSampleSize() {
return this.sampleSize;
}
diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/MediaProperties.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/MediaProperties.java
index 37fb72a..5e31c9b 100644
--- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/MediaProperties.java
+++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/config/MediaProperties.java
@@ -25,14 +25,6 @@ public class MediaProperties {
* 最大视频高度
*/
private Integer maxHeight;
- /**
- * 最小视频码率
- */
- private Integer minBitrate;
- /**
- * 最大视频码率
- */
- private Integer maxBitrate;
/**
* 最小视频帧率
*/
@@ -42,11 +34,19 @@ public class MediaProperties {
*/
private Integer maxFrameRate;
/**
- * 最小音频采样数
+ * 最小视频码率
+ */
+ private Integer minVideoBitrate;
+ /**
+ * 最大视频码率
+ */
+ private Integer maxVideoBitrate;
+ /**
+ * 最小音频采样位数
*/
private Integer minSampleSize;
/**
- * 最大音频采样数
+ * 最大音频采样位数
*/
private Integer maxSampleSize;
/**
@@ -57,6 +57,14 @@ public class MediaProperties {
* 最大音频采样率
*/
private Integer maxSampleRate;
+ /**
+ * 最小音频码率
+ */
+ private Integer minAudioBitrate;
+ /**
+ * 最大音频码率
+ */
+ private Integer maxAudioBitrate;
/**
* 音频默认配置
*/
@@ -106,22 +114,6 @@ public class MediaProperties {
this.maxHeight = maxHeight;
}
- public Integer getMinBitrate() {
- return minBitrate;
- }
-
- public void setMinBitrate(Integer minBitrate) {
- this.minBitrate = minBitrate;
- }
-
- public Integer getMaxBitrate() {
- return maxBitrate;
- }
-
- public void setMaxBitrate(Integer maxBitrate) {
- this.maxBitrate = maxBitrate;
- }
-
public Integer getMinFrameRate() {
return minFrameRate;
}
@@ -138,6 +130,22 @@ public class MediaProperties {
this.maxFrameRate = maxFrameRate;
}
+ public Integer getMinVideoBitrate() {
+ return minVideoBitrate;
+ }
+
+ public void setMinVideoBitrate(Integer minVideoBitrate) {
+ this.minVideoBitrate = minVideoBitrate;
+ }
+
+ public Integer getMaxVideoBitrate() {
+ return maxVideoBitrate;
+ }
+
+ public void setMaxVideoBitrate(Integer maxVideoBitrate) {
+ this.maxVideoBitrate = maxVideoBitrate;
+ }
+
public Integer getMinSampleSize() {
return minSampleSize;
}
@@ -170,6 +178,22 @@ public class MediaProperties {
this.maxSampleRate = maxSampleRate;
}
+ public Integer getMinAudioBitrate() {
+ return minAudioBitrate;
+ }
+
+ public void setMinAudioBitrate(Integer minAudioBitrate) {
+ this.minAudioBitrate = minAudioBitrate;
+ }
+
+ public Integer getMaxAudioBitrate() {
+ return maxAudioBitrate;
+ }
+
+ public void setMaxAudioBitrate(Integer maxAudioBitrate) {
+ this.maxAudioBitrate = maxAudioBitrate;
+ }
+
public MediaAudioProperties getAudio() {
return audio;
}
@@ -201,4 +225,5 @@ public class MediaProperties {
public void setVideos(Map videos) {
this.videos = videos;
}
+
}
diff --git a/taoyao-client-media/src/Taoyao.js b/taoyao-client-media/src/Taoyao.js
index d074c4d..3ed8457 100644
--- a/taoyao-client-media/src/Taoyao.js
+++ b/taoyao-client-media/src/Taoyao.js
@@ -1507,7 +1507,7 @@ class Taoyao {
interval: 2000,
// 范围：-127~0
threshold: -80,
- // 采样数量
+ // 监控数量
maxEntries: 2,
});
// 采样监控
diff --git a/taoyao-client-web/src/components/Config.js b/taoyao-client-web/src/components/Config.js
index 473fdbd..b7b38e9 100644
--- a/taoyao-client-web/src/components/Config.js
+++ b/taoyao-client-web/src/components/Config.js
@@ -8,7 +8,7 @@ const defaultAudioConfig = {
volume: 0.5,
// 延迟大小（单位毫秒）：500毫秒以内较好
latency: 0.4,
- // 采样数：8|16|32
+ // 采样位数：8|16|32
sampleSize: { min: 8, ideal: 16, max: 32 },
// 采样率：8000|16000|32000|48000
sampleRate: { min: 8000, ideal: 32000, max: 48000 },
diff --git a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/MediaAudioProperties.java b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/MediaAudioProperties.java
index 0d930a8..82cd16e 100644
--- a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/MediaAudioProperties.java
+++ b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/MediaAudioProperties.java
@@ -7,6 +7,8 @@ import lombok.Setter;
/**
* 音频配置
*
+ * 比特率 = 采样率 * 采样位数 * 声道数 / 8 / 1024
+ *
* @author acgist
*/
@Getter
@@ -30,7 +32,9 @@ public class MediaAudioProperties {
@Schema(title = "格式", description = "格式", example = "G722|PCMA|PCMU|OPUS")
private Format format;
- @Schema(title = "采样数", description = "采样数", example = "8|16|32")
+ @Schema(title = "比特率", description = "比特率", example = "96|128|256")
+ private Integer bitrate;
+ @Schema(title = "采样位数", description = "采样位数", example = "8|16|32")
private Integer sampleSize;
@Schema(title = "采样率", description = "采样率", example = "8000|16000|32000|48000")
private Integer sampleRate;
diff --git a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/MediaProperties.java b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/MediaProperties.java
index 517c1fd..3a18368 100644
--- a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/MediaProperties.java
+++ b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/MediaProperties.java
@@ -27,22 +27,26 @@ public class MediaProperties {
private Integer minHeight;
@Schema(title = "最大视频高度", description = "最大视频高度")
private Integer maxHeight;
- @Schema(title = "最小视频码率", description = "最小视频码率")
- private Integer minBitrate;
- @Schema(title = "最大视频码率", description = "最大视频码率")
- private Integer maxBitrate;
@Schema(title = "最小视频帧率", description = "最小视频帧率")
private Integer minFrameRate;
@Schema(title = "最大视频帧率", description = "最大视频帧率")
private Integer maxFrameRate;
- @Schema(title = "最小音频采样数", description = "最小音频采样数")
+ @Schema(title = "最小视频码率", description = "最小视频码率")
+ private Integer minVideoBitrate;
+ @Schema(title = "最大视频码率", description = "最大视频码率")
+ private Integer maxVideoBitrate;
+ @Schema(title = "最小音频采样位数", description = "最小音频采样位数")
private Integer minSampleSize;
- @Schema(title = "最大音频采样数", description = "最大音频采样数")
+ @Schema(title = "最大音频采样位数", description = "最大音频采样位数")
private Integer maxSampleSize;
@Schema(title = "最小音频采样率", description = "最小音频采样率")
private Integer minSampleRate;
@Schema(title = "最大音频采样率", description = "最大音频采样率")
private Integer maxSampleRate;
+ @Schema(title = "最小音频码率", description = "最小音频码率")
+ private Integer minAudioBitrate;
+ @Schema(title = "最大音频码率", description = "最大音频码率")
+ private Integer maxAudioBitrate;
@Schema(title = "音频默认配置", description = "音频默认配置")
private MediaAudioProperties audio;
@Schema(title = "视频默认配置", description = "视频默认配置")
diff --git a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/MediaVideoProperties.java b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/MediaVideoProperties.java
index efe76ff..3e93a31 100644
--- a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/MediaVideoProperties.java
+++ b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/MediaVideoProperties.java
@@ -6,6 +6,10 @@ import lombok.Setter;
/**
* 视频配置
+ * 原始数据 = 宽 * 高 * 3 / 2 * 8 * 帧率 / 1024 / 1024
+ * 视频编码 = 压缩
+ * 8 = 颜色位数
+ * 3 / 2 = YUV | RGB
*
* @author acgist
*/
diff --git a/taoyao-signal-server/taoyao-server/src/main/resources/application.yml b/taoyao-signal-server/taoyao-server/src/main/resources/application.yml
index 856cbaf..8ca5334 100644
--- a/taoyao-signal-server/taoyao-server/src/main/resources/application.yml
+++ b/taoyao-signal-server/taoyao-server/src/main/resources/application.yml
@@ -67,28 +67,34 @@ taoyao:
max-client-index: 99999
# 媒体配置
media:
+ # =============== 视频配置 ===============
# 宽度
min-width: 720
max-width: 4096
# 高度
min-height: 480
max-height: 2160
- # 码率
- min-bitrate: 800
- max-bitrate: 1600
# 帧率
min-frame-rate: 15
max-frame-rate: 45
- # 采样数
+ # 视频码率
+ min-video-bitrate: 800
+ max-video-bitrate: 1600
+ # =============== 音频配置 ===============
+ # 采样位数
min-sample-size: 8
max-sample-size: 32
# 采样率
min-sample-rate: 8000
max-sample-rate: 48000
+ # 音频码率
+ min-audio-bitrate: 96
+ max-audio-bitrate: 256
# ABR CBR VBR
# 默认音频
audio:
format: OPUS
+ bitrate: 96
sample-size: 32
sample-rate: 44100
# 默认视频
@@ -101,16 +107,19 @@ taoyao:
# 超清
fd-audio:
format: OPUS
+ bitrate: 256
sample-size: 32
sample-rate: 48000
# 高清
hd-audio:
format: OPUS
+ bitrate: 128
sample-size: 24
sample-rate: 44100
# 标清
sd-audio:
format: OPUS
+ bitrate: 96
sample-size: 24
sample-rate: 32000
videos: