qq27800734/taoyao
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[*] 优化

Browse Source
This commit is contained in:
acgist
2023-05-24 08:12:22 +08:00
parent 1d8507862b
commit 45f3c24a61
3 changed files with 94 additions and 46 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
docs/TODO.md
View File

@@ -38,6 +38,7 @@
* 分辨率调整 * 分辨率调整
* 服务端录制 * 服务端录制
* 安卓预览按钮 * 安卓预览按钮
* 安装内存抖动
* 降低视频录制大小 * 降低视频录制大小
* 防止重复邀请拉取 * 防止重复邀请拉取
* 码率等等参数配置验证 * 码率等等参数配置验证

3
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java
View File

@@ -139,7 +139,8 @@ public class MixerProcesser extends Thread implements JavaAudioDeviceModule.Samp
@Override @Override
public void run() { public void run() {
long pts = System.nanoTime(); // long pts = System.nanoTime();
long pts = System.currentTimeMillis();
int recordSize = 0; int recordSize = 0;
int mixDataLength = 0; int mixDataLength = 0;
byte[] mixData = null; byte[] mixData = null;

136
taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java
View File

@@ -7,7 +7,6 @@ import android.media.MediaMuxer;
import android.os.Environment; import android.os.Environment;
import android.os.Handler; import android.os.Handler;
import android.os.HandlerThread; import android.os.HandlerThread;
import android.os.SystemClock;
import android.util.Log; import android.util.Log;
import com.acgist.taoyao.boot.utils.DateUtils; import com.acgist.taoyao.boot.utils.DateUtils;
@@ -46,11 +45,11 @@ public class RecordClient extends Client implements VideoSink {
private static final long WAIT_TIME_US = WAIT_TIME_MS * 1000; private static final long WAIT_TIME_US = WAIT_TIME_MS * 1000;
/** /**
* 音频准备录制 * 音频录制准备完成
*/ */
private volatile boolean audioActive; private volatile boolean audioActive;
/** /**
* 视频准备录制 * 视频录制准备完成
*/ */
private volatile boolean videoActive; private volatile boolean videoActive;
/** /**
@@ -127,15 +126,37 @@ public class RecordClient extends Client implements VideoSink {
* 是否已经开始录制 * 是否已经开始录制
* 不能使用多线程wait/notify录制音频没有结束 * 不能使用多线程wait/notify录制音频没有结束
*/ */
private boolean start = false; private boolean muxerActive = false;
/** /**
* 媒体合成器 * 媒体合成器
*/ */
private MediaMuxer mediaMuxer; private MediaMuxer mediaMuxer;
/**
* WebRTC VideoTrack
*/
private VideoTrack videoTrack; private VideoTrack videoTrack;
/**
* WebRTC混音处理器
*/
private MixerProcesser mixerProcesser; private MixerProcesser mixerProcesser;
/**
* WebRTC音频设备模块
*/
private JavaAudioDeviceModule javaAudioDeviceModule; private JavaAudioDeviceModule javaAudioDeviceModule;
/**
* @param audioBitRate 音频比特率
* @param sampleRate 采样率
* @param channelCount 通道数量
* @param videoBitRate 视频比特率
* @param frameRate 帧率
* @param iFrameInterval 关键帧频率
* @param width 视频宽度
* @param height 视频高度
* @param path 文件目录
* @param taoyao 信令
* @param mainHandler MainHandler
*/
public RecordClient( public RecordClient(
int audioBitRate, int sampleRate, int channelCount, int audioBitRate, int sampleRate, int channelCount,
int videoBitRate, int frameRate, int iFrameInterval, int videoBitRate, int frameRate, int iFrameInterval,
@@ -158,22 +179,40 @@ public class RecordClient extends Client implements VideoSink {
this.videoActive = false; this.videoActive = false;
} }
public void start() { /**
* 开始录制
*
* @return 录制文件路径
*/
public String start() {
synchronized (this) { synchronized (this) {
if(this.init) { if(this.init) {
return; return this.filepath;
} }
Log.i(RecordClient.class.getSimpleName(), "录制视频文件：" + this.filepath); Log.i(RecordClient.class.getSimpleName(), "录制视频文件：" + this.filepath);
super.init(); super.init();
this.mediaManager.newClient(); this.mediaManager.newClient();
if ( this.initMediaMuxer();
this.audioThread == null || !this.audioThread.isAlive() || this.initAudioThread(MediaFormat.MIMETYPE_AUDIO_AAC);
this.videoThread == null || !this.videoThread.isAlive() this.initVideoThread(MediaFormat.MIMETYPE_VIDEO_AVC);
) { }
this.initMediaMuxer(); return this.filepath;
this.initAudioThread(MediaFormat.MIMETYPE_AUDIO_AAC); }
this.initVideoThread(MediaFormat.MIMETYPE_VIDEO_AVC);
} /**
* 加载MediaMuxer
*/
private void initMediaMuxer() {
try {
this.mediaMuxer = new MediaMuxer(
this.filepath,
MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4
);
// 设置方向
// this.mediaMuxer.setLocation();
// this.mediaMuxer.setOrientationHint();
} catch (IOException e) {
Log.e(MediaManager.class.getSimpleName(), "加载媒体合成器异常", e);
} }
} }
@@ -198,12 +237,15 @@ public class RecordClient extends Client implements VideoSink {
} catch (Exception e) { } catch (Exception e) {
Log.e(RecordClient.class.getSimpleName(), "加载音频录制线程异常", e); Log.e(RecordClient.class.getSimpleName(), "加载音频录制线程异常", e);
} }
this.audioThread = new HandlerThread("AudioRecoderThread"); this.audioThread = new HandlerThread("AudioRecordThread");
this.audioThread.start(); this.audioThread.start();
this.audioHandler = new Handler(this.audioThread.getLooper()); this.audioHandler = new Handler(this.audioThread.getLooper());
this.audioHandler.post(this::audioCodec); this.audioHandler.post(this::audioCodec);
} }
/**
* 音频编码
*/
private void audioCodec() { private void audioCodec() {
long pts = 0L; long pts = 0L;
int trackIndex = -1; int trackIndex = -1;
@@ -222,12 +264,12 @@ public class RecordClient extends Client implements VideoSink {
if (!this.close && this.videoActive) { if (!this.close && this.videoActive) {
Log.i(RecordClient.class.getSimpleName(), "开始录制文件：" + this.filename); Log.i(RecordClient.class.getSimpleName(), "开始录制文件：" + this.filename);
this.mediaMuxer.start(); this.mediaMuxer.start();
this.start = true; this.muxerActive = true;
} else { } else {
} }
} }
} else if (outputIndex >= 0) { } else if (outputIndex >= 0) {
if(!this.start) { if(!this.muxerActive) {
// 还没开始直接丢弃数据防止通道阻塞 // 还没开始直接丢弃数据防止通道阻塞
this.audioCodec.releaseOutputBuffer(outputIndex, false); this.audioCodec.releaseOutputBuffer(outputIndex, false);
continue; continue;
@@ -240,7 +282,6 @@ public class RecordClient extends Client implements VideoSink {
outputBuffer.limit(bufferInfo.offset + bufferInfo.size); outputBuffer.limit(bufferInfo.offset + bufferInfo.size);
bufferInfo.presentationTimeUs -= pts; bufferInfo.presentationTimeUs -= pts;
this.mediaMuxer.writeSampleData(trackIndex, outputBuffer, bufferInfo); this.mediaMuxer.writeSampleData(trackIndex, outputBuffer, bufferInfo);
// TODO：验证第二个参数作用是否复用
this.audioCodec.releaseOutputBuffer(outputIndex, false); this.audioCodec.releaseOutputBuffer(outputIndex, false);
// Log.d(RecordClient.class.getSimpleName(), "录制音频帧（时间戳）：" + (bufferInfo.presentationTimeUs / 1_000_000F)); // Log.d(RecordClient.class.getSimpleName(), "录制音频帧（时间戳）：" + (bufferInfo.presentationTimeUs / 1_000_000F));
// if (bufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME == MediaCodec.BUFFER_FLAG_KEY_FRAME) { // if (bufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME == MediaCodec.BUFFER_FLAG_KEY_FRAME) {
@@ -253,12 +294,11 @@ public class RecordClient extends Client implements VideoSink {
break; break;
} }
} else { } else {
// WARN
} }
} }
synchronized (this) { synchronized (this) {
if (this.audioCodec != null && this.audioActive) { if (this.audioCodec != null && this.audioActive) {
Log.i(RecordClient.class.getSimpleName(), "结束录制音频"); Log.i(RecordClient.class.getSimpleName(), "结束录制音频：" + this.filename);
this.audioCodec.stop(); this.audioCodec.stop();
this.audioCodec.release(); this.audioCodec.release();
this.audioCodec = null; this.audioCodec = null;
@@ -266,6 +306,7 @@ public class RecordClient extends Client implements VideoSink {
this.audioActive = false; this.audioActive = false;
if (this.mediaMuxer != null && !this.videoActive) { if (this.mediaMuxer != null && !this.videoActive) {
Log.i(RecordClient.class.getSimpleName(), "结束录制文件：" + this.filename); Log.i(RecordClient.class.getSimpleName(), "结束录制文件：" + this.filename);
this.muxerActive = false;
// this.mediaMuxer.stop(); // this.mediaMuxer.stop();
this.mediaMuxer.release(); this.mediaMuxer.release();
this.mediaMuxer = null; this.mediaMuxer = null;
@@ -291,12 +332,15 @@ public class RecordClient extends Client implements VideoSink {
} catch (Exception e) { } catch (Exception e) {
Log.e(RecordClient.class.getSimpleName(), "加载视频录制线程异常", e); Log.e(RecordClient.class.getSimpleName(), "加载视频录制线程异常", e);
} }
this.videoThread = new HandlerThread("VideoRecoderThread"); this.videoThread = new HandlerThread("VideoRecordThread");
this.videoThread.start(); this.videoThread.start();
this.videoHandler = new Handler(this.videoThread.getLooper()); this.videoHandler = new Handler(this.videoThread.getLooper());
this.videoHandler.post(this::videoCodec); this.videoHandler.post(this::videoCodec);
} }
/**
* 视频编码
*/
private void videoCodec() { private void videoCodec() {
long pts = 0L; long pts = 0L;
int trackIndex = -1; int trackIndex = -1;
@@ -305,7 +349,6 @@ public class RecordClient extends Client implements VideoSink {
this.videoActive = true; this.videoActive = true;
final MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo(); final MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
while (!this.close) { while (!this.close) {
// TODO：验证是否其他类型也要releaseOutputBuffer
outputIndex = this.videoCodec.dequeueOutputBuffer(bufferInfo, WAIT_TIME_US); outputIndex = this.videoCodec.dequeueOutputBuffer(bufferInfo, WAIT_TIME_US);
if (outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { if (outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
// } else if (outputIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { // } else if (outputIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
@@ -316,12 +359,12 @@ public class RecordClient extends Client implements VideoSink {
if (!this.close && this.audioActive) { if (!this.close && this.audioActive) {
Log.i(RecordClient.class.getSimpleName(), "开始录制文件：" + this.filename); Log.i(RecordClient.class.getSimpleName(), "开始录制文件：" + this.filename);
this.mediaMuxer.start(); this.mediaMuxer.start();
this.start = true; this.muxerActive = true;
} else { } else {
} }
} }
} else if (outputIndex >= 0) { } else if (outputIndex >= 0) {
if(!this.start) { if(!this.muxerActive) {
// 还没开始直接丢弃数据防止通道阻塞 // 还没开始直接丢弃数据防止通道阻塞
this.videoCodec.releaseOutputBuffer(outputIndex, false); this.videoCodec.releaseOutputBuffer(outputIndex, false);
continue; continue;
@@ -347,12 +390,11 @@ public class RecordClient extends Client implements VideoSink {
break; break;
} }
} else { } else {
// WARN
} }
} }
synchronized (this) { synchronized (this) {
if (this.videoCodec != null && this.videoActive) { if (this.videoCodec != null && this.videoActive) {
Log.i(RecordClient.class.getSimpleName(), "结束录制视频"); Log.i(RecordClient.class.getSimpleName(), "结束录制视频：" + this.filename);
this.videoCodec.stop(); this.videoCodec.stop();
this.videoCodec.release(); this.videoCodec.release();
this.videoCodec = null; this.videoCodec = null;
@@ -360,6 +402,7 @@ public class RecordClient extends Client implements VideoSink {
this.videoActive = false; this.videoActive = false;
if (this.mediaMuxer != null && !this.audioActive) { if (this.mediaMuxer != null && !this.audioActive) {
Log.i(RecordClient.class.getSimpleName(), "结束录制文件：" + this.filename); Log.i(RecordClient.class.getSimpleName(), "结束录制文件：" + this.filename);
this.muxerActive = false;
// this.mediaMuxer.stop(); // this.mediaMuxer.stop();
this.mediaMuxer.release(); this.mediaMuxer.release();
this.mediaMuxer = null; this.mediaMuxer = null;
@@ -367,19 +410,13 @@ public class RecordClient extends Client implements VideoSink {
} }
} }
private void initMediaMuxer() { /**
try { * 加载录制来源
this.mediaMuxer = new MediaMuxer( *
this.filepath, * @param videoSource 视频来源
MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4 * @param javaAudioDeviceModule 音频设备模块
); * @param peerConnectionFactory PeerConnectionFactory
// 设置方向 */
// this.mediaMuxer.setOrientationHint();
} catch (IOException e) {
Log.e(MediaManager.class.getSimpleName(), "加载媒体合成器异常", e);
}
}
public void record(VideoSource videoSource, JavaAudioDeviceModule javaAudioDeviceModule, PeerConnectionFactory peerConnectionFactory) { public void record(VideoSource videoSource, JavaAudioDeviceModule javaAudioDeviceModule, PeerConnectionFactory peerConnectionFactory) {
// 音频 // 音频
if(javaAudioDeviceModule != null) { if(javaAudioDeviceModule != null) {
@@ -403,10 +440,10 @@ public class RecordClient extends Client implements VideoSink {
return; return;
} }
super.close(); super.close();
this.start = false;
Log.i(RecordClient.class.getSimpleName(), "结束录制：" + this.filepath); Log.i(RecordClient.class.getSimpleName(), "结束录制：" + this.filepath);
if(this.javaAudioDeviceModule != null) { if(this.javaAudioDeviceModule != null) {
this.javaAudioDeviceModule.removeMixerProcesser(); this.javaAudioDeviceModule.removeMixerProcesser();
this.javaAudioDeviceModule = null;
} }
if(this.mixerProcesser != null) { if(this.mixerProcesser != null) {
this.mixerProcesser.close(); this.mixerProcesser.close();
@@ -434,10 +471,16 @@ public class RecordClient extends Client implements VideoSink {
} }
} }
/**
* @return 文件名称
*/
public String getFilename() { public String getFilename() {
return this.filename; return this.filename;
} }
/**
* @return 文件路径
*/
public String getFilepath() { public String getFilepath() {
return this.filepath; return this.filepath;
} }
@@ -450,6 +493,7 @@ public class RecordClient extends Client implements VideoSink {
if(this.close || !this.audioActive || !this.videoActive) { if(this.close || !this.audioActive || !this.videoActive) {
return; return;
} }
// Log.d(RecordClient.class.getSimpleName(), "音频信息：" + pts);
final int index = this.audioCodec.dequeueInputBuffer(WAIT_TIME_US); final int index = this.audioCodec.dequeueInputBuffer(WAIT_TIME_US);
if (index < 0) { if (index < 0) {
return; return;
@@ -464,19 +508,21 @@ public class RecordClient extends Client implements VideoSink {
if (this.close || !this.audioActive || !this.videoActive) { if (this.close || !this.audioActive || !this.videoActive) {
return; return;
} }
// Log.d(RecordClient.class.getSimpleName(), "视频信息：" + videoFrame.getRotatedWidth() + " - " + videoFrame.getRotatedHeight()); // Log.d(RecordClient.class.getSimpleName(), "视频信息：" + videoFrame.getTimestampNs() + " - " + videoFrame.getRotatedWidth() + " - " + videoFrame.getRotatedHeight());
final int index = this.videoCodec.dequeueInputBuffer(WAIT_TIME_US); final int index = this.videoCodec.dequeueInputBuffer(WAIT_TIME_US);
if(index < 0) { if(index < 0) {
return; return;
} }
// 内存抖动未提
final ByteBuffer inputBuffer = this.videoCodec.getInputBuffer(index); final ByteBuffer inputBuffer = this.videoCodec.getInputBuffer(index);
final VideoFrame.I420Buffer i420 = videoFrame.getBuffer().toI420(); final VideoFrame.I420Buffer i420 = videoFrame.getBuffer().toI420();
YuvHelper.I420ToNV12(i420.getDataY(), i420.getStrideY(), i420.getDataU(), i420.getStrideU(), i420.getDataV(), i420.getStrideV(), inputBuffer, i420.getWidth(), i420.getHeight()); YuvHelper.I420ToNV12(
i420.getDataY(), i420.getStrideY(),
i420.getDataU(), i420.getStrideU(),
i420.getDataV(), i420.getStrideV(),
inputBuffer, i420.getWidth(), i420.getHeight()
);
i420.release(); i420.release();
this.videoCodec.queueInputBuffer(index, 0, this.yuvSize, videoFrame.getTimestampNs(), 0); this.videoCodec.queueInputBuffer(index, 0, this.yuvSize, videoFrame.getTimestampNs(), 0);
// final int videoFrameSize = videoFrame.getRotatedWidth() * videoFrame.getRotatedHeight() * 3 / 2;
// this.videoCodec.queueInputBuffer(index, 0, this.videoFrameSize, videoFrame.getTimestampNs(), 0);
} }
} }