From 052bb99b2051480ca349723f2a660adf09d01916 Mon Sep 17 00:00:00 2001
From: acgist <289547414@qq.com>
Date: Thu, 18 May 2023 08:19:29 +0800
Subject: [PATCH] =?UTF-8?q?[*]=20=E5=85=BC=E5=AE=B9=E5=AE=89=E5=8D=939?=
=?UTF-8?q?=E3=80=81=E8=A7=A3=E5=86=B3=E5=BD=95=E5=83=8F=E6=B5=8F=E8=A7=88?=
=?UTF-8?q?=E5=99=A8=E6=97=A0=E6=B3=95=E6=92=AD=E6=94=BE=E9=97=AE=E9=A2=98?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
docs/TODO.md | 6 ++-
taoyao-client-android/taoyao/README.md | 5 +++
.../taoyao/boot/build.gradle | 2 +-
.../taoyao/client/build.gradle | 2 +-
.../acgist/taoyao/client/signal/Taoyao.java | 11 ++++-
.../taoyao/media/build.gradle | 2 +-
.../taoyao/media/client/RecordClient.java | 41 +++++++++++--------
.../media/video/WatermarkProcesser.java | 7 +++-
taoyao-client-media/src/Config.js | 8 ++++
taoyao-client-web/src/App.vue | 2 +-
.../src/components/LocalClient.vue | 2 +-
.../src/components/RemoteClient.vue | 2 +-
.../src/components/SessionClient.vue | 2 +-
taoyao-client-web/src/components/Taoyao.js | 4 +-
.../boot/config/MediaVideoProperties.java | 2 +-
.../java/com/acgist/taoyao/rtp/RtpTest.java | 6 ++-
.../media/MediaTransportPlainProtocol.java | 4 +-
17 files changed, 74 insertions(+), 34 deletions(-)
diff --git a/docs/TODO.md b/docs/TODO.md
index e250f80..255420b 100644
--- a/docs/TODO.md
+++ b/docs/TODO.md
@@ -20,7 +20,7 @@
* 过时方法
* 缩进调整：`tab->space`
-* 方法调整：`getter/setter`
+* 方法改造：`getter/setter`
* 注释优化：详细描述、单位描述
* 日志优化：
* 添加测试：
@@ -30,13 +30,17 @@
## 计划任务
+* 发布版本：1.0.0
* 开机自启
* 录制底噪
+* 性能测试
+* 稳定性测试
* 分辨率调整
* 服务端录制
* 安卓预览按钮
* 降低视频录制大小
* 防止重复邀请拉取
+* 码率等等参数配置验证
* 查询消费者生产者信息
* 浏览器WebRTC监控页面关闭：`chrome://webrtc-internals/`
diff --git a/taoyao-client-android/taoyao/README.md b/taoyao-client-android/taoyao/README.md
index fe21cec..9acda3e 100644
--- a/taoyao-client-android/taoyao/README.md
+++ b/taoyao-client-android/taoyao/README.md
@@ -1,5 +1,10 @@
# 桃夭安卓
+## 版本
+
+* SDK：28~32
+* Andoird：9~12
+
## WebRTC源码修改
### JavaAudioDeviceModule
diff --git a/taoyao-client-android/taoyao/boot/build.gradle b/taoyao-client-android/taoyao/boot/build.gradle
index a04ef28..79a0ce0 100644
--- a/taoyao-client-android/taoyao/boot/build.gradle
+++ b/taoyao-client-android/taoyao/boot/build.gradle
@@ -6,7 +6,7 @@ android {
namespace 'com.acgist.client.boot'
compileSdk 32
defaultConfig {
- minSdk 30
+ minSdk 28
targetSdk 32
versionCode 100
versionName "1.0.0"
diff --git a/taoyao-client-android/taoyao/client/build.gradle b/taoyao-client-android/taoyao/client/build.gradle
index 7583a69..0fda6f5 100644
--- a/taoyao-client-android/taoyao/client/build.gradle
+++ b/taoyao-client-android/taoyao/client/build.gradle
@@ -6,7 +6,7 @@ android {
namespace 'com.acgist.taoyao.client'
compileSdk 32
defaultConfig {
- minSdk 30
+ minSdk 28
targetSdk 32
versionCode 100
versionName "1.0.0"
diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java
index ada7a37..e2a87b0 100644
--- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java
+++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/signal/Taoyao.java
@@ -9,6 +9,7 @@ import android.location.LocationManager;
import android.net.wifi.WifiInfo;
import android.net.wifi.WifiManager;
import android.os.BatteryManager;
+import android.os.Build;
import android.os.Handler;
import android.os.HandlerThread;
import android.os.PowerManager;
@@ -1254,7 +1255,15 @@ public final class Taoyao implements ITaoyao {
if(wifiInfo == null) {
return -1;
}
- return this.wifiManager.calculateSignalLevel(wifiInfo.getRssi()) / this.wifiManager.getMaxSignalLevel() * 100;
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
+ return this.wifiManager.calculateSignalLevel(wifiInfo.getRssi()) / this.wifiManager.getMaxSignalLevel() * 100;
+ } else {
+ // 0 ~ -50 : 优秀
+ // -50 ~ -70 : 良好
+ // -70 ~ -100: 较差
+ final int rssi = wifiInfo.getRssi();
+ return rssi <= -100 ? 0 : (100 + rssi);
+ }
}
/**
diff --git a/taoyao-client-android/taoyao/media/build.gradle b/taoyao-client-android/taoyao/media/build.gradle
index 623360f..f162ae8 100644
--- a/taoyao-client-android/taoyao/media/build.gradle
+++ b/taoyao-client-android/taoyao/media/build.gradle
@@ -10,7 +10,7 @@ android {
compileSdk 32
ndkVersion "23.1.7779620"
defaultConfig {
- minSdk 30
+ minSdk 28
targetSdk 32
versionCode 100
versionName "1.0.0"
diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java
index a66907e..50a8cec 100644
--- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java
+++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java
@@ -7,6 +7,7 @@ import android.media.MediaMuxer;
import android.os.Environment;
import android.os.Handler;
import android.os.HandlerThread;
+import android.os.SystemClock;
import android.util.Log;
import com.acgist.taoyao.boot.utils.DateUtils;
@@ -122,6 +123,11 @@ public class RecordClient extends Client implements VideoSink {
* 视频Handler
*/
private Handler videoHandler;
+ /**
+ * 是否已经开始录制
+ * 不能使用多线程wait/notify录制音频没有结束
+ */
+ private boolean start = false;
/**
* 媒体合成器
*/
@@ -216,16 +222,16 @@ public class RecordClient extends Client implements VideoSink {
if (!this.close && this.videoActive) {
Log.i(RecordClient.class.getSimpleName(), "开始录制文件：" + this.filename);
this.mediaMuxer.start();
- this.notifyAll();
- } else if (!this.close) {
- try {
- this.wait();
- } catch (InterruptedException e) {
- Log.e(RecordClient.class.getSimpleName(), "录制线程等待异常", e);
- }
+ this.start = true;
+ } else {
}
}
} else if (outputIndex >= 0) {
+ if(!this.start) {
+ // 还没开始直接丢弃数据防止通道阻塞
+ this.audioCodec.releaseOutputBuffer(outputIndex, false);
+ continue;
+ }
if(pts == 0L) {
pts = bufferInfo.presentationTimeUs;
}
@@ -234,6 +240,7 @@ public class RecordClient extends Client implements VideoSink {
outputBuffer.limit(bufferInfo.offset + bufferInfo.size);
bufferInfo.presentationTimeUs -= pts;
this.mediaMuxer.writeSampleData(trackIndex, outputBuffer, bufferInfo);
+ // TODO：验证第二个参数作用是否复用
this.audioCodec.releaseOutputBuffer(outputIndex, false);
// Log.d(RecordClient.class.getSimpleName(), "录制音频帧（时间戳）：" + (bufferInfo.presentationTimeUs / 1_000_000F));
// if (bufferInfo.flags & MediaCodec.BUFFER_FLAG_KEY_FRAME == MediaCodec.BUFFER_FLAG_KEY_FRAME) {
@@ -298,6 +305,7 @@ public class RecordClient extends Client implements VideoSink {
this.videoActive = true;
final MediaCodec.BufferInfo bufferInfo = new MediaCodec.BufferInfo();
while (!this.close) {
+ // TODO：验证是否其他类型也要releaseOutputBuffer
outputIndex = this.videoCodec.dequeueOutputBuffer(bufferInfo, WAIT_TIME_US);
if (outputIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
// } else if (outputIndex == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) {
@@ -308,17 +316,16 @@ public class RecordClient extends Client implements VideoSink {
if (!this.close && this.audioActive) {
Log.i(RecordClient.class.getSimpleName(), "开始录制文件：" + this.filename);
this.mediaMuxer.start();
- this.notifyAll();
- } else if (!this.close) {
- try {
- this.wait();
- } catch (InterruptedException e) {
- Log.e(RecordClient.class.getSimpleName(), "录制线程等待异常", e);
- }
+ this.start = true;
} else {
}
}
} else if (outputIndex >= 0) {
+ if(!this.start) {
+ // 还没开始直接丢弃数据防止通道阻塞
+ this.videoCodec.releaseOutputBuffer(outputIndex, false);
+ continue;
+ }
if(pts == 0L) {
pts = bufferInfo.presentationTimeUs / 1000;
}
@@ -396,6 +403,7 @@ public class RecordClient extends Client implements VideoSink {
return;
}
super.close();
+ this.start = false;
Log.i(RecordClient.class.getSimpleName(), "结束录制：" + this.filepath);
if(this.javaAudioDeviceModule != null) {
this.javaAudioDeviceModule.removeMixerProcesser();
@@ -422,7 +430,6 @@ public class RecordClient extends Client implements VideoSink {
Log.i(RecordClient.class.getSimpleName(), "删除没有录制数据文件：" + this.filepath);
file.delete();
}
- this.notifyAll();
this.mediaManager.closeClient();
}
}
@@ -440,7 +447,7 @@ public class RecordClient extends Client implements VideoSink {
* @param data PCM数据
*/
public void onPcm(long pts, byte[] data) {
- if(this.close || !this.audioActive) {
+ if(this.close || !this.audioActive || !this.videoActive) {
return;
}
final int index = this.audioCodec.dequeueInputBuffer(WAIT_TIME_US);
@@ -454,7 +461,7 @@ public class RecordClient extends Client implements VideoSink {
@Override
public void onFrame(VideoFrame videoFrame) {
- if (this.close || !this.videoActive) {
+ if (this.close || !this.audioActive || !this.videoActive) {
return;
}
// Log.d(RecordClient.class.getSimpleName(), "视频信息：" + videoFrame.getRotatedWidth() + " - " + videoFrame.getRotatedHeight());
diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/WatermarkProcesser.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/WatermarkProcesser.java
index a38d0a6..7492813 100644
--- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/WatermarkProcesser.java
+++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/WatermarkProcesser.java
@@ -4,6 +4,7 @@ import android.graphics.Bitmap;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.Paint;
+import android.os.Build;
import org.webrtc.VideoFrame;
@@ -88,7 +89,11 @@ public class WatermarkProcesser extends VideoProcesser {
final boolean[][] matrix = new boolean[width][height];
for (int j = 0; j < height; j++) {
for (int i = 0; i < width; i++) {
- matrix[i][j] = bitmap.getColor(i, j).toArgb() != 0;
+ if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.Q) {
+ matrix[i][j] = bitmap.getColor(i, j).toArgb() != 0;
+ } else {
+ matrix[i][j] = bitmap.getPixel(i, j) != 0;
+ }
}
}
MATRICES[source] = new WatermarkMatrix(width, height, matrix);
diff --git a/taoyao-client-media/src/Config.js b/taoyao-client-media/src/Config.js
index d7b3899..a431449 100644
--- a/taoyao-client-media/src/Config.js
+++ b/taoyao-client-media/src/Config.js
@@ -73,6 +73,8 @@ module.exports = {
clockRate: 90000,
parameters: {
"x-google-start-bitrate": 1000,
+ // "x-google-min-bitrate": 800,
+ // "x-google-max-bitrate": 1600,
},
},
{
@@ -82,6 +84,8 @@ module.exports = {
parameters: {
"profile-id": 2,
"x-google-start-bitrate": 1000,
+ // "x-google-min-bitrate": 800,
+ // "x-google-max-bitrate": 1600,
},
},
{
@@ -93,6 +97,8 @@ module.exports = {
"profile-level-id": "4d0032",
"level-asymmetry-allowed": 1,
"x-google-start-bitrate": 1000,
+ // "x-google-min-bitrate": 800,
+ // "x-google-max-bitrate": 1600,
},
},
{
@@ -104,6 +110,8 @@ module.exports = {
"profile-level-id": "42e01f",
"level-asymmetry-allowed": 1,
"x-google-start-bitrate": 1000,
+ // "x-google-min-bitrate": 800,
+ // "x-google-max-bitrate": 1600,
},
},
],
diff --git a/taoyao-client-web/src/App.vue b/taoyao-client-web/src/App.vue
index 570e7b4..dbbb2dc 100644
--- a/taoyao-client-web/src/App.vue
+++ b/taoyao-client-web/src/App.vue
@@ -220,7 +220,7 @@ export default {
.client{float:left;width:50vw;height:50vh;box-shadow:0 0 1px 0px rgba(0,0,0,0.4);}
.client audio{display:none;}
.client video{width:100%;height:100%;}
-.client .mic{background:linear-gradient(to top, var(--el-color-primary) 10%, transparent 0%);}
+.client .mic{background:linear-gradient(to top, var(--el-color-primary) 100%, transparent 0%);}
.client .title{position:absolute;top:0;left:0;text-align:center;width:100%;}
.client .buttons{width:100%;bottom:0;left:0;text-align:center;position:absolute;padding:0.8rem 0;background:rgba(0,0,0,0.4);}
.client .buttons .el-button{margin:0 6px;}
diff --git a/taoyao-client-web/src/components/LocalClient.vue b/taoyao-client-web/src/components/LocalClient.vue
index cd0f640..1e05a38 100644
--- a/taoyao-client-web/src/components/LocalClient.vue
+++ b/taoyao-client-web/src/components/LocalClient.vue
@@ -107,5 +107,5 @@ export default {
};
diff --git a/taoyao-client-web/src/components/RemoteClient.vue b/taoyao-client-web/src/components/RemoteClient.vue
index 97e8b22..4d4aeab 100644
--- a/taoyao-client-web/src/components/RemoteClient.vue
+++ b/taoyao-client-web/src/components/RemoteClient.vue
@@ -114,5 +114,5 @@ export default {
};
diff --git a/taoyao-client-web/src/components/SessionClient.vue b/taoyao-client-web/src/components/SessionClient.vue
index 48e44d8..3ce61fd 100644
--- a/taoyao-client-web/src/components/SessionClient.vue
+++ b/taoyao-client-web/src/components/SessionClient.vue
@@ -105,5 +105,5 @@ export default {
};
diff --git a/taoyao-client-web/src/components/Taoyao.js b/taoyao-client-web/src/components/Taoyao.js
index 49b0008..9deb422 100644
--- a/taoyao-client-web/src/components/Taoyao.js
+++ b/taoyao-client-web/src/components/Taoyao.js
@@ -244,7 +244,7 @@ class Session {
// 远程终端名称
name;
// 音量
- volume = 100;
+ volume = "100%";
// 是否关闭
closed;
// 远程终端ID
@@ -382,7 +382,7 @@ class RemoteClient {
// 终端标识
clientId;
// 音量
- volume = 0;
+ volume = "100%";
// 代理对象
proxy;
// 数据消费者
diff --git a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/MediaVideoProperties.java b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/MediaVideoProperties.java
index 25c22b9..b55bd22 100644
--- a/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/MediaVideoProperties.java
+++ b/taoyao-signal-server/taoyao-boot/src/main/java/com/acgist/taoyao/boot/config/MediaVideoProperties.java
@@ -10,7 +10,7 @@ import lombok.Setter;
* 视频编码 = 压缩
* 8 = 颜色位数
* 3 / 2 = YUV | RGB
- * 码率（比特率）：单位（kbps | kbit/s | kb/s）
+ * 码率（比特率）：单位（kbps | kbit/s）
* 码率 = 文件字节大小 * 8 / 秒数 / 1000
* 1Byte = 8bit
*
diff --git a/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/rtp/RtpTest.java b/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/rtp/RtpTest.java
index f5c3e2e..0feee5e 100644
--- a/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/rtp/RtpTest.java
+++ b/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/rtp/RtpTest.java
@@ -82,11 +82,13 @@ public class RtpTest {
// {"header":{"v":"1.0.0","id":1215310510002009,"signal":"room::enter"},"body":{"roomId":"8260e615-3081-4bfc-96a8-574f4dd780d9"}}
// {"header":{"v":"1.0.0","id":1215310510002010,"signal":"media::transport::plain"},"body":{"roomId":"8260e615-3081-4bfc-96a8-574f4dd780d9","rtcpMux":false,"comedia":true}}
// {"header":{"v":"1.0.0","id":1215375110006012,"signal":"media::produce"},"body":{"kind":"video","roomId":"8260e615-3081-4bfc-96a8-574f4dd780d9","transportId":"14dc9307-bf9c-4442-a9ad-ce6a97623ef4","appData":{},"rtpParameters":{"codecs":[{"mimeType":"video/vp8","clockRate":90000,"payloadType":102,"rtcpFeedback":[]}],"encodings":[{"ssrc":123123}]}}}
+ // ffmpeg直接播放
+// ffmpeg -re -i video.mp4 -c:a copy -vn -map 0:1 -f rtp rtp://localhost:6666 > taoyao.sdp
+// ffmpeg -re -i video.mp4 -c:v copy -an -map 0:0 -f rtp rtp://localhost:6666 > taoyao.sdp
// ffplay -protocol_whitelist "file,udp,rtp" taoyao.sdp
-// ffmpeg -re -i video.mp4 -vcodec copy -map 0:0 -f rtp rtp://localhost:6666 > taoyao.sdp
// ffmpeg不支持rtcpMux
// ffmpeg -re -i video.mp4 -c:v libvpx -map 0:0 -f tee "[select=v:f=rtp:ssrc=123123:payload_type=102]rtp://192.168.1.110:40793?rtcpport=47218"
-// ffmpeg -re -i video.mp4 -vcodec vp8 -map 0:0 -f tee "[select=v:f=rtp:ssrc=123123:payload_type=102]rtp://192.168.1.110:40793?rtcpport=47218"
+// ffmpeg -re -i video.mp4 -c:v vp8 -map 0:0 -f tee "[select=v:f=rtp:ssrc=123123:payload_type=102]rtp://192.168.1.110:40793?rtcpport=47218"
final Scanner scanner = new Scanner(System.in);
do {
if(StringUtils.isEmpty(line)) {
diff --git a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/media/MediaTransportPlainProtocol.java b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/media/MediaTransportPlainProtocol.java
index 865ae58..2e2ff59 100644
--- a/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/media/MediaTransportPlainProtocol.java
+++ b/taoyao-signal-server/taoyao-signal/src/main/java/com/acgist/taoyao/signal/protocol/media/MediaTransportPlainProtocol.java
@@ -23,9 +23,9 @@ import lombok.extern.slf4j.Slf4j;
/**
* 创建RTP输入通道信令
* 注意：
- * 3. ffmpeg不支持rtcpMux
+ * 1. ffmpeg不支持rtcpMux
* 2. comedia必须开启srtp功能
- * 1. 如果关闭comedia不会自动升级双向通道
+ * 3. 如果关闭comedia不会自动升级双向通道
*
* @author acgist
*/