From 60ba25bdb4ee615ccbe82c9ae573f0d4f5ce5202 Mon Sep 17 00:00:00 2001 From: acgist <289547414@qq.com> Date: Wed, 3 May 2023 16:24:38 +0800 Subject: [PATCH] =?UTF-8?q?[+]=20=E6=B0=B4=E5=8D=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 6 +- taoyao-client-android/README.md | 5 +- .../acgist/taoyao/client/MainActivity.java | 1 + .../client/src/main/res/values/settings.xml | 2 + .../com/acgist/taoyao/media/MediaManager.java | 57 ++++++-- .../media/audio/AudioChangerProcesser.java | 10 -- .../{AudioMixer.java => MixerProcesser.java} | 4 +- .../taoyao/media/client/PhotographClient.java | 1 + .../taoyao/media/client/RecordClient.java | 6 + .../taoyao/media/client/SessionClient.java | 2 + .../taoyao/media/video/AiProcesser.java | 22 +-- .../taoyao/media/video/BeautyProcesser.java | 29 ---- .../taoyao/media/video/VideoProcesser.java | 33 +++++ .../media/video/WatermarkProcesser.java | 125 ++++++++++++++++-- .../java/com/acgist/taoyao/rtp/RtpTest.java | 2 +- 15 files changed, 219 insertions(+), 86 deletions(-) delete mode 100644 taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/AudioChangerProcesser.java rename taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/{AudioMixer.java => MixerProcesser.java} (78%) delete mode 100644 taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/BeautyProcesser.java create mode 100644 taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/VideoProcesser.java diff --git a/README.md b/README.md index 897ff1d..5ab8611 100644 --- a/README.md +++ b/README.md @@ -45,10 +45,8 @@ |控制|支持|完成|部分控制信令| |拍照|支持|完成|拍照| |录像|支持|完成|录制| -|变声|支持|暂未实现|变声器| -|水印|支持|暂未实现|视频水印| -|美颜|支持|暂未实现|视频美颜| -|AI识别|支持|暂未实现|视频AI识别| +|混音|支持|暂未实现|多路混音| +|水印|支持|完成|视频水印| > 注意：Web终端不支持同时进入多个视频房间，安卓终端支持同时进入多个视频房间。 diff --git a/taoyao-client-android/README.md b/taoyao-client-android/README.md index 6c5d658..559efd8 100644 --- a/taoyao-client-android/README.md +++ b/taoyao-client-android/README.md @@ -18,9 +18,8 @@ ## 视频旋转 -1. 应用旋转（横屏|竖屏） -2. 代码旋转 -3. 镜头物理旋转 +1. 应用旋转：横屏竖屏 +2. 物理旋转：旋转镜头 ## 学习资料 diff --git a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java index c3b434e..14b55e0 100644 --- a/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java +++ b/taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java @@ -130,6 +130,7 @@ public class MainActivity extends AppCompatActivity implements Serializable { resources.getInteger(R.integer.iFrameInterval), resources.getString(R.string.storagePathImage), resources.getString(R.string.storagePathVideo), + resources.getString(R.string.watermark), VideoSourceType.valueOf(resources.getString(R.string.videoSourceType)) ); final Display display = this.getWindow().getContext().getDisplay(); diff --git a/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml b/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml index ed60c05..45473aa 100644 --- a/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml +++ b/taoyao-client-android/taoyao/client/src/main/res/values/settings.xml @@ -44,4 +44,6 @@ 1 1 + + "'TAOYAO' yyyy-MM-dd HH:mm:ss" \ No newline at end of file diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java index 8ed0ab1..c4c1da4 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java @@ -14,7 +14,10 @@ import com.acgist.taoyao.media.config.MediaProperties; import com.acgist.taoyao.media.config.MediaVideoProperties; import com.acgist.taoyao.media.config.WebrtcProperties; import com.acgist.taoyao.media.signal.ITaoyao; +import com.acgist.taoyao.media.video.VideoProcesser; +import com.acgist.taoyao.media.video.WatermarkProcesser; +import org.apache.commons.lang3.StringUtils; import org.webrtc.AudioSource; import org.webrtc.AudioTrack; import org.webrtc.Camera2Enumerator; @@ -27,7 +30,6 @@ import org.webrtc.EglBase; import org.webrtc.MediaConstraints; import org.webrtc.MediaStream; import org.webrtc.PeerConnectionFactory; -import org.webrtc.RendererCommon; import org.webrtc.ScreenCapturerAndroid; import org.webrtc.SurfaceTextureHelper; import org.webrtc.SurfaceViewRenderer; @@ -88,6 +90,10 @@ public final class MediaManager { * 关键帧频率 */ private int iFrameInterval; + /** + * 水印 + */ + private String watermark; /** * 视频来源类型 */ @@ -148,6 +154,10 @@ public final class MediaManager { * PeerConnectionFactory */ private PeerConnectionFactory peerConnectionFactory; + /** + * 视频处理 + */ + private VideoProcesser videoProcesser; static { // // 设置采样 @@ -190,14 +200,14 @@ public final class MediaManager { /** * @param mainHandler Handler - * @param context 上下文 + * @param context 上下文 */ public void initContext( Handler mainHandler, Context context, int imageQuantity, String audioQuantity, String videoQuantity, int channelCount, int iFrameInterval, String imagePath, String videoPath, - VideoSourceType videoSourceType + String watermark, VideoSourceType videoSourceType ) { this.mainHandler = mainHandler; this.context = context; @@ -208,6 +218,7 @@ public final class MediaManager { this.iFrameInterval = iFrameInterval; this.imagePath = imagePath; this.videoPath = videoPath; + this.watermark = watermark; this.videoSourceType = videoSourceType; } @@ -320,6 +331,7 @@ public final class MediaManager { }); this.initAudio(); this.initVideo(); + this.initWatermark(); } private JavaAudioDeviceModule javaAudioDeviceModule() { @@ -419,7 +431,7 @@ public final class MediaManager { // 忽略其他摄像头 } } - this.initVideoTrack(); + this.initVideoSource(); } private void initSharePromise() { @@ -433,13 +445,13 @@ public final class MediaManager { */ public void initScreen(Intent intent) { this.videoCapturer = new ScreenCapturerAndroid(intent, new ScreenCallback()); - this.initVideoTrack(); + this.initVideoSource(); } /** * 加载视频 */ - private void initVideoTrack() { + private void initVideoSource() { // 加载视频 this.surfaceTextureHelper = SurfaceTextureHelper.create("MediaVideoThread", this.eglContext); // this.surfaceTextureHelper.setTextureSize(); @@ -456,12 +468,21 @@ public final class MediaManager { // this.shareVideoSource.setVideoProcessor(); } + private void initWatermark() { + if(StringUtils.isNotEmpty(this.watermark)) { + final MediaVideoProperties mediaVideoProperties = this.mediaProperties.getVideos().get(this.videoQuantity); + if(this.videoProcesser == null) { + this.videoProcesser = new WatermarkProcesser(this.watermark, mediaVideoProperties.getWidth(), mediaVideoProperties.getHeight()); + } else { + this.videoProcesser = new WatermarkProcesser(this.watermark, mediaVideoProperties.getWidth(), mediaVideoProperties.getHeight(), this.videoProcesser); + } + } + } + /** * 更新配置 * - * @param mediaProperties 媒体配置 - * @param mediaAudioProperties 音频配置 - * @param mediaVideoProperties 视频配置 + * @param mediaProperties 媒体配置 */ public void updateMediaConfig(MediaProperties mediaProperties) { this.mediaProperties = mediaProperties; @@ -707,6 +728,10 @@ public final class MediaManager { this.peerConnectionFactory.dispose(); this.peerConnectionFactory = null; } + if(this.videoProcesser != null) { + this.videoProcesser.close(); + this.videoProcesser = null; + } } /** @@ -741,8 +766,18 @@ public final class MediaManager { @Override public void onFrameCaptured(VideoFrame videoFrame) { // 注意：VideoFrame必须释放，多线程环境需要调用retain和release方法。 - this.mainObserver.onFrameCaptured(videoFrame); - this.shareObserver.onFrameCaptured(videoFrame); + if(MediaManager.this.videoProcesser == null) { + this.mainObserver.onFrameCaptured(videoFrame); + this.shareObserver.onFrameCaptured(videoFrame); + } else { + final VideoFrame.I420Buffer i420Buffer = videoFrame.getBuffer().toI420(); + MediaManager.this.videoProcesser.process(i420Buffer); + final VideoFrame processVideoFrame = new VideoFrame(i420Buffer.cropAndScale(0, 0, i420Buffer.getWidth(), i420Buffer.getHeight(), i420Buffer.getWidth(), i420Buffer.getHeight()), videoFrame.getRotation(), videoFrame.getTimestampNs()); + i420Buffer.release(); + this.mainObserver.onFrameCaptured(processVideoFrame); + this.shareObserver.onFrameCaptured(processVideoFrame); + processVideoFrame.release(); + } } } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/AudioChangerProcesser.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/AudioChangerProcesser.java deleted file mode 100644 index d2e765f..0000000 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/AudioChangerProcesser.java +++ /dev/null @@ -1,10 +0,0 @@ -package com.acgist.taoyao.media.audio; - -/** - * 变声处理器 - * - * @author acgist - */ -public class AudioChangerProcesser { - -} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/AudioMixer.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java similarity index 78% rename from taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/AudioMixer.java rename to taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java index 2b6af8f..b8f73c8 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/AudioMixer.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java @@ -1,13 +1,13 @@ package com.acgist.taoyao.media.audio; /** - * 混音 + * 混音处理器 * * WebRtcAudioTrack#AudioTrackThread ：远程音频 * WebRtcAudioRecord#AudioRecordThread：本地音频 * * @author acgist */ -public class AudioMixer { +public class MixerProcesser { } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java index d0ad0d7..adc85f1 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java @@ -135,6 +135,7 @@ public class PhotographClient implements VideoSink { // matrix.setRotate(90); // final Bitmap matrixBitmap = Bitmap.createBitmap(bitmap, 0, 0, width, height, matrix, false); bitmap.compress(Bitmap.CompressFormat.JPEG, this.quantity, output); + bitmap.recycle(); } catch (Exception e) { Log.e(PhotographClient.class.getSimpleName(), "拍照异常", e); } finally { diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java index 85e33b3..28d0367 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java @@ -32,7 +32,13 @@ import java.time.LocalDateTime; */ public class RecordClient extends Client implements VideoSink, JavaAudioDeviceModule.SamplesReadyCallback { + /** + * 等待时间（毫秒） + */ private static final long WAIT_TIME_MS = 50; + /** + * 等待时间（纳秒） + */ private static final long WAIT_TIME_US = WAIT_TIME_MS * 1000; /** diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java index d515390..4878590 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java @@ -377,6 +377,7 @@ public class SessionClient extends Client { public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) { Log.d(SessionClient.class.getSimpleName(), "PCIce连接状态改变：" + iceConnectionState); SessionClient.this.logState(); + // disconnected：暂时连接不上可能自我恢复 } @Override @@ -432,6 +433,7 @@ public class SessionClient extends Client { public void onRenegotiationNeeded() { Log.d(SessionClient.class.getSimpleName(), "重新协商媒体：" + SessionClient.this.sessionId); if(SessionClient.this.peerConnection.connectionState() == PeerConnection.PeerConnectionState.CONNECTED) { +// SessionClient.this.peerConnection.restartIce(); // TODO：重新协商 // SessionClient.this.offer(); } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/AiProcesser.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/AiProcesser.java index 1dc85e5..9bc4ba1 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/AiProcesser.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/AiProcesser.java @@ -1,30 +1,18 @@ package com.acgist.taoyao.media.video; import org.webrtc.VideoFrame; -import org.webrtc.VideoProcessor; -import org.webrtc.VideoSink; /** - * AI处理器 + * AI识别处理器 + * + * 建议不要每帧识别，如果没有识别出来结果可以复用识别结果。 * * @author acgist */ -public class AiProcesser implements VideoProcessor { +public class AiProcesser extends VideoProcesser { @Override - public void setSink(VideoSink videoSink) { - } - - @Override - public void onCapturerStarted(boolean status) { - } - - @Override - public void onCapturerStopped() { - } - - @Override - public void onFrameCaptured(VideoFrame videoFrame) { + protected void doProcess(VideoFrame.I420Buffer i420Buffer) { } } diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/BeautyProcesser.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/BeautyProcesser.java deleted file mode 100644 index 61887e3..0000000 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/BeautyProcesser.java +++ /dev/null @@ -1,29 +0,0 @@ -package com.acgist.taoyao.media.video; - -import org.webrtc.VideoFrame; -import org.webrtc.VideoProcessor; -import org.webrtc.VideoSink; - -/** - * 美颜处理器 - * - * @author acgist - */ -public class BeautyProcesser implements VideoProcessor { - - @Override - public void setSink(VideoSink videoSink) { - } - - @Override - public void onCapturerStarted(boolean status) { - } - - @Override - public void onCapturerStopped() { - } - - @Override - public void onFrameCaptured(VideoFrame videoFrame) { - } -} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/VideoProcesser.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/VideoProcesser.java new file mode 100644 index 0000000..d25dfa5 --- /dev/null +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/VideoProcesser.java @@ -0,0 +1,33 @@ +package com.acgist.taoyao.media.video; + +import org.webrtc.VideoFrame; + +import java.io.Closeable; + +/** + * 视频处理器 + */ +public abstract class VideoProcesser implements Closeable { + + protected VideoProcesser next; + + public void process(VideoFrame.I420Buffer i420Buffer) { + this.doProcess(i420Buffer); + if(this.next == null) { + // 忽略 + } else { + this.next.process(i420Buffer); + } + } + + protected abstract void doProcess(VideoFrame.I420Buffer i420Buffer); + + public void close() { + if(this.next == null) { + // 忽略 + } else { + this.next.close(); + } + } + +} diff --git a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/WatermarkProcesser.java b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/WatermarkProcesser.java index 25fd9c6..2ffd9ba 100644 --- a/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/WatermarkProcesser.java +++ b/taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/WatermarkProcesser.java @@ -1,30 +1,137 @@ package com.acgist.taoyao.media.video; +import android.graphics.Bitmap; +import android.graphics.Canvas; +import android.graphics.Color; +import android.graphics.Paint; + import org.webrtc.VideoFrame; -import org.webrtc.VideoProcessor; -import org.webrtc.VideoSink; + +import java.nio.ByteBuffer; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.Timer; +import java.util.TimerTask; /** * 水印处理器 * + * 性能优化： + * 没有水印：20~25波动 + * 没有定时：26~32波动 + * 定时水印：28~32 + * * @author acgist */ -public class WatermarkProcesser implements VideoProcessor { +public class WatermarkProcesser extends VideoProcesser { - @Override - public void setSink(VideoSink videoSink) { + private static final WatermarkMatrix[] MATRICES = new WatermarkMatrix[256]; + + private final String format; + private final int width; + private final int height; + private final Timer timer; + private final WatermarkMatrix[] watermark; + + public WatermarkProcesser(String format, int width, int height) { + this.format = format; + this.width = width; + this.height = height; + final String date = LocalDateTime.now().format(DateTimeFormatter.ofPattern(format)); + this.watermark = new WatermarkMatrix[date.length()]; + this.timer = new Timer("Watermark-Timer", true); + this.init(); + } + + public WatermarkProcesser(String format, int width, int height, VideoProcesser videoProcesser) { + this(format, width, height); + this.next = videoProcesser; + } + + private void init() { + final String source = "-: 0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"; + final char[] chars = source.toCharArray(); + for (char value : chars) { + this.build(value); + } + this.timer.schedule(new TimerTask() { + @Override + public void run() { + int index = 0; + final char[] chars = LocalDateTime.now().format(DateTimeFormatter.ofPattern(WatermarkProcesser.this.format)).toCharArray(); + for (char value : chars) { + WatermarkProcesser.this.watermark[index] = MATRICES[value]; + index++; + } + } + }, 1000, 1000); + } + + private void build(char source) { + // TODO：优化复用bitmap + final String target = Character.toString(source); + final Paint paint = new Paint(); + paint.setColor(Color.WHITE); + paint.setDither(true); + paint.setTextSize(40.0F); + paint.setTextAlign(Paint.Align.LEFT); + paint.setFilterBitmap(true); + final Paint.FontMetricsInt box = paint.getFontMetricsInt(); + final int width = (int) paint.measureText(target); + final int height = box.descent - box.ascent; + final Bitmap bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888); + final Canvas canvas = new Canvas(bitmap); + canvas.drawText(target, 0, box.leading - box.ascent, paint); + canvas.save(); + final boolean[][] matrix = new boolean[width][height]; + for (int j = 0; j < height; j++) { + for (int i = 0; i < width; i++) { + matrix[i][j] = bitmap.getColor(i, j).toArgb() != 0; + } + } + MATRICES[source] = new WatermarkMatrix(width, height, matrix); + bitmap.recycle(); } @Override - public void onCapturerStarted(boolean status) { + protected void doProcess(VideoFrame.I420Buffer i420Buffer) { + int widthPos = 0; + int heightPos = 0; + final ByteBuffer buffer = i420Buffer.getDataY(); + for (WatermarkMatrix matrix : watermark) { + if(matrix == null) { + continue; + } + for (int height = 0; height < matrix.height; height++) { + for (int width = 0; width < matrix.width; width++) { + if(matrix.matrix[width][height]) { + buffer.put(this.width * height + width + widthPos, (byte) 0); + } + } + } + widthPos += matrix.width; + heightPos += matrix.height; + } } @Override - public void onCapturerStopped() { + public void close() { + super.close(); + this.timer.cancel(); } - @Override - public void onFrameCaptured(VideoFrame videoFrame) { + static class WatermarkMatrix { + + int width; + int height; + boolean[][] matrix; + + public WatermarkMatrix(int width, int height, boolean[][] matrix) { + this.width = width; + this.height = height; + this.matrix = matrix; + } + } } diff --git a/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/rtp/RtpTest.java b/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/rtp/RtpTest.java index 87edd73..f5c3e2e 100644 --- a/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/rtp/RtpTest.java +++ b/taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/rtp/RtpTest.java @@ -20,7 +20,7 @@ import lombok.extern.slf4j.Slf4j; @Slf4j public class RtpTest { - + @Test void testSocket() throws Exception { final Socket socket = new Socket();