[+] 水印

README.md

@@ -45,10 +45,8 @@
 |控制|支持|完成|部分控制信令|
 |拍照|支持|完成|拍照|
 |录像|支持|完成|录制|
-|变声|支持|暂未实现|变声器|
+|混音|支持|暂未实现|多路混音|
-|水印|支持|暂未实现|视频水印|
+|水印|支持|完成|视频水印|
-|美颜|支持|暂未实现|视频美颜|
-|AI识别|支持|暂未实现|视频AI识别|
 
 > 注意：Web终端不支持同时进入多个视频房间，安卓终端支持同时进入多个视频房间。
 

taoyao-client-android/README.md

@@ -18,9 +18,8 @@
 
 ## 视频旋转
 
-1. 应用旋转（横屏|竖屏）
+1. 应用旋转：横屏竖屏
-2. 代码旋转
+2. 物理旋转：旋转镜头
-3. 镜头物理旋转
 
 ## 学习资料
 

taoyao-client-android/taoyao/client/src/main/java/com/acgist/taoyao/client/MainActivity.java

@@ -130,6 +130,7 @@ public class MainActivity extends AppCompatActivity implements Serializable {
             resources.getInteger(R.integer.iFrameInterval),
             resources.getString(R.string.storagePathImage),
             resources.getString(R.string.storagePathVideo),
+            resources.getString(R.string.watermark),
             VideoSourceType.valueOf(resources.getString(R.string.videoSourceType))
         );
         final Display display = this.getWindow().getContext().getDisplay();

taoyao-client-android/taoyao/client/src/main/res/values/settings.xml

@@ -44,4 +44,6 @@
     <integer name="channelCount">1</integer>
     <!-- 视频关键帧频率 -->
     <integer name="iFrameInterval">1</integer>
+    <!-- 水印 -->
+    <string name="watermark">"'TAOYAO' yyyy-MM-dd HH:mm:ss"</string>
 </resources>

taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/MediaManager.java

@@ -14,7 +14,10 @@ import com.acgist.taoyao.media.config.MediaProperties;
 import com.acgist.taoyao.media.config.MediaVideoProperties;
 import com.acgist.taoyao.media.config.WebrtcProperties;
 import com.acgist.taoyao.media.signal.ITaoyao;
+import com.acgist.taoyao.media.video.VideoProcesser;
+import com.acgist.taoyao.media.video.WatermarkProcesser;
 
+import org.apache.commons.lang3.StringUtils;
 import org.webrtc.AudioSource;
 import org.webrtc.AudioTrack;
 import org.webrtc.Camera2Enumerator;
@@ -27,7 +30,6 @@ import org.webrtc.EglBase;
 import org.webrtc.MediaConstraints;
 import org.webrtc.MediaStream;
 import org.webrtc.PeerConnectionFactory;
-import org.webrtc.RendererCommon;
 import org.webrtc.ScreenCapturerAndroid;
 import org.webrtc.SurfaceTextureHelper;
 import org.webrtc.SurfaceViewRenderer;
@@ -88,6 +90,10 @@ public final class MediaManager {
      * 关键帧频率
      */
     private int iFrameInterval;
+    /**
+     * 水印
+     */
+    private String watermark;
     /**
      * 视频来源类型
      */
@@ -148,6 +154,10 @@ public final class MediaManager {
      * PeerConnectionFactory
      */
     private PeerConnectionFactory peerConnectionFactory;
+    /**
+     * 视频处理
+     */
+    private VideoProcesser videoProcesser;
 
     static {
 //      // 设置采样
@@ -190,14 +200,14 @@ public final class MediaManager {
 
     /**
      * @param mainHandler   Handler
-     * @param context      上下文
+     * @param context       上下文
      */
     public void initContext(
         Handler mainHandler, Context context,
         int imageQuantity, String audioQuantity, String videoQuantity,
         int channelCount, int iFrameInterval,
         String imagePath, String videoPath,
-        VideoSourceType videoSourceType
+        String watermark, VideoSourceType videoSourceType
     ) {
         this.mainHandler = mainHandler;
         this.context   = context;
@@ -208,6 +218,7 @@ public final class MediaManager {
         this.iFrameInterval = iFrameInterval;
         this.imagePath = imagePath;
         this.videoPath = videoPath;
+        this.watermark = watermark;
         this.videoSourceType = videoSourceType;
     }
 
@@ -320,6 +331,7 @@ public final class MediaManager {
         });
         this.initAudio();
         this.initVideo();
+        this.initWatermark();
     }
 
     private JavaAudioDeviceModule javaAudioDeviceModule() {
@@ -419,7 +431,7 @@ public final class MediaManager {
                 // 忽略其他摄像头
             }
         }
-        this.initVideoTrack();
+        this.initVideoSource();
     }
 
     private void initSharePromise() {
@@ -433,13 +445,13 @@ public final class MediaManager {
      */
     public void initScreen(Intent intent) {
         this.videoCapturer = new ScreenCapturerAndroid(intent, new ScreenCallback());
-        this.initVideoTrack();
+        this.initVideoSource();
     }
 
     /**
      * 加载视频
      */
-    private void initVideoTrack() {
+    private void initVideoSource() {
         // 加载视频
         this.surfaceTextureHelper = SurfaceTextureHelper.create("MediaVideoThread", this.eglContext);
 //      this.surfaceTextureHelper.setTextureSize();
@@ -456,12 +468,21 @@ public final class MediaManager {
 //      this.shareVideoSource.setVideoProcessor();
     }
 
+    private void initWatermark() {
+        if(StringUtils.isNotEmpty(this.watermark)) {
+            final MediaVideoProperties mediaVideoProperties = this.mediaProperties.getVideos().get(this.videoQuantity);
+            if(this.videoProcesser == null) {
+                this.videoProcesser = new WatermarkProcesser(this.watermark, mediaVideoProperties.getWidth(), mediaVideoProperties.getHeight());
+            } else {
+                this.videoProcesser = new WatermarkProcesser(this.watermark, mediaVideoProperties.getWidth(), mediaVideoProperties.getHeight(), this.videoProcesser);
+            }
+        }
+    }
+
     /**
      * 更新配置
      *
-     * @param mediaProperties      媒体配置
+     * @param mediaProperties 媒体配置
-     * @param mediaAudioProperties 音频配置
-     * @param mediaVideoProperties 视频配置
      */
     public void updateMediaConfig(MediaProperties mediaProperties) {
         this.mediaProperties = mediaProperties;
@@ -707,6 +728,10 @@ public final class MediaManager {
             this.peerConnectionFactory.dispose();
             this.peerConnectionFactory = null;
         }
+        if(this.videoProcesser != null) {
+            this.videoProcesser.close();
+            this.videoProcesser = null;
+        }
     }
 
     /**
@@ -741,8 +766,18 @@ public final class MediaManager {
         @Override
         public void onFrameCaptured(VideoFrame videoFrame) {
             // 注意：VideoFrame必须释放，多线程环境需要调用retain和release方法。
-            this.mainObserver.onFrameCaptured(videoFrame);
+            if(MediaManager.this.videoProcesser == null) {
-            this.shareObserver.onFrameCaptured(videoFrame);
+                this.mainObserver.onFrameCaptured(videoFrame);
+                this.shareObserver.onFrameCaptured(videoFrame);
+            } else {
+                final VideoFrame.I420Buffer i420Buffer = videoFrame.getBuffer().toI420();
+                MediaManager.this.videoProcesser.process(i420Buffer);
+                final VideoFrame processVideoFrame = new VideoFrame(i420Buffer.cropAndScale(0, 0, i420Buffer.getWidth(), i420Buffer.getHeight(), i420Buffer.getWidth(), i420Buffer.getHeight()), videoFrame.getRotation(), videoFrame.getTimestampNs());
+                i420Buffer.release();
+                this.mainObserver.onFrameCaptured(processVideoFrame);
+                this.shareObserver.onFrameCaptured(processVideoFrame);
+                processVideoFrame.release();
+            }
         }
 
     }

taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/AudioChangerProcesser.java

@@ -1,10 +0,0 @@
-package com.acgist.taoyao.media.audio;
-
-/**
- * 变声处理器
- *
- * @author acgist
- */
-public class AudioChangerProcesser {
-
-}

taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/audio/MixerProcesser.java

@@ -1,13 +1,13 @@
 package com.acgist.taoyao.media.audio;
 
 /**
- * 混音
+ * 混音处理器
  *
  * WebRtcAudioTrack#AudioTrackThread  ：远程音频
  * WebRtcAudioRecord#AudioRecordThread：本地音频
  *
  * @author acgist
  */
-public class AudioMixer {
+public class MixerProcesser {
 
 }

taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/PhotographClient.java

@@ -135,6 +135,7 @@ public class PhotographClient implements VideoSink {
 //          matrix.setRotate(90);
 //          final Bitmap matrixBitmap = Bitmap.createBitmap(bitmap, 0, 0, width, height, matrix, false);
             bitmap.compress(Bitmap.CompressFormat.JPEG, this.quantity, output);
+            bitmap.recycle();
         } catch (Exception e) {
             Log.e(PhotographClient.class.getSimpleName(), "拍照异常", e);
         } finally {

taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/RecordClient.java

@@ -32,7 +32,13 @@ import java.time.LocalDateTime;
  */
 public class RecordClient extends Client implements VideoSink, JavaAudioDeviceModule.SamplesReadyCallback {
 
+    /**
+     * 等待时间（毫秒）
+     */
     private static final long WAIT_TIME_MS = 50;
+    /**
+     * 等待时间（纳秒）
+     */
     private static final long WAIT_TIME_US = WAIT_TIME_MS * 1000;
 
     /**

taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/client/SessionClient.java

@@ -377,6 +377,7 @@ public class SessionClient extends Client {
             public void onIceConnectionChange(PeerConnection.IceConnectionState iceConnectionState) {
                 Log.d(SessionClient.class.getSimpleName(), "PCIce连接状态改变：" + iceConnectionState);
                 SessionClient.this.logState();
+                // disconnected：暂时连接不上可能自我恢复
             }
 
             @Override
@@ -432,6 +433,7 @@ public class SessionClient extends Client {
             public void onRenegotiationNeeded() {
                 Log.d(SessionClient.class.getSimpleName(), "重新协商媒体：" + SessionClient.this.sessionId);
                 if(SessionClient.this.peerConnection.connectionState() == PeerConnection.PeerConnectionState.CONNECTED) {
+//                    SessionClient.this.peerConnection.restartIce();
                 // TODO：重新协商
 //                  SessionClient.this.offer();
                 }

taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/AiProcesser.java

@@ -1,30 +1,18 @@
 package com.acgist.taoyao.media.video;
 
 import org.webrtc.VideoFrame;
-import org.webrtc.VideoProcessor;
-import org.webrtc.VideoSink;
 
 /**
- * AI处理器
+ * AI识别处理器
+ *
+ * 建议不要每帧识别，如果没有识别出来结果可以复用识别结果。
  *
  * @author acgist
  */
-public class AiProcesser implements VideoProcessor {
+public class AiProcesser extends VideoProcesser {
 
     @Override
-    public void setSink(VideoSink videoSink) {
+    protected void doProcess(VideoFrame.I420Buffer i420Buffer) {
-    }
-
-    @Override
-    public void onCapturerStarted(boolean status) {
-    }
-
-    @Override
-    public void onCapturerStopped() {
-    }
-
-    @Override
-    public void onFrameCaptured(VideoFrame videoFrame) {
     }
 
 }

taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/BeautyProcesser.java

@@ -1,29 +0,0 @@
-package com.acgist.taoyao.media.video;
-
-import org.webrtc.VideoFrame;
-import org.webrtc.VideoProcessor;
-import org.webrtc.VideoSink;
-
-/**
- * 美颜处理器
- *
- * @author acgist
- */
-public class BeautyProcesser implements VideoProcessor {
-
-    @Override
-    public void setSink(VideoSink videoSink) {
-    }
-
-    @Override
-    public void onCapturerStarted(boolean status) {
-    }
-
-    @Override
-    public void onCapturerStopped() {
-    }
-
-    @Override
-    public void onFrameCaptured(VideoFrame videoFrame) {
-    }
-}

taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/VideoProcesser.java

@@ -0,0 +1,33 @@
+package com.acgist.taoyao.media.video;
+
+import org.webrtc.VideoFrame;
+
+import java.io.Closeable;
+
+/**
+ * 视频处理器
+ */
+public abstract class VideoProcesser implements Closeable {
+
+    protected VideoProcesser next;
+
+    public void process(VideoFrame.I420Buffer i420Buffer) {
+        this.doProcess(i420Buffer);
+        if(this.next == null) {
+            // 忽略
+        } else {
+            this.next.process(i420Buffer);
+        }
+    }
+
+    protected abstract void doProcess(VideoFrame.I420Buffer i420Buffer);
+
+    public void close() {
+        if(this.next == null) {
+            // 忽略
+        } else {
+            this.next.close();
+        }
+    }
+
+}

taoyao-client-android/taoyao/media/src/main/java/com/acgist/taoyao/media/video/WatermarkProcesser.java

@@ -1,30 +1,137 @@
 package com.acgist.taoyao.media.video;
 
+import android.graphics.Bitmap;
+import android.graphics.Canvas;
+import android.graphics.Color;
+import android.graphics.Paint;
+
 import org.webrtc.VideoFrame;
-import org.webrtc.VideoProcessor;
+
-import org.webrtc.VideoSink;
+import java.nio.ByteBuffer;
+import java.time.LocalDateTime;
+import java.time.format.DateTimeFormatter;
+import java.util.Timer;
+import java.util.TimerTask;
 
 /**
  * 水印处理器
  *
+ * 性能优化：
+ * 没有水印：20~25波动
+ * 没有定时：26~32波动
+ * 定时水印：28~32
+ *
  * @author acgist
  */
-public class WatermarkProcesser implements VideoProcessor {
+public class WatermarkProcesser extends VideoProcesser {
 
-    @Override
+    private static final WatermarkMatrix[] MATRICES = new WatermarkMatrix[256];
-    public void setSink(VideoSink videoSink) {
+
+    private final String format;
+    private final int width;
+    private final int height;
+    private final Timer timer;
+    private final WatermarkMatrix[] watermark;
+
+    public WatermarkProcesser(String format, int width, int height) {
+        this.format = format;
+        this.width = width;
+        this.height = height;
+        final String date = LocalDateTime.now().format(DateTimeFormatter.ofPattern(format));
+        this.watermark = new WatermarkMatrix[date.length()];
+        this.timer = new Timer("Watermark-Timer", true);
+        this.init();
+    }
+
+    public WatermarkProcesser(String format, int width, int height, VideoProcesser videoProcesser) {
+        this(format, width, height);
+        this.next = videoProcesser;
+    }
+
+    private void init() {
+        final String source = "-: 0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
+        final char[] chars = source.toCharArray();
+        for (char value : chars) {
+            this.build(value);
+        }
+        this.timer.schedule(new TimerTask() {
+            @Override
+            public void run() {
+                int index = 0;
+                final char[] chars = LocalDateTime.now().format(DateTimeFormatter.ofPattern(WatermarkProcesser.this.format)).toCharArray();
+                for (char value : chars) {
+                    WatermarkProcesser.this.watermark[index] = MATRICES[value];
+                    index++;
+                }
+            }
+        }, 1000, 1000);
+    }
+
+    private void build(char source) {
+        // TODO：优化复用bitmap
+        final String target = Character.toString(source);
+        final Paint paint = new Paint();
+        paint.setColor(Color.WHITE);
+        paint.setDither(true);
+        paint.setTextSize(40.0F);
+        paint.setTextAlign(Paint.Align.LEFT);
+        paint.setFilterBitmap(true);
+        final Paint.FontMetricsInt box = paint.getFontMetricsInt();
+        final int width = (int) paint.measureText(target);
+        final int height = box.descent - box.ascent;
+        final Bitmap bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888);
+        final Canvas canvas = new Canvas(bitmap);
+        canvas.drawText(target, 0, box.leading - box.ascent, paint);
+        canvas.save();
+        final boolean[][] matrix = new boolean[width][height];
+        for (int j = 0; j < height; j++) {
+            for (int i = 0; i < width; i++) {
+                matrix[i][j] = bitmap.getColor(i, j).toArgb() != 0;
+            }
+        }
+        MATRICES[source] = new WatermarkMatrix(width, height, matrix);
+        bitmap.recycle();
     }
 
     @Override
-    public void onCapturerStarted(boolean status) {
+    protected void doProcess(VideoFrame.I420Buffer i420Buffer) {
+        int widthPos = 0;
+        int heightPos = 0;
+        final ByteBuffer buffer = i420Buffer.getDataY();
+        for (WatermarkMatrix matrix : watermark) {
+            if(matrix == null) {
+                continue;
+            }
+            for (int height = 0; height < matrix.height; height++) {
+                for (int width = 0; width < matrix.width; width++) {
+                    if(matrix.matrix[width][height]) {
+                        buffer.put(this.width * height + width + widthPos, (byte) 0);
+                    }
+                }
+            }
+            widthPos += matrix.width;
+            heightPos += matrix.height;
+        }
     }
 
     @Override
-    public void onCapturerStopped() {
+    public void close() {
+        super.close();
+        this.timer.cancel();
     }
 
-    @Override
+    static class WatermarkMatrix {
-    public void onFrameCaptured(VideoFrame videoFrame) {
+
+        int width;
+        int height;
+        boolean[][] matrix;
+
+        public WatermarkMatrix(int width, int height, boolean[][] matrix) {
+            this.width = width;
+            this.height = height;
+            this.matrix = matrix;
+        }
+
     }
 
 }

taoyao-signal-server/taoyao-server/src/test/java/com/acgist/taoyao/rtp/RtpTest.java

@@ -20,7 +20,7 @@ import lombok.extern.slf4j.Slf4j;
 
 @Slf4j
 public class RtpTest {
-	
+    
 	@Test
 	void testSocket() throws Exception {
 		final Socket socket = new Socket();