Rebuild to use openCV. Alarm not yet implemented

app/build.gradle

@@ -37,6 +37,12 @@ dependencies {
     androidTestImplementation 'androidx.test.ext:junit:1.1.5'
     androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1'
 
+
+    // OpenCV (from: https://github.com/QuickBirdEng/opencv-android)
+    def opencv_version = "4.5.3.0"
+    implementation "com.quickbirdstudios:opencv:${opencv_version}"
+
+
     // Required for CameraX
     def camerax_version = "1.2.2"
     implementation "androidx.camera:camera-core:${camerax_version}"

app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java

@@ -1,9 +1,12 @@
 package com.example.ueberwachungssystem.Detection;
 
 import android.content.Context;
+import android.graphics.Bitmap;
 import android.graphics.ImageFormat;
 import android.media.Image;
-import android.util.Size;
+import android.os.CountDownTimer;
+import android.util.Log;
+import android.widget.ImageView;
 
 import androidx.annotation.NonNull;
 import androidx.camera.core.CameraSelector;
@@ -11,13 +14,24 @@ import androidx.camera.core.ExperimentalGetImage;
 import androidx.camera.core.ImageAnalysis;
 import androidx.camera.core.Preview;
 import androidx.camera.lifecycle.ProcessCameraProvider;
-import androidx.camera.view.PreviewView;
 import androidx.core.content.ContextCompat;
 import androidx.lifecycle.LifecycleOwner;
 
 import com.google.common.util.concurrent.ListenableFuture;
 
+import org.opencv.android.OpenCVLoader;
+import org.opencv.android.Utils;
+import org.opencv.core.Core;
+import org.opencv.core.CvType;
+import org.opencv.core.Mat;
+import org.opencv.core.MatOfPoint;
+import org.opencv.core.Scalar;
+import org.opencv.imgproc.Imgproc;
+
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
 import java.util.concurrent.ExecutionException;
 
 
@@ -28,10 +42,6 @@ import java.util.concurrent.ExecutionException;
  * USE FROM MAIN ACTIVITY:
  *
  * VideoDetector vd = new VideoDetector(this);
- * vd.setPreview(previewView);  //THIS IS OPTIONAL!
- * vd.setOnDetectionListener(new Detector.OnDetectionListener{...});
- * vd.startDetection();
- * vd.stopDetection
  *
  * */
 
@@ -44,12 +54,28 @@ public class VideoDetector extends Detector {
     // Camera Provider
     private ProcessCameraProvider cameraProvider;
     private Boolean isDetectionRunning = false;
-    // Preview Camera Image
-    private PreviewView previewView = null;
-    // Detect Violation
+    // Detection
+    private Mat previousImage = null;
+    private Mat lastThresh;
+    private Mat currentOut;
+
+    public ImageView imageView1 = null;
+    public ImageView imageView2 = null;
+
+
+    private int frameCnt = 0;
+
+
+
+    // Parameters
     private static final float PIXEL_THRESHOLD = 30f;  // Luminosity (brightness channel of YUV_420_888)
+    private static final int BLUR_KERNEL_SIZE = 5;
+    private static final int DILATE_KERNEL_SIZE = 5;
+    private static final float CONTOUR_THRESHOLD = 100;
     private static final float ALARM_THRESHOLD = 0.2f;   // Percent of pixels changed
-    private ByteBuffer previousBuffer = null;
+    private static final long START_DELAY = 1;   // milliseconds
+    private static final android.util.Size IMAGE_RES = new android.util.Size(640, 480);
+
 
 
 
@@ -70,6 +96,15 @@ public class VideoDetector extends Detector {
     public void startDetection() {
         if (isDetectionRunning)
             return;
+
+        // Open CV startup check
+        if (!OpenCVLoader.initDebug()) {
+            Log.e("OpenCV", "Unable to load OpenCV!");
+            return;
+        } else
+            Log.d("OpenCV", "OpenCV loaded Successfully!");
+
+
         // Request Camera Provider
         final ListenableFuture<ProcessCameraProvider> cameraProviderFuture = ProcessCameraProvider.getInstance(context);
         //Check for Camera availability
@@ -78,14 +113,12 @@ public class VideoDetector extends Detector {
                 cameraProvider = cameraProviderFuture.get();
                 bindAnalysis(cameraProvider);
                 isDetectionRunning = true;
-                previousBuffer = null;
             } catch (ExecutionException | InterruptedException e) {
                 // No errors need to be handled for this Future. This should never be reached.
             }
         },ContextCompat.getMainExecutor(context));
     }
 
-
     /**
      * Stops the Video Detection
      * */
@@ -98,15 +131,6 @@ public class VideoDetector extends Detector {
     }
 
 
-    /**
-     * Set PreviewView to show video feed while detecting
-     * this is optional and does not need to be called
-     * */
-    public void setPreviewView(PreviewView previewView) {
-        this.previewView = previewView;
-    }
-
-
     /**
      * Binds the Luminosity Analyzer (configure and run Analysis)
      * @param cameraProvider: CameraProvider of Context passed by Constructor
@@ -114,7 +138,7 @@ public class VideoDetector extends Detector {
     private void bindAnalysis(@NonNull ProcessCameraProvider cameraProvider) {
         // Configure and create Image Analysis
         ImageAnalysis.Builder builder = new ImageAnalysis.Builder();
-        builder.setTargetResolution(new Size(640, 480));
+        builder.setTargetResolution(IMAGE_RES);
         builder.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST);
         builder.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_YUV_420_888);
         ImageAnalysis imageAnalysis = builder.build();
@@ -125,15 +149,13 @@ public class VideoDetector extends Detector {
                 Image image = imageProxy.getImage();
                 assert image != null;
 
-                // Changed Pixel Detection
-                int pixelChanged = getChangedPixelCount(image);
-                int width = image.getWidth();
-                int height = image.getHeight();
+                Mat mat = extractYChannel(image);
 
-                float percentPixelChanged = (float) 100f * pixelChanged / (width * height);
+                debugMat(mat, imageView2);
 
-                if (percentPixelChanged > ALARM_THRESHOLD)
-                    reportViolation("0", "Video", percentPixelChanged);
+                //mat = alternativeProcessImage(mat);
+                mat = processImage(mat);
+                debugMat(mat, imageView1);
             }
             imageProxy.close();
         });
@@ -141,56 +163,119 @@ public class VideoDetector extends Detector {
         Preview preview = new Preview.Builder().build();
         // Specify which Camera to use
         CameraSelector cameraSelector = new CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_BACK).build();
-        // Update PreviewView if set
-        if (previewView != null)
-            preview.setSurfaceProvider(previewView.getSurfaceProvider());
+        // Delay till start
+        new CountDownTimer((long)(START_DELAY * 1000), 1000){
+            @Override
+            public void onTick(long millisUntilFinished) {}
+            @Override
+            public void onFinish() {
+                cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, imageAnalysis, preview);
+            }
+        }.start();
+    }
+
+
+    private Mat extractYChannel(@NonNull Image img) {
+        ByteBuffer yBuffer = img.getPlanes()[0].getBuffer();
+        byte[] yData = new byte[yBuffer.remaining()];
+        yBuffer.get(yData);
 
-        cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, imageAnalysis, preview);
+        Mat yMat = new Mat(img.getHeight(), img.getWidth(), CvType.CV_8UC1);
+        yMat.put(0, 0, yData);
+
+        return yMat;
     }
 
+    private Mat processImage(Mat currentImage){
+        if (previousImage == null) {
+            previousImage = currentImage;
+            return null;
+        }
 
+        Mat mat = new Mat();
+        currentImage = addGaussianBlur(currentImage, BLUR_KERNEL_SIZE);
 
-    /**
-     * Calculate Amount of Pixels changed using a threshold
-     * @param image
-     * */
-    private int getChangedPixelCount(Image image) {
-        Image.Plane[] planes = image.getPlanes();
-        ByteBuffer buffer = planes[0].getBuffer();
-
-        if (previousBuffer == null) {
-            previousBuffer = ByteBuffer.allocate(buffer.capacity());
-            buffer.rewind();
-            previousBuffer.put(buffer);
-            previousBuffer.rewind();
-            return 0;
+        mat = thresholdPixels(currentImage, previousImage, PIXEL_THRESHOLD);
+        mat = dilateNonZero(mat, DILATE_KERNEL_SIZE);
+        mat = thresholdContourArea(mat, CONTOUR_THRESHOLD);
+
+        previousImage = currentImage.clone();
+        return mat;
+    }
+
+    private Mat alternativeProcessImage(Mat currentImage){
+        if (previousImage == null) {
+            previousImage = currentImage;
+            return null;
+        } else if (lastThresh == null) {
+            lastThresh = thresholdPixels(currentImage, previousImage, PIXEL_THRESHOLD);
+            return null;
         }
 
-        int width = image.getWidth();
-        int height = image.getHeight();
+        while (frameCnt < 20){
+            Mat thresh = thresholdPixels(currentImage, previousImage, PIXEL_THRESHOLD);
+            Core.bitwise_or(thresh, lastThresh, lastThresh);
+            frameCnt++;
+            return currentOut;
+        }
+        return lastThresh;
+    }
 
-        int yRowStride = image.getPlanes()[0].getRowStride();
-        int yPixelStride = image.getPlanes()[0].getPixelStride();
 
-        int changedPixelCount = 0;
+    private Mat thresholdPixels(Mat inputMat, Mat previousImage, float luminosityThreshold){
+        Mat diffImage = new Mat();
+        Core.absdiff(inputMat, previousImage, diffImage);
 
-        // Count changed pixels
-        for (int y = 0; y < height; ++y) {
-            for (int x = 0; x < width; x++) {
-                int index = (y * yRowStride) + (x * yPixelStride);
-                int luminosity = (buffer.get(index) & 0xff);
-                int previousLuminosity = (previousBuffer.get(index) & 0xff);
-                int diff = Math.abs(luminosity - previousLuminosity);
-                if (diff > PIXEL_THRESHOLD)
-                    changedPixelCount++;
+        Mat binaryMat = new Mat();
+        Imgproc.threshold(diffImage, binaryMat, luminosityThreshold, 255, Imgproc.THRESH_BINARY);
+
+        return binaryMat;
+    }
+
+    private Mat dilateNonZero(Mat inputMat, int kernelSize){
+        Mat dilatedMat = new Mat();
+        Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, new org.opencv.core.Size(kernelSize, kernelSize));
+        Imgproc.dilate(inputMat, dilatedMat, kernel);
+        return dilatedMat;
+    }
+
+    private Mat addGaussianBlur(Mat inputMat, int kernelSize){
+        Mat outputMat = new Mat();
+        Imgproc.GaussianBlur(inputMat, outputMat, new org.opencv.core.Size(kernelSize, kernelSize), 0);
+        return outputMat;
+    }
+
+    private Mat thresholdContourArea(Mat inputMat, float areaThreshold){
+        List<MatOfPoint> contours = new ArrayList<>();
+        Mat hierarchy = new Mat();
+        Imgproc.findContours(inputMat, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);
+
+        Mat outputMat = new Mat(inputMat.size(), inputMat.type(), new Scalar(0));
+        // Iterate over the contours and draw only the larger contours on the outputMat
+        for (MatOfPoint contour : contours) {
+            double contourArea = Imgproc.contourArea(contour);
+            if (contourArea > areaThreshold) {
+                Imgproc.drawContours(outputMat, Collections.singletonList(contour), 0, new Scalar(255), -1);
             }
         }
+        // Apply the outputMat as a mask to the dilatedImage
+        Mat maskedImage = new Mat();
+        inputMat.copyTo(maskedImage, outputMat);
+        return outputMat;
+    }
+
+    private int countNonZeroPixels(Mat inputImage) {
+        return Core.countNonZero(inputImage);
+    }
+
+    private void debugMat(Mat mat, ImageView imageView) {
+        if (imageView == null || mat == null)
+            return;
 
-        // Reset and copy Byte Buffer
-        buffer.rewind();
-        previousBuffer.rewind();
-        previousBuffer.put(buffer);
+        Bitmap bitmap = Bitmap.createBitmap(mat.cols(), mat.rows(), Bitmap.Config.ARGB_8888);
+        Utils.matToBitmap(mat, bitmap);
 
-        return changedPixelCount;
+        // Display the bitmap in an ImageView
+        imageView.setImageBitmap(bitmap);
     }
-}
+}

app/src/main/java/com/example/ueberwachungssystem/MainActivity.java

@@ -11,6 +11,7 @@ import android.Manifest;
 import android.content.pm.PackageManager;
 import android.os.Bundle;
 import android.view.View;
+import android.widget.ImageView;
 import android.widget.TextView;
 import android.widget.Toast;
 import android.widget.ToggleButton;
@@ -30,37 +31,20 @@ public class MainActivity extends AppCompatActivity {
         super.onCreate(savedInstanceState);
         setContentView(R.layout.activity_main);
 
-        TextView textView = findViewById(R.id.textView);
-        PreviewView previewView = findViewById(R.id.previewView);
+        ImageView imageView = findViewById(R.id.imageView);
+        ImageView ogiv = findViewById(R.id.ogiv);
 
 
         VideoDetector vd = new VideoDetector(this);
-        vd.setPreviewView(previewView);
+        vd.imageView1 = imageView;
+        vd.imageView2 = ogiv;
         vd.setOnDetectionListener(new Detector.OnDetectionListener(){
             @Override
             public void onDetection(@NonNull DetectionReport detectionReport) {
                 detectionReport.log("OnDetection");
-                textView.setText(detectionReport.toString());
-            }
-        });
-        //vd.startDetection();
-
-
-        ToggleButton toggleButton = findViewById(R.id.previewButton);
-        toggleButton.setOnClickListener(new View.OnClickListener() {
-            @Override
-            public void onClick(View v) {
-                getCameraAccess();
-                if (isCameraAccessAllowed() && toggleButton.isChecked())
-                {
-                    vd.startDetection();
-                }
-                else {
-                    vd.stopDetection();
-                    textView.setText("");
-                }
             }
         });
+        vd.startDetection();
     }
 
     private boolean isCameraAccessAllowed() {

app/src/main/res/layout/activity_main.xml

@@ -5,33 +5,28 @@
     android:layout_width="match_parent"
     android:layout_height="match_parent"
     android:layout_gravity="center"
-    android:gravity="center"
+    android:gravity="top"
     android:orientation="vertical"
     tools:context=".MainActivity">
 
-    <TextView
-        android:id="@+id/textView"
-        android:layout_width="wrap_content"
-        android:layout_height="wrap_content"
-        android:text=""
-        android:textSize="15dp"
-        app:layout_constraintBottom_toBottomOf="parent"
-        app:layout_constraintEnd_toEndOf="parent"
-        app:layout_constraintStart_toStartOf="parent"
-        app:layout_constraintTop_toTopOf="parent" />
 
+    <androidx.camera.view.PreviewView
+        android:id="@+id/previewView"
+        android:layout_width="match_parent"
+        android:layout_height="1dp"
+        android:backgroundTint="@android:color/black"/>
 
-    <ToggleButton
-        android:id="@+id/previewButton"
+    <ImageView
+        android:id="@+id/ogiv"
         android:layout_width="match_parent"
         android:layout_height="wrap_content"
-        android:text="ToggleButton" />
-
+        tools:srcCompat="@tools:sample/avatars" />
 
-    <androidx.camera.view.PreviewView
-        android:id="@+id/previewView"
+    <ImageView
+        android:id="@+id/imageView"
         android:layout_width="match_parent"
-        android:layout_height="match_parent"
-        android:backgroundTint="@android:color/black"/>
+        android:layout_height="wrap_content"
+        tools:srcCompat="@tools:sample/avatars" />
+
 
 </LinearLayout>