diff --git a/app/build.gradle b/app/build.gradle index 4d7dcc8..02a2cc2 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -37,6 +37,12 @@ dependencies { androidTestImplementation 'androidx.test.ext:junit:1.1.5' androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1' + + // OpenCV (from: https://github.com/QuickBirdEng/opencv-android) + def opencv_version = "4.5.3.0" + implementation "com.quickbirdstudios:opencv:${opencv_version}" + + // Required for CameraX def camerax_version = "1.2.2" implementation "androidx.camera:camera-core:${camerax_version}" diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index bfec02c..bb08b03 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -1,9 +1,12 @@ package com.example.ueberwachungssystem.Detection; import android.content.Context; +import android.graphics.Bitmap; import android.graphics.ImageFormat; import android.media.Image; -import android.util.Size; +import android.os.CountDownTimer; +import android.util.Log; +import android.widget.ImageView; import androidx.annotation.NonNull; import androidx.camera.core.CameraSelector; @@ -11,13 +14,24 @@ import androidx.camera.core.ExperimentalGetImage; import androidx.camera.core.ImageAnalysis; import androidx.camera.core.Preview; import androidx.camera.lifecycle.ProcessCameraProvider; -import androidx.camera.view.PreviewView; import androidx.core.content.ContextCompat; import androidx.lifecycle.LifecycleOwner; import com.google.common.util.concurrent.ListenableFuture; +import org.opencv.android.OpenCVLoader; +import org.opencv.android.Utils; +import org.opencv.core.Core; +import org.opencv.core.CvType; +import org.opencv.core.Mat; +import org.opencv.core.MatOfPoint; +import org.opencv.core.Scalar; +import org.opencv.imgproc.Imgproc; + import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.concurrent.ExecutionException; @@ -28,10 +42,6 @@ import java.util.concurrent.ExecutionException; * USE FROM MAIN ACTIVITY: * * VideoDetector vd = new VideoDetector(this); - * vd.setPreview(previewView); //THIS IS OPTIONAL! - * vd.setOnDetectionListener(new Detector.OnDetectionListener{...}); - * vd.startDetection(); - * vd.stopDetection * * */ @@ -44,12 +54,28 @@ public class VideoDetector extends Detector { // Camera Provider private ProcessCameraProvider cameraProvider; private Boolean isDetectionRunning = false; - // Preview Camera Image - private PreviewView previewView = null; - // Detect Violation + // Detection + private Mat previousImage = null; + private Mat lastThresh; + private Mat currentOut; + + public ImageView imageView1 = null; + public ImageView imageView2 = null; + + + private int frameCnt = 0; + + + + // Parameters private static final float PIXEL_THRESHOLD = 30f; // Luminosity (brightness channel of YUV_420_888) + private static final int BLUR_KERNEL_SIZE = 5; + private static final int DILATE_KERNEL_SIZE = 5; + private static final float CONTOUR_THRESHOLD = 100; private static final float ALARM_THRESHOLD = 0.2f; // Percent of pixels changed - private ByteBuffer previousBuffer = null; + private static final long START_DELAY = 1; // milliseconds + private static final android.util.Size IMAGE_RES = new android.util.Size(640, 480); + @@ -70,6 +96,15 @@ public class VideoDetector extends Detector { public void startDetection() { if (isDetectionRunning) return; + + // Open CV startup check + if (!OpenCVLoader.initDebug()) { + Log.e("OpenCV", "Unable to load OpenCV!"); + return; + } else + Log.d("OpenCV", "OpenCV loaded Successfully!"); + + // Request Camera Provider final ListenableFuture cameraProviderFuture = ProcessCameraProvider.getInstance(context); //Check for Camera availability @@ -78,14 +113,12 @@ public class VideoDetector extends Detector { cameraProvider = cameraProviderFuture.get(); bindAnalysis(cameraProvider); isDetectionRunning = true; - previousBuffer = null; } catch (ExecutionException | InterruptedException e) { // No errors need to be handled for this Future. This should never be reached. } },ContextCompat.getMainExecutor(context)); } - /** * Stops the Video Detection * */ @@ -98,15 +131,6 @@ public class VideoDetector extends Detector { } - /** - * Set PreviewView to show video feed while detecting - * this is optional and does not need to be called - * */ - public void setPreviewView(PreviewView previewView) { - this.previewView = previewView; - } - - /** * Binds the Luminosity Analyzer (configure and run Analysis) * @param cameraProvider: CameraProvider of Context passed by Constructor @@ -114,7 +138,7 @@ public class VideoDetector extends Detector { private void bindAnalysis(@NonNull ProcessCameraProvider cameraProvider) { // Configure and create Image Analysis ImageAnalysis.Builder builder = new ImageAnalysis.Builder(); - builder.setTargetResolution(new Size(640, 480)); + builder.setTargetResolution(IMAGE_RES); builder.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST); builder.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_YUV_420_888); ImageAnalysis imageAnalysis = builder.build(); @@ -125,15 +149,13 @@ public class VideoDetector extends Detector { Image image = imageProxy.getImage(); assert image != null; - // Changed Pixel Detection - int pixelChanged = getChangedPixelCount(image); - int width = image.getWidth(); - int height = image.getHeight(); + Mat mat = extractYChannel(image); - float percentPixelChanged = (float) 100f * pixelChanged / (width * height); + debugMat(mat, imageView2); - if (percentPixelChanged > ALARM_THRESHOLD) - reportViolation("0", "Video", percentPixelChanged); + //mat = alternativeProcessImage(mat); + mat = processImage(mat); + debugMat(mat, imageView1); } imageProxy.close(); }); @@ -141,56 +163,119 @@ public class VideoDetector extends Detector { Preview preview = new Preview.Builder().build(); // Specify which Camera to use CameraSelector cameraSelector = new CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_BACK).build(); - // Update PreviewView if set - if (previewView != null) - preview.setSurfaceProvider(previewView.getSurfaceProvider()); - - cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, imageAnalysis, preview); + // Delay till start + new CountDownTimer((long)(START_DELAY * 1000), 1000){ + @Override + public void onTick(long millisUntilFinished) {} + @Override + public void onFinish() { + cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, imageAnalysis, preview); + } + }.start(); } + private Mat extractYChannel(@NonNull Image img) { + ByteBuffer yBuffer = img.getPlanes()[0].getBuffer(); + byte[] yData = new byte[yBuffer.remaining()]; + yBuffer.get(yData); - /** - * Calculate Amount of Pixels changed using a threshold - * @param image - * */ - private int getChangedPixelCount(Image image) { - Image.Plane[] planes = image.getPlanes(); - ByteBuffer buffer = planes[0].getBuffer(); + Mat yMat = new Mat(img.getHeight(), img.getWidth(), CvType.CV_8UC1); + yMat.put(0, 0, yData); - if (previousBuffer == null) { - previousBuffer = ByteBuffer.allocate(buffer.capacity()); - buffer.rewind(); - previousBuffer.put(buffer); - previousBuffer.rewind(); - return 0; + return yMat; + } + + private Mat processImage(Mat currentImage){ + if (previousImage == null) { + previousImage = currentImage; + return null; } - int width = image.getWidth(); - int height = image.getHeight(); + Mat mat = new Mat(); + currentImage = addGaussianBlur(currentImage, BLUR_KERNEL_SIZE); - int yRowStride = image.getPlanes()[0].getRowStride(); - int yPixelStride = image.getPlanes()[0].getPixelStride(); + mat = thresholdPixels(currentImage, previousImage, PIXEL_THRESHOLD); + mat = dilateNonZero(mat, DILATE_KERNEL_SIZE); + mat = thresholdContourArea(mat, CONTOUR_THRESHOLD); - int changedPixelCount = 0; + previousImage = currentImage.clone(); + return mat; + } - // Count changed pixels - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; x++) { - int index = (y * yRowStride) + (x * yPixelStride); - int luminosity = (buffer.get(index) & 0xff); - int previousLuminosity = (previousBuffer.get(index) & 0xff); - int diff = Math.abs(luminosity - previousLuminosity); - if (diff > PIXEL_THRESHOLD) - changedPixelCount++; + private Mat alternativeProcessImage(Mat currentImage){ + if (previousImage == null) { + previousImage = currentImage; + return null; + } else if (lastThresh == null) { + lastThresh = thresholdPixels(currentImage, previousImage, PIXEL_THRESHOLD); + return null; + } + + while (frameCnt < 20){ + Mat thresh = thresholdPixels(currentImage, previousImage, PIXEL_THRESHOLD); + Core.bitwise_or(thresh, lastThresh, lastThresh); + frameCnt++; + return currentOut; + } + return lastThresh; + } + + + private Mat thresholdPixels(Mat inputMat, Mat previousImage, float luminosityThreshold){ + Mat diffImage = new Mat(); + Core.absdiff(inputMat, previousImage, diffImage); + + Mat binaryMat = new Mat(); + Imgproc.threshold(diffImage, binaryMat, luminosityThreshold, 255, Imgproc.THRESH_BINARY); + + return binaryMat; + } + + private Mat dilateNonZero(Mat inputMat, int kernelSize){ + Mat dilatedMat = new Mat(); + Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, new org.opencv.core.Size(kernelSize, kernelSize)); + Imgproc.dilate(inputMat, dilatedMat, kernel); + return dilatedMat; + } + + private Mat addGaussianBlur(Mat inputMat, int kernelSize){ + Mat outputMat = new Mat(); + Imgproc.GaussianBlur(inputMat, outputMat, new org.opencv.core.Size(kernelSize, kernelSize), 0); + return outputMat; + } + + private Mat thresholdContourArea(Mat inputMat, float areaThreshold){ + List contours = new ArrayList<>(); + Mat hierarchy = new Mat(); + Imgproc.findContours(inputMat, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE); + + Mat outputMat = new Mat(inputMat.size(), inputMat.type(), new Scalar(0)); + // Iterate over the contours and draw only the larger contours on the outputMat + for (MatOfPoint contour : contours) { + double contourArea = Imgproc.contourArea(contour); + if (contourArea > areaThreshold) { + Imgproc.drawContours(outputMat, Collections.singletonList(contour), 0, new Scalar(255), -1); } } - - // Reset and copy Byte Buffer - buffer.rewind(); - previousBuffer.rewind(); - previousBuffer.put(buffer); - - return changedPixelCount; + // Apply the outputMat as a mask to the dilatedImage + Mat maskedImage = new Mat(); + inputMat.copyTo(maskedImage, outputMat); + return outputMat; } -} + + private int countNonZeroPixels(Mat inputImage) { + return Core.countNonZero(inputImage); + } + + private void debugMat(Mat mat, ImageView imageView) { + if (imageView == null || mat == null) + return; + + Bitmap bitmap = Bitmap.createBitmap(mat.cols(), mat.rows(), Bitmap.Config.ARGB_8888); + Utils.matToBitmap(mat, bitmap); + + // Display the bitmap in an ImageView + imageView.setImageBitmap(bitmap); + } +} \ No newline at end of file diff --git a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java index 9a09a49..9bfbed8 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java +++ b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java @@ -11,6 +11,7 @@ import android.Manifest; import android.content.pm.PackageManager; import android.os.Bundle; import android.view.View; +import android.widget.ImageView; import android.widget.TextView; import android.widget.Toast; import android.widget.ToggleButton; @@ -30,37 +31,20 @@ public class MainActivity extends AppCompatActivity { super.onCreate(savedInstanceState); setContentView(R.layout.activity_main); - TextView textView = findViewById(R.id.textView); - PreviewView previewView = findViewById(R.id.previewView); + ImageView imageView = findViewById(R.id.imageView); + ImageView ogiv = findViewById(R.id.ogiv); VideoDetector vd = new VideoDetector(this); - vd.setPreviewView(previewView); + vd.imageView1 = imageView; + vd.imageView2 = ogiv; vd.setOnDetectionListener(new Detector.OnDetectionListener(){ @Override public void onDetection(@NonNull DetectionReport detectionReport) { detectionReport.log("OnDetection"); - textView.setText(detectionReport.toString()); - } - }); - //vd.startDetection(); - - - ToggleButton toggleButton = findViewById(R.id.previewButton); - toggleButton.setOnClickListener(new View.OnClickListener() { - @Override - public void onClick(View v) { - getCameraAccess(); - if (isCameraAccessAllowed() && toggleButton.isChecked()) - { - vd.startDetection(); - } - else { - vd.stopDetection(); - textView.setText(""); - } } }); + vd.startDetection(); } private boolean isCameraAccessAllowed() { diff --git a/app/src/main/res/layout/activity_main.xml b/app/src/main/res/layout/activity_main.xml index ebe18b4..d613b0c 100644 --- a/app/src/main/res/layout/activity_main.xml +++ b/app/src/main/res/layout/activity_main.xml @@ -5,33 +5,28 @@ android:layout_width="match_parent" android:layout_height="match_parent" android:layout_gravity="center" - android:gravity="center" + android:gravity="top" android:orientation="vertical" tools:context=".MainActivity"> - - - - - + + + + + \ No newline at end of file