From d32e0a11f597a6f3eedb8448a0d1eeac5e5d4c51 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Sat, 17 Jun 2023 11:20:56 +0200 Subject: [PATCH] Fixed bug in Detector and VideoDetector --- app/src/main/AndroidManifest.xml | 2 + .../Detection/Detector.java | 5 +- .../Detection/VideoDetector.java | 320 ++++++++++-------- .../ueberwachungssystem/MainActivity.java | 32 +- app/src/main/res/layout/activity_main.xml | 10 +- 5 files changed, 211 insertions(+), 158 deletions(-) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 36b1460..b5bb284 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -5,6 +5,8 @@ + + cameraProviderFuture = ProcessCameraProvider.getInstance(context); //Check for Camera availability cameraProviderFuture.addListener(() -> { try { cameraProvider = cameraProviderFuture.get(); - bindAnalysis(cameraProvider); + bindCameraProvider(cameraProvider); isDetectionRunning = true; } catch (ExecutionException | InterruptedException e) { // No errors need to be handled for this Future. This should never be reached. } - },ContextCompat.getMainExecutor(context)); + }, ContextCompat.getMainExecutor(context)); } - /** - * Stops the Video Detection - * */ + /** Stops the Video Detection */ @Override public void stopDetection() { - if (!isDetectionRunning) + if (!isDetectionRunning || imageAnalysis == null) return; - cameraProvider.unbindAll(); + cameraProvider.unbind(imageAnalysis); isDetectionRunning = false; + allowReportViolation = false; } - /** - * Permission handling - */ + /** Permission handling */ private boolean isCameraAccessAllowed() { return ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED; } @@ -150,70 +150,88 @@ public class VideoDetector extends Detector { } - /** - * Binds the Luminosity Analyzer (configure and run Analysis) - * @param cameraProvider: CameraProvider of Context passed by Constructor - * */ - private void bindAnalysis(@NonNull ProcessCameraProvider cameraProvider) { + /** Binds the Luminosity Analyzer (configure and run Analysis) */ + private void bindCameraProvider(@NonNull ProcessCameraProvider cameraProvider) { + // Create Preview + //Preview preview = new Preview.Builder().build(); + // Specify which Camera to use + CameraSelector cameraSelector = new CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_BACK).build(); + + cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, imageAnalysis); + + // Delay till violation is allowed + startViolationTimer(); + } + + + /** Start delay until Violation Report is allowed */ + private void startViolationTimer() { + new CountDownTimer((long) (START_DELAY), 100) { + @Override + public void onTick(long millisUntilFinished) { + } + + @Override + public void onFinish() { + allowReportViolation = true; + } + }.start(); + } + + + private ImageAnalysis setupImageAnalysis() { // Configure and create Image Analysis ImageAnalysis.Builder builder = new ImageAnalysis.Builder(); builder.setTargetResolution(IMAGE_RES); builder.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST); builder.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_YUV_420_888); ImageAnalysis imageAnalysis = builder.build(); - // Set Analyzer imageAnalysis.setAnalyzer(ContextCompat.getMainExecutor(context), imageProxy -> { if (imageProxy.getFormat() == ImageFormat.YUV_420_888) { Image image = imageProxy.getImage(); assert image != null; - Mat currentMat = extractYChannel(image); - - Mat mat = currentMat.clone(); - debugMat(mat, imageView2); - - Mat processed = processImage(mat); - - debugMat(processed, imageView1); - - int n = countNonZeroPixels(processed); + // Violation Handling + Mat processed = processImage(image); + int n = OpenCVHelper.countNonZeroPixels(processed); int pixelCount = image.getWidth() * image.getHeight(); - float percentChanged = (float)n / pixelCount; - // report violation - if (percentChanged * 100 > ALARM_THRESHOLD) { - reportViolation("Video", n); + float percentChanged = (float) n / pixelCount; + + // Violation Condition + if (percentChanged * 100 > ALARM_THRESHOLD) { + if (allowReportViolation) + reportViolation("Video", n); } } imageProxy.close(); }); - // Create Preview - Preview preview = new Preview.Builder().build(); - // Specify which Camera to use - CameraSelector cameraSelector = new CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_BACK).build(); - // Delay till start - new CountDownTimer((long)(START_DELAY), 100){ - @Override - public void onTick(long millisUntilFinished) {} - @Override - public void onFinish() { - cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, imageAnalysis, preview); - } - }.start(); + return imageAnalysis; + } + + @SuppressLint("RestrictedApi") + private VideoCapture setupVideoCapture() { + return new VideoCapture.Builder() + .setVideoFrameRate(30) + .setTargetRotation(Surface.ROTATION_0) + .build(); } - /** - * Process Image to be used for Motion Detection - * - * @param image: OpenCV Mat file that should be processed - */ - private Mat processImage(Mat image){ + /** Process Image to be used for Motion Detection */ + private Mat processImage(Image image){ + + // Image Transformation + Mat imageMat = OpenCVHelper.extractYChannel(image); + + // Show Input Image + if (inputImageView != null) + OpenCVHelper.debugMat(imageMat, inputImageView); // Preprocess Image - Mat preprocessed = image.clone(); - preprocessed = addGaussianBlur(preprocessed, new Size(21, 21)); - preprocessed = addBlur(preprocessed, new Size(3, 3)); + Mat preprocessed = imageMat; + preprocessed = OpenCVHelper.addGaussianBlur(preprocessed, new Size(21, 21)); + preprocessed = OpenCVHelper.addBlur(preprocessed, new Size(3, 3)); if (previousImage == null) { previousImage = preprocessed; @@ -222,91 +240,111 @@ public class VideoDetector extends Detector { // Process Image Mat processed = preprocessed.clone(); - processed = thresholdPixels(processed, previousImage, 25); - processed = dilateBinaryMat(processed, new Size(3,3)); - processed = dilateBinaryMat(processed, new Size(3,3)); - processed = thresholdContourArea(processed, 500); + processed = OpenCVHelper.thresholdPixels(processed, previousImage, 25); + processed = OpenCVHelper.dilateBinaryMat(processed, new Size(3,3)); + processed = OpenCVHelper.dilateBinaryMat(processed, new Size(3,3)); + processed = OpenCVHelper.thresholdContourArea(processed, 500); + + // Output previousImage = preprocessed.clone(); + // Show Output Image + if (outputImageView != null) + OpenCVHelper.debugMat(processed, outputImageView); return processed; } - - /** OpenCV helper methods **/ - private Mat addGaussianBlur(Mat inputMat, Size kernelSize){ - Mat outputMat = new Mat(); - Imgproc.GaussianBlur(inputMat, outputMat, kernelSize, 0); - return outputMat; + public void debugProcessing(ImageView inputImageView, ImageView outputImageView){ + this.inputImageView = inputImageView; + this.outputImageView = outputImageView; } - private Mat addBlur(Mat inputMat, Size kernelSize){ - Mat outputMat = new Mat(); - Imgproc.blur(inputMat, outputMat, kernelSize); - return outputMat; - } - private Mat extractYChannel(@NonNull Image img) { - ByteBuffer yBuffer = img.getPlanes()[0].getBuffer(); - byte[] yData = new byte[yBuffer.remaining()]; - yBuffer.get(yData); - Mat yMat = new Mat(img.getHeight(), img.getWidth(), CvType.CV_8UC1); - yMat.put(0, 0, yData); - return yMat; - } - private Mat thresholdPixels(Mat inputMat, Mat previousImage, int threshold){ - Mat diffImage = new Mat(); - Core.absdiff(inputMat, previousImage, diffImage); - Mat binaryMat = new Mat(); - Imgproc.threshold(diffImage, binaryMat, threshold, 255, Imgproc.THRESH_BINARY); - return binaryMat; - } - private Mat thresholdContourArea(Mat inputMat, float areaThreshold){ - List contours = new ArrayList<>(); - Mat hierarchy = new Mat(); - Imgproc.findContours(inputMat, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE); - Mat outputMat = new Mat(inputMat.size(), inputMat.type(), new Scalar(0)); - // Iterate over the contours and draw only the larger contours on the outputMat - for (MatOfPoint contour : contours) { - double contourArea = Imgproc.contourArea(contour); - if (contourArea > areaThreshold) { - Imgproc.drawContours(outputMat, Collections.singletonList(contour), 0, new Scalar(255), -1); - } + + + private static class OpenCVHelper{ + private OpenCVHelper() {} + + /** OpenCV helper methods **/ + private static Mat addGaussianBlur(Mat inputMat, Size kernelSize){ + Mat outputMat = new Mat(); + Imgproc.GaussianBlur(inputMat, outputMat, kernelSize, 0); + return outputMat; } - // Apply the outputMat as a mask to the dilatedImage - Mat maskedImage = new Mat(); - inputMat.copyTo(maskedImage, outputMat); - return outputMat; - } - private Mat dilateBinaryMat(Mat inputMat, Size kernelSize){ - Mat dilatedMat = new Mat(); - Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, kernelSize); - Imgproc.dilate(inputMat, dilatedMat, kernel); - return dilatedMat; - } + private static Mat addBlur(Mat inputMat, Size kernelSize){ + Mat outputMat = new Mat(); + Imgproc.blur(inputMat, outputMat, kernelSize); + return outputMat; + } - private int countNonZeroPixels(Mat inputImage) { - if (inputImage != null) - return Core.countNonZero(inputImage); - else - return 0; - } + private static Mat extractYChannel(@NonNull Image img) { + ByteBuffer yBuffer = img.getPlanes()[0].getBuffer(); + byte[] yData = new byte[yBuffer.remaining()]; + yBuffer.get(yData); + + Mat yMat = new Mat(img.getHeight(), img.getWidth(), CvType.CV_8UC1); + yMat.put(0, 0, yData); + + return yMat; + } + + private static Mat thresholdPixels(Mat inputMat, Mat previousImage, int threshold){ + Mat diffImage = new Mat(); + Core.absdiff(inputMat, previousImage, diffImage); + Mat binaryMat = new Mat(); + Imgproc.threshold(diffImage, binaryMat, threshold, 255, Imgproc.THRESH_BINARY); + return binaryMat; + } + + private static Mat thresholdContourArea(Mat inputMat, float areaThreshold){ + List contours = new ArrayList<>(); + Mat hierarchy = new Mat(); + Imgproc.findContours(inputMat, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE); + + Mat outputMat = new Mat(inputMat.size(), inputMat.type(), new Scalar(0)); + // Iterate over the contours and draw only the larger contours on the outputMat + for (MatOfPoint contour : contours) { + double contourArea = Imgproc.contourArea(contour); + if (contourArea > areaThreshold) { + Imgproc.drawContours(outputMat, Collections.singletonList(contour), 0, new Scalar(255), -1); + } + } + // Apply the outputMat as a mask to the dilatedImage + Mat maskedImage = new Mat(); + inputMat.copyTo(maskedImage, outputMat); + return outputMat; + } + + private static Mat dilateBinaryMat(Mat inputMat, Size kernelSize){ + Mat dilatedMat = new Mat(); + Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, kernelSize); + Imgproc.dilate(inputMat, dilatedMat, kernel); + return dilatedMat; + } + + private static int countNonZeroPixels(Mat inputImage) { + if (inputImage != null) + return Core.countNonZero(inputImage); + else + return 0; + } + private static void debugMat(Mat mat, ImageView imageView) { + if (imageView == null || mat == null) + return; - private void debugMat(Mat mat, ImageView imageView) { - if (imageView == null || mat == null) - return; + Bitmap bitmap = Bitmap.createBitmap(mat.cols(), mat.rows(), Bitmap.Config.ARGB_8888); + Utils.matToBitmap(mat, bitmap); - Bitmap bitmap = Bitmap.createBitmap(mat.cols(), mat.rows(), Bitmap.Config.ARGB_8888); - Utils.matToBitmap(mat, bitmap); - - // Display the bitmap in an ImageView - imageView.setImageBitmap(bitmap); + // Display the bitmap in an ImageView + imageView.setImageBitmap(bitmap); + } } } \ No newline at end of file diff --git a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java index 3efe3ab..fb73468 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java +++ b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java @@ -2,18 +2,11 @@ package com.example.ueberwachungssystem; import androidx.annotation.NonNull; import androidx.appcompat.app.AppCompatActivity; -import androidx.camera.view.PreviewView; -import androidx.core.app.ActivityCompat; -import androidx.core.content.ContextCompat; import androidx.camera.core.ExperimentalGetImage; -import android.Manifest; -import android.content.pm.PackageManager; import android.os.Bundle; import android.view.View; import android.widget.ImageView; -import android.widget.TextView; -import android.widget.Toast; import android.widget.ToggleButton; import com.example.ueberwachungssystem.Detection.DetectionReport; @@ -32,19 +25,34 @@ public class MainActivity extends AppCompatActivity { setContentView(R.layout.activity_main); - ImageView imageView = findViewById(R.id.imageView); - ImageView ogiv = findViewById(R.id.ogiv); + ImageView inputImageView = findViewById(R.id.inputImageView); + ImageView outputImageView = findViewById(R.id.outputImageView); VideoDetector vd = new VideoDetector(this); - vd.imageView1 = imageView; - vd.imageView2 = ogiv; + vd.debugProcessing(inputImageView, outputImageView); vd.setOnDetectionListener(new Detector.OnDetectionListener(){ @Override public void onDetection(@NonNull DetectionReport detectionReport) { detectionReport.log("OnDetection"); } }); - vd.startDetection(); + + + + ToggleButton toggleButton = findViewById(R.id.toggleButton); + toggleButton.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View v) { + if (toggleButton.isChecked()) + { + vd.startDetection(); + } + else { + vd.stopDetection(); + } + } + }); + } } \ No newline at end of file diff --git a/app/src/main/res/layout/activity_main.xml b/app/src/main/res/layout/activity_main.xml index d613b0c..aa0ac96 100644 --- a/app/src/main/res/layout/activity_main.xml +++ b/app/src/main/res/layout/activity_main.xml @@ -16,14 +16,20 @@ android:layout_height="1dp" android:backgroundTint="@android:color/black"/> + +