From f98079d43184298d30acea11cb3a86e3c859ec15 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Fri, 9 Jun 2023 09:09:50 +0200 Subject: [PATCH 01/28] Readded main and main xml --- .../ueberwachungssystem/MainActivity.java | 74 +++++++++++++++++++ app/src/main/res/layout/activity_main.xml | 25 ++++++- 2 files changed, 96 insertions(+), 3 deletions(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java index f4fdae7..9a09a49 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java +++ b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java @@ -1,14 +1,88 @@ package com.example.ueberwachungssystem; +import androidx.annotation.NonNull; import androidx.appcompat.app.AppCompatActivity; +import androidx.camera.view.PreviewView; +import androidx.core.app.ActivityCompat; +import androidx.core.content.ContextCompat; +import androidx.camera.core.ExperimentalGetImage; +import android.Manifest; +import android.content.pm.PackageManager; import android.os.Bundle; +import android.view.View; +import android.widget.TextView; +import android.widget.Toast; +import android.widget.ToggleButton; +import com.example.ueberwachungssystem.Detection.DetectionReport; +import com.example.ueberwachungssystem.Detection.Detector; +import com.example.ueberwachungssystem.Detection.VideoDetector; + +@ExperimentalGetImage public class MainActivity extends AppCompatActivity { + private static final int CAMERA_PERMISSION_REQUEST_CODE = 101; + + @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); setContentView(R.layout.activity_main); + + TextView textView = findViewById(R.id.textView); + PreviewView previewView = findViewById(R.id.previewView); + + + VideoDetector vd = new VideoDetector(this); + vd.setPreviewView(previewView); + vd.setOnDetectionListener(new Detector.OnDetectionListener(){ + @Override + public void onDetection(@NonNull DetectionReport detectionReport) { + detectionReport.log("OnDetection"); + textView.setText(detectionReport.toString()); + } + }); + //vd.startDetection(); + + + ToggleButton toggleButton = findViewById(R.id.previewButton); + toggleButton.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View v) { + getCameraAccess(); + if (isCameraAccessAllowed() && toggleButton.isChecked()) + { + vd.startDetection(); + } + else { + vd.stopDetection(); + textView.setText(""); + } + } + }); + } + + private boolean isCameraAccessAllowed() { + return ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED; + } + + private void getCameraAccess() { + if (!isCameraAccessAllowed()) + ActivityCompat.requestPermissions(this, new String[]{android.Manifest.permission.CAMERA}, CAMERA_PERMISSION_REQUEST_CODE); + } + + @Override + public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults); + + if (requestCode == CAMERA_PERMISSION_REQUEST_CODE && grantResults.length > 0) { + boolean cameraRights = grantResults[0] == PackageManager.PERMISSION_GRANTED; + if (cameraRights) { + Toast.makeText(this, "camera permission granted", Toast.LENGTH_LONG).show(); + } else { + Toast.makeText(this, "camera permission denied", Toast.LENGTH_LONG).show(); + } + } } } \ No newline at end of file diff --git a/app/src/main/res/layout/activity_main.xml b/app/src/main/res/layout/activity_main.xml index 17eab17..ebe18b4 100644 --- a/app/src/main/res/layout/activity_main.xml +++ b/app/src/main/res/layout/activity_main.xml @@ -1,18 +1,37 @@ - - \ No newline at end of file + + + + + + + \ No newline at end of file From 7d29ed1ca90c2a64d2c4895d8db7aedb8efb0861 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Fri, 9 Jun 2023 20:56:14 +0200 Subject: [PATCH 02/28] Rebuild to use openCV. Alarm not yet implemented --- app/build.gradle | 6 + .../Detection/VideoDetector.java | 223 ++++++++++++------ .../ueberwachungssystem/MainActivity.java | 28 +-- app/src/main/res/layout/activity_main.xml | 35 ++- 4 files changed, 181 insertions(+), 111 deletions(-) diff --git a/app/build.gradle b/app/build.gradle index 4d7dcc8..02a2cc2 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -37,6 +37,12 @@ dependencies { androidTestImplementation 'androidx.test.ext:junit:1.1.5' androidTestImplementation 'androidx.test.espresso:espresso-core:3.5.1' + + // OpenCV (from: https://github.com/QuickBirdEng/opencv-android) + def opencv_version = "4.5.3.0" + implementation "com.quickbirdstudios:opencv:${opencv_version}" + + // Required for CameraX def camerax_version = "1.2.2" implementation "androidx.camera:camera-core:${camerax_version}" diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index bfec02c..bb08b03 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -1,9 +1,12 @@ package com.example.ueberwachungssystem.Detection; import android.content.Context; +import android.graphics.Bitmap; import android.graphics.ImageFormat; import android.media.Image; -import android.util.Size; +import android.os.CountDownTimer; +import android.util.Log; +import android.widget.ImageView; import androidx.annotation.NonNull; import androidx.camera.core.CameraSelector; @@ -11,13 +14,24 @@ import androidx.camera.core.ExperimentalGetImage; import androidx.camera.core.ImageAnalysis; import androidx.camera.core.Preview; import androidx.camera.lifecycle.ProcessCameraProvider; -import androidx.camera.view.PreviewView; import androidx.core.content.ContextCompat; import androidx.lifecycle.LifecycleOwner; import com.google.common.util.concurrent.ListenableFuture; +import org.opencv.android.OpenCVLoader; +import org.opencv.android.Utils; +import org.opencv.core.Core; +import org.opencv.core.CvType; +import org.opencv.core.Mat; +import org.opencv.core.MatOfPoint; +import org.opencv.core.Scalar; +import org.opencv.imgproc.Imgproc; + import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; import java.util.concurrent.ExecutionException; @@ -28,10 +42,6 @@ import java.util.concurrent.ExecutionException; * USE FROM MAIN ACTIVITY: * * VideoDetector vd = new VideoDetector(this); - * vd.setPreview(previewView); //THIS IS OPTIONAL! - * vd.setOnDetectionListener(new Detector.OnDetectionListener{...}); - * vd.startDetection(); - * vd.stopDetection * * */ @@ -44,12 +54,28 @@ public class VideoDetector extends Detector { // Camera Provider private ProcessCameraProvider cameraProvider; private Boolean isDetectionRunning = false; - // Preview Camera Image - private PreviewView previewView = null; - // Detect Violation + // Detection + private Mat previousImage = null; + private Mat lastThresh; + private Mat currentOut; + + public ImageView imageView1 = null; + public ImageView imageView2 = null; + + + private int frameCnt = 0; + + + + // Parameters private static final float PIXEL_THRESHOLD = 30f; // Luminosity (brightness channel of YUV_420_888) + private static final int BLUR_KERNEL_SIZE = 5; + private static final int DILATE_KERNEL_SIZE = 5; + private static final float CONTOUR_THRESHOLD = 100; private static final float ALARM_THRESHOLD = 0.2f; // Percent of pixels changed - private ByteBuffer previousBuffer = null; + private static final long START_DELAY = 1; // milliseconds + private static final android.util.Size IMAGE_RES = new android.util.Size(640, 480); + @@ -70,6 +96,15 @@ public class VideoDetector extends Detector { public void startDetection() { if (isDetectionRunning) return; + + // Open CV startup check + if (!OpenCVLoader.initDebug()) { + Log.e("OpenCV", "Unable to load OpenCV!"); + return; + } else + Log.d("OpenCV", "OpenCV loaded Successfully!"); + + // Request Camera Provider final ListenableFuture cameraProviderFuture = ProcessCameraProvider.getInstance(context); //Check for Camera availability @@ -78,14 +113,12 @@ public class VideoDetector extends Detector { cameraProvider = cameraProviderFuture.get(); bindAnalysis(cameraProvider); isDetectionRunning = true; - previousBuffer = null; } catch (ExecutionException | InterruptedException e) { // No errors need to be handled for this Future. This should never be reached. } },ContextCompat.getMainExecutor(context)); } - /** * Stops the Video Detection * */ @@ -98,15 +131,6 @@ public class VideoDetector extends Detector { } - /** - * Set PreviewView to show video feed while detecting - * this is optional and does not need to be called - * */ - public void setPreviewView(PreviewView previewView) { - this.previewView = previewView; - } - - /** * Binds the Luminosity Analyzer (configure and run Analysis) * @param cameraProvider: CameraProvider of Context passed by Constructor @@ -114,7 +138,7 @@ public class VideoDetector extends Detector { private void bindAnalysis(@NonNull ProcessCameraProvider cameraProvider) { // Configure and create Image Analysis ImageAnalysis.Builder builder = new ImageAnalysis.Builder(); - builder.setTargetResolution(new Size(640, 480)); + builder.setTargetResolution(IMAGE_RES); builder.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST); builder.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_YUV_420_888); ImageAnalysis imageAnalysis = builder.build(); @@ -125,15 +149,13 @@ public class VideoDetector extends Detector { Image image = imageProxy.getImage(); assert image != null; - // Changed Pixel Detection - int pixelChanged = getChangedPixelCount(image); - int width = image.getWidth(); - int height = image.getHeight(); + Mat mat = extractYChannel(image); - float percentPixelChanged = (float) 100f * pixelChanged / (width * height); + debugMat(mat, imageView2); - if (percentPixelChanged > ALARM_THRESHOLD) - reportViolation("0", "Video", percentPixelChanged); + //mat = alternativeProcessImage(mat); + mat = processImage(mat); + debugMat(mat, imageView1); } imageProxy.close(); }); @@ -141,56 +163,119 @@ public class VideoDetector extends Detector { Preview preview = new Preview.Builder().build(); // Specify which Camera to use CameraSelector cameraSelector = new CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_BACK).build(); - // Update PreviewView if set - if (previewView != null) - preview.setSurfaceProvider(previewView.getSurfaceProvider()); - - cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, imageAnalysis, preview); + // Delay till start + new CountDownTimer((long)(START_DELAY * 1000), 1000){ + @Override + public void onTick(long millisUntilFinished) {} + @Override + public void onFinish() { + cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, imageAnalysis, preview); + } + }.start(); } + private Mat extractYChannel(@NonNull Image img) { + ByteBuffer yBuffer = img.getPlanes()[0].getBuffer(); + byte[] yData = new byte[yBuffer.remaining()]; + yBuffer.get(yData); - /** - * Calculate Amount of Pixels changed using a threshold - * @param image - * */ - private int getChangedPixelCount(Image image) { - Image.Plane[] planes = image.getPlanes(); - ByteBuffer buffer = planes[0].getBuffer(); + Mat yMat = new Mat(img.getHeight(), img.getWidth(), CvType.CV_8UC1); + yMat.put(0, 0, yData); - if (previousBuffer == null) { - previousBuffer = ByteBuffer.allocate(buffer.capacity()); - buffer.rewind(); - previousBuffer.put(buffer); - previousBuffer.rewind(); - return 0; + return yMat; + } + + private Mat processImage(Mat currentImage){ + if (previousImage == null) { + previousImage = currentImage; + return null; } - int width = image.getWidth(); - int height = image.getHeight(); + Mat mat = new Mat(); + currentImage = addGaussianBlur(currentImage, BLUR_KERNEL_SIZE); - int yRowStride = image.getPlanes()[0].getRowStride(); - int yPixelStride = image.getPlanes()[0].getPixelStride(); + mat = thresholdPixels(currentImage, previousImage, PIXEL_THRESHOLD); + mat = dilateNonZero(mat, DILATE_KERNEL_SIZE); + mat = thresholdContourArea(mat, CONTOUR_THRESHOLD); - int changedPixelCount = 0; + previousImage = currentImage.clone(); + return mat; + } - // Count changed pixels - for (int y = 0; y < height; ++y) { - for (int x = 0; x < width; x++) { - int index = (y * yRowStride) + (x * yPixelStride); - int luminosity = (buffer.get(index) & 0xff); - int previousLuminosity = (previousBuffer.get(index) & 0xff); - int diff = Math.abs(luminosity - previousLuminosity); - if (diff > PIXEL_THRESHOLD) - changedPixelCount++; + private Mat alternativeProcessImage(Mat currentImage){ + if (previousImage == null) { + previousImage = currentImage; + return null; + } else if (lastThresh == null) { + lastThresh = thresholdPixels(currentImage, previousImage, PIXEL_THRESHOLD); + return null; + } + + while (frameCnt < 20){ + Mat thresh = thresholdPixels(currentImage, previousImage, PIXEL_THRESHOLD); + Core.bitwise_or(thresh, lastThresh, lastThresh); + frameCnt++; + return currentOut; + } + return lastThresh; + } + + + private Mat thresholdPixels(Mat inputMat, Mat previousImage, float luminosityThreshold){ + Mat diffImage = new Mat(); + Core.absdiff(inputMat, previousImage, diffImage); + + Mat binaryMat = new Mat(); + Imgproc.threshold(diffImage, binaryMat, luminosityThreshold, 255, Imgproc.THRESH_BINARY); + + return binaryMat; + } + + private Mat dilateNonZero(Mat inputMat, int kernelSize){ + Mat dilatedMat = new Mat(); + Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, new org.opencv.core.Size(kernelSize, kernelSize)); + Imgproc.dilate(inputMat, dilatedMat, kernel); + return dilatedMat; + } + + private Mat addGaussianBlur(Mat inputMat, int kernelSize){ + Mat outputMat = new Mat(); + Imgproc.GaussianBlur(inputMat, outputMat, new org.opencv.core.Size(kernelSize, kernelSize), 0); + return outputMat; + } + + private Mat thresholdContourArea(Mat inputMat, float areaThreshold){ + List contours = new ArrayList<>(); + Mat hierarchy = new Mat(); + Imgproc.findContours(inputMat, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE); + + Mat outputMat = new Mat(inputMat.size(), inputMat.type(), new Scalar(0)); + // Iterate over the contours and draw only the larger contours on the outputMat + for (MatOfPoint contour : contours) { + double contourArea = Imgproc.contourArea(contour); + if (contourArea > areaThreshold) { + Imgproc.drawContours(outputMat, Collections.singletonList(contour), 0, new Scalar(255), -1); } } - - // Reset and copy Byte Buffer - buffer.rewind(); - previousBuffer.rewind(); - previousBuffer.put(buffer); - - return changedPixelCount; + // Apply the outputMat as a mask to the dilatedImage + Mat maskedImage = new Mat(); + inputMat.copyTo(maskedImage, outputMat); + return outputMat; } -} + + private int countNonZeroPixels(Mat inputImage) { + return Core.countNonZero(inputImage); + } + + private void debugMat(Mat mat, ImageView imageView) { + if (imageView == null || mat == null) + return; + + Bitmap bitmap = Bitmap.createBitmap(mat.cols(), mat.rows(), Bitmap.Config.ARGB_8888); + Utils.matToBitmap(mat, bitmap); + + // Display the bitmap in an ImageView + imageView.setImageBitmap(bitmap); + } +} \ No newline at end of file diff --git a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java index 9a09a49..9bfbed8 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java +++ b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java @@ -11,6 +11,7 @@ import android.Manifest; import android.content.pm.PackageManager; import android.os.Bundle; import android.view.View; +import android.widget.ImageView; import android.widget.TextView; import android.widget.Toast; import android.widget.ToggleButton; @@ -30,37 +31,20 @@ public class MainActivity extends AppCompatActivity { super.onCreate(savedInstanceState); setContentView(R.layout.activity_main); - TextView textView = findViewById(R.id.textView); - PreviewView previewView = findViewById(R.id.previewView); + ImageView imageView = findViewById(R.id.imageView); + ImageView ogiv = findViewById(R.id.ogiv); VideoDetector vd = new VideoDetector(this); - vd.setPreviewView(previewView); + vd.imageView1 = imageView; + vd.imageView2 = ogiv; vd.setOnDetectionListener(new Detector.OnDetectionListener(){ @Override public void onDetection(@NonNull DetectionReport detectionReport) { detectionReport.log("OnDetection"); - textView.setText(detectionReport.toString()); - } - }); - //vd.startDetection(); - - - ToggleButton toggleButton = findViewById(R.id.previewButton); - toggleButton.setOnClickListener(new View.OnClickListener() { - @Override - public void onClick(View v) { - getCameraAccess(); - if (isCameraAccessAllowed() && toggleButton.isChecked()) - { - vd.startDetection(); - } - else { - vd.stopDetection(); - textView.setText(""); - } } }); + vd.startDetection(); } private boolean isCameraAccessAllowed() { diff --git a/app/src/main/res/layout/activity_main.xml b/app/src/main/res/layout/activity_main.xml index ebe18b4..d613b0c 100644 --- a/app/src/main/res/layout/activity_main.xml +++ b/app/src/main/res/layout/activity_main.xml @@ -5,33 +5,28 @@ android:layout_width="match_parent" android:layout_height="match_parent" android:layout_gravity="center" - android:gravity="center" + android:gravity="top" android:orientation="vertical" tools:context=".MainActivity"> - - - - - + + + + + \ No newline at end of file From e98ae1026712edf4cddc19457bb4f071234810d9 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Mon, 12 Jun 2023 19:03:52 +0200 Subject: [PATCH 03/28] refactor --- .../ueberwachungssystem/Detection/VideoDetector.java | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index bb08b03..6362e9d 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -211,14 +211,7 @@ public class VideoDetector extends Detector { lastThresh = thresholdPixels(currentImage, previousImage, PIXEL_THRESHOLD); return null; } - - while (frameCnt < 20){ - Mat thresh = thresholdPixels(currentImage, previousImage, PIXEL_THRESHOLD); - Core.bitwise_or(thresh, lastThresh, lastThresh); - frameCnt++; - return currentOut; - } - return lastThresh; + return null; } From bbd333426d398af02e0666362bcb81d1a8a065d0 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Tue, 13 Jun 2023 20:55:30 +0200 Subject: [PATCH 04/28] unfinished changes in Detector --- .../Detection/DetectionReport.java | 13 ++++++++----- .../ueberwachungssystem/Detection/Detector.java | 5 ++--- .../Detection/VideoDetector.java | 1 - 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectionReport.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectionReport.java index 2dc8cba..653e57b 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectionReport.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectionReport.java @@ -9,13 +9,16 @@ public class DetectionReport { public String timeStamp; public String detectionType; public float detectedValue; - public String detectorID; + public boolean detectionState; + //public String detectorID; - public DetectionReport(String detectorID, String detectionType, float detectedAmplitude) { + public DetectionReport(boolean detectionState, String detectionType, float detectedAmplitude) { this.timeStamp = String.valueOf(Calendar.getInstance().getTime()); this.detectionType = detectionType; this.detectedValue = detectedAmplitude; - this.detectorID = detectorID; + this.detectionState = detectionState; + + //this.detectorID = detectorID; } @@ -24,9 +27,9 @@ public class DetectionReport { String time = "Time: " + "[" + this.timeStamp + "]"; String type = "Type: " + "[" + this.detectionType + "]"; String value = "Value: " + "[" + this.detectedValue + "]"; - String id = "ID: " + "[" + this.detectorID + "]"; + //String id = "ID: " + "[" + this.detectorID + "]"; - return String.join("\t", time, type, value, id); + return String.join("\t", time, type, value); } /** Debug Report */ diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java index 0b726a7..38f55f0 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java @@ -20,16 +20,15 @@ abstract public class Detector { /** Triggers onDetectionListener - call this to trigger violation/alarm */ - public void reportViolation(String detectorID, String detectionType, float amplitude) { + public void reportViolation(String detectionType, float amplitude) { if (listener != null) { - DetectionReport detectionReport = new DetectionReport(detectorID, detectionType, amplitude); + DetectionReport detectionReport = new DetectionReport(true, detectionType, amplitude); listener.onDetection(detectionReport); } else { throw new IllegalStateException("No listener set for violation reporting"); } } - /** Starts Detection (abstract method: needs to be overridden in child class) */ public abstract void startDetection(); diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index 6362e9d..72a548c 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -66,7 +66,6 @@ public class VideoDetector extends Detector { private int frameCnt = 0; - // Parameters private static final float PIXEL_THRESHOLD = 30f; // Luminosity (brightness channel of YUV_420_888) private static final int BLUR_KERNEL_SIZE = 5; From 8551abfc5d1e12caba7c138a70fa51391b48dde3 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Wed, 14 Jun 2023 16:38:33 +0200 Subject: [PATCH 05/28] Changde Detector class --- .../Detection/DetectionReport.java | 5 +-- .../Detection/Detector.java | 41 +++++++++++++++++-- .../Detection/VideoDetector.java | 11 ++++- .../ueberwachungssystem/MainActivity.java | 2 + 4 files changed, 51 insertions(+), 8 deletions(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectionReport.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectionReport.java index 653e57b..96f9dbf 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectionReport.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectionReport.java @@ -10,7 +10,6 @@ public class DetectionReport { public String detectionType; public float detectedValue; public boolean detectionState; - //public String detectorID; public DetectionReport(boolean detectionState, String detectionType, float detectedAmplitude) { this.timeStamp = String.valueOf(Calendar.getInstance().getTime()); @@ -24,12 +23,12 @@ public class DetectionReport { /** Get Detection Report in String format */ public String toString() { + String state = "State: " + "[" + this.detectionState + "]"; String time = "Time: " + "[" + this.timeStamp + "]"; String type = "Type: " + "[" + this.detectionType + "]"; String value = "Value: " + "[" + this.detectedValue + "]"; - //String id = "ID: " + "[" + this.detectorID + "]"; - return String.join("\t", time, type, value); + return String.join("\t", state, time, type, value); } /** Debug Report */ diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java index 38f55f0..30ced55 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java @@ -1,10 +1,18 @@ package com.example.ueberwachungssystem.Detection; +import android.os.CountDownTimer; + import androidx.annotation.NonNull; abstract public class Detector { private OnDetectionListener listener; + private boolean isDetecting = false; + private boolean extendDetection = false; + + // Countdown parameters + private int COUNTDOWN_TIME = 1000; // milliseconds + private int COUNTDOWN_POLLING_TIME = 100; // milliseconds /** Constructor - takes context of current activity */ public Detector() {} @@ -18,17 +26,44 @@ abstract public class Detector { this.listener = listener; } - /** Triggers onDetectionListener - call this to trigger violation/alarm */ public void reportViolation(String detectionType, float amplitude) { if (listener != null) { - DetectionReport detectionReport = new DetectionReport(true, detectionType, amplitude); - listener.onDetection(detectionReport); + if (!isDetecting) { + isDetecting = true; + DetectionReport detectionReport = new DetectionReport(true, detectionType, amplitude); + listener.onDetection(detectionReport); + startDetectionTimer(detectionType, amplitude); + } else { + extendDetection = true; + } } else { + isDetecting = false; + extendDetection = false; throw new IllegalStateException("No listener set for violation reporting"); } } + private void startDetectionTimer(String detectionType, float amplitude) { + isDetecting = true; + new CountDownTimer((long) COUNTDOWN_TIME, COUNTDOWN_POLLING_TIME) { + @Override + public void onTick(long millisUntilFinished) { + if (extendDetection) { + extendDetection = false; + startDetectionTimer(detectionType, amplitude); + this.cancel(); + } + } + @Override + public void onFinish() { + isDetecting = false; + DetectionReport detectionReport = new DetectionReport(false, detectionType, amplitude); + listener.onDetection(detectionReport); + } + }.start(); + } + /** Starts Detection (abstract method: needs to be overridden in child class) */ public abstract void startDetection(); diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index 72a548c..3b99ef4 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -72,7 +72,7 @@ public class VideoDetector extends Detector { private static final int DILATE_KERNEL_SIZE = 5; private static final float CONTOUR_THRESHOLD = 100; private static final float ALARM_THRESHOLD = 0.2f; // Percent of pixels changed - private static final long START_DELAY = 1; // milliseconds + private static final long START_DELAY = 1000; // milliseconds private static final android.util.Size IMAGE_RES = new android.util.Size(640, 480); @@ -155,15 +155,22 @@ public class VideoDetector extends Detector { //mat = alternativeProcessImage(mat); mat = processImage(mat); debugMat(mat, imageView1); + + if (frameCnt <= 5) { + reportViolation("Video", 0); + frameCnt++; + } } imageProxy.close(); }); + + // Create Preview Preview preview = new Preview.Builder().build(); // Specify which Camera to use CameraSelector cameraSelector = new CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_BACK).build(); // Delay till start - new CountDownTimer((long)(START_DELAY * 1000), 1000){ + new CountDownTimer((long)(START_DELAY), 1000){ @Override public void onTick(long millisUntilFinished) {} @Override diff --git a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java index 9bfbed8..6206490 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java +++ b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java @@ -31,6 +31,8 @@ public class MainActivity extends AppCompatActivity { super.onCreate(savedInstanceState); setContentView(R.layout.activity_main); + getCameraAccess(); + ImageView imageView = findViewById(R.id.imageView); ImageView ogiv = findViewById(R.id.ogiv); From 50b579705e687a8e6edbcecf9340e59d35cc1568 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Wed, 14 Jun 2023 19:43:18 +0200 Subject: [PATCH 06/28] Detector based on OpenCV working --- .../Detection/Detector.java | 2 +- .../Detection/VideoDetector.java | 86 ++++++++----------- 2 files changed, 37 insertions(+), 51 deletions(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java index 30ced55..dc6b0cf 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java @@ -11,7 +11,7 @@ abstract public class Detector { private boolean extendDetection = false; // Countdown parameters - private int COUNTDOWN_TIME = 1000; // milliseconds + private int COUNTDOWN_TIME = 3000; // milliseconds private int COUNTDOWN_POLLING_TIME = 100; // milliseconds /** Constructor - takes context of current activity */ diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index 3b99ef4..d91c3c2 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -37,12 +37,8 @@ import java.util.concurrent.ExecutionException; /** * Video Detector inherits some methods from abstract Detector class (more info there) - * - * * USE FROM MAIN ACTIVITY: - * * VideoDetector vd = new VideoDetector(this); - * * */ @@ -56,22 +52,18 @@ public class VideoDetector extends Detector { private Boolean isDetectionRunning = false; // Detection private Mat previousImage = null; - private Mat lastThresh; - private Mat currentOut; public ImageView imageView1 = null; public ImageView imageView2 = null; - private int frameCnt = 0; - // Parameters - private static final float PIXEL_THRESHOLD = 30f; // Luminosity (brightness channel of YUV_420_888) + private static final float PIXEL_THRESHOLD = 40f; // Luminosity (brightness channel of YUV_420_888) private static final int BLUR_KERNEL_SIZE = 5; private static final int DILATE_KERNEL_SIZE = 5; - private static final float CONTOUR_THRESHOLD = 100; - private static final float ALARM_THRESHOLD = 0.2f; // Percent of pixels changed + private static final float CONTOUR_THRESHOLD = 250; + private static final float ALARM_THRESHOLD = 0.5f; // Percent of pixels changed private static final long START_DELAY = 1000; // milliseconds private static final android.util.Size IMAGE_RES = new android.util.Size(640, 480); @@ -87,7 +79,6 @@ public class VideoDetector extends Detector { this.context = context; } - /** * Starts the Video Detection * */ @@ -148,17 +139,23 @@ public class VideoDetector extends Detector { Image image = imageProxy.getImage(); assert image != null; - Mat mat = extractYChannel(image); + if (image != null) { - debugMat(mat, imageView2); + Mat mat = extractYChannel(image); - //mat = alternativeProcessImage(mat); - mat = processImage(mat); - debugMat(mat, imageView1); + debugMat(mat, imageView2); + mat = processImage(mat); + debugMat(mat, imageView1); - if (frameCnt <= 5) { - reportViolation("Video", 0); - frameCnt++; + int n = 0; + n = countNonZeroPixels(mat); + + int pixelCount = image.getWidth() * image.getHeight(); + float percentChanged = (float)n / pixelCount; + // report violation + if (percentChanged * 100 > ALARM_THRESHOLD) { + reportViolation("Video", n); + } } } imageProxy.close(); @@ -170,7 +167,7 @@ public class VideoDetector extends Detector { // Specify which Camera to use CameraSelector cameraSelector = new CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_BACK).build(); // Delay till start - new CountDownTimer((long)(START_DELAY), 1000){ + new CountDownTimer((long)(START_DELAY), 100){ @Override public void onTick(long millisUntilFinished) {} @Override @@ -180,6 +177,20 @@ public class VideoDetector extends Detector { }.start(); } + private Mat processImage(Mat image){ + if (previousImage == null) { + previousImage = image; + return null; + } + + Mat mat = addGaussianBlur(image, BLUR_KERNEL_SIZE); + mat = thresholdPixels(mat, previousImage, PIXEL_THRESHOLD); + mat = dilateNonZero(mat, DILATE_KERNEL_SIZE); + mat = thresholdContourArea(mat, CONTOUR_THRESHOLD); + + previousImage = image.clone(); + return mat; + } private Mat extractYChannel(@NonNull Image img) { ByteBuffer yBuffer = img.getPlanes()[0].getBuffer(); @@ -192,34 +203,6 @@ public class VideoDetector extends Detector { return yMat; } - private Mat processImage(Mat currentImage){ - if (previousImage == null) { - previousImage = currentImage; - return null; - } - - Mat mat = new Mat(); - currentImage = addGaussianBlur(currentImage, BLUR_KERNEL_SIZE); - - mat = thresholdPixels(currentImage, previousImage, PIXEL_THRESHOLD); - mat = dilateNonZero(mat, DILATE_KERNEL_SIZE); - mat = thresholdContourArea(mat, CONTOUR_THRESHOLD); - - previousImage = currentImage.clone(); - return mat; - } - - private Mat alternativeProcessImage(Mat currentImage){ - if (previousImage == null) { - previousImage = currentImage; - return null; - } else if (lastThresh == null) { - lastThresh = thresholdPixels(currentImage, previousImage, PIXEL_THRESHOLD); - return null; - } - return null; - } - private Mat thresholdPixels(Mat inputMat, Mat previousImage, float luminosityThreshold){ Mat diffImage = new Mat(); @@ -264,7 +247,10 @@ public class VideoDetector extends Detector { } private int countNonZeroPixels(Mat inputImage) { - return Core.countNonZero(inputImage); + if (inputImage != null) + return Core.countNonZero(inputImage); + else + return 0; } private void debugMat(Mat mat, ImageView imageView) { From 7ef0df48dd33996c7229c9a6c5e905d9ac2214da Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Wed, 14 Jun 2023 20:14:06 +0200 Subject: [PATCH 07/28] Refactored Detector --- .../Detection/VideoDetector.java | 93 +++++++++++++++++-- .../ueberwachungssystem/MainActivity.java | 24 ----- 2 files changed, 86 insertions(+), 31 deletions(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index d91c3c2..dd52256 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -1,6 +1,9 @@ package com.example.ueberwachungssystem.Detection; +import android.Manifest; +import android.app.Activity; import android.content.Context; +import android.content.pm.PackageManager; import android.graphics.Bitmap; import android.graphics.ImageFormat; import android.media.Image; @@ -14,6 +17,7 @@ import androidx.camera.core.ExperimentalGetImage; import androidx.camera.core.ImageAnalysis; import androidx.camera.core.Preview; import androidx.camera.lifecycle.ProcessCameraProvider; +import androidx.core.app.ActivityCompat; import androidx.core.content.ContextCompat; import androidx.lifecycle.LifecycleOwner; @@ -47,17 +51,20 @@ public class VideoDetector extends Detector { // Calling Activity private final Context context; + // Permission handling + private static final int CAMERA_PERMISSION_REQUEST_CODE = 3691; // Camera Provider private ProcessCameraProvider cameraProvider; - private Boolean isDetectionRunning = false; - // Detection - private Mat previousImage = null; + private boolean isDetectionRunning = false; + // Debugging + private Mat previousImage = null; public ImageView imageView1 = null; public ImageView imageView2 = null; + // Parameters private static final float PIXEL_THRESHOLD = 40f; // Luminosity (brightness channel of YUV_420_888) private static final int BLUR_KERNEL_SIZE = 5; @@ -79,6 +86,15 @@ public class VideoDetector extends Detector { this.context = context; } + + /** + * Get State of the Detector + */ + public boolean isRunning() { + return isDetectionRunning; + } + + /** * Starts the Video Detection * */ @@ -87,6 +103,10 @@ public class VideoDetector extends Detector { if (isDetectionRunning) return; + if (!isCameraAccessAllowed()){ + getCameraAccess(); + } + // Open CV startup check if (!OpenCVLoader.initDebug()) { Log.e("OpenCV", "Unable to load OpenCV!"); @@ -121,6 +141,19 @@ public class VideoDetector extends Detector { } + /** + * Permission handling + */ + private boolean isCameraAccessAllowed() { + return ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED; + } + + private void getCameraAccess() { + if (!isCameraAccessAllowed()) + ActivityCompat.requestPermissions((Activity) context, new String[]{android.Manifest.permission.CAMERA}, CAMERA_PERMISSION_REQUEST_CODE); + } + + /** * Binds the Luminosity Analyzer (configure and run Analysis) * @param cameraProvider: CameraProvider of Context passed by Constructor @@ -177,6 +210,12 @@ public class VideoDetector extends Detector { }.start(); } + + /** + * Process Image to be used for Motion Detection + * + * @param image: OpenCV Mat file that should be processed + */ private Mat processImage(Mat image){ if (previousImage == null) { previousImage = image; @@ -184,7 +223,7 @@ public class VideoDetector extends Detector { } Mat mat = addGaussianBlur(image, BLUR_KERNEL_SIZE); - mat = thresholdPixels(mat, previousImage, PIXEL_THRESHOLD); + mat = thresholdPixels(mat, previousImage); mat = dilateNonZero(mat, DILATE_KERNEL_SIZE); mat = thresholdContourArea(mat, CONTOUR_THRESHOLD); @@ -192,6 +231,12 @@ public class VideoDetector extends Detector { return mat; } + /** + * Convert Android Image to OpenCV Mat file with only y-Channel + * + * @param img: Android Image + * @return yChannel of Image in OpenCV Mat Format + */ private Mat extractYChannel(@NonNull Image img) { ByteBuffer yBuffer = img.getPlanes()[0].getBuffer(); byte[] yData = new byte[yBuffer.remaining()]; @@ -203,17 +248,28 @@ public class VideoDetector extends Detector { return yMat; } - - private Mat thresholdPixels(Mat inputMat, Mat previousImage, float luminosityThreshold){ + /** + * Threshold all Pixels + * @param inputMat: Input Image + * @param previousImage: previous Image (openCV Mat) + * @return Binary Mat Image + */ + private Mat thresholdPixels(Mat inputMat, Mat previousImage){ Mat diffImage = new Mat(); Core.absdiff(inputMat, previousImage, diffImage); Mat binaryMat = new Mat(); - Imgproc.threshold(diffImage, binaryMat, luminosityThreshold, 255, Imgproc.THRESH_BINARY); + Imgproc.threshold(diffImage, binaryMat, VideoDetector.PIXEL_THRESHOLD, 255, Imgproc.THRESH_BINARY); return binaryMat; } + /** + * Dilate Pixels of Binary Image + * @param inputMat: Input Image + * @param kernelSize: kernel Size + * @return binary Image with dilated Pixels + */ private Mat dilateNonZero(Mat inputMat, int kernelSize){ Mat dilatedMat = new Mat(); Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, new org.opencv.core.Size(kernelSize, kernelSize)); @@ -221,12 +277,25 @@ public class VideoDetector extends Detector { return dilatedMat; } + /** + * Add Gaussian Blur to OpenCV Mat Image + * @param inputMat: Input Image + * @param kernelSize: kernel Size + * @return Mat Image + */ private Mat addGaussianBlur(Mat inputMat, int kernelSize){ Mat outputMat = new Mat(); Imgproc.GaussianBlur(inputMat, outputMat, new org.opencv.core.Size(kernelSize, kernelSize), 0); return outputMat; } + + /** + * Filter Contour Areas by Size + * @param inputMat: Input Image + * @param areaThreshold: Threshold + * @return Binary Mat Image + */ private Mat thresholdContourArea(Mat inputMat, float areaThreshold){ List contours = new ArrayList<>(); Mat hierarchy = new Mat(); @@ -246,6 +315,11 @@ public class VideoDetector extends Detector { return outputMat; } + /** + * Count all Pixels that are non Zero + * @param inputImage: Input Image in OpenCV Mat Format + * @return Count of Pixels that are non zero + */ private int countNonZeroPixels(Mat inputImage) { if (inputImage != null) return Core.countNonZero(inputImage); @@ -253,6 +327,11 @@ public class VideoDetector extends Detector { return 0; } + /** + * Show OpenCV Mat on ImageView + * @param mat: current OpenCV Mat Image + * @param imageView: ImageView xml-Element + */ private void debugMat(Mat mat, ImageView imageView) { if (imageView == null || mat == null) return; diff --git a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java index 6206490..3efe3ab 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java +++ b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java @@ -31,7 +31,6 @@ public class MainActivity extends AppCompatActivity { super.onCreate(savedInstanceState); setContentView(R.layout.activity_main); - getCameraAccess(); ImageView imageView = findViewById(R.id.imageView); ImageView ogiv = findViewById(R.id.ogiv); @@ -48,27 +47,4 @@ public class MainActivity extends AppCompatActivity { }); vd.startDetection(); } - - private boolean isCameraAccessAllowed() { - return ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED; - } - - private void getCameraAccess() { - if (!isCameraAccessAllowed()) - ActivityCompat.requestPermissions(this, new String[]{android.Manifest.permission.CAMERA}, CAMERA_PERMISSION_REQUEST_CODE); - } - - @Override - public void onRequestPermissionsResult(int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) { - super.onRequestPermissionsResult(requestCode, permissions, grantResults); - - if (requestCode == CAMERA_PERMISSION_REQUEST_CODE && grantResults.length > 0) { - boolean cameraRights = grantResults[0] == PackageManager.PERMISSION_GRANTED; - if (cameraRights) { - Toast.makeText(this, "camera permission granted", Toast.LENGTH_LONG).show(); - } else { - Toast.makeText(this, "camera permission denied", Toast.LENGTH_LONG).show(); - } - } - } } \ No newline at end of file From fb8bdcd89504418e48d7a8a93f8679675bb15224 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Thu, 15 Jun 2023 12:40:54 +0200 Subject: [PATCH 08/28] Fixed up Video detector --- .../Detection/VideoDetector.java | 137 +++++++----------- 1 file changed, 52 insertions(+), 85 deletions(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index dd52256..17b0f13 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -30,6 +30,7 @@ import org.opencv.core.CvType; import org.opencv.core.Mat; import org.opencv.core.MatOfPoint; import org.opencv.core.Scalar; +import org.opencv.core.Size; import org.opencv.imgproc.Imgproc; import java.nio.ByteBuffer; @@ -66,10 +67,6 @@ public class VideoDetector extends Detector { // Parameters - private static final float PIXEL_THRESHOLD = 40f; // Luminosity (brightness channel of YUV_420_888) - private static final int BLUR_KERNEL_SIZE = 5; - private static final int DILATE_KERNEL_SIZE = 5; - private static final float CONTOUR_THRESHOLD = 250; private static final float ALARM_THRESHOLD = 0.5f; // Percent of pixels changed private static final long START_DELAY = 1000; // milliseconds private static final android.util.Size IMAGE_RES = new android.util.Size(640, 480); @@ -140,7 +137,6 @@ public class VideoDetector extends Detector { isDetectionRunning = false; } - /** * Permission handling */ @@ -172,29 +168,25 @@ public class VideoDetector extends Detector { Image image = imageProxy.getImage(); assert image != null; - if (image != null) { + Mat currentMat = extractYChannel(image); - Mat mat = extractYChannel(image); + Mat mat = currentMat.clone(); + debugMat(mat, imageView2); - debugMat(mat, imageView2); - mat = processImage(mat); - debugMat(mat, imageView1); + Mat processed = processImage(mat); - int n = 0; - n = countNonZeroPixels(mat); + debugMat(processed, imageView1); - int pixelCount = image.getWidth() * image.getHeight(); - float percentChanged = (float)n / pixelCount; - // report violation - if (percentChanged * 100 > ALARM_THRESHOLD) { - reportViolation("Video", n); - } + int n = countNonZeroPixels(processed); + int pixelCount = image.getWidth() * image.getHeight(); + float percentChanged = (float)n / pixelCount; + // report violation + if (percentChanged * 100 > ALARM_THRESHOLD) { + reportViolation("Video", n); } } imageProxy.close(); }); - - // Create Preview Preview preview = new Preview.Builder().build(); // Specify which Camera to use @@ -217,26 +209,42 @@ public class VideoDetector extends Detector { * @param image: OpenCV Mat file that should be processed */ private Mat processImage(Mat image){ + + // Preprocess Image + Mat preprocessed = image.clone(); + preprocessed = addGaussianBlur(preprocessed, new Size(21, 21)); + preprocessed = addBlur(preprocessed, new Size(3, 3)); + if (previousImage == null) { - previousImage = image; + previousImage = preprocessed; return null; } - Mat mat = addGaussianBlur(image, BLUR_KERNEL_SIZE); - mat = thresholdPixels(mat, previousImage); - mat = dilateNonZero(mat, DILATE_KERNEL_SIZE); - mat = thresholdContourArea(mat, CONTOUR_THRESHOLD); + // Process Image + Mat processed = preprocessed.clone(); + processed = thresholdPixels(processed, previousImage, 25); + processed = dilateBinaryMat(processed, new Size(3,3)); + processed = dilateBinaryMat(processed, new Size(3,3)); + processed = thresholdContourArea(processed, 500); - previousImage = image.clone(); - return mat; + previousImage = preprocessed.clone(); + return processed; + } + + + /** OpenCV helper methods **/ + private Mat addGaussianBlur(Mat inputMat, Size kernelSize){ + Mat outputMat = new Mat(); + Imgproc.GaussianBlur(inputMat, outputMat, kernelSize, 0); + return outputMat; + } + + private Mat addBlur(Mat inputMat, Size kernelSize){ + Mat outputMat = new Mat(); + Imgproc.blur(inputMat, outputMat, kernelSize); + return outputMat; } - /** - * Convert Android Image to OpenCV Mat file with only y-Channel - * - * @param img: Android Image - * @return yChannel of Image in OpenCV Mat Format - */ private Mat extractYChannel(@NonNull Image img) { ByteBuffer yBuffer = img.getPlanes()[0].getBuffer(); byte[] yData = new byte[yBuffer.remaining()]; @@ -248,54 +256,14 @@ public class VideoDetector extends Detector { return yMat; } - /** - * Threshold all Pixels - * @param inputMat: Input Image - * @param previousImage: previous Image (openCV Mat) - * @return Binary Mat Image - */ - private Mat thresholdPixels(Mat inputMat, Mat previousImage){ + private Mat thresholdPixels(Mat inputMat, Mat previousImage, int threshold){ Mat diffImage = new Mat(); Core.absdiff(inputMat, previousImage, diffImage); - Mat binaryMat = new Mat(); - Imgproc.threshold(diffImage, binaryMat, VideoDetector.PIXEL_THRESHOLD, 255, Imgproc.THRESH_BINARY); - + Imgproc.threshold(diffImage, binaryMat, threshold, 255, Imgproc.THRESH_BINARY); return binaryMat; } - /** - * Dilate Pixels of Binary Image - * @param inputMat: Input Image - * @param kernelSize: kernel Size - * @return binary Image with dilated Pixels - */ - private Mat dilateNonZero(Mat inputMat, int kernelSize){ - Mat dilatedMat = new Mat(); - Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, new org.opencv.core.Size(kernelSize, kernelSize)); - Imgproc.dilate(inputMat, dilatedMat, kernel); - return dilatedMat; - } - - /** - * Add Gaussian Blur to OpenCV Mat Image - * @param inputMat: Input Image - * @param kernelSize: kernel Size - * @return Mat Image - */ - private Mat addGaussianBlur(Mat inputMat, int kernelSize){ - Mat outputMat = new Mat(); - Imgproc.GaussianBlur(inputMat, outputMat, new org.opencv.core.Size(kernelSize, kernelSize), 0); - return outputMat; - } - - - /** - * Filter Contour Areas by Size - * @param inputMat: Input Image - * @param areaThreshold: Threshold - * @return Binary Mat Image - */ private Mat thresholdContourArea(Mat inputMat, float areaThreshold){ List contours = new ArrayList<>(); Mat hierarchy = new Mat(); @@ -315,11 +283,13 @@ public class VideoDetector extends Detector { return outputMat; } - /** - * Count all Pixels that are non Zero - * @param inputImage: Input Image in OpenCV Mat Format - * @return Count of Pixels that are non zero - */ + private Mat dilateBinaryMat(Mat inputMat, Size kernelSize){ + Mat dilatedMat = new Mat(); + Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, kernelSize); + Imgproc.dilate(inputMat, dilatedMat, kernel); + return dilatedMat; + } + private int countNonZeroPixels(Mat inputImage) { if (inputImage != null) return Core.countNonZero(inputImage); @@ -327,11 +297,8 @@ public class VideoDetector extends Detector { return 0; } - /** - * Show OpenCV Mat on ImageView - * @param mat: current OpenCV Mat Image - * @param imageView: ImageView xml-Element - */ + + private void debugMat(Mat mat, ImageView imageView) { if (imageView == null || mat == null) return; From d32e0a11f597a6f3eedb8448a0d1eeac5e5d4c51 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Sat, 17 Jun 2023 11:20:56 +0200 Subject: [PATCH 09/28] Fixed bug in Detector and VideoDetector --- app/src/main/AndroidManifest.xml | 2 + .../Detection/Detector.java | 5 +- .../Detection/VideoDetector.java | 320 ++++++++++-------- .../ueberwachungssystem/MainActivity.java | 32 +- app/src/main/res/layout/activity_main.xml | 10 +- 5 files changed, 211 insertions(+), 158 deletions(-) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 36b1460..b5bb284 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -5,6 +5,8 @@ + + cameraProviderFuture = ProcessCameraProvider.getInstance(context); //Check for Camera availability cameraProviderFuture.addListener(() -> { try { cameraProvider = cameraProviderFuture.get(); - bindAnalysis(cameraProvider); + bindCameraProvider(cameraProvider); isDetectionRunning = true; } catch (ExecutionException | InterruptedException e) { // No errors need to be handled for this Future. This should never be reached. } - },ContextCompat.getMainExecutor(context)); + }, ContextCompat.getMainExecutor(context)); } - /** - * Stops the Video Detection - * */ + /** Stops the Video Detection */ @Override public void stopDetection() { - if (!isDetectionRunning) + if (!isDetectionRunning || imageAnalysis == null) return; - cameraProvider.unbindAll(); + cameraProvider.unbind(imageAnalysis); isDetectionRunning = false; + allowReportViolation = false; } - /** - * Permission handling - */ + /** Permission handling */ private boolean isCameraAccessAllowed() { return ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED; } @@ -150,70 +150,88 @@ public class VideoDetector extends Detector { } - /** - * Binds the Luminosity Analyzer (configure and run Analysis) - * @param cameraProvider: CameraProvider of Context passed by Constructor - * */ - private void bindAnalysis(@NonNull ProcessCameraProvider cameraProvider) { + /** Binds the Luminosity Analyzer (configure and run Analysis) */ + private void bindCameraProvider(@NonNull ProcessCameraProvider cameraProvider) { + // Create Preview + //Preview preview = new Preview.Builder().build(); + // Specify which Camera to use + CameraSelector cameraSelector = new CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_BACK).build(); + + cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, imageAnalysis); + + // Delay till violation is allowed + startViolationTimer(); + } + + + /** Start delay until Violation Report is allowed */ + private void startViolationTimer() { + new CountDownTimer((long) (START_DELAY), 100) { + @Override + public void onTick(long millisUntilFinished) { + } + + @Override + public void onFinish() { + allowReportViolation = true; + } + }.start(); + } + + + private ImageAnalysis setupImageAnalysis() { // Configure and create Image Analysis ImageAnalysis.Builder builder = new ImageAnalysis.Builder(); builder.setTargetResolution(IMAGE_RES); builder.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST); builder.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_YUV_420_888); ImageAnalysis imageAnalysis = builder.build(); - // Set Analyzer imageAnalysis.setAnalyzer(ContextCompat.getMainExecutor(context), imageProxy -> { if (imageProxy.getFormat() == ImageFormat.YUV_420_888) { Image image = imageProxy.getImage(); assert image != null; - Mat currentMat = extractYChannel(image); - - Mat mat = currentMat.clone(); - debugMat(mat, imageView2); - - Mat processed = processImage(mat); - - debugMat(processed, imageView1); - - int n = countNonZeroPixels(processed); + // Violation Handling + Mat processed = processImage(image); + int n = OpenCVHelper.countNonZeroPixels(processed); int pixelCount = image.getWidth() * image.getHeight(); - float percentChanged = (float)n / pixelCount; - // report violation - if (percentChanged * 100 > ALARM_THRESHOLD) { - reportViolation("Video", n); + float percentChanged = (float) n / pixelCount; + + // Violation Condition + if (percentChanged * 100 > ALARM_THRESHOLD) { + if (allowReportViolation) + reportViolation("Video", n); } } imageProxy.close(); }); - // Create Preview - Preview preview = new Preview.Builder().build(); - // Specify which Camera to use - CameraSelector cameraSelector = new CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_BACK).build(); - // Delay till start - new CountDownTimer((long)(START_DELAY), 100){ - @Override - public void onTick(long millisUntilFinished) {} - @Override - public void onFinish() { - cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, imageAnalysis, preview); - } - }.start(); + return imageAnalysis; + } + + @SuppressLint("RestrictedApi") + private VideoCapture setupVideoCapture() { + return new VideoCapture.Builder() + .setVideoFrameRate(30) + .setTargetRotation(Surface.ROTATION_0) + .build(); } - /** - * Process Image to be used for Motion Detection - * - * @param image: OpenCV Mat file that should be processed - */ - private Mat processImage(Mat image){ + /** Process Image to be used for Motion Detection */ + private Mat processImage(Image image){ + + // Image Transformation + Mat imageMat = OpenCVHelper.extractYChannel(image); + + // Show Input Image + if (inputImageView != null) + OpenCVHelper.debugMat(imageMat, inputImageView); // Preprocess Image - Mat preprocessed = image.clone(); - preprocessed = addGaussianBlur(preprocessed, new Size(21, 21)); - preprocessed = addBlur(preprocessed, new Size(3, 3)); + Mat preprocessed = imageMat; + preprocessed = OpenCVHelper.addGaussianBlur(preprocessed, new Size(21, 21)); + preprocessed = OpenCVHelper.addBlur(preprocessed, new Size(3, 3)); if (previousImage == null) { previousImage = preprocessed; @@ -222,91 +240,111 @@ public class VideoDetector extends Detector { // Process Image Mat processed = preprocessed.clone(); - processed = thresholdPixels(processed, previousImage, 25); - processed = dilateBinaryMat(processed, new Size(3,3)); - processed = dilateBinaryMat(processed, new Size(3,3)); - processed = thresholdContourArea(processed, 500); + processed = OpenCVHelper.thresholdPixels(processed, previousImage, 25); + processed = OpenCVHelper.dilateBinaryMat(processed, new Size(3,3)); + processed = OpenCVHelper.dilateBinaryMat(processed, new Size(3,3)); + processed = OpenCVHelper.thresholdContourArea(processed, 500); + + // Output previousImage = preprocessed.clone(); + // Show Output Image + if (outputImageView != null) + OpenCVHelper.debugMat(processed, outputImageView); return processed; } - - /** OpenCV helper methods **/ - private Mat addGaussianBlur(Mat inputMat, Size kernelSize){ - Mat outputMat = new Mat(); - Imgproc.GaussianBlur(inputMat, outputMat, kernelSize, 0); - return outputMat; + public void debugProcessing(ImageView inputImageView, ImageView outputImageView){ + this.inputImageView = inputImageView; + this.outputImageView = outputImageView; } - private Mat addBlur(Mat inputMat, Size kernelSize){ - Mat outputMat = new Mat(); - Imgproc.blur(inputMat, outputMat, kernelSize); - return outputMat; - } - private Mat extractYChannel(@NonNull Image img) { - ByteBuffer yBuffer = img.getPlanes()[0].getBuffer(); - byte[] yData = new byte[yBuffer.remaining()]; - yBuffer.get(yData); - Mat yMat = new Mat(img.getHeight(), img.getWidth(), CvType.CV_8UC1); - yMat.put(0, 0, yData); - return yMat; - } - private Mat thresholdPixels(Mat inputMat, Mat previousImage, int threshold){ - Mat diffImage = new Mat(); - Core.absdiff(inputMat, previousImage, diffImage); - Mat binaryMat = new Mat(); - Imgproc.threshold(diffImage, binaryMat, threshold, 255, Imgproc.THRESH_BINARY); - return binaryMat; - } - private Mat thresholdContourArea(Mat inputMat, float areaThreshold){ - List contours = new ArrayList<>(); - Mat hierarchy = new Mat(); - Imgproc.findContours(inputMat, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE); - Mat outputMat = new Mat(inputMat.size(), inputMat.type(), new Scalar(0)); - // Iterate over the contours and draw only the larger contours on the outputMat - for (MatOfPoint contour : contours) { - double contourArea = Imgproc.contourArea(contour); - if (contourArea > areaThreshold) { - Imgproc.drawContours(outputMat, Collections.singletonList(contour), 0, new Scalar(255), -1); - } + + + private static class OpenCVHelper{ + private OpenCVHelper() {} + + /** OpenCV helper methods **/ + private static Mat addGaussianBlur(Mat inputMat, Size kernelSize){ + Mat outputMat = new Mat(); + Imgproc.GaussianBlur(inputMat, outputMat, kernelSize, 0); + return outputMat; } - // Apply the outputMat as a mask to the dilatedImage - Mat maskedImage = new Mat(); - inputMat.copyTo(maskedImage, outputMat); - return outputMat; - } - private Mat dilateBinaryMat(Mat inputMat, Size kernelSize){ - Mat dilatedMat = new Mat(); - Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, kernelSize); - Imgproc.dilate(inputMat, dilatedMat, kernel); - return dilatedMat; - } + private static Mat addBlur(Mat inputMat, Size kernelSize){ + Mat outputMat = new Mat(); + Imgproc.blur(inputMat, outputMat, kernelSize); + return outputMat; + } - private int countNonZeroPixels(Mat inputImage) { - if (inputImage != null) - return Core.countNonZero(inputImage); - else - return 0; - } + private static Mat extractYChannel(@NonNull Image img) { + ByteBuffer yBuffer = img.getPlanes()[0].getBuffer(); + byte[] yData = new byte[yBuffer.remaining()]; + yBuffer.get(yData); + + Mat yMat = new Mat(img.getHeight(), img.getWidth(), CvType.CV_8UC1); + yMat.put(0, 0, yData); + + return yMat; + } + + private static Mat thresholdPixels(Mat inputMat, Mat previousImage, int threshold){ + Mat diffImage = new Mat(); + Core.absdiff(inputMat, previousImage, diffImage); + Mat binaryMat = new Mat(); + Imgproc.threshold(diffImage, binaryMat, threshold, 255, Imgproc.THRESH_BINARY); + return binaryMat; + } + + private static Mat thresholdContourArea(Mat inputMat, float areaThreshold){ + List contours = new ArrayList<>(); + Mat hierarchy = new Mat(); + Imgproc.findContours(inputMat, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE); + + Mat outputMat = new Mat(inputMat.size(), inputMat.type(), new Scalar(0)); + // Iterate over the contours and draw only the larger contours on the outputMat + for (MatOfPoint contour : contours) { + double contourArea = Imgproc.contourArea(contour); + if (contourArea > areaThreshold) { + Imgproc.drawContours(outputMat, Collections.singletonList(contour), 0, new Scalar(255), -1); + } + } + // Apply the outputMat as a mask to the dilatedImage + Mat maskedImage = new Mat(); + inputMat.copyTo(maskedImage, outputMat); + return outputMat; + } + + private static Mat dilateBinaryMat(Mat inputMat, Size kernelSize){ + Mat dilatedMat = new Mat(); + Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, kernelSize); + Imgproc.dilate(inputMat, dilatedMat, kernel); + return dilatedMat; + } + + private static int countNonZeroPixels(Mat inputImage) { + if (inputImage != null) + return Core.countNonZero(inputImage); + else + return 0; + } + private static void debugMat(Mat mat, ImageView imageView) { + if (imageView == null || mat == null) + return; - private void debugMat(Mat mat, ImageView imageView) { - if (imageView == null || mat == null) - return; + Bitmap bitmap = Bitmap.createBitmap(mat.cols(), mat.rows(), Bitmap.Config.ARGB_8888); + Utils.matToBitmap(mat, bitmap); - Bitmap bitmap = Bitmap.createBitmap(mat.cols(), mat.rows(), Bitmap.Config.ARGB_8888); - Utils.matToBitmap(mat, bitmap); - - // Display the bitmap in an ImageView - imageView.setImageBitmap(bitmap); + // Display the bitmap in an ImageView + imageView.setImageBitmap(bitmap); + } } } \ No newline at end of file diff --git a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java index 3efe3ab..fb73468 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java +++ b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java @@ -2,18 +2,11 @@ package com.example.ueberwachungssystem; import androidx.annotation.NonNull; import androidx.appcompat.app.AppCompatActivity; -import androidx.camera.view.PreviewView; -import androidx.core.app.ActivityCompat; -import androidx.core.content.ContextCompat; import androidx.camera.core.ExperimentalGetImage; -import android.Manifest; -import android.content.pm.PackageManager; import android.os.Bundle; import android.view.View; import android.widget.ImageView; -import android.widget.TextView; -import android.widget.Toast; import android.widget.ToggleButton; import com.example.ueberwachungssystem.Detection.DetectionReport; @@ -32,19 +25,34 @@ public class MainActivity extends AppCompatActivity { setContentView(R.layout.activity_main); - ImageView imageView = findViewById(R.id.imageView); - ImageView ogiv = findViewById(R.id.ogiv); + ImageView inputImageView = findViewById(R.id.inputImageView); + ImageView outputImageView = findViewById(R.id.outputImageView); VideoDetector vd = new VideoDetector(this); - vd.imageView1 = imageView; - vd.imageView2 = ogiv; + vd.debugProcessing(inputImageView, outputImageView); vd.setOnDetectionListener(new Detector.OnDetectionListener(){ @Override public void onDetection(@NonNull DetectionReport detectionReport) { detectionReport.log("OnDetection"); } }); - vd.startDetection(); + + + + ToggleButton toggleButton = findViewById(R.id.toggleButton); + toggleButton.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View v) { + if (toggleButton.isChecked()) + { + vd.startDetection(); + } + else { + vd.stopDetection(); + } + } + }); + } } \ No newline at end of file diff --git a/app/src/main/res/layout/activity_main.xml b/app/src/main/res/layout/activity_main.xml index d613b0c..aa0ac96 100644 --- a/app/src/main/res/layout/activity_main.xml +++ b/app/src/main/res/layout/activity_main.xml @@ -16,14 +16,20 @@ android:layout_height="1dp" android:backgroundTint="@android:color/black"/> + + From 76442888f7c215c630d070675fe6c887d0c842a3 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Sat, 17 Jun 2023 19:44:08 +0200 Subject: [PATCH 10/28] Working Video Detector + Recorder WIP: Use Case logic --- app/src/main/AndroidManifest.xml | 1 - .../Detection/Detector.java | 4 + .../Detection/VideoDetector.java | 147 +++++++++++++----- .../ueberwachungssystem/MainActivity.java | 19 +-- app/src/main/res/layout/activity_main.xml | 2 +- 5 files changed, 121 insertions(+), 52 deletions(-) diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index b5bb284..3ea0c3b 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -4,7 +4,6 @@ - cameraProviderFuture = ProcessCameraProvider.getInstance(context); - //Check for Camera availability cameraProviderFuture.addListener(() -> { try { cameraProvider = cameraProviderFuture.get(); - bindCameraProvider(cameraProvider); - isDetectionRunning = true; - } catch (ExecutionException | InterruptedException e) { - // No errors need to be handled for this Future. This should never be reached. - } + isDetecting = true; + bindCameraProvider(UseCase.ImageAnalysis); + } catch (ExecutionException | InterruptedException e) {} }, ContextCompat.getMainExecutor(context)); } /** Stops the Video Detection */ @Override public void stopDetection() { - if (!isDetectionRunning || imageAnalysis == null) + if (!isDetecting || imageAnalysis == null) return; cameraProvider.unbind(imageAnalysis); - isDetectionRunning = false; + cameraProvider.unbind(preview); + isDetecting = false; allowReportViolation = false; } /** Permission handling */ - private boolean isCameraAccessAllowed() { - return ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED; + private boolean hasPermissions() { + return ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED && + ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED; + } + private void getPermissions() { + if (!hasPermissions()) + ActivityCompat.requestPermissions((Activity) context, new String[]{android.Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO}, PERMISSION_REQUEST_CODE); } - private void getCameraAccess() { - if (!isCameraAccessAllowed()) - ActivityCompat.requestPermissions((Activity) context, new String[]{android.Manifest.permission.CAMERA}, CAMERA_PERMISSION_REQUEST_CODE); - } /** Binds the Luminosity Analyzer (configure and run Analysis) */ - private void bindCameraProvider(@NonNull ProcessCameraProvider cameraProvider) { - // Create Preview - //Preview preview = new Preview.Builder().build(); + private void bindCameraProvider(UseCase useCase) { // Specify which Camera to use CameraSelector cameraSelector = new CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_BACK).build(); - - cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, imageAnalysis); - - // Delay till violation is allowed - startViolationTimer(); + if(useCase == UseCase.ImageAnalysis && !cameraProvider.isBound(videoCapture)) { + cameraProvider.unbindAll(); + cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, imageAnalysis, preview); + startViolationTimer(); + } + if(useCase == UseCase.VideoCapture) { + if(cameraProvider.isBound(imageAnalysis)) { + cameraProvider.unbindAll(); + cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, imageAnalysis, videoCapture); + } else { + cameraProvider.unbindAll(); + cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, videoCapture); + } + } } @@ -170,7 +192,6 @@ public class VideoDetector extends Detector { @Override public void onTick(long millisUntilFinished) { } - @Override public void onFinish() { allowReportViolation = true; @@ -212,15 +233,63 @@ public class VideoDetector extends Detector { @SuppressLint("RestrictedApi") private VideoCapture setupVideoCapture() { return new VideoCapture.Builder() - .setVideoFrameRate(30) .setTargetRotation(Surface.ROTATION_0) .build(); } + @SuppressLint("RestrictedApi") + public void startRecording() { + // Check States + if (isRecording){ + extendDetection(); + return; + } + + // Return On Request Permissions + if (!hasPermissions()) { + getPermissions(); + return; + } + + final ListenableFuture cameraProviderFuture = ProcessCameraProvider.getInstance(context); + cameraProviderFuture.addListener(() -> { + try { + cameraProvider = cameraProviderFuture.get(); + isRecording = true; + bindCameraProvider(UseCase.VideoCapture); + + File vidFile = new File(context.getFilesDir() + "/" + outputName); + if (ActivityCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) { + return; + } + videoCapture.startRecording( + new VideoCapture.OutputFileOptions.Builder(vidFile).build(), + context.getMainExecutor(), + new VideoCapture.OnVideoSavedCallback() { + @Override + public void onVideoSaved(@NonNull VideoCapture.OutputFileResults outputFileResults) { + Toast.makeText(context, "recording saved", Toast.LENGTH_SHORT).show(); + } + @Override + public void onError(int videoCaptureError, @NonNull String message, @Nullable Throwable cause) { + Toast.makeText(context, "recording failed", Toast.LENGTH_SHORT).show(); + } + } + ); + } catch (ExecutionException | InterruptedException e) {} + }, ContextCompat.getMainExecutor(context)); + } + + @SuppressLint("RestrictedApi") + public void stopRecording(){ + videoCapture.stopRecording(); + cameraProvider.unbind(videoCapture); + if (isDetecting) + bindCameraProvider(UseCase.ImageAnalysis); + } /** Process Image to be used for Motion Detection */ private Mat processImage(Image image){ - // Image Transformation Mat imageMat = OpenCVHelper.extractYChannel(image); @@ -245,7 +314,6 @@ public class VideoDetector extends Detector { processed = OpenCVHelper.dilateBinaryMat(processed, new Size(3,3)); processed = OpenCVHelper.thresholdContourArea(processed, 500); - // Output previousImage = preprocessed.clone(); // Show Output Image @@ -254,13 +322,16 @@ public class VideoDetector extends Detector { return processed; } - public void debugProcessing(ImageView inputImageView, ImageView outputImageView){ + public void debugProcessing(@NonNull ImageView inputImageView, @NonNull ImageView outputImageView){ this.inputImageView = inputImageView; this.outputImageView = outputImageView; } - - + public void setPreviewView(@NonNull PreviewView previewView) { + // Create Preview + if (this.preview != null) + this.preview.setSurfaceProvider(previewView.getSurfaceProvider()); + } diff --git a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java index fb73468..5e33822 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java +++ b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java @@ -3,6 +3,7 @@ package com.example.ueberwachungssystem; import androidx.annotation.NonNull; import androidx.appcompat.app.AppCompatActivity; import androidx.camera.core.ExperimentalGetImage; +import androidx.camera.view.PreviewView; import android.os.Bundle; import android.view.View; @@ -16,9 +17,6 @@ import com.example.ueberwachungssystem.Detection.VideoDetector; @ExperimentalGetImage public class MainActivity extends AppCompatActivity { - private static final int CAMERA_PERMISSION_REQUEST_CODE = 101; - - @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); @@ -27,16 +25,12 @@ public class MainActivity extends AppCompatActivity { ImageView inputImageView = findViewById(R.id.inputImageView); ImageView outputImageView = findViewById(R.id.outputImageView); + PreviewView previewView = findViewById(R.id.previewView); VideoDetector vd = new VideoDetector(this); + vd.setPreviewView(previewView); vd.debugProcessing(inputImageView, outputImageView); - vd.setOnDetectionListener(new Detector.OnDetectionListener(){ - @Override - public void onDetection(@NonNull DetectionReport detectionReport) { - detectionReport.log("OnDetection"); - } - }); @@ -46,13 +40,14 @@ public class MainActivity extends AppCompatActivity { public void onClick(View v) { if (toggleButton.isChecked()) { - vd.startDetection(); + //vd.startDetection(); + vd.startRecording(); } else { - vd.stopDetection(); + //vd.stopDetection(); + vd.stopRecording(); } } }); - } } \ No newline at end of file diff --git a/app/src/main/res/layout/activity_main.xml b/app/src/main/res/layout/activity_main.xml index aa0ac96..57571c5 100644 --- a/app/src/main/res/layout/activity_main.xml +++ b/app/src/main/res/layout/activity_main.xml @@ -13,7 +13,7 @@ Date: Sun, 18 Jun 2023 10:55:21 +0200 Subject: [PATCH 11/28] Removed Preview Use Case, Audio still not working --- .../Detection/Detector.java | 14 ++-- .../Detection/VideoDetector.java | 70 ++++++++++++------- .../ueberwachungssystem/MainActivity.java | 10 ++- 3 files changed, 61 insertions(+), 33 deletions(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java index 65eb239..fb8588b 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java @@ -8,7 +8,7 @@ import androidx.annotation.NonNull; abstract public class Detector { private OnDetectionListener listener; private boolean isDetecting = false; - private boolean extendDetection = false; + private boolean extendViolation = false; // Countdown parameters private final int COUNTDOWN_TIME = 5000; // milliseconds @@ -35,11 +35,11 @@ abstract public class Detector { listener.onDetection(detectionReport); startDetectionTimer(detectionType, amplitude); } else { - extendDetection = true; + extendViolation = true; } } else { isDetecting = false; - extendDetection = false; + extendViolation = false; } } @@ -48,8 +48,8 @@ abstract public class Detector { new CountDownTimer((long) COUNTDOWN_TIME, COUNTDOWN_POLLING_TIME) { @Override public void onTick(long millisUntilFinished) { - if (extendDetection) { - extendDetection = false; + if (extendViolation) { + extendViolation = false; startDetectionTimer(detectionType, amplitude); this.cancel(); } @@ -63,8 +63,8 @@ abstract public class Detector { }.start(); } - public void extendDetection(){ - this.extendDetection = true; + public void extendViolation(){ + this.extendViolation = true; } /** Starts Detection (abstract method: needs to be overridden in child class) */ diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index 10d321b..738f927 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -19,6 +19,7 @@ import androidx.annotation.Nullable; import androidx.camera.core.CameraSelector; import androidx.camera.core.ExperimentalGetImage; import androidx.camera.core.ImageAnalysis; +import androidx.camera.core.ImageProxy; import androidx.camera.core.Preview; import androidx.camera.core.VideoCapture; import androidx.camera.lifecycle.ProcessCameraProvider; @@ -140,6 +141,8 @@ public class VideoDetector extends Detector { bindCameraProvider(UseCase.ImageAnalysis); } catch (ExecutionException | InterruptedException e) {} }, ContextCompat.getMainExecutor(context)); + + startViolationTimer(); } /** Stops the Video Detection */ @@ -148,7 +151,6 @@ public class VideoDetector extends Detector { if (!isDetecting || imageAnalysis == null) return; cameraProvider.unbind(imageAnalysis); - cameraProvider.unbind(preview); isDetecting = false; allowReportViolation = false; } @@ -169,20 +171,8 @@ public class VideoDetector extends Detector { private void bindCameraProvider(UseCase useCase) { // Specify which Camera to use CameraSelector cameraSelector = new CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_BACK).build(); - if(useCase == UseCase.ImageAnalysis && !cameraProvider.isBound(videoCapture)) { - cameraProvider.unbindAll(); - cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, imageAnalysis, preview); - startViolationTimer(); - } - if(useCase == UseCase.VideoCapture) { - if(cameraProvider.isBound(imageAnalysis)) { - cameraProvider.unbindAll(); - cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, imageAnalysis, videoCapture); - } else { - cameraProvider.unbindAll(); - cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, videoCapture); - } - } + cameraProvider.unbindAll(); + cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, imageAnalysis, videoCapture); } @@ -214,7 +204,7 @@ public class VideoDetector extends Detector { assert image != null; // Violation Handling - Mat processed = processImage(image); + Mat processed = processImage(imageProxy); int n = OpenCVHelper.countNonZeroPixels(processed); int pixelCount = image.getWidth() * image.getHeight(); float percentChanged = (float) n / pixelCount; @@ -241,7 +231,7 @@ public class VideoDetector extends Detector { public void startRecording() { // Check States if (isRecording){ - extendDetection(); + extendViolation(); return; } @@ -268,10 +258,12 @@ public class VideoDetector extends Detector { new VideoCapture.OnVideoSavedCallback() { @Override public void onVideoSaved(@NonNull VideoCapture.OutputFileResults outputFileResults) { + isRecording = false; Toast.makeText(context, "recording saved", Toast.LENGTH_SHORT).show(); } @Override public void onError(int videoCaptureError, @NonNull String message, @Nullable Throwable cause) { + isRecording = false; Toast.makeText(context, "recording failed", Toast.LENGTH_SHORT).show(); } } @@ -284,14 +276,16 @@ public class VideoDetector extends Detector { public void stopRecording(){ videoCapture.stopRecording(); cameraProvider.unbind(videoCapture); - if (isDetecting) - bindCameraProvider(UseCase.ImageAnalysis); + isRecording = false; } /** Process Image to be used for Motion Detection */ - private Mat processImage(Image image){ + private Mat processImage(ImageProxy imageProxy){ + if (imageProxy == null) + return null; + // Image Transformation - Mat imageMat = OpenCVHelper.extractYChannel(image); + Mat imageMat = OpenCVHelper.extractYChannel(imageProxy); // Show Input Image if (inputImageView != null) @@ -335,9 +329,6 @@ public class VideoDetector extends Detector { - - - private static class OpenCVHelper{ private OpenCVHelper() {} @@ -354,7 +345,10 @@ public class VideoDetector extends Detector { return outputMat; } - private static Mat extractYChannel(@NonNull Image img) { + private static Mat extractYChannel(@NonNull ImageProxy imgProxy) { + Image img = imgProxy.getImage(); + + assert img != null; ByteBuffer yBuffer = img.getPlanes()[0].getBuffer(); byte[] yData = new byte[yBuffer.remaining()]; yBuffer.get(yData); @@ -373,6 +367,32 @@ public class VideoDetector extends Detector { return binaryMat; } + private static Mat imageProxyToGrayscaleMat(ImageProxy imageProxy) { + // Step 1: Extract the image data from ImageProxy + ImageProxy.PlaneProxy[] planes = imageProxy.getPlanes(); + ByteBuffer yBuffer = planes[0].getBuffer(); + byte[] yData = new byte[yBuffer.remaining()]; + yBuffer.get(yData); + + // Step 2: Convert the image data to NV21 format + int width = imageProxy.getWidth(); + int height = imageProxy.getHeight(); + byte[] nv21Data = new byte[width * height * 3 / 2]; + // Assuming the image format is YUV_420_888 + System.arraycopy(yData, 0, nv21Data, 0, yData.length); + for (int i = yData.length; i < nv21Data.length; i += 2) { + nv21Data[i] = yData[i + 1]; + nv21Data[i + 1] = yData[i]; + } + + // Step 3: Create a grayscale Mat from the NV21 data + Mat grayscaleMat = new Mat(height, width, CvType.CV_8UC1); + grayscaleMat.put(0, 0, nv21Data); + + return grayscaleMat; + } + + private static Mat thresholdContourArea(Mat inputMat, float areaThreshold){ List contours = new ArrayList<>(); Mat hierarchy = new Mat(); diff --git a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java index 5e33822..1bfd3d8 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java +++ b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java @@ -6,6 +6,7 @@ import androidx.camera.core.ExperimentalGetImage; import androidx.camera.view.PreviewView; import android.os.Bundle; +import android.util.Log; import android.view.View; import android.widget.ImageView; import android.widget.ToggleButton; @@ -29,8 +30,15 @@ public class MainActivity extends AppCompatActivity { VideoDetector vd = new VideoDetector(this); - vd.setPreviewView(previewView); + //vd.setPreviewView(previewView); vd.debugProcessing(inputImageView, outputImageView); + vd.setOnDetectionListener(new Detector.OnDetectionListener() { + @Override + public void onDetection(@NonNull DetectionReport detectionReport) { + Log.d("onDetection", detectionReport.toString()); + } + }); + vd.startDetection(); From 057c7fc1399ccfe3db86900485e2917936bf5cb2 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Sun, 18 Jun 2023 13:03:38 +0200 Subject: [PATCH 12/28] Added Audio Recorder --- .../Detection/AudioRecorder.java | 83 +++++++++++++++++++ 1 file changed, 83 insertions(+) create mode 100644 app/src/main/java/com/example/ueberwachungssystem/Detection/AudioRecorder.java diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/AudioRecorder.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/AudioRecorder.java new file mode 100644 index 0000000..0abe0f5 --- /dev/null +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/AudioRecorder.java @@ -0,0 +1,83 @@ +package com.example.ueberwachungssystem.Detection; + +import android.content.Context; +import android.media.MediaPlayer; +import android.media.MediaRecorder; + +import java.io.File; +import java.io.IOException; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; + +public class AudioRecorder { + private final Context context; + private MediaRecorder mediaRecorder = null; + private boolean isRecording = false; + private File outputDir; // Default: in app files directory + + + public AudioRecorder (Context context) { + this.context = context; + this.outputDir = context.getFilesDir(); + } + + public void startRecording() { + // Handle logic + if (outputDir == null) + return; + if (isRecording) + return; + isRecording = true; + + // Setup Audio Recorder for output Format: 3GP + mediaRecorder = new MediaRecorder(); + mediaRecorder.setAudioSource(MediaRecorder.AudioSource.MIC); + mediaRecorder.setOutputFormat(MediaRecorder.OutputFormat.THREE_GPP); + mediaRecorder.setOutputFile(outputDir + "/" + generateFileName() + ".3gp"); + mediaRecorder.setAudioEncoder(MediaRecorder.AudioEncoder.AMR_NB); + try { + mediaRecorder.prepare(); + } catch (IOException e) { + e.printStackTrace(); + } + mediaRecorder.start(); + } + + public void stopRecording() { + if (mediaRecorder != null) { + mediaRecorder.stop(); + mediaRecorder.reset(); + mediaRecorder.release(); + mediaRecorder = null; + isRecording = false; + } + } + + public boolean isRecording(){ + return isRecording; + } + + public void setOutputDir(File outputDir) { + this.outputDir = outputDir; + } + + private String generateFileName(){ + // Get the current timestamp + LocalDateTime currentTime = LocalDateTime.now(); + // Define the format for the timestamp + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss"); + // Return the timestamp as a string + return currentTime.format(formatter); + } + + public void playAudio() { + MediaPlayer mp = new MediaPlayer(); + try { + mp.setDataSource(context.getFilesDir() + "/audio.3gp"); + mp.prepare(); + mp.start(); + } catch (Exception e) { + e.printStackTrace(); + } + } +} From 6954d381430f44256d7de46bba1864a12acb9fde Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Sun, 18 Jun 2023 13:43:10 +0200 Subject: [PATCH 13/28] Moved OpenCV Helper to its own Java File --- .../Detection/Detector.java | 1 + .../Detection/OpenCVHelper.java | 109 +++++++++++++ .../Detection/VideoDetector.java | 143 +++--------------- .../ueberwachungssystem/MainActivity.java | 6 + 4 files changed, 133 insertions(+), 126 deletions(-) create mode 100644 app/src/main/java/com/example/ueberwachungssystem/Detection/OpenCVHelper.java diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java index fb8588b..c8ad62c 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java @@ -3,6 +3,7 @@ package com.example.ueberwachungssystem.Detection; import android.os.CountDownTimer; import androidx.annotation.NonNull; +import androidx.camera.core.ExperimentalGetImage; abstract public class Detector { diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/OpenCVHelper.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/OpenCVHelper.java new file mode 100644 index 0000000..7a6cb28 --- /dev/null +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/OpenCVHelper.java @@ -0,0 +1,109 @@ +package com.example.ueberwachungssystem.Detection; + +import android.graphics.Bitmap; +import android.media.Image; +import android.widget.ImageView; + +import androidx.annotation.NonNull; +import androidx.camera.core.ExperimentalGetImage; +import androidx.camera.core.ImageProxy; + +import org.opencv.android.Utils; +import org.opencv.core.Core; +import org.opencv.core.CvType; +import org.opencv.core.Mat; +import org.opencv.core.MatOfPoint; +import org.opencv.core.Scalar; +import org.opencv.core.Size; +import org.opencv.imgproc.Imgproc; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + + +@ExperimentalGetImage +public class OpenCVHelper { + + /** OpenCV helper methods **/ + public static Mat addGaussianBlur(Mat inputMat, Size kernelSize){ + Mat outputMat = new Mat(); + Imgproc.GaussianBlur(inputMat, outputMat, kernelSize, 0); + return outputMat; + } + + public static Mat addBlur(Mat inputMat, Size kernelSize){ + Mat outputMat = new Mat(); + Imgproc.blur(inputMat, outputMat, kernelSize); + return outputMat; + } + + public static Mat extractYChannel(@NonNull ImageProxy imgProxy) { + Image img = imgProxy.getImage(); + + assert img != null; + ByteBuffer yBuffer = img.getPlanes()[0].getBuffer(); + byte[] yData = new byte[yBuffer.remaining()]; + yBuffer.get(yData); + + Mat yMat = new Mat(img.getHeight(), img.getWidth(), CvType.CV_8UC1); + yMat.put(0, 0, yData); + + return yMat; + } + + public static Mat thresholdPixels(Mat inputMat, Mat previousImage, int threshold){ + Mat diffImage = new Mat(); + Core.absdiff(inputMat, previousImage, diffImage); + Mat binaryMat = new Mat(); + Imgproc.threshold(diffImage, binaryMat, threshold, 255, Imgproc.THRESH_BINARY); + return binaryMat; + } + + + public static Mat thresholdContourArea(Mat inputMat, float areaThreshold){ + List contours = new ArrayList<>(); + Mat hierarchy = new Mat(); + Imgproc.findContours(inputMat, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE); + + Mat outputMat = new Mat(inputMat.size(), inputMat.type(), new Scalar(0)); + // Iterate over the contours and draw only the larger contours on the outputMat + for (MatOfPoint contour : contours) { + double contourArea = Imgproc.contourArea(contour); + if (contourArea > areaThreshold) { + Imgproc.drawContours(outputMat, Collections.singletonList(contour), 0, new Scalar(255), -1); + } + } + // Apply the outputMat as a mask to the dilatedImage + Mat maskedImage = new Mat(); + inputMat.copyTo(maskedImage, outputMat); + return outputMat; + } + + public static Mat dilateBinaryMat(Mat inputMat, Size kernelSize){ + Mat dilatedMat = new Mat(); + Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, kernelSize); + Imgproc.dilate(inputMat, dilatedMat, kernel); + return dilatedMat; + } + + public static int countNonZeroPixels(Mat inputImage) { + if (inputImage != null) + return Core.countNonZero(inputImage); + else + return 0; + } + + + public static void debugMat(Mat mat, ImageView imageView) { + if (imageView == null || mat == null) + return; + + Bitmap bitmap = Bitmap.createBitmap(mat.cols(), mat.rows(), Bitmap.Config.ARGB_8888); + Utils.matToBitmap(mat, bitmap); + + // Display the bitmap in an ImageView + imageView.setImageBitmap(bitmap); + } +} diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index 738f927..17ed3b0 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -5,7 +5,6 @@ import android.annotation.SuppressLint; import android.app.Activity; import android.content.Context; import android.content.pm.PackageManager; -import android.graphics.Bitmap; import android.graphics.ImageFormat; import android.media.Image; import android.os.CountDownTimer; @@ -31,20 +30,12 @@ import androidx.lifecycle.LifecycleOwner; import com.google.common.util.concurrent.ListenableFuture; import org.opencv.android.OpenCVLoader; -import org.opencv.android.Utils; -import org.opencv.core.Core; -import org.opencv.core.CvType; import org.opencv.core.Mat; -import org.opencv.core.MatOfPoint; -import org.opencv.core.Scalar; import org.opencv.core.Size; -import org.opencv.imgproc.Imgproc; import java.io.File; -import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.concurrent.ExecutionException; @@ -109,10 +100,13 @@ public class VideoDetector extends Detector { this.preview = new Preview.Builder().build(); } - /** Get State of the Detector */ - public boolean isRunning() { + /** Get States */ + public boolean isDetecting() { return isDetecting; } + public boolean isRecording(){ + return isRecording; + } /** Starts the Video Detection */ @@ -190,6 +184,7 @@ public class VideoDetector extends Detector { } + /** Setup Use Cases */ private ImageAnalysis setupImageAnalysis() { // Configure and create Image Analysis ImageAnalysis.Builder builder = new ImageAnalysis.Builder(); @@ -234,7 +229,6 @@ public class VideoDetector extends Detector { extendViolation(); return; } - // Return On Request Permissions if (!hasPermissions()) { getPermissions(); @@ -248,7 +242,7 @@ public class VideoDetector extends Detector { isRecording = true; bindCameraProvider(UseCase.VideoCapture); - File vidFile = new File(context.getFilesDir() + "/" + outputName); + File vidFile = new File(context.getFilesDir() + "/" + generateFileName() + ".mp4"); if (ActivityCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) { return; } @@ -268,7 +262,7 @@ public class VideoDetector extends Detector { } } ); - } catch (ExecutionException | InterruptedException e) {} + } catch (ExecutionException | InterruptedException ignored) {} }, ContextCompat.getMainExecutor(context)); } @@ -327,115 +321,12 @@ public class VideoDetector extends Detector { this.preview.setSurfaceProvider(previewView.getSurfaceProvider()); } - - - private static class OpenCVHelper{ - private OpenCVHelper() {} - - /** OpenCV helper methods **/ - private static Mat addGaussianBlur(Mat inputMat, Size kernelSize){ - Mat outputMat = new Mat(); - Imgproc.GaussianBlur(inputMat, outputMat, kernelSize, 0); - return outputMat; - } - - private static Mat addBlur(Mat inputMat, Size kernelSize){ - Mat outputMat = new Mat(); - Imgproc.blur(inputMat, outputMat, kernelSize); - return outputMat; - } - - private static Mat extractYChannel(@NonNull ImageProxy imgProxy) { - Image img = imgProxy.getImage(); - - assert img != null; - ByteBuffer yBuffer = img.getPlanes()[0].getBuffer(); - byte[] yData = new byte[yBuffer.remaining()]; - yBuffer.get(yData); - - Mat yMat = new Mat(img.getHeight(), img.getWidth(), CvType.CV_8UC1); - yMat.put(0, 0, yData); - - return yMat; - } - - private static Mat thresholdPixels(Mat inputMat, Mat previousImage, int threshold){ - Mat diffImage = new Mat(); - Core.absdiff(inputMat, previousImage, diffImage); - Mat binaryMat = new Mat(); - Imgproc.threshold(diffImage, binaryMat, threshold, 255, Imgproc.THRESH_BINARY); - return binaryMat; - } - - private static Mat imageProxyToGrayscaleMat(ImageProxy imageProxy) { - // Step 1: Extract the image data from ImageProxy - ImageProxy.PlaneProxy[] planes = imageProxy.getPlanes(); - ByteBuffer yBuffer = planes[0].getBuffer(); - byte[] yData = new byte[yBuffer.remaining()]; - yBuffer.get(yData); - - // Step 2: Convert the image data to NV21 format - int width = imageProxy.getWidth(); - int height = imageProxy.getHeight(); - byte[] nv21Data = new byte[width * height * 3 / 2]; - // Assuming the image format is YUV_420_888 - System.arraycopy(yData, 0, nv21Data, 0, yData.length); - for (int i = yData.length; i < nv21Data.length; i += 2) { - nv21Data[i] = yData[i + 1]; - nv21Data[i + 1] = yData[i]; - } - - // Step 3: Create a grayscale Mat from the NV21 data - Mat grayscaleMat = new Mat(height, width, CvType.CV_8UC1); - grayscaleMat.put(0, 0, nv21Data); - - return grayscaleMat; - } - - - private static Mat thresholdContourArea(Mat inputMat, float areaThreshold){ - List contours = new ArrayList<>(); - Mat hierarchy = new Mat(); - Imgproc.findContours(inputMat, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE); - - Mat outputMat = new Mat(inputMat.size(), inputMat.type(), new Scalar(0)); - // Iterate over the contours and draw only the larger contours on the outputMat - for (MatOfPoint contour : contours) { - double contourArea = Imgproc.contourArea(contour); - if (contourArea > areaThreshold) { - Imgproc.drawContours(outputMat, Collections.singletonList(contour), 0, new Scalar(255), -1); - } - } - // Apply the outputMat as a mask to the dilatedImage - Mat maskedImage = new Mat(); - inputMat.copyTo(maskedImage, outputMat); - return outputMat; - } - - private static Mat dilateBinaryMat(Mat inputMat, Size kernelSize){ - Mat dilatedMat = new Mat(); - Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, kernelSize); - Imgproc.dilate(inputMat, dilatedMat, kernel); - return dilatedMat; - } - - private static int countNonZeroPixels(Mat inputImage) { - if (inputImage != null) - return Core.countNonZero(inputImage); - else - return 0; - } - - - private static void debugMat(Mat mat, ImageView imageView) { - if (imageView == null || mat == null) - return; - - Bitmap bitmap = Bitmap.createBitmap(mat.cols(), mat.rows(), Bitmap.Config.ARGB_8888); - Utils.matToBitmap(mat, bitmap); - - // Display the bitmap in an ImageView - imageView.setImageBitmap(bitmap); - } + private String generateFileName(){ + // Get the current timestamp + LocalDateTime currentTime = LocalDateTime.now(); + // Define the format for the timestamp + DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMdd_HHmmss"); + // Return the timestamp as a string + return currentTime.format(formatter); } } \ No newline at end of file diff --git a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java index 1bfd3d8..503c022 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java +++ b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java @@ -11,6 +11,7 @@ import android.view.View; import android.widget.ImageView; import android.widget.ToggleButton; +import com.example.ueberwachungssystem.Detection.AudioRecorder; import com.example.ueberwachungssystem.Detection.DetectionReport; import com.example.ueberwachungssystem.Detection.Detector; import com.example.ueberwachungssystem.Detection.VideoDetector; @@ -41,6 +42,9 @@ public class MainActivity extends AppCompatActivity { vd.startDetection(); + AudioRecorder audioRecorder = new AudioRecorder(this); + + ToggleButton toggleButton = findViewById(R.id.toggleButton); toggleButton.setOnClickListener(new View.OnClickListener() { @@ -50,10 +54,12 @@ public class MainActivity extends AppCompatActivity { { //vd.startDetection(); vd.startRecording(); + audioRecorder.startRecording(); } else { //vd.stopDetection(); vd.stopRecording(); + audioRecorder.stopRecording(); } } }); From 493fa8ca832f163e841d87e41191c2c362466eb4 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Sun, 18 Jun 2023 18:12:03 +0200 Subject: [PATCH 14/28] Fixed Grey Stripes and Rotation Problems --- .../Detection/VideoDetector.java | 31 +++++++++++++++---- .../ueberwachungssystem/MainActivity.java | 1 + app/src/main/res/layout/activity_main.xml | 7 +++-- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index 17ed3b0..3925962 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -6,15 +6,21 @@ import android.app.Activity; import android.content.Context; import android.content.pm.PackageManager; import android.graphics.ImageFormat; +import android.graphics.Rect; +import android.hardware.SensorManager; import android.media.Image; import android.os.CountDownTimer; import android.util.Log; +import android.view.Display; +import android.view.OrientationEventListener; import android.view.Surface; +import android.view.WindowManager; import android.widget.ImageView; import android.widget.Toast; import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import androidx.camera.core.AspectRatio; import androidx.camera.core.CameraSelector; import androidx.camera.core.ExperimentalGetImage; import androidx.camera.core.ImageAnalysis; @@ -57,8 +63,8 @@ public class VideoDetector extends Detector { // Camera Provider private ProcessCameraProvider cameraProvider; - private final ImageAnalysis imageAnalysis; - private final VideoCapture videoCapture; + private ImageAnalysis imageAnalysis; + private VideoCapture videoCapture; private final Preview preview; // Logic @@ -73,9 +79,7 @@ public class VideoDetector extends Detector { private ImageView inputImageView = null; private ImageView outputImageView = null; - - // Recording - private final String outputName = "video.mp4"; + private int rotation = 0; // Parameters @@ -120,6 +124,9 @@ public class VideoDetector extends Detector { getPermissions(); return; } + + imageAnalysis = setupImageAnalysis(); + // Open CV startup check if (!OpenCVLoader.initDebug()) { Log.e("OpenCV", "Unable to load OpenCV!"); @@ -191,6 +198,7 @@ public class VideoDetector extends Detector { builder.setTargetResolution(IMAGE_RES); builder.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST); builder.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_YUV_420_888); + builder.setTargetRotation(Surface.ROTATION_90); ImageAnalysis imageAnalysis = builder.build(); // Set Analyzer imageAnalysis.setAnalyzer(ContextCompat.getMainExecutor(context), imageProxy -> { @@ -200,6 +208,7 @@ public class VideoDetector extends Detector { // Violation Handling Mat processed = processImage(imageProxy); + int n = OpenCVHelper.countNonZeroPixels(processed); int pixelCount = image.getWidth() * image.getHeight(); float percentChanged = (float) n / pixelCount; @@ -217,8 +226,9 @@ public class VideoDetector extends Detector { @SuppressLint("RestrictedApi") private VideoCapture setupVideoCapture() { + int rotation = getRotation(); return new VideoCapture.Builder() - .setTargetRotation(Surface.ROTATION_0) + .setTargetRotation(rotation) .build(); } @@ -235,6 +245,8 @@ public class VideoDetector extends Detector { return; } + videoCapture = setupVideoCapture(); + final ListenableFuture cameraProviderFuture = ProcessCameraProvider.getInstance(context); cameraProviderFuture.addListener(() -> { try { @@ -307,6 +319,7 @@ public class VideoDetector extends Detector { // Show Output Image if (outputImageView != null) OpenCVHelper.debugMat(processed, outputImageView); + return processed; } @@ -329,4 +342,10 @@ public class VideoDetector extends Detector { // Return the timestamp as a string return currentTime.format(formatter); } + + private int getRotation() { + WindowManager windowManager = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE); + Display display = windowManager.getDefaultDisplay(); + return display.getRotation(); + } } \ No newline at end of file diff --git a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java index 503c022..72fbefe 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java +++ b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java @@ -53,6 +53,7 @@ public class MainActivity extends AppCompatActivity { if (toggleButton.isChecked()) { //vd.startDetection(); + vd.stopDetection(); vd.startRecording(); audioRecorder.startRecording(); } diff --git a/app/src/main/res/layout/activity_main.xml b/app/src/main/res/layout/activity_main.xml index 57571c5..c5abe55 100644 --- a/app/src/main/res/layout/activity_main.xml +++ b/app/src/main/res/layout/activity_main.xml @@ -24,15 +24,18 @@ + \ No newline at end of file From e088b0afa3d8c964f8fdad866389b41dce8dae03 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Sun, 18 Jun 2023 20:17:19 +0200 Subject: [PATCH 15/28] Working Recorder and Video Detector --- .../Detection/AudioRecorder.java | 11 - .../Detection/Detector.java | 2 +- .../Detection/VideoDetector.java | 214 +++++++++--------- .../ueberwachungssystem/MainActivity.java | 6 +- 4 files changed, 112 insertions(+), 121 deletions(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/AudioRecorder.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/AudioRecorder.java index 0abe0f5..e794c72 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/AudioRecorder.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/AudioRecorder.java @@ -69,15 +69,4 @@ public class AudioRecorder { // Return the timestamp as a string return currentTime.format(formatter); } - - public void playAudio() { - MediaPlayer mp = new MediaPlayer(); - try { - mp.setDataSource(context.getFilesDir() + "/audio.3gp"); - mp.prepare(); - mp.start(); - } catch (Exception e) { - e.printStackTrace(); - } - } } diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java index c8ad62c..826878b 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/Detector.java @@ -12,7 +12,7 @@ abstract public class Detector { private boolean extendViolation = false; // Countdown parameters - private final int COUNTDOWN_TIME = 5000; // milliseconds + private final int COUNTDOWN_TIME = 10000; // milliseconds private final int COUNTDOWN_POLLING_TIME = 100; // milliseconds /** Constructor - takes context of current activity */ diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index 3925962..9c0ebda 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -6,13 +6,10 @@ import android.app.Activity; import android.content.Context; import android.content.pm.PackageManager; import android.graphics.ImageFormat; -import android.graphics.Rect; -import android.hardware.SensorManager; import android.media.Image; import android.os.CountDownTimer; import android.util.Log; import android.view.Display; -import android.view.OrientationEventListener; import android.view.Surface; import android.view.WindowManager; import android.widget.ImageView; @@ -20,7 +17,6 @@ import android.widget.Toast; import androidx.annotation.NonNull; import androidx.annotation.Nullable; -import androidx.camera.core.AspectRatio; import androidx.camera.core.CameraSelector; import androidx.camera.core.ExperimentalGetImage; import androidx.camera.core.ImageAnalysis; @@ -54,7 +50,6 @@ import java.util.concurrent.ExecutionException; @ExperimentalGetImage public class VideoDetector extends Detector { - // Calling Activity private final Context context; @@ -65,7 +60,7 @@ public class VideoDetector extends Detector { private ProcessCameraProvider cameraProvider; private ImageAnalysis imageAnalysis; private VideoCapture videoCapture; - private final Preview preview; + //private Preview preview; // Logic private boolean isDetecting = false; @@ -79,21 +74,17 @@ public class VideoDetector extends Detector { private ImageView inputImageView = null; private ImageView outputImageView = null; + // Recorder + private File outputDir; // Default: in app files directory private int rotation = 0; // Parameters private static final float ALARM_THRESHOLD = 0.5f; // Percent of pixels changed - private static final long START_DELAY = 20000; // milliseconds + private static final float START_DELAY = 20000; // milliseconds private static final android.util.Size IMAGE_RES = new android.util.Size(640, 480); - private enum UseCase { - ImageAnalysis, - Preview, - VideoCapture - }; - /** Constructor */ public VideoDetector(Context context) { @@ -101,7 +92,8 @@ public class VideoDetector extends Detector { this.context = context; this.imageAnalysis = setupImageAnalysis(); this.videoCapture = setupVideoCapture(); - this.preview = new Preview.Builder().build(); + this.outputDir = context.getFilesDir(); + //this.preview = new Preview.Builder().build(); } /** Get States */ @@ -124,9 +116,8 @@ public class VideoDetector extends Detector { getPermissions(); return; } - + // Configure Image Analysis imageAnalysis = setupImageAnalysis(); - // Open CV startup check if (!OpenCVLoader.initDebug()) { Log.e("OpenCV", "Unable to load OpenCV!"); @@ -139,11 +130,58 @@ public class VideoDetector extends Detector { try { cameraProvider = cameraProviderFuture.get(); isDetecting = true; - bindCameraProvider(UseCase.ImageAnalysis); + bindCameraProvider(); } catch (ExecutionException | InterruptedException e) {} }, ContextCompat.getMainExecutor(context)); + // Disable Violation Calling for Setup Time + startViolationTimer(START_DELAY); + } - startViolationTimer(); + /** Starts the Recorder */ + @SuppressLint("RestrictedApi") + public void startRecording() { + // Check States + if (isRecording){ + extendViolation(); + return; + } + // Return On Request Permissions + if (!hasPermissions()) { + getPermissions(); + return; + } + + videoCapture = setupVideoCapture(); + + final ListenableFuture cameraProviderFuture = ProcessCameraProvider.getInstance(context); + cameraProviderFuture.addListener(() -> { + try { + cameraProvider = cameraProviderFuture.get(); + isRecording = true; + bindCameraProvider(); + + File vidFile = new File(context.getFilesDir() + "/" + generateFileName() + ".mp4"); + if (ActivityCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) { + return; + } + videoCapture.startRecording( + new VideoCapture.OutputFileOptions.Builder(vidFile).build(), + context.getMainExecutor(), + new VideoCapture.OnVideoSavedCallback() { + @Override + public void onVideoSaved(@NonNull VideoCapture.OutputFileResults outputFileResults) { + isRecording = false; + Toast.makeText(context, "recording saved", Toast.LENGTH_SHORT).show(); + } + @Override + public void onError(int videoCaptureError, @NonNull String message, @Nullable Throwable cause) { + isRecording = false; + Toast.makeText(context, "recording failed", Toast.LENGTH_SHORT).show(); + } + } + ); + } catch (ExecutionException | InterruptedException ignored) {} + }, ContextCompat.getMainExecutor(context)); } /** Stops the Video Detection */ @@ -156,41 +194,22 @@ public class VideoDetector extends Detector { allowReportViolation = false; } - /** Permission handling */ - private boolean hasPermissions() { - return ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED && - ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED; - } - private void getPermissions() { - if (!hasPermissions()) - ActivityCompat.requestPermissions((Activity) context, new String[]{android.Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO}, PERMISSION_REQUEST_CODE); + /** Stops the Recording */ + @SuppressLint("RestrictedApi") + public void stopRecording(){ + videoCapture.stopRecording(); + cameraProvider.unbind(videoCapture); + isRecording = false; } - - - /** Binds the Luminosity Analyzer (configure and run Analysis) */ - private void bindCameraProvider(UseCase useCase) { + /** Bind Camera Provider */ + private void bindCameraProvider() { // Specify which Camera to use CameraSelector cameraSelector = new CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_BACK).build(); cameraProvider.unbindAll(); cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, imageAnalysis, videoCapture); } - - /** Start delay until Violation Report is allowed */ - private void startViolationTimer() { - new CountDownTimer((long) (START_DELAY), 100) { - @Override - public void onTick(long millisUntilFinished) { - } - @Override - public void onFinish() { - allowReportViolation = true; - } - }.start(); - } - - /** Setup Use Cases */ private ImageAnalysis setupImageAnalysis() { // Configure and create Image Analysis @@ -226,114 +245,63 @@ public class VideoDetector extends Detector { @SuppressLint("RestrictedApi") private VideoCapture setupVideoCapture() { - int rotation = getRotation(); + int rotation = getDisplayRotation(); return new VideoCapture.Builder() .setTargetRotation(rotation) .build(); } - @SuppressLint("RestrictedApi") - public void startRecording() { - // Check States - if (isRecording){ - extendViolation(); - return; - } - // Return On Request Permissions - if (!hasPermissions()) { - getPermissions(); - return; - } - - videoCapture = setupVideoCapture(); - - final ListenableFuture cameraProviderFuture = ProcessCameraProvider.getInstance(context); - cameraProviderFuture.addListener(() -> { - try { - cameraProvider = cameraProviderFuture.get(); - isRecording = true; - bindCameraProvider(UseCase.VideoCapture); - - File vidFile = new File(context.getFilesDir() + "/" + generateFileName() + ".mp4"); - if (ActivityCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED) { - return; - } - videoCapture.startRecording( - new VideoCapture.OutputFileOptions.Builder(vidFile).build(), - context.getMainExecutor(), - new VideoCapture.OnVideoSavedCallback() { - @Override - public void onVideoSaved(@NonNull VideoCapture.OutputFileResults outputFileResults) { - isRecording = false; - Toast.makeText(context, "recording saved", Toast.LENGTH_SHORT).show(); - } - @Override - public void onError(int videoCaptureError, @NonNull String message, @Nullable Throwable cause) { - isRecording = false; - Toast.makeText(context, "recording failed", Toast.LENGTH_SHORT).show(); - } - } - ); - } catch (ExecutionException | InterruptedException ignored) {} - }, ContextCompat.getMainExecutor(context)); - } - - @SuppressLint("RestrictedApi") - public void stopRecording(){ - videoCapture.stopRecording(); - cameraProvider.unbind(videoCapture); - isRecording = false; - } - /** Process Image to be used for Motion Detection */ private Mat processImage(ImageProxy imageProxy){ if (imageProxy == null) return null; - // Image Transformation Mat imageMat = OpenCVHelper.extractYChannel(imageProxy); - // Show Input Image if (inputImageView != null) OpenCVHelper.debugMat(imageMat, inputImageView); - // Preprocess Image Mat preprocessed = imageMat; preprocessed = OpenCVHelper.addGaussianBlur(preprocessed, new Size(21, 21)); preprocessed = OpenCVHelper.addBlur(preprocessed, new Size(3, 3)); - + // Set Previous Image if (previousImage == null) { previousImage = preprocessed; return null; } - // Process Image Mat processed = preprocessed.clone(); processed = OpenCVHelper.thresholdPixels(processed, previousImage, 25); processed = OpenCVHelper.dilateBinaryMat(processed, new Size(3,3)); processed = OpenCVHelper.dilateBinaryMat(processed, new Size(3,3)); processed = OpenCVHelper.thresholdContourArea(processed, 500); - // Output previousImage = preprocessed.clone(); // Show Output Image if (outputImageView != null) OpenCVHelper.debugMat(processed, outputImageView); - return processed; } + + /** Debug input and result of processing */ public void debugProcessing(@NonNull ImageView inputImageView, @NonNull ImageView outputImageView){ this.inputImageView = inputImageView; this.outputImageView = outputImageView; } - public void setPreviewView(@NonNull PreviewView previewView) { + + /** + private void setPreviewView(@NonNull PreviewView previewView) { // Create Preview if (this.preview != null) this.preview.setSurfaceProvider(previewView.getSurfaceProvider()); } + */ + + + /** Generate File Name */ private String generateFileName(){ // Get the current timestamp LocalDateTime currentTime = LocalDateTime.now(); @@ -343,9 +311,39 @@ public class VideoDetector extends Detector { return currentTime.format(formatter); } - private int getRotation() { + + /** Get current Display Rotation */ + private int getDisplayRotation() { WindowManager windowManager = (WindowManager) context.getSystemService(Context.WINDOW_SERVICE); Display display = windowManager.getDefaultDisplay(); return display.getRotation(); } + + + /** Start delay until Violation Report is allowed */ + private void startViolationTimer(float setupTime) { + new CountDownTimer((long) (START_DELAY), 100) { + @Override + public void onTick(long millisUntilFinished) { + } + @Override + public void onFinish() { + allowReportViolation = true; + } + }.start(); + } + + /** Permission handling */ + private boolean hasPermissions() { + return ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED && + ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED; + } + private void getPermissions() { + if (!hasPermissions()) + ActivityCompat.requestPermissions((Activity) context, new String[]{android.Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO}, PERMISSION_REQUEST_CODE); + } + + public void setOutputDir(File outputDir) { + this.outputDir = outputDir; + } } \ No newline at end of file diff --git a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java index 72fbefe..a75571b 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java +++ b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java @@ -37,6 +37,10 @@ public class MainActivity extends AppCompatActivity { @Override public void onDetection(@NonNull DetectionReport detectionReport) { Log.d("onDetection", detectionReport.toString()); + if (detectionReport.detectionState) + vd.startRecording(); + else + vd.stopRecording(); } }); vd.startDetection(); @@ -54,7 +58,7 @@ public class MainActivity extends AppCompatActivity { { //vd.startDetection(); vd.stopDetection(); - vd.startRecording(); + //vd.startRecording(); audioRecorder.startRecording(); } else { From ab41217f0e5d25c31766de981961fe3e024948bb Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Mon, 19 Jun 2023 13:13:23 +0200 Subject: [PATCH 16/28] Added DetectionService --- app/build.gradle | 2 + app/src/main/AndroidManifest.xml | 1 + .../Detection/DetectorService.java | 134 ++++++++++++++++++ .../ueberwachungssystem/MainActivity.java | 65 +++++---- 4 files changed, 175 insertions(+), 27 deletions(-) create mode 100644 app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java diff --git a/app/build.gradle b/app/build.gradle index 02a2cc2..083ba82 100644 --- a/app/build.gradle +++ b/app/build.gradle @@ -42,6 +42,8 @@ dependencies { def opencv_version = "4.5.3.0" implementation "com.quickbirdstudios:opencv:${opencv_version}" + implementation "androidx.lifecycle:lifecycle-extensions:2.0.0" + // Required for CameraX def camerax_version = "1.2.2" diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml index 3ea0c3b..3fe9720 100644 --- a/app/src/main/AndroidManifest.xml +++ b/app/src/main/AndroidManifest.xml @@ -24,6 +24,7 @@ + \ No newline at end of file diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java new file mode 100644 index 0000000..8834374 --- /dev/null +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java @@ -0,0 +1,134 @@ +package com.example.ueberwachungssystem.Detection; + +import android.content.Intent; +import android.os.Binder; +import android.os.IBinder; +import android.util.Log; +import android.widget.ImageView; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.camera.core.ExperimentalGetImage; +import androidx.lifecycle.LifecycleService; + +@ExperimentalGetImage +public class DetectorService extends LifecycleService { + + public TestBinder testBinder = new TestBinder(); + private DetectorService.OnDetectionListener listener; + + VideoDetector videoDetector = null; + AudioRecorder audioRecorder = null; + + + @Override + public int onStartCommand(Intent intent, int flags, int startId) { + videoDetector = new VideoDetector(this); + videoDetector.setOnDetectionListener(new Detector.OnDetectionListener() { + @Override + public void onDetection(@NonNull DetectionReport detectionReport) { + passToServiceListener(detectionReport); + } + }); + + audioRecorder = new AudioRecorder(this); + return super.onStartCommand(intent, flags, startId); + } + + /** Service methods */ + public class TestBinder extends Binder { + public DetectorService getBoundService() { + // Return an instance of the TestService + return DetectorService.this; + } + } + @Nullable + @Override + public IBinder onBind(Intent intent) { + super.onBind(intent); + return testBinder; + } + + + /** Video Detection */ + public void startVideoDetection() { + if(videoDetector != null) + videoDetector.startDetection(); + } + public void stopVideoDetection() { + if(videoDetector != null) + videoDetector.stopDetection(); + } + public boolean isVideoDetectionRunning() { + if(videoDetector != null) + return videoDetector.isDetecting(); + return false; + } + public void debugVideoProcessing(ImageView input, ImageView output) { + if(videoDetector != null) + videoDetector.debugProcessing(input, output); + } + + /** Audio Detection */ + public void startAudioDetection() { + + } + public void stopAudioDetection() { + + } + + /** Motion Detection */ + public void startMotionDetection() { + + } + public void stopMotionDetection() { + + } + + /** Video Recording */ + public void startVideoRecording() { + if(videoDetector != null) + videoDetector.startRecording(); + } + public void stopVideoRecording() { + if(videoDetector != null) + videoDetector.stopRecording(); + } + public boolean isVideoRecordingRunning() { + if(videoDetector != null) + return videoDetector.isRecording(); + return false; + } + + /** Audio Recording */ + public void startAudioRecording() { + if(audioRecorder != null) + audioRecorder.startRecording(); + } + public void stopAudioRecording() { + if(audioRecorder != null) + audioRecorder.stopRecording(); + } + public boolean isAudioRecordingRunning() { + if(videoDetector != null) + return audioRecorder.isRecording(); + return false; + } + + + /** pass Detection Report to Service Detection Listener and trigger it */ + public void passToServiceListener(DetectionReport detectionReport) { + if (listener != null) { + listener.onDetection(detectionReport); + } + } + + + /** On Detection Listener - runs when violation is reported */ + public interface OnDetectionListener { + void onDetection(@NonNull DetectionReport detectionReport); + } + public void setOnDetectionListener(@NonNull DetectorService.OnDetectionListener listener) { + this.listener = listener; + } +} diff --git a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java index a75571b..cdecc38 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java +++ b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java @@ -5,48 +5,43 @@ import androidx.appcompat.app.AppCompatActivity; import androidx.camera.core.ExperimentalGetImage; import androidx.camera.view.PreviewView; +import android.content.ComponentName; +import android.content.Context; +import android.content.Intent; +import android.content.ServiceConnection; import android.os.Bundle; +import android.os.IBinder; import android.util.Log; import android.view.View; import android.widget.ImageView; import android.widget.ToggleButton; -import com.example.ueberwachungssystem.Detection.AudioRecorder; import com.example.ueberwachungssystem.Detection.DetectionReport; -import com.example.ueberwachungssystem.Detection.Detector; -import com.example.ueberwachungssystem.Detection.VideoDetector; +import com.example.ueberwachungssystem.Detection.DetectorService; @ExperimentalGetImage public class MainActivity extends AppCompatActivity { + private DetectorService detectorService = new DetectorService(); + private ImageView inputImageView; + private ImageView outputImageView; + @Override protected void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); setContentView(R.layout.activity_main); - ImageView inputImageView = findViewById(R.id.inputImageView); - ImageView outputImageView = findViewById(R.id.outputImageView); + inputImageView = findViewById(R.id.inputImageView); + outputImageView = findViewById(R.id.outputImageView); PreviewView previewView = findViewById(R.id.previewView); - VideoDetector vd = new VideoDetector(this); - //vd.setPreviewView(previewView); - vd.debugProcessing(inputImageView, outputImageView); - vd.setOnDetectionListener(new Detector.OnDetectionListener() { - @Override - public void onDetection(@NonNull DetectionReport detectionReport) { - Log.d("onDetection", detectionReport.toString()); - if (detectionReport.detectionState) - vd.startRecording(); - else - vd.stopRecording(); - } - }); - vd.startDetection(); + Intent serviceIntent = new Intent(this, DetectorService.class); + bindService(serviceIntent, serviceConnection, Context.BIND_AUTO_CREATE); + startService(serviceIntent); - AudioRecorder audioRecorder = new AudioRecorder(this); @@ -56,17 +51,33 @@ public class MainActivity extends AppCompatActivity { public void onClick(View v) { if (toggleButton.isChecked()) { - //vd.startDetection(); - vd.stopDetection(); - //vd.startRecording(); - audioRecorder.startRecording(); + detectorService.startVideoRecording(); + //detectorService.startAudioRecording(); } else { - //vd.stopDetection(); - vd.stopRecording(); - audioRecorder.stopRecording(); + //detectorService.stopAudioRecording(); + detectorService.stopVideoRecording(); } } }); } + + private ServiceConnection serviceConnection = new ServiceConnection() { + @Override + public void onServiceConnected(ComponentName name, IBinder service) { + DetectorService.TestBinder binder = (DetectorService.TestBinder) service; + detectorService = binder.getBoundService(); + + detectorService.startVideoDetection(); + detectorService.debugVideoProcessing(inputImageView, outputImageView); + detectorService.setOnDetectionListener(new DetectorService.OnDetectionListener() { + @Override + public void onDetection(@NonNull DetectionReport detectionReport) { + Log.d("onDetection", detectionReport.toString()); + } + }); + } + @Override + public void onServiceDisconnected(ComponentName name) {} + }; } \ No newline at end of file From 847e90f5acd77451e197a3d941499e730d864d13 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Mon, 19 Jun 2023 13:17:17 +0200 Subject: [PATCH 17/28] Fixed Bug in VideoDetector --- .../ueberwachungssystem/Detection/VideoDetector.java | 3 +++ .../com/example/ueberwachungssystem/MainActivity.java | 10 +++------- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index 9c0ebda..5f48f76 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -197,6 +197,9 @@ public class VideoDetector extends Detector { /** Stops the Recording */ @SuppressLint("RestrictedApi") public void stopRecording(){ + if(!isRecording) + return; + videoCapture.stopRecording(); cameraProvider.unbind(videoCapture); isRecording = false; diff --git a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java index a75571b..b45baa7 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java +++ b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java @@ -37,10 +37,6 @@ public class MainActivity extends AppCompatActivity { @Override public void onDetection(@NonNull DetectionReport detectionReport) { Log.d("onDetection", detectionReport.toString()); - if (detectionReport.detectionState) - vd.startRecording(); - else - vd.stopRecording(); } }); vd.startDetection(); @@ -57,14 +53,14 @@ public class MainActivity extends AppCompatActivity { if (toggleButton.isChecked()) { //vd.startDetection(); - vd.stopDetection(); + //vd.stopDetection(); //vd.startRecording(); - audioRecorder.startRecording(); + //audioRecorder.startRecording(); } else { //vd.stopDetection(); vd.stopRecording(); - audioRecorder.stopRecording(); + //audioRecorder.stopRecording(); } } }); From 9b2d57ad37b2b5b97b45fe804dc3a238ee64819d Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Mon, 19 Jun 2023 13:35:02 +0200 Subject: [PATCH 18/28] Refactor --- .../com/example/ueberwachungssystem/Detection/VideoDetector.java | 1 - 1 file changed, 1 deletion(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index 5f48f76..fc339cf 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -76,7 +76,6 @@ public class VideoDetector extends Detector { // Recorder private File outputDir; // Default: in app files directory - private int rotation = 0; // Parameters From 0a3600d0d6f6a43335956570c616c3f75234dfa5 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Mon, 19 Jun 2023 14:27:04 +0200 Subject: [PATCH 19/28] Refactor --- .../Detection/DetectorService.java | 18 ++++++++++++++++++ .../Detection/VideoDetector.java | 3 +-- .../ueberwachungssystem/MainActivity.java | 15 +++++++++++++++ 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java index 8834374..8f1bb82 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java @@ -1,6 +1,9 @@ package com.example.ueberwachungssystem.Detection; +import android.Manifest; +import android.app.Activity; import android.content.Intent; +import android.content.pm.PackageManager; import android.os.Binder; import android.os.IBinder; import android.util.Log; @@ -9,6 +12,8 @@ import android.widget.ImageView; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.camera.core.ExperimentalGetImage; +import androidx.core.app.ActivityCompat; +import androidx.core.content.ContextCompat; import androidx.lifecycle.LifecycleService; @ExperimentalGetImage @@ -16,6 +21,7 @@ public class DetectorService extends LifecycleService { public TestBinder testBinder = new TestBinder(); private DetectorService.OnDetectionListener listener; + private boolean isServiceRunning = false; VideoDetector videoDetector = null; AudioRecorder audioRecorder = null; @@ -23,6 +29,8 @@ public class DetectorService extends LifecycleService { @Override public int onStartCommand(Intent intent, int flags, int startId) { + if (isServiceRunning) + return START_NOT_STICKY; videoDetector = new VideoDetector(this); videoDetector.setOnDetectionListener(new Detector.OnDetectionListener() { @Override @@ -32,9 +40,19 @@ public class DetectorService extends LifecycleService { }); audioRecorder = new AudioRecorder(this); + + + isServiceRunning = true; + return super.onStartCommand(intent, flags, startId); } + @Override + public void onDestroy() { + super.onDestroy(); + isServiceRunning = false; + } + /** Service methods */ public class TestBinder extends Binder { public DetectorService getBoundService() { diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index 9c0ebda..5df58d6 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -76,13 +76,12 @@ public class VideoDetector extends Detector { // Recorder private File outputDir; // Default: in app files directory - private int rotation = 0; // Parameters private static final float ALARM_THRESHOLD = 0.5f; // Percent of pixels changed private static final float START_DELAY = 20000; // milliseconds - private static final android.util.Size IMAGE_RES = new android.util.Size(640, 480); + private static final android.util.Size IMAGE_RES = new android.util.Size(480, 360); diff --git a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java index cdecc38..c4ff843 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java +++ b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java @@ -4,11 +4,16 @@ import androidx.annotation.NonNull; import androidx.appcompat.app.AppCompatActivity; import androidx.camera.core.ExperimentalGetImage; import androidx.camera.view.PreviewView; +import androidx.core.app.ActivityCompat; +import androidx.core.content.ContextCompat; +import android.Manifest; +import android.app.Activity; import android.content.ComponentName; import android.content.Context; import android.content.Intent; import android.content.ServiceConnection; +import android.content.pm.PackageManager; import android.os.Bundle; import android.os.IBinder; import android.util.Log; @@ -80,4 +85,14 @@ public class MainActivity extends AppCompatActivity { @Override public void onServiceDisconnected(ComponentName name) {} }; + + + private boolean hasPermissions() { + return ContextCompat.checkSelfPermission(this, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED && + ContextCompat.checkSelfPermission(this, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED; + } + private void getPermissions() { + if (!hasPermissions()) + ActivityCompat.requestPermissions((Activity) this, new String[]{android.Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO}, 12345); + } } \ No newline at end of file From 4185cb5200a603c946422eb2f157cce28bf4d253 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Mon, 19 Jun 2023 14:34:07 +0200 Subject: [PATCH 20/28] Proper display of Toasts in Audio Recorder --- .../ueberwachungssystem/Detection/AudioRecorder.java | 2 ++ .../ueberwachungssystem/Detection/VideoDetector.java | 4 ++-- .../java/com/example/ueberwachungssystem/MainActivity.java | 6 +++--- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/AudioRecorder.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/AudioRecorder.java index e794c72..aa16846 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/AudioRecorder.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/AudioRecorder.java @@ -3,6 +3,7 @@ package com.example.ueberwachungssystem.Detection; import android.content.Context; import android.media.MediaPlayer; import android.media.MediaRecorder; +import android.widget.Toast; import java.io.File; import java.io.IOException; @@ -50,6 +51,7 @@ public class AudioRecorder { mediaRecorder.release(); mediaRecorder = null; isRecording = false; + Toast.makeText(context, "audio recording saved", Toast.LENGTH_SHORT).show(); } } diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index fc339cf..ec7350f 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -170,12 +170,12 @@ public class VideoDetector extends Detector { @Override public void onVideoSaved(@NonNull VideoCapture.OutputFileResults outputFileResults) { isRecording = false; - Toast.makeText(context, "recording saved", Toast.LENGTH_SHORT).show(); + Toast.makeText(context, "video recording saved", Toast.LENGTH_SHORT).show(); } @Override public void onError(int videoCaptureError, @NonNull String message, @Nullable Throwable cause) { isRecording = false; - Toast.makeText(context, "recording failed", Toast.LENGTH_SHORT).show(); + Toast.makeText(context, "video recording failed", Toast.LENGTH_SHORT).show(); } } ); diff --git a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java index b45baa7..8487e19 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java +++ b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java @@ -54,13 +54,13 @@ public class MainActivity extends AppCompatActivity { { //vd.startDetection(); //vd.stopDetection(); - //vd.startRecording(); - //audioRecorder.startRecording(); + vd.startRecording(); + audioRecorder.startRecording(); } else { //vd.stopDetection(); vd.stopRecording(); - //audioRecorder.stopRecording(); + audioRecorder.stopRecording(); } } }); From bc5227ba4f9770b384078d8614b962ca041a69d9 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Mon, 19 Jun 2023 14:37:49 +0200 Subject: [PATCH 21/28] Refactor of DetectorService --- .../Detection/DetectorService.java | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java index 8f1bb82..49b46c0 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java @@ -16,6 +16,8 @@ import androidx.core.app.ActivityCompat; import androidx.core.content.ContextCompat; import androidx.lifecycle.LifecycleService; +import java.io.File; + @ExperimentalGetImage public class DetectorService extends LifecycleService { @@ -117,6 +119,10 @@ public class DetectorService extends LifecycleService { return videoDetector.isRecording(); return false; } + public void setVideoRecordingDir(File outputDir) { + if (videoDetector != null) + videoDetector.setOutputDir(outputDir); + } /** Audio Recording */ public void startAudioRecording() { @@ -132,6 +138,12 @@ public class DetectorService extends LifecycleService { return audioRecorder.isRecording(); return false; } + public void setAudioRecordingDir(File outputDir) { + if (audioRecorder != null) + audioRecorder.setOutputDir(outputDir); + } + + /** pass Detection Report to Service Detection Listener and trigger it */ From 7dee1243788342386191a1415b06e207fa4c9ba2 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Mon, 19 Jun 2023 14:41:41 +0200 Subject: [PATCH 22/28] Refactor in DetectorService class --- .../Detection/DetectorService.java | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java index 49b46c0..6d30379 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java @@ -1,19 +1,13 @@ package com.example.ueberwachungssystem.Detection; -import android.Manifest; -import android.app.Activity; import android.content.Intent; -import android.content.pm.PackageManager; import android.os.Binder; import android.os.IBinder; -import android.util.Log; import android.widget.ImageView; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.camera.core.ExperimentalGetImage; -import androidx.core.app.ActivityCompat; -import androidx.core.content.ContextCompat; import androidx.lifecycle.LifecycleService; import java.io.File; @@ -21,7 +15,7 @@ import java.io.File; @ExperimentalGetImage public class DetectorService extends LifecycleService { - public TestBinder testBinder = new TestBinder(); + public ServiceBinder serviceBinder = new ServiceBinder(); private DetectorService.OnDetectionListener listener; private boolean isServiceRunning = false; @@ -56,7 +50,7 @@ public class DetectorService extends LifecycleService { } /** Service methods */ - public class TestBinder extends Binder { + public class ServiceBinder extends Binder { public DetectorService getBoundService() { // Return an instance of the TestService return DetectorService.this; @@ -66,7 +60,7 @@ public class DetectorService extends LifecycleService { @Override public IBinder onBind(Intent intent) { super.onBind(intent); - return testBinder; + return serviceBinder; } From ea0e88ed895ea44ba24bcea442cec63f63c8eeed Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Mon, 19 Jun 2023 14:51:04 +0200 Subject: [PATCH 23/28] Refactor in DetectorService class --- .../com/example/ueberwachungssystem/Detection/VideoDetector.java | 1 - 1 file changed, 1 deletion(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index 0392220..1f1a759 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -141,7 +141,6 @@ public class VideoDetector extends Detector { public void startRecording() { // Check States if (isRecording){ - extendViolation(); return; } // Return On Request Permissions From 5d31f0bb465fba4b73da19e2b425ad6cdd7f24dc Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Mon, 19 Jun 2023 15:47:40 +0200 Subject: [PATCH 24/28] Adapted Parameters --- .../Detection/VideoDetector.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index 1f1a759..51faec1 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -79,9 +79,11 @@ public class VideoDetector extends Detector { // Parameters - private static final float ALARM_THRESHOLD = 0.5f; // Percent of pixels changed + private static final float ALARM_THRESHOLD = 0f; // Percent of pixels changed + private static final float AREA_THRESHOLD = 10f; + private static final int DILATE_ITERATIONS = 2; private static final float START_DELAY = 20000; // milliseconds - private static final android.util.Size IMAGE_RES = new android.util.Size(480, 360); + private static final android.util.Size IMAGE_RES = new android.util.Size(640, 480); @@ -236,7 +238,7 @@ public class VideoDetector extends Detector { // Violation Condition if (percentChanged * 100 > ALARM_THRESHOLD) { if (allowReportViolation) - reportViolation("Video", n); + reportViolation("Video", percentChanged); } } imageProxy.close(); @@ -273,9 +275,11 @@ public class VideoDetector extends Detector { // Process Image Mat processed = preprocessed.clone(); processed = OpenCVHelper.thresholdPixels(processed, previousImage, 25); - processed = OpenCVHelper.dilateBinaryMat(processed, new Size(3,3)); - processed = OpenCVHelper.dilateBinaryMat(processed, new Size(3,3)); - processed = OpenCVHelper.thresholdContourArea(processed, 500); + + for(int i = 0; i < DILATE_ITERATIONS; i++) + processed = OpenCVHelper.dilateBinaryMat(processed, new Size(3,3)); + + processed = OpenCVHelper.thresholdContourArea(processed, AREA_THRESHOLD); // Output previousImage = preprocessed.clone(); // Show Output Image From bff6abbd29124f1243381b755bb30a580a571c1f Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Tue, 20 Jun 2023 11:00:26 +0200 Subject: [PATCH 25/28] Removed Permission Handling from Video Detector, Added PermissionHandler.java class --- .../Detection/VideoDetector.java | 21 ------ .../ueberwachungssystem/MainActivity.java | 69 +++++++++++-------- .../PermissionHandler.java | 35 ++++++++++ 3 files changed, 75 insertions(+), 50 deletions(-) create mode 100644 app/src/main/java/com/example/ueberwachungssystem/PermissionHandler.java diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index 51faec1..809ea92 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -112,11 +112,6 @@ public class VideoDetector extends Detector { // Check States if (isDetecting) return; - // Return On Request Permissions - if (!hasPermissions()) { - getPermissions(); - return; - } // Configure Image Analysis imageAnalysis = setupImageAnalysis(); // Open CV startup check @@ -145,11 +140,6 @@ public class VideoDetector extends Detector { if (isRecording){ return; } - // Return On Request Permissions - if (!hasPermissions()) { - getPermissions(); - return; - } videoCapture = setupVideoCapture(); @@ -324,7 +314,6 @@ public class VideoDetector extends Detector { return display.getRotation(); } - /** Start delay until Violation Report is allowed */ private void startViolationTimer(float setupTime) { new CountDownTimer((long) (START_DELAY), 100) { @@ -338,16 +327,6 @@ public class VideoDetector extends Detector { }.start(); } - /** Permission handling */ - private boolean hasPermissions() { - return ContextCompat.checkSelfPermission(context, Manifest.permission.CAMERA) == PackageManager.PERMISSION_GRANTED && - ContextCompat.checkSelfPermission(context, Manifest.permission.RECORD_AUDIO) == PackageManager.PERMISSION_GRANTED; - } - private void getPermissions() { - if (!hasPermissions()) - ActivityCompat.requestPermissions((Activity) context, new String[]{android.Manifest.permission.CAMERA, Manifest.permission.RECORD_AUDIO}, PERMISSION_REQUEST_CODE); - } - public void setOutputDir(File outputDir) { this.outputDir = outputDir; } diff --git a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java index 8487e19..18de78f 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java +++ b/app/src/main/java/com/example/ueberwachungssystem/MainActivity.java @@ -30,39 +30,50 @@ public class MainActivity extends AppCompatActivity { PreviewView previewView = findViewById(R.id.previewView); - VideoDetector vd = new VideoDetector(this); - //vd.setPreviewView(previewView); - vd.debugProcessing(inputImageView, outputImageView); - vd.setOnDetectionListener(new Detector.OnDetectionListener() { - @Override - public void onDetection(@NonNull DetectionReport detectionReport) { - Log.d("onDetection", detectionReport.toString()); - } - }); - vd.startDetection(); + PermissionHandler permissionHandler = new PermissionHandler(this); - AudioRecorder audioRecorder = new AudioRecorder(this); + permissionHandler.getPermissions(); - - - ToggleButton toggleButton = findViewById(R.id.toggleButton); - toggleButton.setOnClickListener(new View.OnClickListener() { - @Override - public void onClick(View v) { - if (toggleButton.isChecked()) - { - //vd.startDetection(); - //vd.stopDetection(); - vd.startRecording(); - audioRecorder.startRecording(); + if (permissionHandler.hasPermissions()) { + AudioRecorder audioRecorder = new AudioRecorder(this); + VideoDetector vd = new VideoDetector(this); + //vd.setPreviewView(previewView); + vd.debugProcessing(inputImageView, outputImageView); + vd.setOnDetectionListener(new Detector.OnDetectionListener() { + @Override + public void onDetection(@NonNull DetectionReport detectionReport) { + Log.d("onDetection", detectionReport.toString()); } - else { - //vd.stopDetection(); - vd.stopRecording(); - audioRecorder.stopRecording(); + }); + vd.startDetection(); + + ToggleButton toggleButton = findViewById(R.id.toggleButton); + toggleButton.setOnClickListener(new View.OnClickListener() { + @Override + public void onClick(View v) { + if (toggleButton.isChecked()) + { + //vd.startDetection(); + //vd.stopDetection(); + vd.startRecording(); + audioRecorder.startRecording(); + } + else { + //vd.stopDetection(); + vd.stopRecording(); + audioRecorder.stopRecording(); + } } - } - }); + }); + + + + + + + + } + } } \ No newline at end of file diff --git a/app/src/main/java/com/example/ueberwachungssystem/PermissionHandler.java b/app/src/main/java/com/example/ueberwachungssystem/PermissionHandler.java new file mode 100644 index 0000000..2b23123 --- /dev/null +++ b/app/src/main/java/com/example/ueberwachungssystem/PermissionHandler.java @@ -0,0 +1,35 @@ +package com.example.ueberwachungssystem; + + +import android.app.Activity; +import android.content.Context; +import android.content.pm.PackageManager; + +import androidx.core.app.ActivityCompat; +import androidx.core.content.ContextCompat; + +public class PermissionHandler { + private final Context context; + private static final int PERMISSION_REQUEST_CODE = 23409; + private static final String[] permissions = new String[]{ + android.Manifest.permission.CAMERA, + android.Manifest.permission.RECORD_AUDIO + }; + + public PermissionHandler(Context context) { + this.context = context; + } + + public boolean hasPermissions() { + boolean permissionState = true; + for (String permission: permissions) { + permissionState = permissionState && ContextCompat.checkSelfPermission(context, permission) == PackageManager.PERMISSION_GRANTED; + } + return permissionState; + } + + public void getPermissions() { + if (!hasPermissions()) + ActivityCompat.requestPermissions((Activity) context, permissions, PERMISSION_REQUEST_CODE); + } +} From aed079377f2e1472c9e8e4b27a7feca8cef9a9e5 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Tue, 20 Jun 2023 11:07:57 +0200 Subject: [PATCH 26/28] Refactor --- .../example/ueberwachungssystem/Detection/VideoDetector.java | 5 ----- 1 file changed, 5 deletions(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index 809ea92..7ef72a1 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -53,9 +53,6 @@ public class VideoDetector extends Detector { // Calling Activity private final Context context; - // Permission handling - private static final int PERMISSION_REQUEST_CODE = 3691; - // Camera Provider private ProcessCameraProvider cameraProvider; private ImageAnalysis imageAnalysis; @@ -285,7 +282,6 @@ public class VideoDetector extends Detector { this.outputImageView = outputImageView; } - /** private void setPreviewView(@NonNull PreviewView previewView) { // Create Preview @@ -295,7 +291,6 @@ public class VideoDetector extends Detector { */ - /** Generate File Name */ private String generateFileName(){ // Get the current timestamp From a493bafb7961bf75c19e963ae4b832182e15fce4 Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Tue, 20 Jun 2023 11:16:26 +0200 Subject: [PATCH 27/28] Refactor --- .../ueberwachungssystem/Detection/DetectorService.java | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java index 6d30379..3205133 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java @@ -34,12 +34,8 @@ public class DetectorService extends LifecycleService { passToServiceListener(detectionReport); } }); - audioRecorder = new AudioRecorder(this); - - isServiceRunning = true; - return super.onStartCommand(intent, flags, startId); } From e01e23eccd60add976cf07fa08be4ad603587c5f Mon Sep 17 00:00:00 2001 From: Bastian Kohler Date: Tue, 20 Jun 2023 14:29:10 +0200 Subject: [PATCH 28/28] Refactor --- .../ueberwachungssystem/Detection/DetectorService.java | 8 +++++++- .../ueberwachungssystem/Detection/VideoDetector.java | 1 - .../example/ueberwachungssystem/PermissionHandler.java | 8 ++++++++ 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java index 3205133..af2d440 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/DetectorService.java @@ -14,7 +14,6 @@ import java.io.File; @ExperimentalGetImage public class DetectorService extends LifecycleService { - public ServiceBinder serviceBinder = new ServiceBinder(); private DetectorService.OnDetectionListener listener; private boolean isServiceRunning = false; @@ -27,6 +26,9 @@ public class DetectorService extends LifecycleService { public int onStartCommand(Intent intent, int flags, int startId) { if (isServiceRunning) return START_NOT_STICKY; + + + // Setup Service classes: videoDetector = new VideoDetector(this); videoDetector.setOnDetectionListener(new Detector.OnDetectionListener() { @Override @@ -34,7 +36,11 @@ public class DetectorService extends LifecycleService { passToServiceListener(detectionReport); } }); + audioRecorder = new AudioRecorder(this); + + + isServiceRunning = true; return super.onStartCommand(intent, flags, startId); } diff --git a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java index 7ef72a1..9018ddf 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java +++ b/app/src/main/java/com/example/ueberwachungssystem/Detection/VideoDetector.java @@ -2,7 +2,6 @@ package com.example.ueberwachungssystem.Detection; import android.Manifest; import android.annotation.SuppressLint; -import android.app.Activity; import android.content.Context; import android.content.pm.PackageManager; import android.graphics.ImageFormat; diff --git a/app/src/main/java/com/example/ueberwachungssystem/PermissionHandler.java b/app/src/main/java/com/example/ueberwachungssystem/PermissionHandler.java index 2b23123..2ffcc72 100644 --- a/app/src/main/java/com/example/ueberwachungssystem/PermissionHandler.java +++ b/app/src/main/java/com/example/ueberwachungssystem/PermissionHandler.java @@ -4,6 +4,7 @@ package com.example.ueberwachungssystem; import android.app.Activity; import android.content.Context; import android.content.pm.PackageManager; +import android.widget.Toast; import androidx.core.app.ActivityCompat; import androidx.core.content.ContextCompat; @@ -32,4 +33,11 @@ public class PermissionHandler { if (!hasPermissions()) ActivityCompat.requestPermissions((Activity) context, permissions, PERMISSION_REQUEST_CODE); } + + public void showPermissionToast() { + if (hasPermissions()) + Toast.makeText(context, "permissions available", Toast.LENGTH_SHORT).show(); + else + Toast.makeText(context, "permissions missing", Toast.LENGTH_SHORT).show(); + } }