|
|
|
|
|
|
|
|
package com.example.ueberwachungssystem.Detection; |
|
|
package com.example.ueberwachungssystem.Detection; |
|
|
|
|
|
|
|
|
import android.content.Context; |
|
|
import android.content.Context; |
|
|
|
|
|
import android.graphics.Bitmap; |
|
|
import android.graphics.ImageFormat; |
|
|
import android.graphics.ImageFormat; |
|
|
import android.media.Image; |
|
|
import android.media.Image; |
|
|
import android.util.Size; |
|
|
|
|
|
|
|
|
import android.os.CountDownTimer; |
|
|
|
|
|
import android.util.Log; |
|
|
|
|
|
import android.widget.ImageView; |
|
|
|
|
|
|
|
|
import androidx.annotation.NonNull; |
|
|
import androidx.annotation.NonNull; |
|
|
import androidx.camera.core.CameraSelector; |
|
|
import androidx.camera.core.CameraSelector; |
|
|
|
|
|
|
|
|
import androidx.camera.core.ImageAnalysis; |
|
|
import androidx.camera.core.ImageAnalysis; |
|
|
import androidx.camera.core.Preview; |
|
|
import androidx.camera.core.Preview; |
|
|
import androidx.camera.lifecycle.ProcessCameraProvider; |
|
|
import androidx.camera.lifecycle.ProcessCameraProvider; |
|
|
import androidx.camera.view.PreviewView; |
|
|
|
|
|
import androidx.core.content.ContextCompat; |
|
|
import androidx.core.content.ContextCompat; |
|
|
import androidx.lifecycle.LifecycleOwner; |
|
|
import androidx.lifecycle.LifecycleOwner; |
|
|
|
|
|
|
|
|
import com.google.common.util.concurrent.ListenableFuture; |
|
|
import com.google.common.util.concurrent.ListenableFuture; |
|
|
|
|
|
|
|
|
|
|
|
import org.opencv.android.OpenCVLoader; |
|
|
|
|
|
import org.opencv.android.Utils; |
|
|
|
|
|
import org.opencv.core.Core; |
|
|
|
|
|
import org.opencv.core.CvType; |
|
|
|
|
|
import org.opencv.core.Mat; |
|
|
|
|
|
import org.opencv.core.MatOfPoint; |
|
|
|
|
|
import org.opencv.core.Scalar; |
|
|
|
|
|
import org.opencv.imgproc.Imgproc; |
|
|
|
|
|
|
|
|
import java.nio.ByteBuffer; |
|
|
import java.nio.ByteBuffer; |
|
|
|
|
|
import java.util.ArrayList; |
|
|
|
|
|
import java.util.Collections; |
|
|
|
|
|
import java.util.List; |
|
|
import java.util.concurrent.ExecutionException; |
|
|
import java.util.concurrent.ExecutionException; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
* USE FROM MAIN ACTIVITY: |
|
|
* USE FROM MAIN ACTIVITY: |
|
|
* |
|
|
* |
|
|
* VideoDetector vd = new VideoDetector(this); |
|
|
* VideoDetector vd = new VideoDetector(this); |
|
|
* vd.setPreview(previewView); //THIS IS OPTIONAL! |
|
|
|
|
|
* vd.setOnDetectionListener(new Detector.OnDetectionListener{...}); |
|
|
|
|
|
* vd.startDetection(); |
|
|
|
|
|
* vd.stopDetection |
|
|
|
|
|
* |
|
|
* |
|
|
* */ |
|
|
* */ |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Camera Provider |
|
|
// Camera Provider |
|
|
private ProcessCameraProvider cameraProvider; |
|
|
private ProcessCameraProvider cameraProvider; |
|
|
private Boolean isDetectionRunning = false; |
|
|
private Boolean isDetectionRunning = false; |
|
|
// Preview Camera Image |
|
|
|
|
|
private PreviewView previewView = null; |
|
|
|
|
|
// Detect Violation |
|
|
|
|
|
|
|
|
// Detection |
|
|
|
|
|
private Mat previousImage = null; |
|
|
|
|
|
private Mat lastThresh; |
|
|
|
|
|
private Mat currentOut; |
|
|
|
|
|
|
|
|
|
|
|
public ImageView imageView1 = null; |
|
|
|
|
|
public ImageView imageView2 = null; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private int frameCnt = 0; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Parameters |
|
|
private static final float PIXEL_THRESHOLD = 30f; // Luminosity (brightness channel of YUV_420_888) |
|
|
private static final float PIXEL_THRESHOLD = 30f; // Luminosity (brightness channel of YUV_420_888) |
|
|
|
|
|
private static final int BLUR_KERNEL_SIZE = 5; |
|
|
|
|
|
private static final int DILATE_KERNEL_SIZE = 5; |
|
|
|
|
|
private static final float CONTOUR_THRESHOLD = 100; |
|
|
private static final float ALARM_THRESHOLD = 0.2f; // Percent of pixels changed |
|
|
private static final float ALARM_THRESHOLD = 0.2f; // Percent of pixels changed |
|
|
private ByteBuffer previousBuffer = null; |
|
|
|
|
|
|
|
|
private static final long START_DELAY = 1; // milliseconds |
|
|
|
|
|
private static final android.util.Size IMAGE_RES = new android.util.Size(640, 480); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
public void startDetection() { |
|
|
public void startDetection() { |
|
|
if (isDetectionRunning) |
|
|
if (isDetectionRunning) |
|
|
return; |
|
|
return; |
|
|
|
|
|
|
|
|
|
|
|
// Open CV startup check |
|
|
|
|
|
if (!OpenCVLoader.initDebug()) { |
|
|
|
|
|
Log.e("OpenCV", "Unable to load OpenCV!"); |
|
|
|
|
|
return; |
|
|
|
|
|
} else |
|
|
|
|
|
Log.d("OpenCV", "OpenCV loaded Successfully!"); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
// Request Camera Provider |
|
|
// Request Camera Provider |
|
|
final ListenableFuture<ProcessCameraProvider> cameraProviderFuture = ProcessCameraProvider.getInstance(context); |
|
|
final ListenableFuture<ProcessCameraProvider> cameraProviderFuture = ProcessCameraProvider.getInstance(context); |
|
|
//Check for Camera availability |
|
|
//Check for Camera availability |
|
|
|
|
|
|
|
|
cameraProvider = cameraProviderFuture.get(); |
|
|
cameraProvider = cameraProviderFuture.get(); |
|
|
bindAnalysis(cameraProvider); |
|
|
bindAnalysis(cameraProvider); |
|
|
isDetectionRunning = true; |
|
|
isDetectionRunning = true; |
|
|
previousBuffer = null; |
|
|
|
|
|
} catch (ExecutionException | InterruptedException e) { |
|
|
} catch (ExecutionException | InterruptedException e) { |
|
|
// No errors need to be handled for this Future. This should never be reached. |
|
|
// No errors need to be handled for this Future. This should never be reached. |
|
|
} |
|
|
} |
|
|
},ContextCompat.getMainExecutor(context)); |
|
|
},ContextCompat.getMainExecutor(context)); |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
/** |
|
|
* Stops the Video Detection |
|
|
* Stops the Video Detection |
|
|
* */ |
|
|
* */ |
|
|
|
|
|
|
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
* Set PreviewView to show video feed while detecting |
|
|
|
|
|
* this is optional and does not need to be called |
|
|
|
|
|
* */ |
|
|
|
|
|
public void setPreviewView(PreviewView previewView) { |
|
|
|
|
|
this.previewView = previewView; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/** |
|
|
/** |
|
|
* Binds the Luminosity Analyzer (configure and run Analysis) |
|
|
* Binds the Luminosity Analyzer (configure and run Analysis) |
|
|
* @param cameraProvider: CameraProvider of Context passed by Constructor |
|
|
* @param cameraProvider: CameraProvider of Context passed by Constructor |
|
|
|
|
|
|
|
|
private void bindAnalysis(@NonNull ProcessCameraProvider cameraProvider) { |
|
|
private void bindAnalysis(@NonNull ProcessCameraProvider cameraProvider) { |
|
|
// Configure and create Image Analysis |
|
|
// Configure and create Image Analysis |
|
|
ImageAnalysis.Builder builder = new ImageAnalysis.Builder(); |
|
|
ImageAnalysis.Builder builder = new ImageAnalysis.Builder(); |
|
|
builder.setTargetResolution(new Size(640, 480)); |
|
|
|
|
|
|
|
|
builder.setTargetResolution(IMAGE_RES); |
|
|
builder.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST); |
|
|
builder.setBackpressureStrategy(ImageAnalysis.STRATEGY_KEEP_ONLY_LATEST); |
|
|
builder.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_YUV_420_888); |
|
|
builder.setOutputImageFormat(ImageAnalysis.OUTPUT_IMAGE_FORMAT_YUV_420_888); |
|
|
ImageAnalysis imageAnalysis = builder.build(); |
|
|
ImageAnalysis imageAnalysis = builder.build(); |
|
|
|
|
|
|
|
|
Image image = imageProxy.getImage(); |
|
|
Image image = imageProxy.getImage(); |
|
|
assert image != null; |
|
|
assert image != null; |
|
|
|
|
|
|
|
|
// Changed Pixel Detection |
|
|
|
|
|
int pixelChanged = getChangedPixelCount(image); |
|
|
|
|
|
int width = image.getWidth(); |
|
|
|
|
|
int height = image.getHeight(); |
|
|
|
|
|
|
|
|
Mat mat = extractYChannel(image); |
|
|
|
|
|
|
|
|
float percentPixelChanged = (float) 100f * pixelChanged / (width * height); |
|
|
|
|
|
|
|
|
debugMat(mat, imageView2); |
|
|
|
|
|
|
|
|
if (percentPixelChanged > ALARM_THRESHOLD) |
|
|
|
|
|
reportViolation("0", "Video", percentPixelChanged); |
|
|
|
|
|
|
|
|
//mat = alternativeProcessImage(mat); |
|
|
|
|
|
mat = processImage(mat); |
|
|
|
|
|
debugMat(mat, imageView1); |
|
|
} |
|
|
} |
|
|
imageProxy.close(); |
|
|
imageProxy.close(); |
|
|
}); |
|
|
}); |
|
|
|
|
|
|
|
|
Preview preview = new Preview.Builder().build(); |
|
|
Preview preview = new Preview.Builder().build(); |
|
|
// Specify which Camera to use |
|
|
// Specify which Camera to use |
|
|
CameraSelector cameraSelector = new CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_BACK).build(); |
|
|
CameraSelector cameraSelector = new CameraSelector.Builder().requireLensFacing(CameraSelector.LENS_FACING_BACK).build(); |
|
|
// Update PreviewView if set |
|
|
|
|
|
if (previewView != null) |
|
|
|
|
|
preview.setSurfaceProvider(previewView.getSurfaceProvider()); |
|
|
|
|
|
|
|
|
// Delay till start |
|
|
|
|
|
new CountDownTimer((long)(START_DELAY * 1000), 1000){ |
|
|
|
|
|
@Override |
|
|
|
|
|
public void onTick(long millisUntilFinished) {} |
|
|
|
|
|
@Override |
|
|
|
|
|
public void onFinish() { |
|
|
|
|
|
cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, imageAnalysis, preview); |
|
|
|
|
|
} |
|
|
|
|
|
}.start(); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
private Mat extractYChannel(@NonNull Image img) { |
|
|
|
|
|
ByteBuffer yBuffer = img.getPlanes()[0].getBuffer(); |
|
|
|
|
|
byte[] yData = new byte[yBuffer.remaining()]; |
|
|
|
|
|
yBuffer.get(yData); |
|
|
|
|
|
|
|
|
cameraProvider.bindToLifecycle((LifecycleOwner) context, cameraSelector, imageAnalysis, preview); |
|
|
|
|
|
|
|
|
Mat yMat = new Mat(img.getHeight(), img.getWidth(), CvType.CV_8UC1); |
|
|
|
|
|
yMat.put(0, 0, yData); |
|
|
|
|
|
|
|
|
|
|
|
return yMat; |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
private Mat processImage(Mat currentImage){ |
|
|
|
|
|
if (previousImage == null) { |
|
|
|
|
|
previousImage = currentImage; |
|
|
|
|
|
return null; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
Mat mat = new Mat(); |
|
|
|
|
|
currentImage = addGaussianBlur(currentImage, BLUR_KERNEL_SIZE); |
|
|
|
|
|
|
|
|
/** |
|
|
|
|
|
* Calculate Amount of Pixels changed using a threshold |
|
|
|
|
|
* @param image |
|
|
|
|
|
* */ |
|
|
|
|
|
private int getChangedPixelCount(Image image) { |
|
|
|
|
|
Image.Plane[] planes = image.getPlanes(); |
|
|
|
|
|
ByteBuffer buffer = planes[0].getBuffer(); |
|
|
|
|
|
|
|
|
|
|
|
if (previousBuffer == null) { |
|
|
|
|
|
previousBuffer = ByteBuffer.allocate(buffer.capacity()); |
|
|
|
|
|
buffer.rewind(); |
|
|
|
|
|
previousBuffer.put(buffer); |
|
|
|
|
|
previousBuffer.rewind(); |
|
|
|
|
|
return 0; |
|
|
|
|
|
|
|
|
mat = thresholdPixels(currentImage, previousImage, PIXEL_THRESHOLD); |
|
|
|
|
|
mat = dilateNonZero(mat, DILATE_KERNEL_SIZE); |
|
|
|
|
|
mat = thresholdContourArea(mat, CONTOUR_THRESHOLD); |
|
|
|
|
|
|
|
|
|
|
|
previousImage = currentImage.clone(); |
|
|
|
|
|
return mat; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
private Mat alternativeProcessImage(Mat currentImage){ |
|
|
|
|
|
if (previousImage == null) { |
|
|
|
|
|
previousImage = currentImage; |
|
|
|
|
|
return null; |
|
|
|
|
|
} else if (lastThresh == null) { |
|
|
|
|
|
lastThresh = thresholdPixels(currentImage, previousImage, PIXEL_THRESHOLD); |
|
|
|
|
|
return null; |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
int width = image.getWidth(); |
|
|
|
|
|
int height = image.getHeight(); |
|
|
|
|
|
|
|
|
while (frameCnt < 20){ |
|
|
|
|
|
Mat thresh = thresholdPixels(currentImage, previousImage, PIXEL_THRESHOLD); |
|
|
|
|
|
Core.bitwise_or(thresh, lastThresh, lastThresh); |
|
|
|
|
|
frameCnt++; |
|
|
|
|
|
return currentOut; |
|
|
|
|
|
} |
|
|
|
|
|
return lastThresh; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
int yRowStride = image.getPlanes()[0].getRowStride(); |
|
|
|
|
|
int yPixelStride = image.getPlanes()[0].getPixelStride(); |
|
|
|
|
|
|
|
|
|
|
|
int changedPixelCount = 0; |
|
|
|
|
|
|
|
|
private Mat thresholdPixels(Mat inputMat, Mat previousImage, float luminosityThreshold){ |
|
|
|
|
|
Mat diffImage = new Mat(); |
|
|
|
|
|
Core.absdiff(inputMat, previousImage, diffImage); |
|
|
|
|
|
|
|
|
// Count changed pixels |
|
|
|
|
|
for (int y = 0; y < height; ++y) { |
|
|
|
|
|
for (int x = 0; x < width; x++) { |
|
|
|
|
|
int index = (y * yRowStride) + (x * yPixelStride); |
|
|
|
|
|
int luminosity = (buffer.get(index) & 0xff); |
|
|
|
|
|
int previousLuminosity = (previousBuffer.get(index) & 0xff); |
|
|
|
|
|
int diff = Math.abs(luminosity - previousLuminosity); |
|
|
|
|
|
if (diff > PIXEL_THRESHOLD) |
|
|
|
|
|
changedPixelCount++; |
|
|
|
|
|
|
|
|
Mat binaryMat = new Mat(); |
|
|
|
|
|
Imgproc.threshold(diffImage, binaryMat, luminosityThreshold, 255, Imgproc.THRESH_BINARY); |
|
|
|
|
|
|
|
|
|
|
|
return binaryMat; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
private Mat dilateNonZero(Mat inputMat, int kernelSize){ |
|
|
|
|
|
Mat dilatedMat = new Mat(); |
|
|
|
|
|
Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_ELLIPSE, new org.opencv.core.Size(kernelSize, kernelSize)); |
|
|
|
|
|
Imgproc.dilate(inputMat, dilatedMat, kernel); |
|
|
|
|
|
return dilatedMat; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
private Mat addGaussianBlur(Mat inputMat, int kernelSize){ |
|
|
|
|
|
Mat outputMat = new Mat(); |
|
|
|
|
|
Imgproc.GaussianBlur(inputMat, outputMat, new org.opencv.core.Size(kernelSize, kernelSize), 0); |
|
|
|
|
|
return outputMat; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
private Mat thresholdContourArea(Mat inputMat, float areaThreshold){ |
|
|
|
|
|
List<MatOfPoint> contours = new ArrayList<>(); |
|
|
|
|
|
Mat hierarchy = new Mat(); |
|
|
|
|
|
Imgproc.findContours(inputMat, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE); |
|
|
|
|
|
|
|
|
|
|
|
Mat outputMat = new Mat(inputMat.size(), inputMat.type(), new Scalar(0)); |
|
|
|
|
|
// Iterate over the contours and draw only the larger contours on the outputMat |
|
|
|
|
|
for (MatOfPoint contour : contours) { |
|
|
|
|
|
double contourArea = Imgproc.contourArea(contour); |
|
|
|
|
|
if (contourArea > areaThreshold) { |
|
|
|
|
|
Imgproc.drawContours(outputMat, Collections.singletonList(contour), 0, new Scalar(255), -1); |
|
|
} |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
// Apply the outputMat as a mask to the dilatedImage |
|
|
|
|
|
Mat maskedImage = new Mat(); |
|
|
|
|
|
inputMat.copyTo(maskedImage, outputMat); |
|
|
|
|
|
return outputMat; |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
private int countNonZeroPixels(Mat inputImage) { |
|
|
|
|
|
return Core.countNonZero(inputImage); |
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
private void debugMat(Mat mat, ImageView imageView) { |
|
|
|
|
|
if (imageView == null || mat == null) |
|
|
|
|
|
return; |
|
|
|
|
|
|
|
|
// Reset and copy Byte Buffer |
|
|
|
|
|
buffer.rewind(); |
|
|
|
|
|
previousBuffer.rewind(); |
|
|
|
|
|
previousBuffer.put(buffer); |
|
|
|
|
|
|
|
|
Bitmap bitmap = Bitmap.createBitmap(mat.cols(), mat.rows(), Bitmap.Config.ARGB_8888); |
|
|
|
|
|
Utils.matToBitmap(mat, bitmap); |
|
|
|
|
|
|
|
|
return changedPixelCount; |
|
|
|
|
|
|
|
|
// Display the bitmap in an ImageView |
|
|
|
|
|
imageView.setImageBitmap(bitmap); |
|
|
} |
|
|
} |
|
|
} |
|
|
|
|
|
|
|
|
} |