From 42965a47335cfa6feb9d1abfb85feea9b2b0fdd8 Mon Sep 17 00:00:00 2001 From: TimoKurz Date: Thu, 19 Feb 2026 17:14:31 +0100 Subject: [PATCH] - added new files - implemented data base writing of AU and ET --- .../camera_stream_AU_and_ET_new.py | 372 ++++++++++++++++++ .../camera_handling/eyeFeature_new.py | 205 ++++++++++ 2 files changed, 577 insertions(+) create mode 100644 dataset_creation/camera_handling/camera_stream_AU_and_ET_new.py create mode 100644 dataset_creation/camera_handling/eyeFeature_new.py diff --git a/dataset_creation/camera_handling/camera_stream_AU_and_ET_new.py b/dataset_creation/camera_handling/camera_stream_AU_and_ET_new.py new file mode 100644 index 0000000..b2368ee --- /dev/null +++ b/dataset_creation/camera_handling/camera_stream_AU_and_ET_new.py @@ -0,0 +1,372 @@ +import warnings + +warnings.filterwarnings( + "ignore", + message=r".*SymbolDatabase\.GetPrototype\(\) is deprecated.*", + category=UserWarning, + module=r"google\.protobuf\.symbol_database" +) +import cv2 +import time +import os +import threading +from datetime import datetime +from feat import Detector +import torch +import mediapipe as mp +import pandas as pd +from pathlib import Path +from eyeFeature_new import compute_features_from_parquet + +# Import your helper functions +# from db_helper import connect_db, disconnect_db, insert_rows_into_table, create_table +import db_helper as db + +# Konfiguration +DB_PATH = Path("~/MSY_FS/databases/database.sqlite").expanduser() +CAMERA_INDEX = 0 +OUTPUT_DIR = "recordings" +VIDEO_DURATION = 50 # Sekunden +START_INTERVAL = 5 # Sekunden bis zum nächsten Start +FPS = 25.0 # Feste FPS + +eye_tracking_features = {} + +if not os.path.exists(OUTPUT_DIR): + os.makedirs(OUTPUT_DIR) + +# Globaler Detector, um ihn nicht bei jedem Video neu laden zu müssen (spart massiv Zeit/Speicher) +print("Initialisiere AU-Detector (bitte warten)...") +detector = Detector(au_model="xgb") + +# ===== MediaPipe FaceMesh Setup ===== +mp_face_mesh = mp.solutions.face_mesh +face_mesh = mp_face_mesh.FaceMesh( + static_image_mode=False, + max_num_faces=1, + refine_landmarks=True, # wichtig für Iris + min_detection_confidence=0.5, + min_tracking_confidence=0.5 +) + +LEFT_IRIS = [474, 475, 476, 477] +RIGHT_IRIS = [469, 470, 471, 472] + +LEFT_EYE_LIDS = (159, 145) +RIGHT_EYE_LIDS = (386, 374) + +EYE_OPEN_THRESHOLD = 6 + +LEFT_EYE_ALL = [33, 7, 163, 144, 145, 153, 154, 155, + 133, 173, 157, 158, 159, 160, 161, 246 +] + +RIGHT_EYE_ALL = [263, 249, 390, 373, 374, 380, 381, 382, + 362, 398, 384, 385, 386, 387, 388, 466 +] + + + +def eye_openness(landmarks, top_idx, bottom_idx, img_height): + top = landmarks[top_idx] + bottom = landmarks[bottom_idx] + return abs(top.y - bottom.y) * img_height + + +def compute_gaze(landmarks, iris_center, eye_indices, w, h): + iris_x, iris_y = iris_center + + eye_points = [] + for idx in eye_indices: + lm = landmarks[idx] + eye_points.append((lm.x * w, lm.y * h)) + + xs = [p[0] for p in eye_points] + ys = [p[1] for p in eye_points] + + eye_left = min(xs) + eye_right = max(xs) + eye_top = min(ys) + eye_bottom = max(ys) + + eye_width = eye_right - eye_left + eye_height = eye_bottom - eye_top + + if eye_width < 1 or eye_height < 1: + return 0.5, 0.5 + + gaze_x = (iris_x - eye_left) / eye_width + gaze_y = (iris_y - eye_top) / eye_height + + return gaze_x, gaze_y + +def extract_aus(path, skip_frames): + + # torch.no_grad() deaktiviert die Gradientenberechnung. + # Das löst den "Can't call numpy() on Tensor that requires grad" Fehler. + with torch.no_grad(): + video_prediction = detector.detect_video( + path, + skip_frames=skip_frames, + face_detection_threshold=0.95 + ) + + # Falls video_prediction oder .aus noch Tensoren sind, + # stellen wir sicher, dass sie korrekt summiert werden. + try: + # Wir nehmen die Summe der Action Units über alle detektierten Frames + res = video_prediction.aus.mean() + return res + except Exception as e: + print(f"Fehler bei der Summenbildung: {e}") + return None + +def startAU_creation(video_path, db_path): + """Diese Funktion läuft nun in einem eigenen Thread.""" + try: + print(f"\n[THREAD START] Analyse läuft für: {video_path}") + # skip_frames berechnen (z.B. alle 5 Sekunden bei 25 FPS = 125) + output = extract_aus(video_path, skip_frames=int(FPS*5)) + + print(f"\n--- Ergebnis für {os.path.basename(video_path)} ---") + print(output) + print("--------------------------------------------------\n") + if output is not None: + # Verbindung für diesen Thread öffnen (SQLite Sicherheit) + conn, cursor = db.connect_db(db_path) + + # Daten vorbereiten: Timestamp + AU Ergebnisse + # Wir wandeln die Series/Dataframe in ein Dictionary um + data_to_insert = output.to_dict() + + data_to_insert = { + f"FACE_{k}_mean": v for k, v in data_to_insert.items() + } + + now = datetime.now() + ticks = int(time.mktime(now.timetuple())) + + data_to_insert['start_time'] = [ticks] + data_to_insert = data_to_insert | eye_tracking_features + + #data_to_insert['start_time'] = [datetime.now().strftime("%Y-%m-%d %H:%M:%S")] + + # Da die AU-Spaltennamen dynamisch sind, stellen wir sicher, dass sie Listen sind + # (insert_rows_into_table erwartet Listen für jeden Key) + final_payload = {k: [v] if not isinstance(v, list) else v for k, v in data_to_insert.items()} + + + db.insert_rows_into_table(conn, cursor, "feature_table", final_payload) + + db.disconnect_db(conn, cursor) + print(f"--- Ergebnis für {os.path.basename(video_path)} in DB gespeichert ---") + os.remove(video_path) + os.remove(video_path.replace(".avi", "_gaze.parquet")) + print(f"Löschen der Datei: {video_path}") + except Exception as e: + print(f"Fehler bei der Analyse von {video_path}: {e}") + + +class VideoRecorder: + def __init__(self, filename, width, height, db_path): + self.gaze_data = [] + self.filename = filename + self.db_path = db_path + fourcc = cv2.VideoWriter_fourcc(*'XVID') + self.out = cv2.VideoWriter(filename, fourcc, FPS, (width, height)) + self.frames_to_record = int(VIDEO_DURATION * FPS) + self.frames_count = 0 + self.is_finished = False + + def write_frame(self, frame): + if self.frames_count < self.frames_to_record: + self.out.write(frame) + self.frames_count += 1 + else: + self.finish() + + def finish(self): + if not self.is_finished: + self.out.release() + self.is_finished = True + abs_path = os.path.abspath(self.filename) + print(f"Video fertig gespeichert: {self.filename}") + + # --- MULTITHREADING HIER --- + # Wir starten die Analyse in einem neuen Thread, damit main() sofort weiter frames lesen kann + analysis_thread = threading.Thread(target=startAU_creation, args=(abs_path, self.db_path)) + analysis_thread.daemon = True # Beendet sich, wenn das Hauptprogramm schließt + analysis_thread.start() + +class GazeRecorder: + def __init__(self, filename): + self.filename = filename + self.frames_to_record = int(VIDEO_DURATION * FPS) + self.frames_count = 0 + self.gaze_data = [] + self.is_finished = False + + def write_frame(self, gaze_row): + if self.frames_count < self.frames_to_record: + self.gaze_data.append(gaze_row) + self.frames_count += 1 + else: + self.finish() + + def finish(self): + if not self.is_finished: + df = pd.DataFrame(self.gaze_data) + df.to_parquet(self.filename, engine="pyarrow", index=False) + print(f"Gaze-Parquet gespeichert: {self.filename}") + features = compute_features_from_parquet(self.filename) + print("Features:", features) + self.is_finished = True + eye_tracking_features = features + +def main(): + cap = cv2.VideoCapture(CAMERA_INDEX) + if not cap.isOpened(): + print("Fehler: Kamera konnte nicht geöffnet werden.") + return + + width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)) + height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT)) + + active_video_recorders = [] + active_gaze_recorders = [] + last_start_time = 0 + + print("Aufnahme läuft. Drücke 'q' zum Beenden.") + + try: + while True: + ret, frame = cap.read() + if not ret: + break + + rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB) + h, w, _ = frame.shape + results = face_mesh.process(rgb) + + left_valid = 0 + right_valid = 0 + left_diameter = None + right_diameter = None + + left_gaze_x = None + left_gaze_y = None + right_gaze_x = None + right_gaze_y = None + + if results.multi_face_landmarks: + face_landmarks = results.multi_face_landmarks[0] + + left_open = eye_openness( + face_landmarks.landmark, + LEFT_EYE_LIDS[0], + LEFT_EYE_LIDS[1], + h + ) + + right_open = eye_openness( + face_landmarks.landmark, + RIGHT_EYE_LIDS[0], + RIGHT_EYE_LIDS[1], + h + ) + + left_valid = 1 if left_open > EYE_OPEN_THRESHOLD else 0 + right_valid = 1 if right_open > EYE_OPEN_THRESHOLD else 0 + + for eye_name, eye_indices in [("left", LEFT_IRIS), ("right", RIGHT_IRIS)]: + iris_points = [] + + for idx in eye_indices: + lm = face_landmarks.landmark[idx] + x_i, y_i = int(lm.x * w), int(lm.y * h) + iris_points.append((x_i, y_i)) + + if len(iris_points) == 4: + cx = int(sum(p[0] for p in iris_points) / 4) + cy = int(sum(p[1] for p in iris_points) / 4) + + radius = max( + ((x - cx) ** 2 + (y - cy) ** 2) ** 0.5 + for (x, y) in iris_points + ) + + diameter = 2 * radius + + cv2.circle(frame, (cx, cy), int(radius), (0, 255, 0), 2) + + if eye_name == "left" and left_valid: + left_diameter = diameter + left_gaze_x, left_gaze_y = compute_gaze( + face_landmarks.landmark, + (cx, cy), + RIGHT_EYE_ALL, + w, h + ) + + elif eye_name == "right" and right_valid: + right_diameter = diameter + right_gaze_x, right_gaze_y = compute_gaze( + face_landmarks.landmark, + (cx, cy), + LEFT_EYE_ALL, + w, h + ) + + gaze_row = { + "timestamp": time.time(), + "EYE_LEFT_GAZE_POINT_ON_DISPLAY_AREA_X": left_gaze_x, + "EYE_LEFT_GAZE_POINT_ON_DISPLAY_AREA_Y": left_gaze_y, + "EYE_RIGHT_GAZE_POINT_ON_DISPLAY_AREA_X": right_gaze_x, + "EYE_RIGHT_GAZE_POINT_ON_DISPLAY_AREA_Y": right_gaze_y, + "EYE_LEFT_PUPIL_VALIDITY": left_valid, + "EYE_RIGHT_PUPIL_VALIDITY": right_valid, + "EYE_LEFT_PUPIL_DIAMETER": left_diameter, + "EYE_RIGHT_PUPIL_DIAMETER": right_diameter + } + + current_time = time.time() + + if current_time - last_start_time >= START_INTERVAL: + timestamp = datetime.now().strftime("%H%M%S") + filename = os.path.join(OUTPUT_DIR, f"rec_{timestamp}.avi") + video_recorder = VideoRecorder(filename, width, height, DB_PATH) + + gaze_filename = filename.replace(".avi", "_gaze.parquet") + gaze_recorder = GazeRecorder(gaze_filename) + + active_video_recorders.append(video_recorder) + active_gaze_recorders.append(gaze_recorder) + + last_start_time = current_time + + for v_rec, g_rec in zip(active_video_recorders[:], active_gaze_recorders[:]): + + v_rec.write_frame(frame) + g_rec.write_frame(gaze_row) + + if v_rec.is_finished: + active_video_recorders.remove(v_rec) + + if g_rec.is_finished: + active_gaze_recorders.remove(g_rec) + + cv2.imshow('Kamera Livestream', frame) + if cv2.waitKey(1) & 0xFF == ord('q'): + break + + time.sleep(1/FPS) + + finally: + face_mesh.close() + cap.release() + cv2.destroyAllWindows() + + print("Programm beendet. Warte ggf. auf laufende Analysen...") + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/dataset_creation/camera_handling/eyeFeature_new.py b/dataset_creation/camera_handling/eyeFeature_new.py new file mode 100644 index 0000000..1df0580 --- /dev/null +++ b/dataset_creation/camera_handling/eyeFeature_new.py @@ -0,0 +1,205 @@ +import numpy as np +import pandas as pd +from pathlib import Path +from sklearn.preprocessing import MinMaxScaler +from scipy.signal import welch +from pygazeanalyser.detectors import fixation_detection, saccade_detection + + +############################################################################## +# KONFIGURATION +############################################################################## + +SAMPLING_RATE = 25 # Hz +MIN_DUR_BLINKS = 2 # x * 40ms + + +############################################################################## +# EYE-TRACKING FUNKTIONEN +############################################################################## + +def clean_eye_df(df): + """Extrahiert nur Eye-Tracking Spalten und entfernt leere Zeilen.""" + eye_cols = [c for c in df.columns if c.startswith("EYE_")] + + if not eye_cols: + return pd.DataFrame() + + df_eye = df[eye_cols].copy() + df_eye = df_eye.replace([np.inf, -np.inf], np.nan) + df_eye = df_eye.dropna(subset=eye_cols, how="all") + + return df_eye.reset_index(drop=True) + + +def extract_gaze_signal(df): + """Extrahiert 2D-Gaze-Positionen, maskiert ungültige Samples und interpoliert.""" + gx_L = df["EYE_LEFT_GAZE_POINT_ON_DISPLAY_AREA_X"].astype(float).copy() + gy_L = df["EYE_LEFT_GAZE_POINT_ON_DISPLAY_AREA_Y"].astype(float).copy() + gx_R = df["EYE_RIGHT_GAZE_POINT_ON_DISPLAY_AREA_X"].astype(float).copy() + gy_R = df["EYE_RIGHT_GAZE_POINT_ON_DISPLAY_AREA_Y"].astype(float).copy() + + val_L = (df["EYE_LEFT_PUPIL_VALIDITY"] == 1) + val_R = (df["EYE_RIGHT_PUPIL_VALIDITY"] == 1) + + # Inf → NaN + for arr in [gx_L, gy_L, gx_R, gy_R]: + arr.replace([np.inf, -np.inf], np.nan, inplace=True) + + # Ungültige maskieren + gx_L[~val_L] = np.nan + gy_L[~val_L] = np.nan + gx_R[~val_R] = np.nan + gy_R[~val_R] = np.nan + + + # Mittelwert beider Augen + gx = np.mean(np.column_stack([gx_L, gx_R]), axis=1) + gy = np.mean(np.column_stack([gy_L, gy_R]), axis=1) + + # Interpolation + gx = pd.Series(gx).interpolate(limit=None, limit_direction="both").bfill().ffill() + gy = pd.Series(gy).interpolate(limit=None, limit_direction="both").bfill().ffill() + + # MinMax Skalierung + xscaler = MinMaxScaler() + gxscale = xscaler.fit_transform(gx.values.reshape(-1, 1)) + + yscaler = MinMaxScaler() + gyscale = yscaler.fit_transform(gy.values.reshape(-1, 1)) + + return np.column_stack((gxscale, gyscale)) + + +def extract_pupil(df): + """Extrahiert Pupillengröße (beide Augen gemittelt).""" + pl = df["EYE_LEFT_PUPIL_DIAMETER"].replace([np.inf, -np.inf], np.nan) + pr = df["EYE_RIGHT_PUPIL_DIAMETER"].replace([np.inf, -np.inf], np.nan) + + vl = df.get("EYE_LEFT_PUPIL_VALIDITY") + vr = df.get("EYE_RIGHT_PUPIL_VALIDITY") + + if vl is None or vr is None: + validity = (~pl.isna() | ~pr.isna()).astype(int).to_numpy() + else: + validity = ((vl == 1) | (vr == 1)).astype(int).to_numpy() + + p = np.mean(np.column_stack([pl, pr]), axis=1) + p = pd.Series(p).interpolate(limit=50, limit_direction="both").bfill().ffill() + + return p.to_numpy(), validity + + +def detect_blinks(pupil_validity, min_duration=5): + """Erkennt Blinks: Validity=0 → Blink.""" + blinks = [] + start = None + + for i, v in enumerate(pupil_validity): + if v == 0 and start is None: + start = i + elif v == 1 and start is not None: + if i - start >= min_duration: + blinks.append([start, i]) + start = None + + return blinks + + +def compute_IPA(pupil, fs=25): + """Index of Pupillary Activity (Duchowski 2018).""" + f, Pxx = welch(pupil, fs=fs, nperseg=int(fs*2)) + hf_band = (f >= 0.6) & (f <= 2.0) + return np.sum(Pxx[hf_band]) + + +def extract_eye_features(df_eye, fs=25, min_dur_blinks=2): + """ + Extrahiert Eye-Tracking Features für ein einzelnes Window. + Gibt Dictionary mit allen Eye-Features zurück. + """ + # Gaze + gaze = extract_gaze_signal(df_eye) + + # Pupille + pupil, pupil_validity = extract_pupil(df_eye) + + + # ---------------------------- + # FIXATIONS + # ---------------------------- + time_ms = np.arange(len(df_eye)) * 1000.0 / fs + + fix, efix = fixation_detection( + x=gaze[:, 0], y=gaze[:, 1], time=time_ms, + missing=0.0, maxdist=0.003, mindur=10 + ) + + fixation_durations = [f[2] for f in efix if np.isfinite(f[2]) and f[2] > 0] + + # Kategorien + F_short = sum(66 <= d <= 150 for d in fixation_durations) + F_medium = sum(300 <= d <= 500 for d in fixation_durations) + F_long = sum(d >= 1000 for d in fixation_durations) + F_hundred = sum(d > 100 for d in fixation_durations) + + # ---------------------------- + # SACCADES + # ---------------------------- + sac, esac = saccade_detection( + x=gaze[:, 0], y=gaze[:, 1], time=time_ms, + missing=0, minlen=12, maxvel=0.2, maxacc=1 + ) + + sac_durations = [s[2] for s in esac] + sac_amplitudes = [((s[5]-s[3])**2 + (s[6]-s[4])**2)**0.5 for s in esac] + + # ---------------------------- + # BLINKS + # ---------------------------- + blinks = detect_blinks(pupil_validity, min_duration=min_dur_blinks) + blink_durations = [(b[1] - b[0]) / fs for b in blinks] + + # ---------------------------- + # PUPIL + # ---------------------------- + if np.all(np.isnan(pupil)): + mean_pupil = np.nan + ipa = np.nan + else: + mean_pupil = np.nanmean(pupil) + ipa = compute_IPA(pupil, fs=fs) + + # Feature Dictionary + return { + "Fix_count_short_66_150": F_short, + "Fix_count_medium_300_500": F_medium, + "Fix_count_long_gt_1000": F_long, + "Fix_count_100": F_hundred, + "Fix_mean_duration": np.mean(fixation_durations) if fixation_durations else 0, + "Fix_median_duration": np.median(fixation_durations) if fixation_durations else 0, + "Sac_count": len(sac), + "Sac_mean_amp": np.mean(sac_amplitudes) if sac_amplitudes else 0, + "Sac_mean_dur": np.mean(sac_durations) if sac_durations else 0, + "Sac_median_dur": np.median(sac_durations) if sac_durations else 0, + "Blink_count": len(blinks), + "Blink_mean_dur": np.mean(blink_durations) if blink_durations else 0, + "Blink_median_dur": np.median(blink_durations) if blink_durations else 0, + "Pupil_mean": mean_pupil, + "Pupil_IPA": ipa + } + +def compute_features_from_parquet(parquet_path): + df = pd.read_parquet(parquet_path) + df_eye = clean_eye_df(df) + + if df_eye.empty: + return None + + features = extract_eye_features( + df_eye, + fs=SAMPLING_RATE, + min_dur_blinks=MIN_DUR_BLINKS + ) + + return features \ No newline at end of file