add note to files on remote directory

- added documentation for eyeTracking data

.gitignore

@@ -8,3 +8,4 @@
 !*.service
 !*.timer
 !*.yaml
+!*.txt

EDA/EDA.ipynb

@@ -23,7 +23,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "file_path = \"adabase-public-0020-v_0_0_2.h5py\""
+    "file_path = \"YOUR_FILE_PATH.h5py\""
    ]
   },
   {
@@ -87,7 +87,7 @@
    "id": "a4731c56",
    "metadata": {},
    "source": [
-    "Actions units"
+    "Insights on actions units"
    ]
   },
   {
@@ -167,7 +167,7 @@
    "id": "332740a8",
    "metadata": {},
    "source": [
-    "Plots"
+    "Example plot of ECG curve"
    ]
   },
   {
@@ -177,7 +177,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# df_signals_ecg = pd.read_hdf(file_path, \"SIGNALS\", mode=\"r\", columns=[\"STUDY\",\"LEVEL\", \"PHASE\", 'RAW_ECG_I'])\n",
     "df_signals_ecg = df_signals[[\"STUDY\",\"LEVEL\", \"PHASE\", 'RAW_ECG_I']]\n",
     "df_signals_ecg.shape"
    ]

EDA/distribution_plots.ipynb

@@ -37,9 +37,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "dataset_path = Path(r\"/home/jovyan/data-paulusjafahrsimulator-gpu/new_datasets/combined_dataset_25hz.parquet\")\n",
-    "# dataset_path = Path(r\"/home/jovyan/data-paulusjafahrsimulator-gpu/new_datasets/60s_combined_dataset_25hz.parquet\")\n",
-    "# dataset_path = Path(r\"/home/jovyan/data-paulusjafahrsimulator-gpu/new_datasets/120s_combined_dataset_25hz.parquet\")"
+    "dataset_path = Path(r\"\") # TODO: enter path to dataset"
    ]
   },
   {

EDA/histogramms.ipynb

@@ -36,7 +36,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "path  = Path(r\"/home/jovyan/data-paulusjafahrsimulator-gpu/new_datasets/50s_25Hz_dataset.parquet\")\n",
+    "path  = Path(r\".parquet\") # TODO: enter path to dataset\n",
     "df = pd.read_parquet(path=path)"
    ]
   },
@@ -192,18 +192,6 @@
    "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.12.10"
   }
  },
  "nbformat": 4,

EDA/login.yaml

@@ -0,0 +1,2 @@
+- url: # enter url
+- password: # enter passwort

EDA/owncloud.ipynb

@@ -1,157 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "aab6b326-a583-47ad-8bb7-723c2fddcc63",
-   "metadata": {
-    "scrolled": true
-   },
-   "outputs": [],
-   "source": [
-    "# %pip install pyocclient\n",
-    "import yaml\n",
-    "import owncloud\n",
-    "import pandas as pd\n",
-    "import time"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "4f42846c-27c3-4394-a40a-e22d73c2902e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "start = time.time()\n",
-    "\n",
-    "with open(\"../login.yaml\") as f:\n",
-    "    cfg = yaml.safe_load(f)\n",
-    "url, password = cfg[0][\"url\"], cfg[1][\"password\"]\n",
-    "file = \"adabase-public-0022-v_0_0_2.h5py\"\n",
-    "oc = owncloud.Client.from_public_link(url, folder_password=password)\n",
-    "\n",
-    "\n",
-    "oc.get_file(file, \"tmp22.h5\")\n",
-    "\n",
-    "end = time.time()\n",
-    "print(end - start)\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "3714dec2-85d0-4f76-af46-ea45ebec2fa3",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "start = time.time()\n",
-    "df_performance =  pd.read_hdf(\"tmp22.h5\", \"PERFORMANCE\")\n",
-    "end = time.time()\n",
-    "print(end - start)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "f50e97d0",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "print(22)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c131c816",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_performance"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "6ae47e52-ad86-4f8d-b929-0080dc99f646",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "start = time.time()\n",
-    "df_4_col =  pd.read_hdf(\"tmp.h5\", \"SIGNALS\", mode=\"r\", columns=[\"STUDY\"], start=0, stop=1)\n",
-    "end = time.time()\n",
-    "print(end - start)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "7c139f3a-ede8-4530-957d-d1bb939f6cb5",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_4_col.head()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "a68d58ea-65f2-46c4-a2b2-8c3447c715d7",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_4_col.shape"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "95aa4523-3784-4ab6-bf92-0227ce60e863",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_4_col.info()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "defbcaf4-ad1b-453f-9b48-ab0ecfc4b5d5",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df_4_col.isna().sum()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "72313895-c478-44a5-9108-00b0bec01bb8",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "base",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.11.5"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

EDA/owncloud_file_access.ipynb

@@ -0,0 +1,112 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "457e7807",
+   "metadata": {},
+   "source": [
+    "# Get data from owncloud"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "dc9ed3f8",
+   "metadata": {},
+   "source": [
+    "Imports"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# %pip install pyocclient\n",
+    "import os\n",
+    "import time\n",
+    "import yaml\n",
+    "import owncloud\n",
+    "import pandas as pd\n"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "68e34abc",
+   "metadata": {},
+   "source": [
+    "Download and save"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "start = time.time()\n",
+    "\n",
+    "# TODO: User input: directory where downloaded files should be saved\n",
+    "save_dir = r\"./downloads\"\n",
+    "\n",
+    "\n",
+    "os.makedirs(save_dir, exist_ok=True)\n",
+    "\n",
+    "# Load credentials\n",
+    "with open(\"login.yaml\", \"r\") as f:\n",
+    "    cfg = yaml.safe_load(f)\n",
+    "\n",
+    "url = cfg[0][\"url\"]\n",
+    "password = cfg[1][\"password\"]\n",
+    "\n",
+    "# Connect to OwnCloud public link\n",
+    "oc = owncloud.Client.from_public_link(url, folder_password=password)\n",
+    "\n",
+    "# List all available files in the shared folder\n",
+    "remote_files = oc.list(\".\")\n",
+    "\n",
+    "# Keep only HDF5 files and sort them by name\n",
+    "hdf5_files = sorted([f.get_name() for f in remote_files if f.get_name().endswith(\".hdf5\")])\n",
+    "\n",
+    "print(f\"Found {len(hdf5_files)} .hdf5 files in OwnCloud\")\n",
+    "\n",
+    "if not hdf5_files:\n",
+    "    print(\"No .hdf5 files found.\")\n",
+    "else:\n",
+    "    for i, remote_name in enumerate(hdf5_files):\n",
+    "        local_name = f\"tmp_{i:04d}.h5\"\n",
+    "        local_path = os.path.join(save_dir, local_name)\n",
+    "\n",
+    "        try:\n",
+    "            oc.get_file(remote_name, local_path)\n",
+    "            print(f\"Downloaded: {remote_name} -> {local_path}\")\n",
+    "        except Exception as e:\n",
+    "            print(f\"Failed to download {remote_name}: {e}\")\n",
+    "\n",
+    "end = time.time()\n",
+    "print(f\"Finished in {end - start:.2f} seconds\")\n"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "base",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}

EDA/researchOnSubjectPerformance.ipynb

@@ -15,6 +15,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "%pip install pyocclient\n",
     "import yaml\n",
     "import owncloud\n",
     "import pandas as pd\n",
@@ -36,101 +37,109 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Load credentials\n",
-    "with open(\"../login.yaml\") as f:\n",
+    "# Load credentials from YAML\n",
+    "with open(\"login.yaml\", \"r\") as f:\n",
     "    cfg = yaml.safe_load(f)\n",
-    "   \n",
-    "url, password = cfg[0][\"url\"], cfg[1][\"password\"]\n",
     "\n",
-    "# Connect once\n",
+    "url = cfg[0][\"url\"]\n",
+    "password = cfg[1][\"password\"]\n",
+    "\n",
+    "# Connect once to the public OwnCloud link\n",
     "oc = owncloud.Client.from_public_link(url, folder_password=password)\n",
-    "# File pattern\n",
-    "# base = \"adabase-public-{num:04d}-v_0_0_2.h5py\"\n",
-    "base = \"{num:04d}-*.h5py\""
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "07c03d07",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "num_files = 2  # number of files to process (min: 1, max: 30)\n",
+    "\n",
+    "num_files = 1  # number of subject IDs to process (min: 1, max: 30)\n",
     "performance_data = []\n",
     "\n",
+    "# Read remote file list once\n",
+    "remote_files = oc.list(\".\")\n",
+    "remote_names = [f.get_name() for f in remote_files]\n",
+    "\n",
     "for i in range(num_files):\n",
-    "    file_pattern = f\"{i:04d}-*\"\n",
-    "    \n",
-    "    # Get list of files matching the pattern\n",
-    "    files = oc.list('.')\n",
-    "    matching_files = [f.get_name() for f in files if f.get_name().startswith(f\"{i:04d}-\")]\n",
-    "    \n",
-    "    if matching_files:\n",
-    "        file_name = matching_files[0]  # Take the first matching file\n",
-    "        local_tmp = f\"tmp_{i:04d}.h5\"\n",
-    "        \n",
-    "        oc.get_file(file_name, local_tmp)\n",
-    "        print(f\"{file_name} geöffnet\")\n",
-    "    else:\n",
-    "        print(f\"Keine Datei gefunden für Muster: {file_pattern}\")\n",
-    "    # file_name = base.format(num=i)\n",
-    "    # local_tmp = f\"tmp_{i:04d}.h5\"\n",
+    "    prefix = f\"{i:04d}-\"\n",
+    "    matching_files = [name for name in remote_names if name.startswith(prefix) and name.endswith(\".hdf5\")]\n",
     "\n",
-    "    # oc.get_file(file_name, local_tmp)\n",
-    "    # print(f\"{file_name} geöffnet\")\n",
-    "\n",
-    "    # check SIGNALS table for AUs\n",
-    "    with pd.HDFStore(local_tmp, mode=\"r\") as store:\n",
-    "        cols = store.select(\"SIGNALS\", start=0, stop=1).columns\n",
-    "    au_cols = [c for c in cols if c.startswith(\"AU\")]\n",
-    "    if not au_cols:\n",
-    "        print(f\"Subject {i} enthält keine AUs\")\n",
+    "    if not matching_files:\n",
+    "        print(f\"No file found for pattern: {prefix}*.hdf5\")\n",
     "        continue\n",
     "\n",
-    "    # load performance table\n",
-    "    with pd.HDFStore(local_tmp, mode=\"r\") as store:\n",
-    "        perf_df = store.select(\"PERFORMANCE\")\n",
+    "    # Take the first matching file, e.g. 0000-AACA.hdf5\n",
+    "    file_name = matching_files[0]\n",
+    "    local_tmp = f\"tmp_{i:04d}.hdf5\"\n",
+    "\n",
+    "    try:\n",
+    "        # Download the file locally\n",
+    "        oc.get_file(file_name, local_tmp)\n",
+    "        print(f\"Downloaded and opened file: {file_name} -> {local_tmp}\")\n",
+    "    except Exception as e:\n",
+    "        print(f\"Failed to download file {file_name}: {e}\")\n",
+    "        continue\n",
+    "\n",
+    "    # Check SIGNALS table for AU columns\n",
+    "    try:\n",
+    "        with pd.HDFStore(local_tmp, mode=\"r\") as store:\n",
+    "            cols = store.select(\"SIGNALS\", start=0, stop=1).columns\n",
+    "    except Exception as e:\n",
+    "        print(f\"Failed to read SIGNALS from {local_tmp}: {e}\")\n",
+    "        continue\n",
+    "\n",
+    "    au_cols = [c for c in cols if c.startswith(\"AU\")]\n",
+    "    if not au_cols:\n",
+    "        print(f\"Subject {i:04d} contains no AU columns\")\n",
+    "        continue\n",
+    "\n",
+    "    # Load PERFORMANCE table\n",
+    "    try:\n",
+    "        with pd.HDFStore(local_tmp, mode=\"r\") as store:\n",
+    "            perf_df = store.select(\"PERFORMANCE\")\n",
+    "    except Exception as e:\n",
+    "        print(f\"Failed to read PERFORMANCE from {local_tmp}: {e}\")\n",
+    "        continue\n",
     "\n",
     "    f1_cols = [c for c in [\"AUDITIVE F1\", \"VISUAL F1\", \"F1\"] if c in perf_df.columns]\n",
     "    if not f1_cols:\n",
-    "        print(f\"Subject {i}: keine F1-Spalten gefunden\")\n",
+    "        print(f\"Subject {i:04d}: no F1 columns found\")\n",
     "        continue\n",
     "\n",
     "    subject_entry = {\"subjectID\": i}\n",
     "    valid_scores = []\n",
     "\n",
-    "    # iterate rows: each (study, level, phase)\n",
+    "    # Iterate through PERFORMANCE rows: each row is one (study, level, phase) combination\n",
     "    for _, row in perf_df.iterrows():\n",
-    "        study, level, phase = row[\"STUDY\"], row[\"LEVEL\"], row[\"PHASE\"]\n",
+    "        study = row[\"STUDY\"]\n",
+    "        level = row[\"LEVEL\"]\n",
+    "        phase = row[\"PHASE\"]\n",
     "        col_name = f\"STUDY_{study}_LEVEL_{level}_PHASE_{phase}\"\n",
     "\n",
-    "        # collect valid F1 values among the three columns\n",
+    "        # Collect non-NaN F1 values from the available F1 columns\n",
     "        scores = [row[c] for c in f1_cols if pd.notna(row[c])]\n",
     "        if scores:\n",
     "            mean_score = float(np.mean(scores))\n",
     "            subject_entry[col_name] = mean_score\n",
     "            valid_scores.extend(scores)\n",
     "\n",
-    "    # compute overall average across all valid combinations\n",
+    "    # Compute overall average across all valid F1 values\n",
     "    if valid_scores:\n",
     "        subject_entry[\"overall_score\"] = float(np.mean(valid_scores))\n",
     "        performance_data.append(subject_entry)\n",
-    "        print(f\"Subject {i}: {len(valid_scores)} gültige Scores, Overall = {subject_entry['overall_score']:.3f}\")\n",
+    "        print(\n",
+    "            f\"Subject {i:04d}: {len(valid_scores)} valid scores, \"\n",
+    "            f\"overall = {subject_entry['overall_score']:.3f}\"\n",
+    "        )\n",
     "    else:\n",
-    "        print(f\"Subject {i}: keine gültigen F1-Scores\")\n",
+    "        print(f\"Subject {i:04d}: no valid F1 scores found\")\n",
     "\n",
-    "# build dataframe\n",
+    "# Build final DataFrame and save CSV\n",
     "if performance_data:\n",
     "    performance_df = pd.DataFrame(performance_data)\n",
     "    combination_cols = sorted([c for c in performance_df.columns if c.startswith(\"STUDY_\")])\n",
     "    final_cols = [\"subjectID\", \"overall_score\"] + combination_cols\n",
     "    performance_df = performance_df[final_cols]\n",
-    "    performance_df.to_csv(\"n_au_performance.csv\", index=False)\n",
+    "    performance_df.to_csv(\"performance.csv\", index=False)\n",
     "\n",
-    "    print(f\"\\nGesamt Subjects mit Action Units: {len(performance_df)}\")\n",
+    "    print(f\"\\nTotal subjects with Action Units: {len(performance_df)}\")\n",
+    "    print(\"Saved results to performance.csv\")\n",
     "else:\n",
-    "    print(\"Keine gültigen Daten gefunden.\")"
+    "    print(\"No valid data found.\")"
    ]
   },
   {
@@ -142,56 +151,11 @@
    "source": [
     "performance_df.head()"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "db95eea7",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "with pd.HDFStore(\"tmp_0000.h5\", mode=\"r\") as store:\n",
-    "    md = store.select(\"META\")\n",
-    "print(\"File 0:\")\n",
-    "print(md)\n",
-    "with pd.HDFStore(\"tmp_0001.h5\", mode=\"r\") as store:\n",
-    "    md = store.select(\"META\")\n",
-    "print(\"File 1\")\n",
-    "print(md)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "8067036b",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "pd.set_option('display.max_columns', None)\n",
-    "pd.set_option('display.max_rows', None)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "f18e7385",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "with pd.HDFStore(\"tmp_0000.h5\", mode=\"r\") as store:\n",
-    "    md = store.select(\"SIGNALS\", start=0, stop=1)\n",
-    "print(\"File 0:\")\n",
-    "md.head()\n",
-    "# with pd.HDFStore(\"tmp_0001.h5\", mode=\"r\",start=0, stop=1) as store:\n",
-    "#     md = store.select(\"SIGNALS\")\n",
-    "# print(\"File 1\")\n",
-    "# print(md.columns)"
-   ]
   }
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "base",
+   "display_name": "310",
    "language": "python",
    "name": "python3"
   },
@@ -205,7 +169,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.11.5"
+   "version": "3.10.19"
   }
  },
  "nbformat": 4,

dataset_creation/AU_creation/AU_creation_service.py

@@ -1,58 +0,0 @@
-from feat import Detector
-from feat.utils.io import get_test_data_path
-from moviepy.video.io.VideoFileClip import VideoFileClip
-import os
-
-def extract_aus(path, model, skip_frames):
-  detector = Detector(au_model=model)
-
-  video_prediction = detector.detect(
-      path, data_type="video", skip_frames=skip_frames, face_detection_threshold=0.95 # alle 5 Sekunden einbeziehen - 24 Frames pro Sekunde
-  )
-
-  return video_prediction.aus.sum()
-
-def split_video(path, chunk_length=120): 
-    video = VideoFileClip(path)
-    duration = int(video.duration)
-
-    subclips_dir = os.path.join(os.dirname(path), "subclips")
-    os.makedirs(subclips_dir, exist_ok=True)
-    paths = []
-
-    for start in range(0, duration, chunk_length):
-        end = min(start + chunk_length, duration)
-
-        subclip = (
-            video
-            .subclip(start, end)
-            .without_audio()      
-            .set_fps(video.fps)          
-        )
-
-        output_path = f"{subclips_dir}_part_{start//chunk_length + 1}.mp4"
-        subclip.write_videofile(
-           output_path, 
-           )
-        paths.append(output_path)
-
-    return output_path
-
-# def start(path):
-#     results = []
-#     clips = split_video(path)
-
-#     for clip in clips:
-#       results.append(extract_aus(clip, 'svm', 25*5))
-#     return results
-
-if __name__ == "__main__":
-  results = []
-  clips = []
-  test_video_path = "AU_creation/YTDown.com_YouTube_Was-ist-los-bei-7-vs-Wild_Media_Gtj9zu_WikU_001_1080p.mp4"
-  clips = split_video(test_video_path)
-
-  for clippath in clips:
-    results.append(extract_aus(clippath, 'svm', 25*5))
-
-  print(results)

dataset_creation/AU_creation/pyfeat_docu.ipynb

@@ -5,27 +5,47 @@
    "id": "3b0c6c82",
    "metadata": {},
    "source": [
-    "Hier entsteht die Dokumentation, wie die Action Units erzeugt wurden.\n",
-    "Daraus wird dann letztendlich ein Skript erstellt, welches automatisch AUs aus Videodateien erstellen soll.\n",
+    "## Action Unit Documentation and Setup\n",
     "\n",
-    "Py-Feat besitzt Dependencies, die ab Python 3.12 nicht mehr verfügbar sind.\n",
-    "Dazu muss ein Kernel mit Python 3.11 erstellt werden.\n",
-    "Folgendes Vorgehen:\n",
-    "1. Seite des Jupyter Labs öffnen\n",
-    "2. Terminal öffnen und folgende Befehle eingeben:\n",
-    "  conda create -n py311 python=3.11\n",
-    "  source ~/.bashrc\n",
-    "  conda activate py311\n",
-    "  conda install jupyter\n",
-    "  python -m ipykernel install --user --name=py311 --display-name \"Python 3.11\"\n",
-    "  pip install py-feat\n",
-    "  pip install \"moviepy<2.0\" (falls benötigt)\n",
-    "3. den Kernel neustarten\n",
-    "4. in VSC den Kernel neu hinzufügen und dann den Kernel mit dem Namen \"Python 3.11\" auswählen.\n",
+    "This documentation outlines the process for generating **Action Units (AUs)** and the eventual creation of a script to automate AU extraction from video files.\n",
     "\n",
-    "Der Code unten zeigt eine beispielhafte Integration der py-feat Bibliothek.\n",
-    "Die Klassifizierung zu 0,1 kommt durch die Wahl des AU-Modells zustande. Dabei wird SVM gewählt. (ADABase Paper)\n",
-    "Gibt die Klassifizierung einen Gleitkommawert zwischen 0 & 1 aus, dann kommt XGB zum Einsatz. (REVELIO Paper)"
+    "### Python Environment Configuration\n",
+    "\n",
+    "**Py-Feat** relies on dependencies that are incompatible with Python 3.12 and later. To ensure functionality, you must set up a dedicated **Python 3.11** kernel.\n",
+    "\n",
+    "#### Setup Instructions:\n",
+    "\n",
+    "1. Open your **Jupyter Lab** interface.\n",
+    "2. Open a **Terminal** and execute the following commands:\n",
+    "```bash\n",
+    "conda create -n py311 python=3.11\n",
+    "source ~/.bashrc\n",
+    "conda activate py311\n",
+    "conda install jupyter\n",
+    "python -m ipykernel install --user --name=py311 --display-name \"Python 3.11\"\n",
+    "pip install py-feat\n",
+    "pip install \"moviepy<2.0\" # Only if required\n",
+    "\n",
+    "```\n",
+    "\n",
+    "\n",
+    "3. **Restart** the kernel.\n",
+    "4. In **VS Code**, refresh your kernel list and select the one labeled **\"Python 3.11\"**.\n",
+    "\n",
+    "---\n",
+    "\n",
+    "### Implementation Details\n",
+    "\n",
+    "The following code demonstrates a sample integration of the `py-feat` library. The classification output format is determined by the specific AU model selected:\n",
+    "\n",
+    "| Model | Output Type | Reference Paper |\n",
+    "| --- | --- | --- |\n",
+    "| **SVM** | Binary (0 or 1) | *ADABase* |\n",
+    "| **XGB** | Floating Point (0.0 - 1.0) | *REVELIO* |\n",
+    "\n",
+    "---\n",
+    "\n",
+    "Would you like me to provide the Python code block to implement the **SVM** or **XGB** detector using these libraries?"
    ]
   },
   {

dataset_creation/camera_handling/camera_stream_AU_and_ET.py

@@ -1,296 +0,0 @@
-import cv2
-import time
-import os
-import threading
-from datetime import datetime
-from feat import Detector
-import torch
-import mediapipe as mp
-import csv
-
-# Konfiguration
-CAMERA_INDEX = 0
-OUTPUT_DIR = "recordings"
-VIDEO_DURATION = 10  # Sekunden
-START_INTERVAL = 5   # Sekunden bis zum nächsten Start
-FPS = 25.0           # Feste FPS
-
-if not os.path.exists(OUTPUT_DIR):
-    os.makedirs(OUTPUT_DIR)
-
-# Globaler Detector, um ihn nicht bei jedem Video neu laden zu müssen (spart massiv Zeit/Speicher)
-print("Initialisiere AU-Detector (bitte warten)...")
-detector = Detector(au_model="xgb") 
-
-# ===== MediaPipe FaceMesh Setup =====
-mp_face_mesh = mp.solutions.face_mesh
-face_mesh = mp_face_mesh.FaceMesh(
-    static_image_mode=False,
-    max_num_faces=1,
-    refine_landmarks=True,  # wichtig für Iris
-    min_detection_confidence=0.5,
-    min_tracking_confidence=0.5
-)
-
-LEFT_IRIS = [474, 475, 476, 477]
-RIGHT_IRIS = [469, 470, 471, 472]
-
-LEFT_EYE_LIDS = (159, 145)
-RIGHT_EYE_LIDS = (386, 374)
-
-LEFT_EYE_GAZE_IDXS = (33, 133, 159, 145)
-RIGHT_EYE_GAZE_IDXS = (263, 362, 386, 374)
-
-EYE_OPEN_THRESHOLD = 6
-
-# CSV vorbereiten
-gaze_csv = open("gaze_data.csv", mode="w", newline="")
-gaze_writer = csv.writer(gaze_csv)
-gaze_writer.writerow([
-    "timestamp",
-    "left_gaze_x",
-    "left_gaze_y",
-    "right_gaze_x",
-    "right_gaze_y",
-    "left_valid",
-    "right_valid",
-    "left_diameter",
-    "right_diameter"
-])
-
-def eye_openness(landmarks, top_idx, bottom_idx, img_height):
-    top = landmarks[top_idx]
-    bottom = landmarks[bottom_idx]
-    return abs(top.y - bottom.y) * img_height
-
-
-def compute_gaze(landmarks, iris_center, indices, w, h):
-    idx1, idx2, top_idx, bottom_idx = indices
-
-    p1 = landmarks[idx1]
-    p2 = landmarks[idx2]
-    top = landmarks[top_idx]
-    bottom = landmarks[bottom_idx]
-
-    x1 = p1.x * w
-    x2 = p2.x * w
-    y_top = top.y * h
-    y_bottom = bottom.y * h
-
-    iris_x, iris_y = iris_center
-
-    eye_left = min(x1, x2)
-    eye_right = max(x1, x2)
-
-    eye_width = eye_right - eye_left
-    eye_height = abs(y_bottom - y_top)
-
-    if eye_width == 0 or eye_height == 0:
-        return 0.5, 0.5
-
-    gaze_x = (iris_x - eye_left) / eye_width
-    gaze_y = (iris_y - min(y_top, y_bottom)) / eye_height
-
-    gaze_x = max(0, min(1, gaze_x))
-    gaze_y = max(0, min(1, gaze_y))
-
-    return gaze_x, gaze_y
-
-def extract_aus(path, skip_frames):
-
-    # torch.no_grad() deaktiviert die Gradientenberechnung.
-    # Das löst den "Can't call numpy() on Tensor that requires grad" Fehler.
-    with torch.no_grad():
-        video_prediction = detector.detect_video(
-            path, 
-            skip_frames=skip_frames, 
-            face_detection_threshold=0.95
-        )
-        
-        # Falls video_prediction oder .aus noch Tensoren sind, 
-        # stellen wir sicher, dass sie korrekt summiert werden.
-        try:
-            # Wir nehmen die Summe der Action Units über alle detektierten Frames
-            res = video_prediction.aus.sum()
-            return res
-        except Exception as e:
-            print(f"Fehler bei der Summenbildung: {e}")
-            return 0
-
-def startAU_creation(video_path):
-    """Diese Funktion läuft nun in einem eigenen Thread."""
-    try:
-        print(f"\n[THREAD START] Analyse läuft für: {video_path}")
-        # skip_frames berechnen (z.B. alle 5 Sekunden bei 25 FPS = 125)
-        output = extract_aus(video_path, skip_frames=int(FPS*5))
-        
-        print(f"\n--- Ergebnis für {os.path.basename(video_path)} ---")
-        print(output)
-        print("--------------------------------------------------\n")
-    except Exception as e:
-        print(f"Fehler bei der Analyse von {video_path}: {e}")
-
-class VideoRecorder:
-    def __init__(self, filename, width, height):
-        self.filename = filename
-        fourcc = cv2.VideoWriter_fourcc(*'XVID')
-        self.out = cv2.VideoWriter(filename, fourcc, FPS, (width, height))
-        self.frames_to_record = int(VIDEO_DURATION * FPS)
-        self.frames_count = 0
-        self.is_finished = False
-
-    def write_frame(self, frame):
-        if self.frames_count < self.frames_to_record:
-            self.out.write(frame)
-            self.frames_count += 1
-        else:
-            self.finish()
-
-    def finish(self):
-        if not self.is_finished:
-            self.out.release()
-            self.is_finished = True
-            abs_path = os.path.abspath(self.filename)
-            print(f"Video fertig gespeichert: {self.filename}")
-
-            # --- MULTITHREADING HIER ---
-            # Wir starten die Analyse in einem neuen Thread, damit main() sofort weiter frames lesen kann
-            analysis_thread = threading.Thread(target=startAU_creation, args=(abs_path,))
-            analysis_thread.daemon = True  # Beendet sich, wenn das Hauptprogramm schließt
-            analysis_thread.start()
-
-def main():
-    cap = cv2.VideoCapture(CAMERA_INDEX)
-    if not cap.isOpened():
-        print("Fehler: Kamera konnte nicht geöffnet werden.")
-        return
-
-    width = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH))
-    height = int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
-    
-    active_recorders = []
-    last_start_time = 0
-    
-    print("Aufnahme läuft. Drücke 'q' zum Beenden.")
-
-    try:
-        while True:
-            ret, frame = cap.read()
-            if not ret:
-                break
-
-            rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
-            h, w, _ = frame.shape
-            results = face_mesh.process(rgb)
-
-            left_valid = 0
-            right_valid = 0
-            left_diameter = None
-            right_diameter = None
-
-            left_gaze_x = None
-            left_gaze_y = None
-            right_gaze_x = None
-            right_gaze_y = None
-
-            if results.multi_face_landmarks:
-                face_landmarks = results.multi_face_landmarks[0]
-
-                left_open = eye_openness(
-                    face_landmarks.landmark,
-                    LEFT_EYE_LIDS[0],
-                    LEFT_EYE_LIDS[1],
-                    h
-                )
-
-                right_open = eye_openness(
-                    face_landmarks.landmark,
-                    RIGHT_EYE_LIDS[0],
-                    RIGHT_EYE_LIDS[1],
-                    h
-                )
-
-                left_valid = 1 if left_open > EYE_OPEN_THRESHOLD else 0
-                right_valid = 1 if right_open > EYE_OPEN_THRESHOLD else 0
-
-                for eye_name, eye_indices in [("left", LEFT_IRIS), ("right", RIGHT_IRIS)]:
-                    iris_points = []
-
-                    for idx in eye_indices:
-                        lm = face_landmarks.landmark[idx]
-                        x_i, y_i = int(lm.x * w), int(lm.y * h)
-                        iris_points.append((x_i, y_i))
-
-                    if len(iris_points) == 4:
-                        cx = int(sum(p[0] for p in iris_points) / 4)
-                        cy = int(sum(p[1] for p in iris_points) / 4)
-
-                        radius = max(
-                            ((x - cx) ** 2 + (y - cy) ** 2) ** 0.5
-                            for (x, y) in iris_points
-                        )
-
-                        diameter = 2 * radius
-
-                        cv2.circle(frame, (cx, cy), int(radius), (0, 255, 0), 2)
-
-                        if eye_name == "left" and left_valid:
-                            left_diameter = diameter
-                            left_gaze_x, left_gaze_y = compute_gaze(
-                                face_landmarks.landmark,
-                                (cx, cy),
-                                LEFT_EYE_GAZE_IDXS,
-                                w, h
-                            )
-
-                        elif eye_name == "right" and right_valid:
-                            right_diameter = diameter
-                            right_gaze_x, right_gaze_y = compute_gaze(
-                                face_landmarks.landmark,
-                                (cx, cy),
-                                RIGHT_EYE_GAZE_IDXS,
-                                w, h
-                            )
-
-            # CSV schreiben
-            gaze_writer.writerow([
-                time.time(),
-                left_gaze_x,
-                left_gaze_y,
-                right_gaze_x,
-                right_gaze_y,
-                left_valid,
-                right_valid,
-                left_diameter,
-                right_diameter
-            ])
-
-            current_time = time.time()
-
-            if current_time - last_start_time >= START_INTERVAL:
-                timestamp = datetime.now().strftime("%H%M%S")
-                filename = os.path.join(OUTPUT_DIR, f"rec_{timestamp}.avi")
-                new_recorder = VideoRecorder(filename, width, height)
-                active_recorders.append(new_recorder)
-                last_start_time = current_time
-
-            for rec in active_recorders[:]:
-                rec.write_frame(frame)
-                if rec.is_finished:
-                    active_recorders.remove(rec)
-
-            cv2.imshow('Kamera Livestream', frame)
-            if cv2.waitKey(1) & 0xFF == ord('q'):
-                break
-
-            time.sleep(1/FPS)
-
-    finally:
-        gaze_csv.close()
-        face_mesh.close()
-        cap.release()
-        cv2.destroyAllWindows()
-        print("Programm beendet. Warte ggf. auf laufende Analysen...")
-
-if __name__ == "__main__":
-    main()

dataset_creation/camera_handling/eyeFeature_kalibrierung.py

@@ -0,0 +1,174 @@
+import cv2
+import mediapipe as mp
+import numpy as np
+import pyautogui
+import pandas as pd
+import time
+from sklearn.pipeline import make_pipeline
+from sklearn.preprocessing import PolynomialFeatures
+from sklearn.linear_model import LinearRegression
+
+# Bildschirmgröße
+screen_w, screen_h = pyautogui.size()
+
+# MediaPipe Setup
+mp_face_mesh = mp.solutions.face_mesh
+face_mesh = mp_face_mesh.FaceMesh(refine_landmarks=True)
+cap = cv2.VideoCapture(0)
+
+# Iris Landmark Indizes
+LEFT_IRIS = [468, 469, 470, 471, 472]
+RIGHT_IRIS = [473, 474, 475, 476, 477]
+
+def get_iris_center(landmarks, indices):
+    points = np.array([[landmarks[i].x, landmarks[i].y] for i in indices])
+    return np.mean(points, axis=0)
+
+# Kalibrierpunkte
+calibration_points = [
+    (0.1,0.1),(0.5,0.1),(0.9,0.1),
+    (0.1,0.5),(0.5,0.5),(0.9,0.5),
+    (0.1,0.9),(0.5,0.9),(0.9,0.9)
+]
+
+left_data = []
+right_data = []
+
+print("Kalibrierung startet...")
+
+for idx, (px, py) in enumerate(calibration_points):
+
+    screen = np.zeros((screen_h, screen_w, 3), dtype=np.uint8)
+
+    for j, (cpx, cpy) in enumerate(calibration_points):
+        cx = int(cpx * screen_w)
+        cy = int(cpy * screen_h)
+
+        if j == idx:
+            color = (0, 0, 255)   
+            radius = 25
+        else:
+            color = (255, 255, 255)   
+            radius = 15
+
+        cv2.circle(screen, (cx, cy), radius, color, -1)
+
+    # Fenster vorbereiten
+    cv2.namedWindow("Calibration", cv2.WINDOW_NORMAL)
+    cv2.imshow("Calibration", screen)
+    cv2.waitKey(1000)
+
+    samples_left = []
+    samples_right = []
+
+    start = time.time()
+    while time.time() - start < 2:
+        ret, frame = cap.read()
+        frame = cv2.flip(frame, 1)
+        rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+        results = face_mesh.process(rgb)
+
+        if results.multi_face_landmarks:
+            mesh = results.multi_face_landmarks[0].landmark
+
+            left_center = get_iris_center(mesh, LEFT_IRIS)
+            right_center = get_iris_center(mesh, RIGHT_IRIS)
+
+            samples_left.append(left_center)
+            samples_right.append(right_center)
+
+    avg_left = np.mean(samples_left, axis=0)
+    avg_right = np.mean(samples_right, axis=0)
+
+    target_x = int(px * screen_w)
+    target_y = int(py * screen_h)
+
+    left_data.append([avg_left[0], avg_left[1], target_x, target_y])
+    right_data.append([avg_right[0], avg_right[1], target_x, target_y])
+
+cv2.destroyWindow("Calibration")
+
+# Training
+def train_model(data):
+    data = np.array(data)
+    X = data[:, :2]
+    yx = data[:, 2]
+    yy = data[:, 3]
+
+    model_x = make_pipeline(PolynomialFeatures(2), LinearRegression())
+    model_y = make_pipeline(PolynomialFeatures(2), LinearRegression())
+
+    model_x.fit(X, yx)
+    model_y.fit(X, yy)
+
+    return model_x, model_y
+
+model_lx, model_ly = train_model(left_data)
+model_rx, model_ry = train_model(right_data)
+
+print("Kalibrierung abgeschlossen. Tracking startet...")
+
+# Datenaufzeichnung
+records = []
+
+while True:
+    ret, frame = cap.read()
+    frame = cv2.flip(frame, 1)
+    rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+    results = face_mesh.process(rgb)
+
+    if results.multi_face_landmarks:
+        mesh = results.multi_face_landmarks[0].landmark
+
+        left_center = get_iris_center(mesh, LEFT_IRIS)
+        right_center = get_iris_center(mesh, RIGHT_IRIS)
+
+        left_input = np.array([left_center])
+        right_input = np.array([right_center])
+
+        lx = model_lx.predict(left_input)[0]
+        ly = model_ly.predict(left_input)[0]
+
+        rx = model_rx.predict(right_input)[0]
+        ry = model_ry.predict(right_input)[0]
+
+        # Pixel-Koordinaten begrenzen
+        lx = np.clip(lx, 0, screen_w)
+        ly = np.clip(ly, 0, screen_h)
+        rx = np.clip(rx, 0, screen_w)
+        ry = np.clip(ry, 0, screen_h)
+
+        # Normierung 0–1
+        lx_norm = lx / screen_w
+        ly_norm = ly / screen_h
+        rx_norm = rx / screen_w
+        ry_norm = ry / screen_h
+
+        records.append([
+            lx_norm, ly_norm,
+            rx_norm, ry_norm
+        ])
+
+        print("L:", int(lx), int(ly), " | R:", int(rx), int(ry))
+
+    cv2.imshow("Tracking", frame)
+
+    key = cv2.waitKey(1) & 0xFF
+    if key == ord('q'):
+        print("q gedrückt – beende Tracking")
+        break
+
+cap.release()
+cv2.destroyAllWindows()
+
+# CSV speichern
+df = pd.DataFrame(records, columns=[
+    "EYE_LEFT_GAZE_POINT_ON_DISPLAY_AREA_X",
+    "EYE_LEFT_GAZE_POINT_ON_DISPLAY_AREA_Y",
+    "EYE_RIGHT_GAZE_POINT_ON_DISPLAY_AREA_X",
+    "EYE_RIGHT_GAZE_POINT_ON_DISPLAY_AREA_Y"
+])
+
+df.to_csv("gaze_data1.csv", index=False)
+
+print("Daten gespeichert als gaze_data1.csv")

model_training/DeepSVDD/deepSVDD.ipynb

@@ -41,7 +41,6 @@
     "import time\n",
     "base_dir = os.path.abspath(os.path.join(os.getcwd(), \"..\"))\n",
     "sys.path.append(base_dir)\n",
-    "print(base_dir)\n",
     "\n",
     "from Fahrsimulator_MSY2526_AI.model_training.tools import evaluation_tools, scaler, mad_outlier_removal, performance_split\n",
     "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
@@ -51,7 +50,7 @@
     "import tensorflow as tf\n",
     "from tensorflow.keras import layers, models, regularizers\n",
     "import pickle\n",
-    "from sklearn.metrics import (roc_auc_score, accuracy_score, precision_score, recall_score, f1_score, confusion_matrix, classification_report, balanced_accuracy_score, ConfusionMatrixDisplay, auc, roc_curve) "
+    "from sklearn.metrics import (accuracy_score, auc, roc_curve, f1_score) "
    ]
   },
   {
@@ -89,15 +88,40 @@
    "id": "f00a477c",
    "metadata": {},
    "source": [
-    "### Data Preprocessing"
+    "### Configuration of paths and data preprocessing"
    ]
   },
   {
-   "cell_type": "markdown",
-   "id": "504c1df7",
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5136fcec",
    "metadata": {},
+   "outputs": [],
    "source": [
-    "Laden der Daten"
+    "# TODO: set path where to save normalizer\n",
+    "normalizer_path=Path('.pkl') # TODO: set manually"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "c2115f65",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "performance_path =  Path(r\".csv\") # TODO: set manually\n",
+    "performance_df = pd.read_csv(performance_path)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "559eb8d2",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "encoder_save_path = Path('.keras') # TODO: set manually\n",
+    "deep_svdd_save_path = Path('.keras') # TODO: set manually"
    ]
   },
   {
@@ -107,8 +131,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "dataset_path = Path(r\"data-paulusjafahrsimulator-gpu/new_datasets/combined_dataset_25hz.parquet\")\n",
-    "# dataset_path = Path(r\"/home/jovyan/data-paulusjafahrsimulator-gpu/new_datasets/120s_combined_dataset_25hz.parquet\")"
+    "dataset_path  = Path(r\".parquet\") # TODO: set manually"
    ]
   },
   {
@@ -121,17 +144,6 @@
     "df = pd.read_parquet(path=dataset_path)"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "c2115f65",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "performance_path =  Path(r\"/home/jovyan/data-paulusjafahrsimulator-gpu/subject_performance/3new_au_performance.csv\")\n",
-    "performance_df = pd.read_csv(performance_path)"
-   ]
-  },
   {
    "cell_type": "markdown",
    "id": "c045c46d",
@@ -215,7 +227,7 @@
     "\n",
     "data = pd.concat([low, high], ignore_index=True)\n",
     "df = data.drop_duplicates()\n",
-    "\n",
+    "df = df.dropna()\n",
     "print(\"Label distribution:\")\n",
     "print(df[\"label\"].value_counts())"
    ]
@@ -275,210 +287,6 @@
     "Normalization"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "acec4a03",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def fit_normalizer(train_data, au_columns, method='standard', scope='global'):\n",
-    "    \"\"\"\n",
-    "    Fit normalization scalers on training data.\n",
-    "    \n",
-    "    Parameters:\n",
-    "    -----------\n",
-    "    train_data : pd.DataFrame\n",
-    "        Training dataframe with AU columns and subjectID\n",
-    "    au_columns : list\n",
-    "        List of AU column names to normalize\n",
-    "    method : str, default='standard'\n",
-    "        Normalization method: 'standard' for StandardScaler or 'minmax' for MinMaxScaler\n",
-    "    scope : str, default='global'\n",
-    "        Normalization scope: 'subject' for per-subject or 'global' for across all subjects\n",
-    "    \n",
-    "    Returns:\n",
-    "    --------\n",
-    "    dict\n",
-    "        Dictionary containing fitted scalers and statistics for new subjects\n",
-    "    \"\"\"\n",
-    "    if method == 'standard':\n",
-    "        Scaler = StandardScaler\n",
-    "    elif method == 'minmax':\n",
-    "        Scaler = MinMaxScaler\n",
-    "    else:\n",
-    "        raise ValueError(\"method must be 'standard' or 'minmax'\")\n",
-    "    \n",
-    "    scalers = {}\n",
-    "    if scope == 'subject':\n",
-    "        # Fit one scaler per subject\n",
-    "        subject_stats = []\n",
-    "        \n",
-    "        for subject in train_data['subjectID'].unique():\n",
-    "            subject_mask = train_data['subjectID'] == subject\n",
-    "            scaler = Scaler()\n",
-    "            scaler.fit(train_data.loc[subject_mask, au_columns].values)\n",
-    "            scalers[subject] = scaler\n",
-    "            \n",
-    "            # Store statistics for averaging\n",
-    "            if method == 'standard':\n",
-    "                subject_stats.append({\n",
-    "                    'mean': scaler.mean_,\n",
-    "                    'std': scaler.scale_\n",
-    "                })\n",
-    "            elif method == 'minmax':\n",
-    "                subject_stats.append({\n",
-    "                    'min': scaler.data_min_,\n",
-    "                    'max': scaler.data_max_\n",
-    "                })\n",
-    "        \n",
-    "        # Calculate average statistics for new subjects\n",
-    "        if method == 'standard':\n",
-    "            avg_mean = np.mean([s['mean'] for s in subject_stats], axis=0)\n",
-    "            avg_std = np.mean([s['std'] for s in subject_stats], axis=0)\n",
-    "            fallback_scaler = StandardScaler()\n",
-    "            fallback_scaler.mean_ = avg_mean\n",
-    "            fallback_scaler.scale_ = avg_std\n",
-    "            fallback_scaler.var_ = avg_std ** 2\n",
-    "            fallback_scaler.n_features_in_ = len(au_columns)\n",
-    "        elif method == 'minmax':\n",
-    "            avg_min = np.mean([s['min'] for s in subject_stats], axis=0)\n",
-    "            avg_max = np.mean([s['max'] for s in subject_stats], axis=0)\n",
-    "            fallback_scaler = MinMaxScaler()\n",
-    "            fallback_scaler.data_min_ = avg_min\n",
-    "            fallback_scaler.data_max_ = avg_max\n",
-    "            fallback_scaler.data_range_ = avg_max - avg_min\n",
-    "            fallback_scaler.scale_ = 1.0 / fallback_scaler.data_range_\n",
-    "            fallback_scaler.min_ = -avg_min * fallback_scaler.scale_\n",
-    "            fallback_scaler.n_features_in_ = len(au_columns)\n",
-    "        \n",
-    "        scalers['_fallback'] = fallback_scaler\n",
-    "    \n",
-    "    elif scope == 'global':\n",
-    "        # Fit one scaler for all subjects\n",
-    "        scaler = Scaler()\n",
-    "        scaler.fit(train_data[au_columns].values)\n",
-    "        scalers['global'] = scaler\n",
-    "    \n",
-    "    else:\n",
-    "        raise ValueError(\"scope must be 'subject' or 'global'\")\n",
-    "    \n",
-    "    return {'scalers': scalers, 'method': method, 'scope': scope}\n",
-    "\n",
-    "def apply_normalizer(data, columns, normalizer_dict):\n",
-    "    \"\"\"\n",
-    "    Apply fitted normalization scalers to data.\n",
-    "    \n",
-    "    Parameters:\n",
-    "    -----------\n",
-    "    data : pd.DataFrame\n",
-    "        Dataframe with AU columns and subjectID\n",
-    "    au_columns : list\n",
-    "        List of AU column names to normalize\n",
-    "    normalizer_dict : dict\n",
-    "        Dictionary containing fitted scalers from fit_normalizer()\n",
-    "    \n",
-    "    Returns:\n",
-    "    --------\n",
-    "    pd.DataFrame\n",
-    "        DataFrame with normalized AU columns\n",
-    "    \"\"\"\n",
-    "    normalized_data = data.copy()\n",
-    "    scalers = normalizer_dict['scalers']\n",
-    "    scope = normalizer_dict['scope']\n",
-    "    normalized_data[columns] = normalized_data[columns].astype(np.float64)\n",
-    "\n",
-    "    if scope == 'subject':\n",
-    "        # Apply per-subject normalization\n",
-    "        for subject in data['subjectID'].unique():\n",
-    "            subject_mask = data['subjectID'] == subject\n",
-    "            \n",
-    "            # Use the subject's scaler if available, otherwise use fallback\n",
-    "            if subject in scalers:\n",
-    "                scaler = scalers[subject]\n",
-    "            else:\n",
-    "                # Use averaged scaler for new subjects\n",
-    "                scaler = scalers['_fallback']\n",
-    "                print(f\"Info: Subject {subject} not in training data. Using averaged scaler from training subjects.\")\n",
-    "            \n",
-    "            normalized_data.loc[subject_mask, columns] = scaler.transform(\n",
-    "                data.loc[subject_mask, columns].values\n",
-    "            )\n",
-    "    \n",
-    "    elif scope == 'global':\n",
-    "        # Apply global normalization\n",
-    "        scaler = scalers['global']\n",
-    "        normalized_data[columns] = scaler.transform(data[columns].values)\n",
-    "    \n",
-    "    return normalized_data\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "53c6ee6f",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def save_normalizer(normalizer_dict, filepath):\n",
-    "    \"\"\"\n",
-    "    Save fitted normalizer to disk.\n",
-    "\n",
-    "    Parameters:\n",
-    "    -----------\n",
-    "    normalizer_dict : dict\n",
-    "        Dictionary containing fitted scalers from fit_normalizer()\n",
-    "    filepath : str\n",
-    "        Path to save the normalizer (e.g., 'normalizer.pkl')\n",
-    "    \"\"\"\n",
-    "    # Create directory if it does not exist\n",
-    "    dirpath = os.path.dirname(filepath)\n",
-    "    if dirpath:\n",
-    "        os.makedirs(dirpath, exist_ok=True)\n",
-    "\n",
-    "    with open(filepath, 'wb') as f:\n",
-    "        pickle.dump(normalizer_dict, f)\n",
-    "\n",
-    "    print(f\"Normalizer saved to {filepath}\")\n",
-    "\n",
-    "def load_normalizer(filepath):\n",
-    "    \"\"\"\n",
-    "    Load fitted normalizer from disk.\n",
-    "    \n",
-    "    Parameters:\n",
-    "    -----------\n",
-    "    filepath : str\n",
-    "        Path to the saved normalizer file\n",
-    "        \n",
-    "    Returns:\n",
-    "    --------\n",
-    "    dict\n",
-    "        Dictionary containing fitted scalers\n",
-    "    \"\"\"\n",
-    "    with open(filepath, 'rb') as f:\n",
-    "        normalizer_dict = pickle.load(f)\n",
-    "    print(f\"Normalizer loaded from {filepath}\")\n",
-    "    return normalizer_dict"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "7280f64f",
-   "metadata": {},
-   "source": [
-    "save Normalizer"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "8420afc2",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "normalizer_path=Path('data-paulusjafahrsimulator-gpu/saved_models/deepsvdd_save/normalizer_min_max_global.pkl')"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -495,8 +303,10 @@
     "print(len(eye_cols))\n",
     "all_signal_columns = face_au_cols+eye_cols\n",
     "print(len(all_signal_columns))\n",
-    "normalizer = fit_normalizer(train_df, all_signal_columns, method='minmax', scope='global')\n",
-    "save_normalizer(normalizer, normalizer_path )"
+    "\n",
+    "# fit and save normalizer\n",
+    "normalizer = scaler.fit_normalizer(train_df, all_signal_columns, method='minmax', scope='global')\n",
+    "scaler.save_normalizer(normalizer, normalizer_path )"
    ]
   },
   {
@@ -506,11 +316,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "normalizer = load_normalizer(normalizer_path)\n",
-    "# 3. Apply normalization to all sets\n",
-    "train_df_norm = apply_normalizer(train_df, all_signal_columns, normalizer)\n",
-    "val_df_norm = apply_normalizer(val_df, all_signal_columns, normalizer)\n",
-    "test_df_norm = apply_normalizer(test_df, all_signal_columns, normalizer)"
+    "normalizer = scaler.load_normalizer(normalizer_path)\n",
+    "# Apply normalization to all sets\n",
+    "train_df_norm = scaler.apply_normalizer(train_df, all_signal_columns, normalizer)\n",
+    "val_df_norm = scaler.apply_normalizer(val_df, all_signal_columns, normalizer)\n",
+    "test_df_norm = scaler.apply_normalizer(test_df, all_signal_columns, normalizer)"
    ]
   },
   {
@@ -566,13 +376,13 @@
     "def build_intermediate_fusion_autoencoder(\n",
     "    input_dim_mod1=15,\n",
     "    input_dim_mod2=20,\n",
-    "    encoder_hidden_dim_mod1=12,    # individuell\n",
-    "    encoder_hidden_dim_mod2=20,    # individuell\n",
-    "    latent_dim=6,                  # Änderung: Bottleneck vergrößert für stabilere Repräsentation\n",
-    "    dropout_rate=0.4,              # Dropout in Hidden Layers\n",
-    "    neg_slope=0.1,\n",
-    "    weight_decay=1e-4,\n",
-    "    decoder_hidden_dims=[16, 32]   # Änderung: Decoder größer für bessere Rekonstruktion\n",
+    "    encoder_hidden_dim_mod1=12,    # TODO: set manually\n",
+    "    encoder_hidden_dim_mod2=20,    # TODO: set manually\n",
+    "    latent_dim=6,                  # TODO: set manually\n",
+    "    dropout_rate=0.4,              # TODO: set manually\n",
+    "    neg_slope=0.1,                 # TODO: set manually\n",
+    "    weight_decay=1e-4,             # TODO: set manually\n",
+    "    decoder_hidden_dims=[16, 32]   # TODO: set manually\n",
     "):\n",
     "    \"\"\"\n",
     "    Verbesserter Intermediate-Fusion Autoencoder für Deep SVDD.\n",
@@ -597,10 +407,10 @@
     "        kernel_regularizer=l2\n",
     "    )(x1_in)\n",
     "    e1 = act(e1)\n",
-    "    e1 = layers.Dropout(dropout_rate)(e1)  # Dropout nur hier\n",
+    "    e1 = layers.Dropout(dropout_rate)(e1) \n",
     "\n",
     "    e1 = layers.Dense(\n",
-    "        16,  # Änderung: Hidden Layer größer für stabilere Fusion\n",
+    "        16,  \n",
     "        use_bias=False,\n",
     "        kernel_regularizer=l2\n",
     "    )(e1)\n",
@@ -613,20 +423,20 @@
     "        kernel_regularizer=l2\n",
     "    )(x2_in)\n",
     "    e2 = act(e2)\n",
-    "    e2 = layers.Dropout(dropout_rate)(e2)  # Dropout nur hier\n",
+    "    e2 = layers.Dropout(dropout_rate)(e2)  \n",
     "\n",
     "    e2 = layers.Dense(\n",
-    "        16,  # Änderung: Hidden Layer größer\n",
+    "        16, \n",
     "        use_bias=False,\n",
     "        kernel_regularizer=l2\n",
     "    )(e2)\n",
     "    e2 = act(e2)\n",
     "\n",
     "    # -------- Intermediate Fusion --------\n",
-    "    fused = layers.Concatenate(name=\"fusion\")([e1, e2])  # 16+16=32 Dimensionen\n",
+    "    fused = layers.Concatenate(name=\"fusion\")([e1, e2])  # 16+16=32 dimensions\n",
     "\n",
     "    # -------- Joint Encoder / Bottleneck --------\n",
-    "      # sinnvoll kleiner als Fusion\n",
+    "\n",
     "    h = layers.Dense(\n",
     "        latent_dim,\n",
     "        use_bias=False,\n",
@@ -637,16 +447,16 @@
     "\n",
     "    z = layers.Dense(\n",
     "        latent_dim,\n",
-    "        activation=None,  # linear, für Deep SVDD\n",
+    "        activation=None,  # linear for Deep SVDD\n",
     "        use_bias=False,\n",
     "        kernel_regularizer=l2,\n",
     "        name=\"latent\"\n",
     "    )(h)\n",
-    "    #  Dropout entfernt direkt vor Bottleneck\n",
+    "\n",
     "\n",
     "    # -------- Decoder --------\n",
     "    d = layers.Dense(\n",
-    "        decoder_hidden_dims[0],  # größerer Decoder\n",
+    "        decoder_hidden_dims[0],  \n",
     "        use_bias=False,\n",
     "        kernel_regularizer=l2\n",
     "    )(z)\n",
@@ -692,10 +502,10 @@
     "model = build_intermediate_fusion_autoencoder(\n",
     "    input_dim_mod1=len(face_au_cols),\n",
     "    input_dim_mod2=len(eye_cols),\n",
-    "    encoder_hidden_dim_mod1=12,   # individuell\n",
-    "    encoder_hidden_dim_mod2=8,   # individuell\n",
+    "    encoder_hidden_dim_mod1=12,   # TODO: set manually\n",
+    "    encoder_hidden_dim_mod2=8,   # TODO: set manually\n",
     "    latent_dim=4,\n",
-    "    dropout_rate=0.7,             # einstellbar\n",
+    "    dropout_rate=0.7,             # TODO: set manually\n",
     "    neg_slope=0.1,\n",
     "    weight_decay=1e-3\n",
     ")\n",
@@ -780,7 +590,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "encoder_save_path =Path('data-paulusjafahrsimulator-gpu/saved_models/deepsvdd_save/encoder_8_deep.keras')\n",
     "encoder.save(encoder_save_path)"
    ]
   },
@@ -876,22 +685,6 @@
     "center = get_center(deep_svdd_net, [X_face, X_eye])"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "da140072",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# def get_radius(nu, dataset):\n",
-    "#     x_face, x_eye = dataset   # <-- zwingend entpacken\n",
-    "\n",
-    "#     dataset_tuple=[x_face, x_eye]\n",
-    "\n",
-    "#     dists = dist_per_sample(deep_svdd_net.predict(dataset_tuple), center)\n",
-    "#     return np.quantile(np.sqrt(dists), 1-nu).astype(np.float32)"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -944,10 +737,9 @@
     "  return get_radius_from_arrays(nu, X_face, X_eye)\n",
     "\n",
     "\n",
-    "nu = 0.25\n",
+    "nu = 0.05 # Set nu respectively\n",
     "\n",
     "train_dataset = tf.data.Dataset.from_tensor_slices((X_face, X_eye)).shuffle(64).batch(64)\n",
-    "# train_dataset = tf.data.Dataset.from_tensor_slices((X_face, X_eye))\n",
     "\n",
     "optimizer = tf.keras.optimizers.Adam(1e-3)\n",
     "train(train_dataset, epochs=150, nu=nu)\n",
@@ -1019,7 +811,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "deep_svdd_save_path =Path('data-paulusjafahrsimulator-gpu/saved_models/deepsvdd_save/deep_svdd_06.keras')\n",
     "deep_svdd_net.save(deep_svdd_save_path)"
    ]
   },
@@ -1104,18 +895,6 @@
    "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.12.10"
   }
  },
  "nbformat": 4,

model_training/IsolationForest/iforest_training.ipynb

@@ -28,7 +28,7 @@
     "sys.path.append(base_dir)\n",
     "print(base_dir)\n",
     "\n",
-    "from Fahrsimulator_MSY2526_AI.model_training.tools import evaluation_tools\n",
+    "from Fahrsimulator_MSY2526_AI.model_training.tools import evaluation_tools, scaler\n",
     "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
     "from sklearn.ensemble import IsolationForest\n",
     "from sklearn.model_selection import GridSearchCV, KFold\n",
@@ -52,7 +52,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "data_path = Path(r\"/home/jovyan/data-paulusjafahrsimulator-gpu/new_datasets/50s_25Hz_dataset.parquet\")"
+    "data_path = Path(r\".parquet\") # TODO: set manually"
    ]
   },
   {
@@ -115,118 +115,6 @@
     "print(f\"high all: {high_all.shape}\")"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "47a0f44d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def fit_normalizer(train_data, au_columns, method='standard', scope='global'):\n",
-    "    \"\"\"\n",
-    "    Fit normalization scalers on training data.\n",
-    "    \n",
-    "    Parameters:\n",
-    "    -----------\n",
-    "    train_data : pd.DataFrame\n",
-    "        Training dataframe with AU columns and subjectID\n",
-    "    au_columns : list\n",
-    "        List of AU column names to normalize\n",
-    "    method : str, default='standard'\n",
-    "        Normalization method: 'standard' for StandardScaler or 'minmax' for MinMaxScaler\n",
-    "    scope : str, default='global'\n",
-    "        Normalization scope: 'subject' for per-subject or 'global' for across all subjects\n",
-    "    \n",
-    "    Returns:\n",
-    "    --------\n",
-    "    dict\n",
-    "        Dictionary containing fitted scalers\n",
-    "    \"\"\"\n",
-    "    # Select scaler based on method\n",
-    "    if method == 'standard':\n",
-    "        Scaler = StandardScaler\n",
-    "    elif method == 'minmax':\n",
-    "        Scaler = MinMaxScaler\n",
-    "    else:\n",
-    "        raise ValueError(\"method must be 'standard' or 'minmax'\")\n",
-    "    \n",
-    "    scalers = {}\n",
-    "    \n",
-    "    if scope == 'subject':\n",
-    "        # Fit one scaler per subject\n",
-    "        for subject in train_data['subjectID'].unique():\n",
-    "            subject_mask = train_data['subjectID'] == subject\n",
-    "            scaler = Scaler()\n",
-    "            scaler.fit(train_data.loc[subject_mask, au_columns])\n",
-    "            scalers[subject] = scaler\n",
-    "    \n",
-    "    elif scope == 'global':\n",
-    "        # Fit one scaler for all subjects\n",
-    "        scaler = Scaler()\n",
-    "        scaler.fit(train_data[au_columns])\n",
-    "        scalers['global'] = scaler\n",
-    "    \n",
-    "    else:\n",
-    "        raise ValueError(\"scope must be 'subject' or 'global'\")\n",
-    "    \n",
-    "    return {'scalers': scalers, 'method': method, 'scope': scope}"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "642d0017",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "def apply_normalizer(data, au_columns, normalizer_dict):\n",
-    "    \"\"\"\n",
-    "    Apply fitted normalization scalers to data.\n",
-    "    \n",
-    "    Parameters:\n",
-    "    -----------\n",
-    "    data : pd.DataFrame\n",
-    "        Dataframe with AU columns and subjectID\n",
-    "    au_columns : list\n",
-    "        List of AU column names to normalize\n",
-    "    normalizer_dict : dict\n",
-    "        Dictionary containing fitted scalers from fit_normalizer()\n",
-    "    \n",
-    "    Returns:\n",
-    "    --------\n",
-    "    pd.DataFrame\n",
-    "        DataFrame with normalized AU columns\n",
-    "    \"\"\"\n",
-    "    normalized_data = data.copy()\n",
-    "    scalers = normalizer_dict['scalers']\n",
-    "    scope = normalizer_dict['scope']\n",
-    "    \n",
-    "    if scope == 'subject':\n",
-    "        # Apply per-subject normalization\n",
-    "        for subject in data['subjectID'].unique():\n",
-    "            subject_mask = data['subjectID'] == subject\n",
-    "            \n",
-    "            # Use the subject's scaler if available, otherwise use a fitted scaler from training\n",
-    "            if subject in scalers:\n",
-    "                scaler = scalers[subject]\n",
-    "            else:\n",
-    "                # For new subjects not seen in training, use the first available scaler\n",
-    "                # (This is a fallback - ideally all test subjects should be in training for subject-level normalization)\n",
-    "                print(f\"Warning: Subject {subject} not found in training data. Using fallback scaler.\")\n",
-    "                scaler = list(scalers.values())[0]\n",
-    "            \n",
-    "            normalized_data.loc[subject_mask, au_columns] = scaler.transform(\n",
-    "                data.loc[subject_mask, au_columns]\n",
-    "            )\n",
-    "    \n",
-    "    elif scope == 'global':\n",
-    "        # Apply global normalization\n",
-    "        scaler = scalers['global']\n",
-    "        normalized_data[au_columns] = scaler.transform(data[au_columns])\n",
-    "    \n",
-    "    return normalized_data"
-   ]
-  },
   {
    "cell_type": "markdown",
    "id": "697b3cf7",
@@ -335,7 +223,7 @@
    "outputs": [],
    "source": [
     "# Fit normalizer on training data\n",
-    "normalizer = fit_normalizer(train_data, cols, method='minmax', scope='global')\n",
+    "normalizer = scaler.fit_normalizer(train_data, cols, method='minmax', scope='global')\n",
     "print(\"Normalizer fitted on training data\")"
    ]
   },
@@ -347,11 +235,11 @@
    "outputs": [],
    "source": [
     "# Apply normalization to all datasets\n",
-    "train_normalized = apply_normalizer(train_data, cols, normalizer)\n",
-    "val_normal_normalized = apply_normalizer(val_normal_data, cols, normalizer)\n",
-    "val_high_normalized = apply_normalizer(val_high_data, cols, normalizer)\n",
-    "test_normal_normalized = apply_normalizer(test_normal_data, cols, normalizer)\n",
-    "test_high_normalized = apply_normalizer(test_high_data, cols, normalizer)\n",
+    "train_normalized = scaler.apply_normalizer(train_data, cols, normalizer)\n",
+    "val_normal_normalized = scaler.apply_normalizer(val_normal_data, cols, normalizer)\n",
+    "val_high_normalized = scaler.apply_normalizer(val_high_data, cols, normalizer)\n",
+    "test_normal_normalized = scaler.apply_normalizer(test_normal_data, cols, normalizer)\n",
+    "test_high_normalized = scaler.apply_normalizer(test_high_data, cols, normalizer)\n",
     "\n",
     "print(\"Normalization applied to all datasets\")"
    ]
@@ -490,18 +378,6 @@
    "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.12.10"
   }
  },
  "nbformat": 4,

model_training/OCSVM/ocsvm_with_AE.ipynb

@@ -20,10 +20,10 @@
     "from pathlib import Path\n",
     "import sys\n",
     "import os\n",
+    "import tensorflow as tf\n",
     "\n",
     "base_dir = os.path.abspath(os.path.join(os.getcwd(), \"..\"))\n",
     "sys.path.append(base_dir)\n",
-    "print(base_dir)\n",
     "\n",
     "from sklearn.pipeline import Pipeline\n",
     "from sklearn.svm import OneClassSVM\n",
@@ -31,7 +31,7 @@
     "import matplotlib.pyplot as plt\n",
     "import tensorflow as tf\n",
     "import pickle\n",
-    "from tools import evaluation_tools, scaler\n",
+    "from Fahrsimulator_MSY2526_AI.model_training.tools import evaluation_tools, scaler\n",
     "from sklearn.metrics import (balanced_accuracy_score, accuracy_score, precision_score, \n",
     "                                recall_score, f1_score, confusion_matrix, classification_report)  "
    ]
@@ -44,6 +44,17 @@
     "### Load data"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "30a4c885",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "enconder_path = Path(\".keras\")\n",
+    "model_path = Path(\".pkl\")"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -51,7 +62,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "data_path = Path(r\"/home/jovyan/data-paulusjafahrsimulator-gpu/first_AU_dataset/output_windowed.parquet\")"
+    "data_path = Path(r\".parquet\")"
    ]
   },
   {
@@ -61,7 +72,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "df = pd.read_parquet(path=data_path)"
+    "df = pd.read_parquet(path=data_path)\n",
+    "df = df.dropna()"
    ]
   },
   {
@@ -150,20 +162,29 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "au_columns = [col for col in low_all.columns if col.startswith('AU')]\n",
+    "au_columns = [col for col in low_all.columns if \"face\" in col.lower()] \n",
+    "\n",
+    "eye_columns = [ \n",
+    "    'Fix_count_short_66_150','Fix_count_medium_300_500','Fix_count_long_gt_1000', \n",
+    "    'Fix_count_100','Fix_mean_duration','Fix_median_duration', \n",
+    "    'Sac_count','Sac_mean_amp','Sac_mean_dur','Sac_median_dur', \n",
+    "    'Blink_count','Blink_mean_dur','Blink_median_dur', \n",
+    "    'Pupil_mean','Pupil_IPA' \n",
+    "] \n",
+    "cols = au_columns +eye_columns\n",
     "\n",
     "# Prepare training data (only normal/low data)\n",
-    "train_data = low_all[low_all['subjectID'].isin(train_subjects)][['subjectID'] + au_columns].copy()\n",
+    "train_data = low_all[low_all['subjectID'].isin(train_subjects)][['subjectID'] + cols].copy()\n",
     "\n",
     "# Prepare validation data (normal and anomaly) \n",
-    "val_normal_data = low_all[low_all['subjectID'].isin(val_subjects)][['subjectID'] + au_columns].copy()\n",
-    "val_high_data = high_all[high_all['subjectID'].isin(val_subjects)][['subjectID'] + au_columns].copy()\n",
-    "val_normal_data = val_normal_data.sample(n=1000, random_state=42)\n",
-    "val_high_data = val_high_data.sample(n=1000, random_state=42)\n",
+    "val_normal_data = low_all[low_all['subjectID'].isin(val_subjects)][['subjectID'] + cols].copy()\n",
+    "val_high_data = high_all[high_all['subjectID'].isin(val_subjects)][['subjectID'] + cols].copy()\n",
+    "val_normal_data = val_normal_data.sample(n=500, random_state=42)\n",
+    "val_high_data = val_high_data.sample(n=500, random_state=42)\n",
     "\n",
     "# Prepare test data (normal and anomaly) - 1000 samples each\n",
-    "test_normal_data = low_all[low_all['subjectID'].isin(test_subjects)][['subjectID'] + au_columns].copy()\n",
-    "test_high_data = high_all[high_all['subjectID'].isin(test_subjects)][['subjectID'] + au_columns].copy()\n",
+    "test_normal_data = low_all[low_all['subjectID'].isin(test_subjects)][['subjectID'] + cols].copy()\n",
+    "test_high_data = high_all[high_all['subjectID'].isin(test_subjects)][['subjectID'] + cols].copy()\n",
     "test_normal_data = test_normal_data.sample(n=500, random_state=42)\n",
     "test_high_data = test_high_data.sample(n=500, random_state=42)\n",
     "\n",
@@ -186,7 +207,7 @@
    "outputs": [],
    "source": [
     "# Cell 3: Fit normalizer on training data\n",
-    "normalizer = scaler.fit_normalizer(train_data, au_columns, method='minmax', scope='global')\n",
+    "normalizer = scaler.fit_normalizer(train_data, cols, method='minmax', scope='global')\n",
     "print(\"Normalizer fitted on training data\")"
    ]
   },
@@ -198,11 +219,11 @@
    "outputs": [],
    "source": [
     "# Cell 4: Apply normalization to all datasets\n",
-    "train_normalized = scaler.apply_normalizer(train_data, au_columns, normalizer)\n",
-    "val_normal_normalized = scaler.apply_normalizer(val_normal_data, au_columns, normalizer)\n",
-    "val_high_normalized = scaler.apply_normalizer(val_high_data, au_columns, normalizer)\n",
-    "test_normal_normalized = scaler.apply_normalizer(test_normal_data, au_columns, normalizer)\n",
-    "test_high_normalized = scaler.apply_normalizer(test_high_data, au_columns, normalizer)\n",
+    "train_normalized = scaler.apply_normalizer(train_data, cols, normalizer)\n",
+    "val_normal_normalized = scaler.apply_normalizer(val_normal_data, cols, normalizer)\n",
+    "val_high_normalized = scaler.apply_normalizer(val_high_data, cols, normalizer)\n",
+    "test_normal_normalized = scaler.apply_normalizer(test_normal_data, cols, normalizer)\n",
+    "test_high_normalized = scaler.apply_normalizer(test_high_data, cols, normalizer)\n",
     "\n",
     "print(\"Normalization applied to all datasets\")"
    ]
@@ -214,13 +235,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Cell 5: Extract AU columns and create labels for grid search\n",
-    "# Extract only AU columns (drop subjectID)\n",
-    "X_train = train_normalized[au_columns].copy()\n",
-    "X_val_normal = val_normal_normalized[au_columns].copy()\n",
-    "X_val_high = val_high_normalized[au_columns].copy()\n",
-    "X_test_high = test_high_normalized[au_columns].copy()\n",
-    "X_test_normal = test_normal_normalized[au_columns].copy()\n",
+    "X_train = train_normalized[cols].copy()\n",
+    "X_val_normal = val_normal_normalized[cols].copy()\n",
+    "X_val_high = val_high_normalized[cols].copy()\n",
+    "X_test_high = test_high_normalized[cols].copy()\n",
+    "X_test_normal = test_normal_normalized[cols].copy()\n",
     "\n",
     "\n",
     "# Create labels for grid search\n",
@@ -241,116 +260,12 @@
     "X_train.shape"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "50fc80dc-fe16-4917-aad6-0dbaa1ce5ef9",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "!pip install keras-tuner --quiet  # nur einmal nötig\n",
-    "\n",
-    "import tensorflow as tf\n",
-    "from tensorflow import keras\n",
-    "from kerastuner import HyperModel\n",
-    "from kerastuner.tuners import RandomSearch\n",
-    "\n",
-    "# 1️⃣ HyperModel definieren\n",
-    "class AutoencoderHyperModel(HyperModel):\n",
-    "    def __init__(self, input_dim):\n",
-    "        self.input_dim = input_dim\n",
-    "\n",
-    "    def build(self, hp):\n",
-    "        reg = hp.Float(\"l2_reg\", min_value=1e-5, max_value=0.01, sampling=\"log\")\n",
-    "        lr = hp.Float(\"learning_rate\", 1e-4, 1e-2, sampling=\"log\")\n",
-    "\n",
-    "        # Encoder\n",
-    "        encoder = keras.Sequential([\n",
-    "            keras.layers.Dense(\n",
-    "                units=hp.Int(\"enc_units1\", min_value=10, max_value=self.input_dim, step=10),\n",
-    "                activation=None,\n",
-    "                kernel_regularizer=keras.regularizers.l2(reg)\n",
-    "            ),\n",
-    "            keras.layers.LeakyReLU(alpha=0.1),\n",
-    "            keras.layers.Dense(\n",
-    "                units=hp.Int(\"enc_units2\", min_value=5, max_value=20, step=1),\n",
-    "                activation=tf.keras.layers.LeakyReLU(alpha=0.1),\n",
-    "                kernel_regularizer=keras.regularizers.l2(reg)\n",
-    "            ),\n",
-    "            keras.layers.Dense(\n",
-    "                units=2,  # Bottleneck\n",
-    "                activation='linear',\n",
-    "                kernel_regularizer=keras.regularizers.l2(reg)\n",
-    "            ),\n",
-    "        ])\n",
-    "\n",
-    "        # Decoder\n",
-    "        decoder = keras.Sequential([\n",
-    "            keras.layers.Dense(\n",
-    "                units=hp.Int(\"dec_units1\", min_value=5, max_value=20, step=1),\n",
-    "                activation=tf.keras.layers.LeakyReLU(alpha=0.1),\n",
-    "                kernel_regularizer=keras.regularizers.l2(reg)\n",
-    "            ),\n",
-    "            keras.layers.Dense(\n",
-    "                units=hp.Int(\"dec_units2\", min_value=10, max_value=self.input_dim, step=10),\n",
-    "                activation=tf.keras.layers.LeakyReLU(alpha=0.1),\n",
-    "                kernel_regularizer=keras.regularizers.l2(reg)\n",
-    "            ),\n",
-    "            keras.layers.Dense(\n",
-    "                units=self.input_dim,\n",
-    "                activation='linear',\n",
-    "                kernel_regularizer=keras.regularizers.l2(reg)\n",
-    "            ),\n",
-    "        ])\n",
-    "\n",
-    "        # Autoencoder\n",
-    "        inputs = keras.Input(shape=(self.input_dim,))\n",
-    "        encoded = encoder(inputs)\n",
-    "        decoded = decoder(encoded)\n",
-    "        autoencoder = keras.Model(inputs, decoded)\n",
-    "\n",
-    "        autoencoder.compile(\n",
-    "            optimizer=keras.optimizers.Adam(learning_rate=lr),\n",
-    "            loss='mse'\n",
-    "        )\n",
-    "\n",
-    "        return autoencoder\n",
-    "\n",
-    "# 2️⃣ RandomSearch-Tuner\n",
-    "hypermodel = AutoencoderHyperModel(input_dim=X_train.shape[1])\n",
-    "\n",
-    "tuner = RandomSearch(\n",
-    "    hypermodel,\n",
-    "    objective='val_loss',\n",
-    "    max_trials=10,            # Anzahl der getesteten Kombinationen\n",
-    "    executions_per_trial=1,   # Anzahl Trainings pro Kombination\n",
-    "    directory='tuner_dir',\n",
-    "    project_name='oc_ae'\n",
-    ")\n",
-    "\n",
-    "# 3️⃣ Hyperparameter-Tuning starten\n",
-    "tuner.search(\n",
-    "    X_train, X_train,\n",
-    "    epochs=100,\n",
-    "    batch_size=64,\n",
-    "    validation_data=(X_val_normal, X_val_normal),\n",
-    "    verbose=0\n",
-    ")\n",
-    "\n",
-    "# 4️⃣ Beste Architektur holen\n",
-    "best_model = tuner.get_best_models(num_models=1)[0]\n",
-    "best_hyperparameters = tuner.get_best_hyperparameters(1)[0]\n",
-    "\n",
-    "print(\"Beste Hyperparameter:\", best_hyperparameters.values)\n"
-   ]
-  },
   {
    "cell_type": "markdown",
    "id": "362c0a6f",
    "metadata": {},
    "source": [
-    "\n",
-    "Beste Hyperparameter: {'l2_reg': 1.3757411430582133e-05, 'learning_rate': 0.007321002854350309, 'enc_units1': 20, 'enc_units2': 16, 'dec_units1': 14, 'dec_units2': 10}"
+    "Build model"
    ]
   },
   {
@@ -360,26 +275,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# reg = 0.1\n",
-    "# encoder = tf.keras.Sequential(\n",
-    "#     [\n",
-    "#         tf.keras.layers.Dense(units=X_train.shape[1], activation='relu', kernel_regularizer=tf.keras.regularizers.l2(reg)),\n",
-    "#         tf.keras.layers.Dense(units=10, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(reg)),\n",
-    "#         tf.keras.layers.Dense(units=5, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(reg)),\n",
-    "        \n",
-    "#     ]\n",
-    "# )\n",
-    "\n",
-    "# decoder = tf.keras.Sequential(\n",
-    "#     [\n",
-    "#         tf.keras.layers.Dense(units=5,activation='relu', kernel_regularizer=tf.keras.regularizers.l2(reg)),\n",
-    "#         tf.keras.layers.Dense(units=10, activation='relu', kernel_regularizer=tf.keras.regularizers.l2(reg)),\n",
-    "#         tf.keras.layers.Dense(units=X_train.shape[1], activation='linear', kernel_regularizer=tf.keras.regularizers.l2(reg))\n",
-    "#     ]\n",
-    "# )\n",
-    "\n",
-    "\n",
-    "\n",
     "reg = 1e-5\n",
     "\n",
     "# ENCODER\n",
@@ -389,14 +284,15 @@
     "        activation=None,\n",
     "        kernel_regularizer=tf.keras.regularizers.l2(reg)\n",
     "    ),\n",
-    "    tf.keras.layers.LeakyReLU(alpha=0.1),\n",
-    "    \n",
+    "    tf.keras.layers.LeakyReLU(negative_slope=0.1),\n",
+    "\n",
     "    tf.keras.layers.Dense(\n",
     "        units=12,\n",
-    "        activation=tf.keras.layers.LeakyReLU(negative_slope=0.1),\n",
+    "        activation=None,\n",
     "        kernel_regularizer=tf.keras.regularizers.l2(reg)\n",
     "    ),\n",
-    "       \n",
+    "    tf.keras.layers.LeakyReLU(negative_slope=0.1),\n",
+    "\n",
     "    tf.keras.layers.Dense(\n",
     "        units=8,\n",
     "        activation='linear',  # Bottleneck stays linear\n",
@@ -408,20 +304,24 @@
     "decoder = tf.keras.Sequential([\n",
     "    tf.keras.layers.Dense(\n",
     "        units=8,\n",
-    "        activation=tf.keras.layers.LeakyReLU(negative_slope=0.1),\n",
+    "        activation=None,\n",
     "        kernel_regularizer=tf.keras.regularizers.l2(reg)\n",
     "    ),\n",
+    "    tf.keras.layers.LeakyReLU(negative_slope=0.1),\n",
+    "\n",
     "    tf.keras.layers.Dense(\n",
     "        units=12,\n",
-    "        activation=tf.keras.layers.LeakyReLU(negative_slope=0.1),\n",
+    "        activation=None,\n",
     "        kernel_regularizer=tf.keras.regularizers.l2(reg)\n",
     "    ),\n",
+    "    tf.keras.layers.LeakyReLU(negative_slope=0.1),\n",
+    "\n",
     "    tf.keras.layers.Dense(\n",
     "        units=X_train.shape[1],\n",
     "        activation='linear',\n",
     "        kernel_regularizer=tf.keras.regularizers.l2(reg)\n",
     "    ),\n",
-    "])\n"
+    "])"
    ]
   },
   {
@@ -456,7 +356,7 @@
    "source": [
     "history = autoencoder.fit(\n",
     "    X_train, X_train,  # Input and target are the same for autoencoder\n",
-    "    epochs=200,\n",
+    "    epochs=50,\n",
     "    batch_size=64,\n",
     "    validation_data=(X_val_normal, X_val_normal),\n",
     "    verbose=1\n",
@@ -470,8 +370,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "save_path = Path(\"/home/jovyan/data-paulusjafahrsimulator-gpu/saved_models/encoder_model_2_neurons_minmax.keras\")\n",
-    "encoder.save(save_path)"
+    "encoder.save(enconder_path)"
    ]
   },
   {
@@ -489,8 +388,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "load_path = Path(\"/home/jovyan/data-paulusjafahrsimulator-gpu/saved_models/encoder_model_2_neurons_minmax.keras\")\n",
-    "encoder = tf.keras.models.load_model(load_path)"
+    "encoder = tf.keras.models.load_model(enconder_path)"
    ]
   },
   {
@@ -541,25 +439,6 @@
     "test_predictions.shape"
    ]
   },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "759118d8-989d-489c-9d35-331454b4795e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "all_zero = {\n",
-    "    \"X_train_encoded\":        np.all(X_train_encoded == 0),\n",
-    "    \"X_val_normal_encoded\":   np.all(X_val_normal_encoded == 0),\n",
-    "    \"X_val_high_encoded\":     np.all(X_val_high_encoded == 0),\n",
-    "    \"X_test_normal_encoded\":  np.all(X_test_normal_encoded == 0),\n",
-    "    \"X_test_high_encoded\":    np.all(X_test_high_encoded == 0),\n",
-    "}\n",
-    "\n",
-    "print(all_zero)\n",
-    "print(X_train_encoded.shape)"
-   ]
-  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -567,81 +446,49 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# fig, axes = plt.subplots(1, 3, figsize=(18, 5))\n",
+    "fig, axes = plt.subplots(1, 3, figsize=(18, 5))\n",
     "\n",
-    "# # Subplot A: Normal\n",
-    "# axes[0].scatter(\n",
-    "#     X_val_normal_encoded[:, 0],\n",
-    "#     X_val_normal_encoded[:, 1],\n",
-    "#     color=\"blue\",\n",
-    "#     label=\"Normal\"\n",
-    "# )\n",
-    "# axes[0].set_title(\"Val Normal (encoded)\")\n",
-    "# axes[0].set_xlabel(\"latent feature 1\")\n",
-    "# axes[0].set_ylabel(\"latent feature 2\")\n",
-    "# axes[0].legend()\n",
-    "\n",
-    "# # Subplot B: High\n",
-    "# axes[1].scatter(\n",
-    "#     X_val_high_encoded[:, 0],\n",
-    "#     X_val_high_encoded[:, 1],\n",
-    "#     color=\"orange\",\n",
-    "#     label=\"High\"\n",
-    "# )\n",
-    "# axes[1].set_title(\"ValHigh (encoded)\")\n",
-    "# axes[1].set_xlabel(\"latent feature 1\")\n",
-    "# axes[1].set_ylabel(\"latent feature 2\")\n",
-    "# axes[1].legend()\n",
-    "\n",
-    "# # Subplot C: Both\n",
-    "# axes[2].scatter(\n",
-    "#     X_val_normal_encoded[:, 0],\n",
-    "#     X_val_normal_encoded[:, 1],\n",
-    "#     color=\"blue\",\n",
-    "#     label=\"Normal\"\n",
-    "# )\n",
-    "# axes[2].scatter(\n",
-    "#     X_val_high_encoded[:, 0],\n",
-    "#     X_val_high_encoded[:, 1],\n",
-    "#     color=\"orange\",\n",
-    "#     label=\"High\"\n",
-    "# )\n",
-    "# axes[2].set_title(\"Normal vs High (encoded)\")\n",
-    "# axes[2].set_xlabel(\"latent feature 1\")\n",
-    "# axes[2].set_ylabel(\"latent feature 2\")\n",
-    "# axes[2].legend()\n",
-    "\n",
-    "\n",
-    "\n",
-    "latent_dim = 8\n",
-    "fig, axes = plt.subplots(2, 4, figsize=(20, 10))\n",
-    "axes = axes.flatten()  # flatten to index easily\n",
-    "\n",
-    "for i in range(latent_dim):\n",
-    "    axes[i].scatter(\n",
-    "        X_val_normal_encoded[:, i],\n",
-    "        [0]*X_val_normal_encoded.shape[0],  # optional: place on a line for 1D visualization\n",
-    "        color='blue',\n",
-    "        label='Normal',\n",
-    "        alpha=0.6\n",
-    "    )\n",
-    "    axes[i].scatter(\n",
-    "        X_val_high_encoded[:, i],\n",
-    "        [0]*X_val_high_encoded.shape[0],  # same for High\n",
-    "        color='orange',\n",
-    "        label='High',\n",
-    "        alpha=0.6\n",
-    "    )\n",
-    "    axes[i].set_title(f'Latent dim {i+1}')\n",
-    "    axes[i].set_xlabel(f'Feature {i+1}')\n",
-    "    axes[i].set_yticks([])  # hide y-axis as it's just a 1D comparison\n",
-    "    axes[i].legend()\n",
-    "    axes[i].grid(True)\n",
-    "\n",
-    "plt.tight_layout()\n",
-    "plt.show()\n",
+    "# Subplot A: Normal\n",
+    "axes[0].scatter(\n",
+    "    X_val_normal_encoded[:, 0],\n",
+    "    X_val_normal_encoded[:, 1],\n",
+    "    color=\"blue\",\n",
+    "    label=\"Normal\"\n",
+    ")\n",
+    "axes[0].set_title(\"Val Normal (encoded)\")\n",
+    "axes[0].set_xlabel(\"latent feature 1\")\n",
+    "axes[0].set_ylabel(\"latent feature 2\")\n",
+    "axes[0].legend()\n",
     "\n",
+    "# Subplot B: High\n",
+    "axes[1].scatter(\n",
+    "    X_val_high_encoded[:, 0],\n",
+    "    X_val_high_encoded[:, 1],\n",
+    "    color=\"orange\",\n",
+    "    label=\"High\"\n",
+    ")\n",
+    "axes[1].set_title(\"ValHigh (encoded)\")\n",
+    "axes[1].set_xlabel(\"latent feature 1\")\n",
+    "axes[1].set_ylabel(\"latent feature 2\")\n",
+    "axes[1].legend()\n",
     "\n",
+    "# Subplot C: Both\n",
+    "axes[2].scatter(\n",
+    "    X_val_normal_encoded[:, 0],\n",
+    "    X_val_normal_encoded[:, 1],\n",
+    "    color=\"blue\",\n",
+    "    label=\"Normal\"\n",
+    ")\n",
+    "axes[2].scatter(\n",
+    "    X_val_high_encoded[:, 0],\n",
+    "    X_val_high_encoded[:, 1],\n",
+    "    color=\"orange\",\n",
+    "    label=\"High\"\n",
+    ")\n",
+    "axes[2].set_title(\"Normal vs High (encoded)\")\n",
+    "axes[2].set_xlabel(\"latent feature 1\")\n",
+    "axes[2].set_ylabel(\"latent feature 2\")\n",
+    "axes[2].legend()\n",
     "\n",
     "plt.tight_layout()\n",
     "plt.show()\n",
@@ -708,11 +555,11 @@
    "outputs": [],
    "source": [
     "# Save\n",
-    "with open('ocsvm_model.pkl', 'wb') as f:\n",
+    "with open(model_path, \"wb\") as f:\n",
     "    pickle.dump(ocsvm, f)\n",
     "\n",
     "# Load later\n",
-    "with open('ocsvm_model.pkl', 'rb') as f:\n",
+    "with open(model_path, \"rb\") as f:\n",
     "    ocsvm_loaded = pickle.load(f)"
    ]
   },
@@ -731,11 +578,6 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# X_combined = np.concatenate([X_train_encoded, X_val_normal_encoded, X_val_high_encoded], axis=0)\n",
-    "# y_combined = np.concatenate([\n",
-    "#     np.ones(X_train_encoded.shape[0]+X_val_normal_encoded.shape[0]),      # normal = 1\n",
-    "#     -np.ones(X_val_high_encoded.shape[0])   # anomaly = -1\n",
-    "# ], axis=0)\n",
     "X_combined = np.concatenate([X_train_encoded, X_val_high_encoded], axis=0)\n",
     "y_combined = np.concatenate([\n",
     "    np.ones(X_train_encoded.shape[0]),      # normal = 1\n",
@@ -856,14 +698,6 @@
    "source": [
     "f1_score(y_true=np.concatenate([y_test_normal, y_test_high]), y_pred=predictions)"
    ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "9e412041-7534-40fe-8486-ee97349a6168",
-   "metadata": {},
-   "outputs": [],
-   "source": []
   }
  ],
  "metadata": {
@@ -871,18 +705,6 @@
    "display_name": "Python 3 (ipykernel)",
    "language": "python",
    "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.12.10"
   }
  },
  "nbformat": 4,

model_training/VAE_SVM/vaesvm.ipynb

@@ -1,877 +0,0 @@
-{
- "cells": [
-  {
-   "cell_type": "markdown",
-   "id": "708c9745",
-   "metadata": {},
-   "source": [
-    "### Imports"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "53b10294",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "import pandas as pd\n",
-    "import numpy as np\n",
-    "from pathlib import Path\n",
-    "import sys\n",
-    "import os\n",
-    "\n",
-    "base_dir = os.path.abspath(os.path.join(os.getcwd(), \"..\"))\n",
-    "sys.path.append(base_dir)\n",
-    "print(base_dir)\n",
-    "\n",
-    "from Fahrsimulator_MSY2526_AI.model_training.tools import evaluation_tools, scaler, mad_outlier_removal\n",
-    "from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
-    "from sklearn.svm import OneClassSVM\n",
-    "from sklearn.model_selection import GridSearchCV, KFold, ParameterGrid, train_test_split\n",
-    "import matplotlib.pyplot as plt\n",
-    "import tensorflow as tf\n",
-    "import pickle\n",
-    "from sklearn.metrics import (roc_auc_score, accuracy_score, precision_score, \n",
-    "                                recall_score, f1_score, confusion_matrix, classification_report)  "
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "68101229",
-   "metadata": {},
-   "source": [
-    "### load Dataset"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "24a765e8",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "dataset_path = Path(r\"/home/jovyan/data-paulusjafahrsimulator-gpu/first_AU_dataset/output_windowed.parquet\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "471001b0",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "df = pd.read_parquet(path=dataset_path)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "0fdecdaa",
-   "metadata": {},
-   "source": [
-    "### Load Performance data and Subject Split"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "692d1b47",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "performance_path =  Path(r\"/home/jovyan/data-paulusjafahrsimulator-gpu/subject_performance/3new_au_performance.csv\")\n",
-    "performance_df = pd.read_csv(performance_path)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ea617e3f",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# Subject IDs aus dem Haupt-Dataset nehmen\n",
-    "subjects_from_df = df[\"subjectID\"].unique()\n",
-    "\n",
-    "# Performance-Subset nur für vorhandene Subjects\n",
-    "perf_filtered = performance_df[\n",
-    "    performance_df[\"subjectID\"].isin(subjects_from_df)\n",
-    "][[\"subjectID\", \"overall_score\"]]\n",
-    "\n",
-    "# Merge: nur Subjects, die sowohl im df als auch im Performance-CSV vorkommen\n",
-    "merged = (\n",
-    "    pd.DataFrame({\"subjectID\": subjects_from_df})\n",
-    "    .merge(perf_filtered, on=\"subjectID\", how=\"inner\")\n",
-    ")\n",
-    "\n",
-    "# Sicherstellen, dass keine Scores fehlen\n",
-    "if merged[\"overall_score\"].isna().any():\n",
-    "    raise ValueError(\"Es fehlen Score-Werte für manche Subjects.\")\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "ae43df8d",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "merged_sorted = merged.sort_values(\"overall_score\", ascending=False).reset_index(drop=True)\n",
-    "\n",
-    "scores = merged_sorted[\"overall_score\"].values\n",
-    "n_total = len(merged_sorted)\n",
-    "n_small = n_total // 3\n",
-    "n_large = n_total - n_small\n",
-    "\n",
-    "# Schritt 1: zufällige Start-Aufteilung\n",
-    "idx = np.arange(n_total)\n",
-    "np.random.shuffle(idx)\n",
-    "\n",
-    "small_idx = idx[:n_small]\n",
-    "large_idx = idx[n_small:]\n",
-    "\n",
-    "def score_diff(small_idx, large_idx):\n",
-    "    return abs(scores[small_idx].mean() - scores[large_idx].mean())\n",
-    "\n",
-    "diff = score_diff(small_idx, large_idx)\n",
-    "threshold = 0.01\n",
-    "max_iter = 100\n",
-    "count = 0\n",
-    "\n",
-    "# Schritt 2: random swaps bis Differenz klein genug\n",
-    "while diff > threshold and count < max_iter:\n",
-    "    # Zwei zufällige Elemente auswählen\n",
-    "    si = np.random.choice(small_idx)\n",
-    "    li = np.random.choice(large_idx)\n",
-    "    \n",
-    "    # Tausch durchführen\n",
-    "    new_small_idx = small_idx.copy()\n",
-    "    new_large_idx = large_idx.copy()\n",
-    "    \n",
-    "    new_small_idx[new_small_idx == si] = li\n",
-    "    new_large_idx[new_large_idx == li] = si\n",
-    "\n",
-    "    # neue Differenz berechnen\n",
-    "    new_diff = score_diff(new_small_idx, new_large_idx)\n",
-    "\n",
-    "    # Swap akzeptieren, wenn es besser wird\n",
-    "    if new_diff < diff:\n",
-    "        small_idx = new_small_idx\n",
-    "        large_idx = new_large_idx\n",
-    "        diff = new_diff\n",
-    "\n",
-    "    count += 1\n",
-    "\n",
-    "# Finalgruppen\n",
-    "group_small = merged_sorted.loc[small_idx].reset_index(drop=True)\n",
-    "group_large = merged_sorted.loc[large_idx].reset_index(drop=True)\n",
-    "\n",
-    "print(\"Finale Score-Differenz:\", diff)\n",
-    "print(\"Größe Gruppe 1:\", len(group_small))\n",
-    "print(\"Größe Gruppe 2:\", len(group_large))\n"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "9d1b414e",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "group_large['overall_score'].mean()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "fa71f9a5",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "group_small['overall_score'].mean()"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "79ecb4a2",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "training_subjects = group_large['subjectID'].values\n",
-    "test_subjects = group_small['subjectID'].values\n",
-    "print(training_subjects)\n",
-    "print(test_subjects)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "4353f87c",
-   "metadata": {},
-   "source": [
-    "### Data cleaning with mad"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "76610052",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "# SET\n",
-    "threshold_mad = 5\n",
-    "column_praefix ='AU'\n",
-    "\n",
-    "au_columns = [col for col in df.columns if col.startswith(column_praefix)]\n",
-    "cleaned_df = mad_outlier_removal.mad_outlier_removal(df,columns=au_columns, threshold=threshold_mad)\n",
-    "print(cleaned_df.shape)\n",
-    "print(df.shape)"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "9a6c1732",
-   "metadata": {},
-   "source": [
-    "#### TO DO\n",
-    "    * pipeline aus Autoencoder und SVM\n",
-    "    * group k fold\n",
-    "    * AE überpüfen, loss dokumentieren"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "877309d9",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "### Variational Autoencoder with Classifier Head\n",
-    "import pandas as pd\n",
-    "import numpy as np\n",
-    "import tensorflow as tf\n",
-    "from tensorflow import keras\n",
-    "from tensorflow.keras import layers, Model\n",
-    "from sklearn.model_selection import GroupKFold\n",
-    "from sklearn.preprocessing import StandardScaler\n",
-    "from sklearn.metrics import (\n",
-    "    accuracy_score, precision_score, recall_score, f1_score, \n",
-    "    roc_auc_score, confusion_matrix, classification_report\n",
-    ")\n",
-    "import matplotlib.pyplot as plt\n",
-    "from collections import defaultdict\n",
-    "\n",
-    "# ============================================================================\n",
-    "# 1. CREATE LABELS\n",
-    "# ============================================================================\n",
-    "\n",
-    "# Low workload: baseline + n-back level 1,4\n",
-    "low_all = cleaned_df[\n",
-    "    ((cleaned_df[\"PHASE\"] == \"baseline\") |\n",
-    "     ((cleaned_df[\"STUDY\"] == \"n-back\") & (cleaned_df[\"PHASE\"] != \"baseline\") & (cleaned_df[\"LEVEL\"].isin([1,4]))))\n",
-    "].copy()\n",
-    "low_all['label'] = 0\n",
-    "print(f\"Low workload samples: {low_all.shape[0]}\")\n",
-    "\n",
-    "# High workload n-back: level 2,3,5,6\n",
-    "high_nback = cleaned_df[\n",
-    "    (cleaned_df[\"STUDY\"]==\"n-back\") &\n",
-    "    (cleaned_df[\"LEVEL\"].isin([2, 3, 5, 6])) &\n",
-    "    (cleaned_df[\"PHASE\"].isin([\"train\", \"test\"]))\n",
-    "].copy()\n",
-    "high_nback['label'] = 1\n",
-    "print(f\"High n-back samples: {high_nback.shape[0]}\")\n",
-    "\n",
-    "# High workload k-drive\n",
-    "high_kdrive = cleaned_df[\n",
-    "    (cleaned_df[\"STUDY\"] == \"k-drive\") & (cleaned_df[\"PHASE\"] != \"baseline\")\n",
-    "].copy()\n",
-    "high_kdrive['label'] = 1\n",
-    "print(f\"High k-drive samples: {high_kdrive.shape[0]}\")\n",
-    "\n",
-    "# Combine all high workload\n",
-    "high_all = pd.concat([high_nback, high_kdrive])\n",
-    "print(f\"Total high workload samples: {high_all.shape[0]}\")\n",
-    "\n",
-    "# Complete labeled dataset\n",
-    "labeled_df = pd.concat([low_all, high_all]).reset_index(drop=True)\n",
-    "print(f\"\\nTotal labeled samples: {labeled_df.shape[0]}\")\n",
-    "print(f\"Class distribution:\\n{labeled_df['label'].value_counts()}\")\n",
-    "\n",
-    "# ============================================================================\n",
-    "# 2. TRAIN/TEST SPLIT BY SUBJECTS\n",
-    "# ============================================================================\n",
-    "\n",
-    "train_df = labeled_df[labeled_df['subjectID'].isin(training_subjects)].copy()\n",
-    "test_df = labeled_df[labeled_df['subjectID'].isin(test_subjects)].copy()\n",
-    "\n",
-    "print(f\"\\nTraining subjects: {training_subjects}\")\n",
-    "print(f\"Test subjects: {test_subjects}\")\n",
-    "print(f\"Train samples: {train_df.shape[0]}, Test samples: {test_df.shape[0]}\")\n",
-    "\n",
-    "# Extract features and labels\n",
-    "au_columns = [col for col in labeled_df.columns if col.startswith('AU')]\n",
-    "print(f\"\\nUsing {len(au_columns)} AU features: {au_columns}\")\n",
-    "\n",
-    "X_train = train_df[au_columns].values\n",
-    "y_train = train_df['label'].values\n",
-    "groups_train = train_df['subjectID'].values\n",
-    "\n",
-    "X_test = test_df[au_columns].values\n",
-    "y_test = test_df['label'].values\n",
-    "\n",
-    "# Normalize features\n",
-    "scaler = StandardScaler()\n",
-    "X_train_scaled = scaler.fit_transform(X_train)\n",
-    "X_test_scaled = scaler.transform(X_test)\n",
-    "\n",
-    "print(f\"\\nTrain class distribution: {np.bincount(y_train)}\")\n",
-    "print(f\"Test class distribution: {np.bincount(y_test)}\")\n",
-    "\n",
-    "# ============================================================================\n",
-    "# 3. VAE WITH CLASSIFIER HEAD MODEL\n",
-    "# ============================================================================\n",
-    "\n",
-    "class Sampling(layers.Layer):\n",
-    "    \"\"\"Reparameterization trick for VAE\"\"\"\n",
-    "    def call(self, inputs):\n",
-    "        z_mean, z_log_var = inputs\n",
-    "        batch = tf.shape(z_mean)[0]\n",
-    "        dim = tf.shape(z_mean)[1]\n",
-    "        epsilon = tf.random.normal(shape=(batch, dim))\n",
-    "        return z_mean + tf.exp(0.5 * z_log_var) * epsilon\n",
-    "\n",
-    "def build_vae_classifier(input_dim, latent_dim, encoder_dims=[32, 16], \n",
-    "                         decoder_dims=[16, 32], classifier_dims=[16]):\n",
-    "    \"\"\"\n",
-    "    Build VAE with classifier head\n",
-    "    \n",
-    "    Args:\n",
-    "        input_dim: Number of input features (20 AUs)\n",
-    "        latent_dim: Dimension of latent space (2-5)\n",
-    "        encoder_dims: Hidden layer sizes for encoder\n",
-    "        decoder_dims: Hidden layer sizes for decoder\n",
-    "        classifier_dims: Hidden layer sizes for classifier\n",
-    "    \"\"\"\n",
-    "    \n",
-    "    # ---- ENCODER ----\n",
-    "    encoder_inputs = keras.Input(shape=(input_dim,), name='encoder_input')\n",
-    "    x = encoder_inputs\n",
-    "    \n",
-    "    for i, dim in enumerate(encoder_dims):\n",
-    "        x = layers.Dense(dim, activation='relu', name=f'encoder_dense_{i}')(x)\n",
-    "        x = layers.BatchNormalization(name=f'encoder_bn_{i}')(x)\n",
-    "        x = layers.Dropout(0.2, name=f'encoder_dropout_{i}')(x)\n",
-    "    \n",
-    "    z_mean = layers.Dense(latent_dim, name='z_mean')(x)\n",
-    "    z_log_var = layers.Dense(latent_dim, name='z_log_var')(x)\n",
-    "    z = Sampling()([z_mean, z_log_var])\n",
-    "    \n",
-    "    encoder = Model(encoder_inputs, [z_mean, z_log_var, z], name='encoder')\n",
-    "    \n",
-    "    # ---- DECODER ----\n",
-    "    latent_inputs = keras.Input(shape=(latent_dim,), name='latent_input')\n",
-    "    x = latent_inputs\n",
-    "    \n",
-    "    for i, dim in enumerate(decoder_dims):\n",
-    "        x = layers.Dense(dim, activation='relu', name=f'decoder_dense_{i}')(x)\n",
-    "        x = layers.BatchNormalization(name=f'decoder_bn_{i}')(x)\n",
-    "    \n",
-    "    decoder_outputs = layers.Dense(input_dim, activation='linear', name='decoder_output')(x)\n",
-    "    decoder = Model(latent_inputs, decoder_outputs, name='decoder')\n",
-    "    \n",
-    "    # ---- CLASSIFIER HEAD ----\n",
-    "    x = latent_inputs\n",
-    "    for i, dim in enumerate(classifier_dims):\n",
-    "        x = layers.Dense(dim, activation='relu', name=f'classifier_dense_{i}')(x)\n",
-    "        x = layers.Dropout(0.3, name=f'classifier_dropout_{i}')(x)\n",
-    "    \n",
-    "    classifier_output = layers.Dense(1, activation='sigmoid', name='classifier_output')(x)\n",
-    "    classifier = Model(latent_inputs, classifier_output, name='classifier')\n",
-    "    \n",
-    "    # ---- FULL MODEL ----\n",
-    "    inputs = keras.Input(shape=(input_dim,), name='vae_input')\n",
-    "    z_mean, z_log_var, z = encoder(inputs)\n",
-    "    reconstructed = decoder(z)\n",
-    "    classification = classifier(z)\n",
-    "    \n",
-    "    model = Model(inputs, [reconstructed, classification], name='vae_classifier')\n",
-    "    \n",
-    "    return model, encoder, decoder, classifier\n",
-    "\n",
-    "# ============================================================================\n",
-    "# 4. CUSTOM TRAINING LOOP WITH COMBINED LOSS\n",
-    "# ============================================================================\n",
-    "\n",
-    "class VAEClassifier(keras.Model):\n",
-    "    def __init__(self, encoder, decoder, classifier, **kwargs):\n",
-    "        super().__init__(**kwargs)\n",
-    "        self.encoder = encoder\n",
-    "        self.decoder = decoder\n",
-    "        self.classifier = classifier\n",
-    "        self.total_loss_tracker = keras.metrics.Mean(name=\"total_loss\")\n",
-    "        self.reconstruction_loss_tracker = keras.metrics.Mean(name=\"reconstruction_loss\")\n",
-    "        self.kl_loss_tracker = keras.metrics.Mean(name=\"kl_loss\")\n",
-    "        self.classification_loss_tracker = keras.metrics.Mean(name=\"classification_loss\")\n",
-    "        self.accuracy_tracker = keras.metrics.BinaryAccuracy(name=\"accuracy\")\n",
-    "        \n",
-    "    @property\n",
-    "    def metrics(self):\n",
-    "        return [\n",
-    "            self.total_loss_tracker,\n",
-    "            self.reconstruction_loss_tracker,\n",
-    "            self.kl_loss_tracker,\n",
-    "            self.classification_loss_tracker,\n",
-    "            self.accuracy_tracker,\n",
-    "        ]\n",
-    "    \n",
-    "    def train_step(self, data):\n",
-    "        x, y = data\n",
-    "        \n",
-    "        with tf.GradientTape() as tape:\n",
-    "            # Forward pass\n",
-    "            z_mean, z_log_var, z = self.encoder(x, training=True)\n",
-    "            reconstruction = self.decoder(z, training=True)\n",
-    "            classification = self.classifier(z, training=True)\n",
-    "            \n",
-    "            # Reconstruction loss (MSE)\n",
-    "            reconstruction_loss = tf.reduce_mean(\n",
-    "            keras.losses.mse(x, reconstruction))\n",
-    "            \n",
-    "            # KL divergence loss\n",
-    "            kl_loss = -0.5 * tf.reduce_mean(\n",
-    "                tf.reduce_sum(\n",
-    "                    1 + z_log_var - tf.square(z_mean) - tf.exp(z_log_var),\n",
-    "                    axis=1\n",
-    "                )\n",
-    "            )\n",
-    "            \n",
-    "            # Classification loss (binary crossentropy)\n",
-    "            # Classification loss (binary crossentropy)\n",
-    "            classification_loss = tf.reduce_mean(\n",
-    "                keras.losses.binary_crossentropy(tf.expand_dims(y, -1), classification)\n",
-    "            )\n",
-    "            \n",
-    "            # Combined loss with weights\n",
-    "            total_loss = reconstruction_loss + kl_loss + classification_loss\n",
-    "        \n",
-    "        # Backpropagation\n",
-    "        grads = tape.gradient(total_loss, self.trainable_weights)\n",
-    "        self.optimizer.apply_gradients(zip(grads, self.trainable_weights))\n",
-    "        \n",
-    "        # Update metrics\n",
-    "        self.total_loss_tracker.update_state(total_loss)\n",
-    "        self.reconstruction_loss_tracker.update_state(reconstruction_loss)\n",
-    "        self.kl_loss_tracker.update_state(kl_loss)\n",
-    "        self.classification_loss_tracker.update_state(classification_loss)\n",
-    "        self.accuracy_tracker.update_state(y, classification)\n",
-    "        \n",
-    "        return {\n",
-    "            \"total_loss\": self.total_loss_tracker.result(),\n",
-    "            \"reconstruction_loss\": self.reconstruction_loss_tracker.result(),\n",
-    "            \"kl_loss\": self.kl_loss_tracker.result(),\n",
-    "            \"classification_loss\": self.classification_loss_tracker.result(),\n",
-    "            \"accuracy\": self.accuracy_tracker.result(),\n",
-    "        }\n",
-    "    \n",
-    "    def test_step(self, data):\n",
-    "        x, y = data\n",
-    "        \n",
-    "        z_mean, z_log_var, z = self.encoder(x, training=False)\n",
-    "        reconstruction = self.decoder(z, training=False)\n",
-    "        classification = self.classifier(z, training=False)\n",
-    "        \n",
-    "        # Reconstruction loss (MSE)\n",
-    "        reconstruction_loss = tf.reduce_mean(\n",
-    "            keras.losses.mse(x, reconstruction))\n",
-    "        kl_loss = -0.5 * tf.reduce_mean(\n",
-    "            tf.reduce_sum(1 + z_log_var - tf.square(z_mean) - tf.exp(z_log_var), axis=1)\n",
-    "        )\n",
-    "        # Classification loss (binary crossentropy)\n",
-    "        classification_loss = tf.reduce_mean(\n",
-    "            keras.losses.binary_crossentropy(tf.expand_dims(y, -1), classification)\n",
-    "        )\n",
-    "        total_loss = reconstruction_loss + kl_loss + classification_loss\n",
-    "        \n",
-    "        self.total_loss_tracker.update_state(total_loss)\n",
-    "        self.reconstruction_loss_tracker.update_state(reconstruction_loss)\n",
-    "        self.kl_loss_tracker.update_state(kl_loss)\n",
-    "        self.classification_loss_tracker.update_state(classification_loss)\n",
-    "        self.accuracy_tracker.update_state(y, classification)\n",
-    "        \n",
-    "        return {\n",
-    "            \"total_loss\": self.total_loss_tracker.result(),\n",
-    "            \"reconstruction_loss\": self.reconstruction_loss_tracker.result(),\n",
-    "            \"kl_loss\": self.kl_loss_tracker.result(),\n",
-    "            \"classification_loss\": self.classification_loss_tracker.result(),\n",
-    "            \"accuracy\": self.accuracy_tracker.result(),\n",
-    "        }\n",
-    "\n",
-    "# ============================================================================\n",
-    "# 5. GROUP K-FOLD CROSS-VALIDATION WITH GRID SEARCH\n",
-    "# ============================================================================\n",
-    "\n",
-    "# Hyperparameter grid\n",
-    "param_grid = {\n",
-    "    'latent_dim': [2, 5],\n",
-    "    'encoder_dims': [[32, 16], [64, 32]],\n",
-    "    'learning_rate': [0.001, 0.005],\n",
-    "    'batch_size': [32, 64],\n",
-    "}\n",
-    "\n",
-    "# Generate all combinations\n",
-    "from itertools import product\n",
-    "keys = param_grid.keys()\n",
-    "values = param_grid.values()\n",
-    "param_combinations = [dict(zip(keys, v)) for v in product(*values)]\n",
-    "\n",
-    "print(f\"\\nTotal hyperparameter combinations: {len(param_combinations)}\")\n",
-    "\n",
-    "# Group K-Fold setup\n",
-    "n_splits = 5\n",
-    "gkf = GroupKFold(n_splits=n_splits)\n",
-    "\n",
-    "# Store results\n",
-    "cv_results = []\n",
-    "\n",
-    "# Grid search with cross-validation\n",
-    "for idx, params in enumerate(param_combinations):\n",
-    "    print(f\"\\n{'='*80}\")\n",
-    "    print(f\"Testing combination {idx+1}/{len(param_combinations)}: {params}\")\n",
-    "    print(f\"{'='*80}\")\n",
-    "    \n",
-    "    fold_results = []\n",
-    "    \n",
-    "    for fold, (train_idx, val_idx) in enumerate(gkf.split(X_train_scaled, y_train, groups_train)):\n",
-    "        print(f\"\\nFold {fold+1}/{n_splits}\")\n",
-    "        \n",
-    "        X_fold_train, X_fold_val = X_train_scaled[train_idx], X_train_scaled[val_idx]\n",
-    "        y_fold_train, y_fold_val = y_train[train_idx], y_train[val_idx]\n",
-    "        \n",
-    "        # Build model\n",
-    "        model, encoder, decoder, classifier = build_vae_classifier(\n",
-    "            input_dim=len(au_columns),\n",
-    "            latent_dim=params['latent_dim'],\n",
-    "            encoder_dims=params['encoder_dims'],\n",
-    "            decoder_dims=list(reversed(params['encoder_dims'])),\n",
-    "            classifier_dims=[16]\n",
-    "        )\n",
-    "        \n",
-    "        vae_classifier = VAEClassifier(encoder, decoder, classifier)\n",
-    "        vae_classifier.compile(optimizer=keras.optimizers.Adam(params['learning_rate']))\n",
-    "        \n",
-    "        # Early stopping\n",
-    "        early_stop = keras.callbacks.EarlyStopping(\n",
-    "            monitor='val_total_loss',\n",
-    "            patience=10,\n",
-    "            restore_best_weights=True,\n",
-    "            mode='min'\n",
-    "        )\n",
-    "        \n",
-    "        # Train\n",
-    "        history = vae_classifier.fit(\n",
-    "            X_fold_train, y_fold_train,\n",
-    "            validation_data=(X_fold_val, y_fold_val),\n",
-    "            epochs=60,\n",
-    "            batch_size=params['batch_size'],\n",
-    "            callbacks=[early_stop],\n",
-    "            verbose=0\n",
-    "        )\n",
-    "        \n",
-    "        # Evaluate on validation fold\n",
-    "        z_mean_val, _, _ = encoder.predict(X_fold_val, verbose=0)\n",
-    "        y_pred_proba = classifier.predict(z_mean_val, verbose=0).flatten()\n",
-    "        y_pred = (y_pred_proba > 0.5).astype(int)\n",
-    "        \n",
-    "        fold_metrics = {\n",
-    "            'accuracy': accuracy_score(y_fold_val, y_pred),\n",
-    "            'precision': precision_score(y_fold_val, y_pred, zero_division=0),\n",
-    "            'recall': recall_score(y_fold_val, y_pred, zero_division=0),\n",
-    "            'f1': f1_score(y_fold_val, y_pred, zero_division=0),\n",
-    "            'roc_auc': roc_auc_score(y_fold_val, y_pred_proba),\n",
-    "            'final_recon_loss': history.history['val_reconstruction_loss'][-1],\n",
-    "            'final_kl_loss': history.history['val_kl_loss'][-1],\n",
-    "            'final_class_loss': history.history['val_classification_loss'][-1],\n",
-    "        }\n",
-    "        \n",
-    "        fold_results.append(fold_metrics)\n",
-    "        print(f\"  Accuracy: {fold_metrics['accuracy']:.4f}, F1: {fold_metrics['f1']:.4f}, AUC: {fold_metrics['roc_auc']:.4f}\")\n",
-    "        \n",
-    "        # Clear session to free memory\n",
-    "        keras.backend.clear_session()\n",
-    "    \n",
-    "    # Average across folds\n",
-    "    avg_results = {\n",
-    "        'params': params,\n",
-    "        'mean_accuracy': np.mean([r['accuracy'] for r in fold_results]),\n",
-    "        'std_accuracy': np.std([r['accuracy'] for r in fold_results]),\n",
-    "        'mean_f1': np.mean([r['f1'] for r in fold_results]),\n",
-    "        'std_f1': np.std([r['f1'] for r in fold_results]),\n",
-    "        'mean_roc_auc': np.mean([r['roc_auc'] for r in fold_results]),\n",
-    "        'std_roc_auc': np.std([r['roc_auc'] for r in fold_results]),\n",
-    "        'mean_recon_loss': np.mean([r['final_recon_loss'] for r in fold_results]),\n",
-    "        'mean_kl_loss': np.mean([r['final_kl_loss'] for r in fold_results]),\n",
-    "        'mean_class_loss': np.mean([r['final_class_loss'] for r in fold_results]),\n",
-    "        'fold_results': fold_results\n",
-    "    }\n",
-    "    \n",
-    "    cv_results.append(avg_results)\n",
-    "    \n",
-    "    print(f\"\\nMean CV Accuracy: {avg_results['mean_accuracy']:.4f} ± {avg_results['std_accuracy']:.4f}\")\n",
-    "    print(f\"Mean CV F1: {avg_results['mean_f1']:.4f} ± {avg_results['std_f1']:.4f}\")\n",
-    "    print(f\"Mean CV AUC: {avg_results['mean_roc_auc']:.4f} ± {avg_results['std_roc_auc']:.4f}\")\n",
-    "\n",
-    "# ============================================================================\n",
-    "# 6. SELECT BEST MODEL AND EVALUATE ON TEST SET\n",
-    "# ============================================================================\n",
-    "\n",
-    "# Find best hyperparameters based on mean F1 score\n",
-    "best_idx = np.argmax([r['mean_f1'] for r in cv_results])\n",
-    "best_params = cv_results[best_idx]['params']\n",
-    "\n",
-    "print(f\"\\n{'='*80}\")\n",
-    "print(\"BEST HYPERPARAMETERS (based on CV F1 score):\")\n",
-    "print(f\"{'='*80}\")\n",
-    "for key, value in best_params.items():\n",
-    "    print(f\"{key}: {value}\")\n",
-    "print(f\"\\nCV Performance:\")\n",
-    "print(f\"  Accuracy: {cv_results[best_idx]['mean_accuracy']:.4f} ± {cv_results[best_idx]['std_accuracy']:.4f}\")\n",
-    "print(f\"  F1 Score: {cv_results[best_idx]['mean_f1']:.4f} ± {cv_results[best_idx]['std_f1']:.4f}\")\n",
-    "print(f\"  ROC-AUC: {cv_results[best_idx]['mean_roc_auc']:.4f} ± {cv_results[best_idx]['std_roc_auc']:.4f}\")\n",
-    "\n",
-    "# Train final model on all training data\n",
-    "print(f\"\\n{'='*80}\")\n",
-    "print(\"TRAINING FINAL MODEL ON ALL TRAINING DATA\")\n",
-    "print(f\"{'='*80}\")\n",
-    "\n",
-    "final_model, final_encoder, final_decoder, final_classifier = build_vae_classifier(\n",
-    "    input_dim=len(au_columns),\n",
-    "    latent_dim=best_params['latent_dim'],\n",
-    "    encoder_dims=best_params['encoder_dims'],\n",
-    "    decoder_dims=list(reversed(best_params['encoder_dims'])),\n",
-    "    classifier_dims=[16]\n",
-    ")\n",
-    "\n",
-    "final_vae_classifier = VAEClassifier(final_encoder, final_decoder, final_classifier)\n",
-    "final_vae_classifier.compile(optimizer=keras.optimizers.Adam(best_params['learning_rate']))\n",
-    "\n",
-    "final_history = final_vae_classifier.fit(\n",
-    "    X_train_scaled, y_train,\n",
-    "    validation_split=0.2,\n",
-    "    epochs=100,\n",
-    "    batch_size=best_params['batch_size'],\n",
-    "    callbacks=[keras.callbacks.EarlyStopping(monitor='val_total_loss', patience=15, restore_best_weights=True, mode='min')],\n",
-    "    verbose=1\n",
-    ")\n",
-    "\n",
-    "# Evaluate on held-out test set\n",
-    "print(f\"\\n{'='*80}\")\n",
-    "print(\"EVALUATION ON HELD-OUT TEST SET\")\n",
-    "print(f\"{'='*80}\")\n",
-    "\n",
-    "z_mean_test, _, _ = final_encoder.predict(X_test_scaled, verbose=0)\n",
-    "y_test_pred_proba = final_classifier.predict(z_mean_test, verbose=0).flatten()\n",
-    "y_test_pred = (y_test_pred_proba > 0.5).astype(int)\n",
-    "\n",
-    "test_metrics = {\n",
-    "    'accuracy': accuracy_score(y_test, y_test_pred),\n",
-    "    'precision': precision_score(y_test, y_test_pred),\n",
-    "    'recall': recall_score(y_test, y_test_pred),\n",
-    "    'f1': f1_score(y_test, y_test_pred),\n",
-    "    'roc_auc': roc_auc_score(y_test, y_test_pred_proba),\n",
-    "}\n",
-    "\n",
-    "print(\"\\nTest Set Performance:\")\n",
-    "for metric, value in test_metrics.items():\n",
-    "    print(f\"  {metric.capitalize()}: {value:.4f}\")\n",
-    "\n",
-    "print(\"\\nConfusion Matrix:\")\n",
-    "print(confusion_matrix(y_test, y_test_pred))\n",
-    "\n",
-    "print(\"\\nClassification Report:\")\n",
-    "print(classification_report(y_test, y_test_pred, target_names=['Low Workload', 'High Workload']))\n",
-    "\n",
-    "# ============================================================================\n",
-    "# 7. VISUALIZATION\n",
-    "# ============================================================================\n",
-    "\n",
-    "# Plot training history\n",
-    "fig, axes = plt.subplots(2, 2, figsize=(15, 10))\n",
-    "\n",
-    "axes[0, 0].plot(final_history.history['reconstruction_loss'], label='Train')\n",
-    "axes[0, 0].plot(final_history.history['val_reconstruction_loss'], label='Val')\n",
-    "axes[0, 0].set_title('Reconstruction Loss')\n",
-    "axes[0, 0].set_xlabel('Epoch')\n",
-    "axes[0, 0].set_ylabel('Loss')\n",
-    "axes[0, 0].legend()\n",
-    "axes[0, 0].grid(True)\n",
-    "\n",
-    "axes[0, 1].plot(final_history.history['kl_loss'], label='Train')\n",
-    "axes[0, 1].plot(final_history.history['val_kl_loss'], label='Val')\n",
-    "axes[0, 1].set_title('KL Divergence Loss')\n",
-    "axes[0, 1].set_xlabel('Epoch')\n",
-    "axes[0, 1].set_ylabel('Loss')\n",
-    "axes[0, 1].legend()\n",
-    "axes[0, 1].grid(True)\n",
-    "\n",
-    "axes[1, 0].plot(final_history.history['classification_loss'], label='Train')\n",
-    "axes[1, 0].plot(final_history.history['val_classification_loss'], label='Val')\n",
-    "axes[1, 0].set_title('Classification Loss')\n",
-    "axes[1, 0].set_xlabel('Epoch')\n",
-    "axes[1, 0].set_ylabel('Loss')\n",
-    "axes[1, 0].legend()\n",
-    "axes[1, 0].grid(True)\n",
-    "\n",
-    "axes[1, 1].plot(final_history.history['accuracy'], label='Train')\n",
-    "axes[1, 1].plot(final_history.history['val_accuracy'], label='Val')\n",
-    "axes[1, 1].set_title('Classification Accuracy')\n",
-    "axes[1, 1].set_xlabel('Epoch')\n",
-    "axes[1, 1].set_ylabel('Accuracy')\n",
-    "axes[1, 1].legend()\n",
-    "axes[1, 1].grid(True)\n",
-    "\n",
-    "plt.tight_layout()\n",
-    "plt.show()\n",
-    "\n",
-    "# Visualize latent space (if 2D or 3D)\n",
-    "if best_params['latent_dim'] == 2:\n",
-    "    z_mean_train, _, _ = final_encoder.predict(X_train_scaled, verbose=0)\n",
-    "    \n",
-    "    plt.figure(figsize=(10, 8))\n",
-    "    scatter = plt.scatter(z_mean_train[:, 0], z_mean_train[:, 1], \n",
-    "                         c=y_train, cmap='RdYlBu', alpha=0.6, edgecolors='k')\n",
-    "    plt.colorbar(scatter, label='Workload (0=Low, 1=High)')\n",
-    "    plt.xlabel('Latent Dimension 1')\n",
-    "    plt.ylabel('Latent Dimension 2')\n",
-    "    plt.title('2D Latent Space Representation (Training Data)')\n",
-    "    plt.grid(True, alpha=0.3)\n",
-    "    plt.show()\n",
-    "    \n",
-    "    # Test set latent space\n",
-    "    plt.figure(figsize=(10, 8))\n",
-    "    scatter = plt.scatter(z_mean_test[:, 0], z_mean_test[:, 1], \n",
-    "                         c=y_test, cmap='RdYlBu', alpha=0.6, edgecolors='k')\n",
-    "    plt.colorbar(scatter, label='Workload (0=Low, 1=High)')\n",
-    "    plt.xlabel('Latent Dimension 1')\n",
-    "    plt.ylabel('Latent Dimension 2')\n",
-    "    plt.title('2D Latent Space Representation (Test Data)')\n",
-    "    plt.grid(True, alpha=0.3)\n",
-    "    plt.show()\n",
-    "\n",
-    "print(\"\\n\" + \"=\"*80)\n",
-    "print(\"TRAINING COMPLETE!\")\n",
-    "print(\"=\"*80)"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "79bcfc58",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "### Save Trained VAE Classifier Model\n",
-    "from pathlib import Path\n",
-    "from datetime import datetime\n",
-    "\n",
-    "# Define save path\n",
-    "model_dir = Path(\"/home/jovyan/data-paulusjafahrsimulator-gpu/trained_models\")\n",
-    "model_dir.mkdir(parents=True, exist_ok=True)\n",
-    "\n",
-    "timestamp = datetime.now().strftime(\"%Y%m%d_%H%M%S\")\n",
-    "model_path = model_dir / f\"vae_classifier_{timestamp}.keras\"\n",
-    "\n",
-    "# Save the complete model\n",
-    "final_vae_classifier.save(model_path)\n",
-    "\n",
-    "print(f\"Model saved to: {model_path}\")"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "d700e517",
-   "metadata": {},
-   "outputs": [],
-   "source": []
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "id": "30d8d100",
-   "metadata": {},
-   "outputs": [],
-   "source": [
-    "### Plot Confusion Matrix for Final Model\n",
-    "from sklearn.metrics import ConfusionMatrixDisplay\n",
-    "x = Path(\"/home/jovyan/data-paulusjafahrsimulator-gpu/trained_models/vae_classifier_20251210_230121.keras\")\n",
-    "# Load the saved model\n",
-    "print(f\"Loading model from: {x}\")\n",
-    "# loaded_vae_classifier = tf.keras.models.load_model(x)\n",
-    "loaded_vae_classifier = final_vae_classifier\n",
-    "print(\"✓ Model loaded successfully!\")\n",
-    "\n",
-    "# Extract encoder and classifier from loaded model\n",
-    "loaded_encoder = loaded_vae_classifier.encoder\n",
-    "loaded_classifier = loaded_vae_classifier.classifier\n",
-    "\n",
-    "# Get predictions on test set\n",
-    "z_mean_test, _, _ = loaded_encoder.predict(X_test_scaled, verbose=0)\n",
-    "y_test_pred_proba = loaded_classifier.predict(z_mean_test, verbose=0).flatten()\n",
-    "y_test_pred = (y_test_pred_proba > 0.5).astype(int)\n",
-    "\n",
-    "# Create and plot confusion matrix\n",
-    "cm = confusion_matrix(y_test, y_test_pred)\n",
-    "disp = ConfusionMatrixDisplay(confusion_matrix=cm, \n",
-    "                               display_labels=['Low Workload', 'High Workload'])\n",
-    "\n",
-    "fig, ax = plt.subplots(figsize=(8, 6))\n",
-    "disp.plot(ax=ax, cmap='Blues', values_format='d')\n",
-    "plt.title('Confusion Matrix - Test Set (Loaded Model)')\n",
-    "plt.tight_layout()\n",
-    "plt.show()\n",
-    "\n",
-    "# Print metrics\n",
-    "print(f\"\\nTest Set Performance (Loaded Model):\")\n",
-    "print(f\"  Accuracy: {accuracy_score(y_test, y_test_pred):.4f}\")\n",
-    "print(f\"  Precision: {precision_score(y_test, y_test_pred):.4f}\")\n",
-    "print(f\"  Recall: {recall_score(y_test, y_test_pred):.4f}\")\n",
-    "print(f\"  F1 Score: {f1_score(y_test, y_test_pred):.4f}\")\n",
-    "print(f\"  ROC-AUC: {roc_auc_score(y_test, y_test_pred_proba):.4f}\")"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "id": "e826a998",
-   "metadata": {},
-   "source": [
-    "TO DO\n",
-    "    * autoencoder langsam anfangen mit 19 schichten\n",
-    "    * dann AE und SVM mit hybridem training wie bei claude?!\n",
-    "    * dataset aus eyetracking verwenden?"
-   ]
-  }
- ],
- "metadata": {
-  "kernelspec": {
-   "display_name": "Python 3 (ipykernel)",
-   "language": "python",
-   "name": "python3"
-  },
-  "language_info": {
-   "codemirror_mode": {
-    "name": "ipython",
-    "version": 3
-   },
-   "file_extension": ".py",
-   "mimetype": "text/x-python",
-   "name": "python",
-   "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython3",
-   "version": "3.12.10"
-  }
- },
- "nbformat": 4,
- "nbformat_minor": 5
-}

model_training/tools/mad_outlier_removal.py

@@ -21,3 +21,42 @@ def mad_outlier_removal(df, columns, threshold=3.5, c=1.4826):
 
     final_mask = np.logical_and.reduce(masks)
     return df_clean[final_mask]
+
+def calculate_mad_params(df, columns):
+    """
+    Calculate median and MAD parameters for each column.
+    This should be run ONLY on the training data.
+    
+    Returns a dictionary: {col: (median, mad)}
+    """
+    params = {}
+    for col in columns:
+        median = df[col].median()
+        mad = np.median(np.abs(df[col] - median))
+        params[col] = (median, mad)
+    return params
+
+def apply_mad_filter(df, params, threshold=3.5):
+    """
+    Apply MAD-based outlier removal using precomputed parameters.
+    Works on training, validation, and test data.
+    
+    df: DataFrame to filter
+    params: dictionary {col: (median, mad)} from training data
+    threshold: cutoff for robust Z-score
+    """
+    df_clean = df.copy()
+
+    for col, (median, mad) in params.items():
+        if mad == 0:
+            continue  # no spread; nothing to remove for this column
+
+        robust_z = 0.6745 * (df_clean[col] - median) / mad
+        outlier_mask = np.abs(robust_z) > threshold
+
+        # Remove values only in this specific column
+        df_clean.loc[outlier_mask, col] = median
+        
+        
+    print(df_clean.shape)
+    return df_clean

predict_pipeline/check_python_version.py

@@ -1,11 +0,0 @@
-# from tools import db_helpers
-import sys
-
-
-def main():
-    print(sys.version)
-    # db_helpers.add_columns_to_table()    
-
-
-if __name__ == "__main__":
-    main()

predict_pipeline/config.yaml

@@ -1,20 +1,20 @@
 database:
-  path: "C:\\repo\\Fahrsimulator_MSY2526_AI\\predict_pipeline\\database.sqlite"
+  path: "/home/edgekit/MSY_FS/databases/database.sqlite"
   table: feature_table
   key: _Id
 
 model:
-  path: "C:\\repo\\Fahrsimulator_MSY2526_AI\\files_for_testing\\xgb_model_3_groupK.joblib"
+  path: "/home/edgekit/MSY_FS/fahrsimulator_msy2526_ai/predict_pipeline/cnn_crossVal_EarlyFusion_V2_0103.keras"
 
 scaler:
-  use_scaling: True
-  path: "C:\\repo\\Fahrsimulator_MSY2526_AI\\predict_pipeline\\normalizer_min_max_global.pkl"
+  use_scaling: true
+  path: "/home/edgekit/MSY_FS/fahrsimulator_msy2526_ai/predict_pipeline/scaler_crossVal_EarlyFusion_V2_0103.joblib"
 
 mqtt:
   enabled: true
-  host: "141.75.215.233"
+  host: "141.75.223.13"
   port: 1883
-  topic: "PREDICTIONS"
+  topic: "PREDICTION"
   client_id: "jetson-board"
   qos: 0
   retain: false

predict_pipeline/feature_extraction.py

@@ -1,9 +0,0 @@
-import sqlite3
-
-def main():
-
-    return 0
-
-
-if __name__ == "__main__":
-    main()

predict_pipeline/fill_db.ipynb

@@ -1,5 +1,13 @@
 {
  "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "fb68b447",
+   "metadata": {},
+   "source": [
+    "## Database creation and filling (for live system) "
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -21,8 +29,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "database_path = Path(r\"/home/edgekit/MSY_FS/databases/rawdata.sqlite\")\n",
-    "parquet_path = Path(r\"/home/edgekit/MSY_FS/fahrsimulator_msy2526_ai/files_for_testing/both_mod_0000.parquet\")"
+    "# TODO: set paths and table name\n",
+    "database_path = Path(r\"database.sqlite\") # this path references an empty, but already created sqlite file\n",
+    "parquet_path = Path(r\"...parquet\") # this path leads to the data that should be used to fill the databse\n",
+    "table_name =  \"XXX\" # name of the new table"
    ]
   },
   {
@@ -56,6 +66,14 @@
     "con, cursor = db_helpers.connect_db(database_path)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "7007c68f",
+   "metadata": {},
+   "source": [
+    "Select a subset to insert into database "
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -69,6 +87,14 @@
     "df_first_100.insert(0, '_Id', df_first_100.index + 1)"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "92171186",
+   "metadata": {},
+   "source": [
+    "Type conversion"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -88,6 +114,14 @@
     "    return \"TEXT\"\n"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "45af9956",
+   "metadata": {},
+   "source": [
+    "Define constraints and primary key"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -109,6 +143,14 @@
     "}\n"
    ]
   },
+  {
+   "cell_type": "markdown",
+   "id": "133e92ee",
+   "metadata": {},
+   "source": [
+    "Create the table"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -119,7 +161,7 @@
     "sql = db_helpers.create_table(\n",
     "    conn=con,\n",
     "    cursor=cursor,\n",
-    "    table_name=\"rawdata\",\n",
+    "    table_name=table_name,\n",
     "    columns=columns,\n",
     "    constraints=constraints,\n",
     "    primary_key=primary_key,\n",
@@ -150,7 +192,7 @@
     "db_helpers.insert_rows_into_table(\n",
     "    conn=con,\n",
     "    cursor=cursor,\n",
-    "    table_name=\"rawdata\",\n",
+    "    table_name=table_name,\n",
     "    columns=columns_to_insert,\n",
     "    commit=True\n",
     ")\n"
@@ -163,7 +205,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "a = db_helpers.get_data_from_table(conn=con, table_name='rawdata',columns_list=['*'])"
+    "request = db_helpers.get_data_from_table(conn=con, table_name='rawdata',columns_list=['*'])"
    ]
   },
   {
@@ -173,7 +215,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "a.head()"
+    "request.head()"
    ]
   },
   {
@@ -189,7 +231,7 @@
  ],
  "metadata": {
   "kernelspec": {
-   "display_name": "MSY_FS_env",
+   "display_name": "310",
    "language": "python",
    "name": "python3"
   },
@@ -203,7 +245,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.12.12"
+   "version": "3.10.19"
   }
  },
  "nbformat": 4,

predict_pipeline/predict.service

@@ -5,7 +5,7 @@ StartLimitIntervalSec=0
 [Service]
 Type=oneshot
 User=edgekit
-ExecStart=~/anaconda3/envs/p310_FS/bin/python /home/edgekit/MSY_FS/fahrsimulator_msy2526_ai/predict_pipeline/predict_sample.py
+ExecStart=/home/edgekit/anaconda3/envs/p310_FS_TF/bin/python /home/edgekit/MSY_FS/fahrsimulator_msy2526_ai/predict_pipeline/predict_sample.py
 
 [Install]
 WantedBy=multi-user.target

predict_pipeline/predict.timer

@@ -2,10 +2,11 @@
 Description=Run predict sample every 5 seconds
 
 [Timer]
-OnBootSec=5
+OnActiveSec=60
 OnUnitActiveSec=5
 AccuracySec=1s
 Unit=predict.service
 
 [Install]
 WantedBy=timers.target
+

predict_pipeline/predict_sample.py

@@ -7,9 +7,9 @@ import sys
 import yaml
 import pickle
 sys.path.append('/home/edgekit/MSY_FS/fahrsimulator_msy2526_ai/tools')
-# sys.path.append(r"c:\\repo\\Fahrsimulator_MSY2526_AI\\tools")
 import db_helpers
 import joblib
+import paho.mqtt.client as mqtt
 
 def _load_serialized(path: Path):
     suffix = path.suffix.lower()
@@ -52,11 +52,12 @@ def callModel(sample, model_path):
     suffix = model_path.suffix.lower()
     if suffix in {".pkl", ".joblib"}:
         model = _load_serialized(model_path)
-    # elif suffix == ".keras":
-       # import tensorflow as tf
-        # model = tf.keras.models.load_model(model_path)
-    # else:
-        # raise ValueError(f"Unsupported model format: {suffix}. Use .pkl, .joblib, or .keras.")
+    elif suffix == ".keras":
+        import tensorflow
+        tensorflow.get_logger().setLevel("ERROR")
+        model = tensorflow.keras.models.load_model(model_path)
+    else:
+        raise ValueError(f"Unsupported model format: {suffix}. Use .pkl, .joblib, or .keras.")
 
     x = np.asarray(sample, dtype=np.float32)
     if x.ndim == 1:
@@ -64,9 +65,7 @@ def callModel(sample, model_path):
 
     if suffix == ".keras":
         x_full = x  
-        # Future model (35 features): keep this call when your new model is active.
-        # prediction = model.predict(x_full[:, :35], verbose=0)
-        prediction = model.predict(x_full[:, :20], verbose=0)
+        prediction = (model.predict(x_full[:, :35], verbose=0) > 0.5).astype(int)
 
     else:
         if hasattr(model, "predict"):
@@ -127,31 +126,27 @@ def sendMessage(config_file_path, message):
     payload = json.dumps(message, ensure_ascii=False)
     print(payload)
 
-    # Later: publish via MQTT using config parameters above.
-    # Example (kept commented intentionally):
-    # import paho.mqtt.client as mqtt
-    # client = mqtt.Client(client_id=mqtt_cfg.get("client_id", "predictor-01"))
-    # if "username" in mqtt_cfg and mqtt_cfg.get("username"):
-    #     client.username_pw_set(mqtt_cfg["username"], mqtt_cfg.get("password"))
-    # client.connect(mqtt_cfg.get("host", "localhost"), int(mqtt_cfg.get("port", 1883)), 60)
-    # client.publish(
-    #     topic=topic,
-    #     payload=payload,
-    #     qos=int(mqtt_cfg.get("qos", 1)),
-    #     retain=bool(mqtt_cfg.get("retain", False)),
-    # )
-    # client.disconnect()
+    # publish via MQTT using config parameters above.    
+    client = mqtt.Client(client_id=mqtt_cfg.get("client_id", "predictor-01"),
+                          callback_api_version=mqtt.CallbackAPIVersion.VERSION2)
+    callback_api_version=mqtt.CallbackAPIVersion.VERSION2
+    if "username" in mqtt_cfg and mqtt_cfg.get("username"):
+        client.username_pw_set(mqtt_cfg["username"], mqtt_cfg.get("password"))
+    client.connect(mqtt_cfg.get("host", "localhost"), int(mqtt_cfg.get("port", 1883)), 60)
+    client.publish(
+        topic=topic,
+        payload=payload,
+        qos=int(mqtt_cfg.get("qos", 1)),
+        retain=bool(mqtt_cfg.get("retain", False)),
+    )
+    client.disconnect()
     return
 
 def replace_nan(sample, config_file_path: Path):
     with config_file_path.open("r", encoding="utf-8") as f:
         cfg = yaml.safe_load(f)
     
-    fallback_list = cfg.get("fallback", [])
-    fallback_map = {}
-    for item in fallback_list:
-        if isinstance(item, dict):
-            fallback_map.update(item)
+    fallback_map = cfg.get("fallback", {})
     
     if sample.empty:
         return False, sample
@@ -162,7 +157,6 @@ def replace_nan(sample, config_file_path: Path):
     if valid and fallback_map:
         sample = sample.fillna(value=fallback_map)
     
-
     return valid, sample
 
 def sample_to_numpy(sample, drop_cols=("_Id", "start_time")):

predict_pipeline/predict_service_timer_documentation.md

@@ -0,0 +1,216 @@
+# Predict Service and Timer Documentation
+
+## Overview
+
+This setup uses **systemd services and timers** to repeatedly execute a
+Python script that performs prediction on the latest sample and sends a
+message.
+
+The systemd unit files are typically stored in:
+
+    /etc/systemd/system/
+
+For this setup, the relevant files are:
+
+    /etc/systemd/system/predict.service
+    /etc/systemd/system/predict.timer
+
+These files define the service execution and the timer scheduling.
+
+-   `predict.service` -- defines how the Python script is executed
+-   `predict.timer` -- schedules the repeated execution of the service
+
+The timer triggers the service **every 5 seconds** after the first
+activation.
+
+------------------------------------------------------------------------
+
+# Systemd Timer
+
+File: `predict.timer`
+
+``` ini
+[Unit]
+Description=Run predict sample every 5 seconds
+
+[Timer]
+OnActiveSec=60
+OnUnitActiveSec=5
+AccuracySec=1s
+Unit=predict.service
+
+[Install]
+WantedBy=timers.target
+```
+
+## Behavior
+
+-   **OnActiveSec=60**\
+    The timer starts **60 seconds after it is activated**.
+
+-   **OnUnitActiveSec=5**\
+    After the service has run once, it will be triggered again **every 5
+    seconds**.
+
+-   **AccuracySec=1s**\
+    Allows systemd to schedule the timer with **1 second precision**.
+
+-   **Unit=predict.service**\
+    Defines which service should be triggered by the timer.
+
+------------------------------------------------------------------------
+
+# Systemd Service
+
+File: `predict.service`
+
+``` ini
+[Unit]
+Description=Predict latest sample and send message
+After=network.target
+StartLimitIntervalSec=0
+
+[Service]
+Type=oneshot
+User=edgekit
+ExecStart=/home/edgekit/anaconda3/envs/p310_FS_TF/bin/python /home/edgekit/MSY_FS/fahrsimulator_msy2526_ai/predict_pipeline/predict_sample.py
+
+[Install]
+WantedBy=multi-user.target
+```
+
+## Behavior
+
+-   **Type=oneshot**\
+    The service runs the script once and then exits.
+
+-   **User=edgekit**\
+    The script is executed under the `edgekit` user.
+
+-   **ExecStart**\
+    Executes the Python script using the specified conda environment.
+
+-   **After=network.target**\
+    Ensures the service only runs after the network is available.
+
+------------------------------------------------------------------------
+
+# Execution Flow
+
+1.  The **timer starts** after it is enabled.
+2.  After **60 seconds**, the first execution happens (this results from the duration of the camera processing initialization)
+3.  The timer triggers `predict.service`.
+4.  The service runs `predict_sample.py`.
+5.  Once the script finishes, the service exits.
+6.  The timer triggers the service again **every 5 seconds**.
+
+------------------------------------------------------------------------
+
+# Debugging and Monitoring
+
+## View Live Output
+
+All `print()` output from the Python script is written to the **systemd
+journal**.
+
+Follow the output live with:
+
+``` bash
+journalctl -u predict.service -f
+```
+
+This command is typically the most useful for debugging.
+
+------------------------------------------------------------------------
+
+# Common Systemd Commands
+
+## Check Service Status
+
+``` bash
+systemctl status predict.service
+```
+
+Shows the last execution result and recent log lines.
+
+------------------------------------------------------------------------
+
+## Check Timer Status
+
+``` bash
+systemctl status predict.timer
+```
+
+Shows when the timer last ran and when it will run next.
+
+------------------------------------------------------------------------
+
+## List All Timers
+
+``` bash
+systemctl list-timers
+```
+
+Displays all active timers and their next scheduled execution.
+
+------------------------------------------------------------------------
+
+# Manual Execution
+
+To run the service manually once:
+
+``` bash
+systemctl start predict.service
+```
+
+------------------------------------------------------------------------
+
+# Restarting the Systemd Units
+
+## Restart the Service
+
+``` bash
+systemctl restart predict.service
+```
+
+## Restart the Timer
+
+``` bash
+systemctl restart predict.timer
+```
+
+------------------------------------------------------------------------
+
+## Reload Systemd After Changes
+
+If `.service` or `.timer` files were modified:
+
+``` bash
+systemctl daemon-reload
+systemctl restart predict.timer
+systemctl restart predict.service
+```
+
+------------------------------------------------------------------------
+
+## Enabling the Timer
+
+To ensure the timer starts automatically on system boot:
+
+``` bash
+systemctl enable predict.timer
+systemctl start predict.timer
+```
+
+------------------------------------------------------------------------
+
+# Summary
+
+-   `predict.timer` schedules periodic execution.
+-   `predict.service` runs the Python prediction script.
+-   The script runs **every 5 seconds** after the initial delay.
+-   Logs and script output are available through:
+
+``` bash
+journalctl -u predict.service -f
+```

predict_pipeline/prediction_env.yaml

@@ -0,0 +1,196 @@
+name: 'prediction_env'
+channels:
+  - defaults
+  - conda-forge
+dependencies:
+  - _py-xgboost-mutex=2.0=cpu_2
+  - absl-py=2.3.1=py310haa95532_0
+  - aom=3.12.1=h00a0c3c_0
+  - arrow-cpp=21.0.0=hcdc3a1c_2
+  - asttokens=3.0.1=pyhd8ed1ab_0
+  - astunparse=1.6.3=py_0
+  - aws-c-auth=0.9.0=h02ab6af_2
+  - aws-c-cal=0.9.2=h02ab6af_1
+  - aws-c-common=0.12.4=h02ab6af_0
+  - aws-c-compression=0.3.1=h02ab6af_2
+  - aws-c-event-stream=0.5.6=h02ab6af_0
+  - aws-c-http=0.10.4=h02ab6af_0
+  - aws-c-io=0.21.4=h02ab6af_0
+  - aws-c-mqtt=0.13.3=h02ab6af_0
+  - aws-c-s3=0.8.7=h02ab6af_0
+  - aws-c-sdkutils=0.2.4=h02ab6af_1
+  - aws-checksums=0.2.7=h02ab6af_1
+  - aws-crt-cpp=0.34.0=h885b0b7_0
+  - aws-sdk-cpp=1.11.638=hf0af688_0
+  - blas=1.0=mkl
+  - brotlicffi=1.2.0.0=py310h885b0b7_0
+  - bzip2=1.0.8=h2bbff1b_6
+  - c-ares=1.34.6=h2c209ce_0
+  - ca-certificates=2026.1.4=h4c7d964_0
+  - cairo=1.18.4=he9e932c_0
+  - certifi=2026.01.04=py310haa95532_0
+  - cffi=2.0.0=py310h02ab6af_1
+  - charset-normalizer=3.4.4=py310haa95532_0
+  - colorama=0.4.6=pyhd8ed1ab_1
+  - comm=0.2.3=pyhe01879c_0
+  - dav1d=1.2.1=h2bbff1b_0
+  - debugpy=1.8.20=py310h699e580_0
+  - decorator=5.2.1=pyhd8ed1ab_0
+  - exceptiongroup=1.3.1=pyhd8ed1ab_0
+  - executing=2.2.1=pyhd8ed1ab_0
+  - expat=2.7.4=hd7fb8db_0
+  - flatbuffers=24.3.25=h21716d4_0
+  - fontconfig=2.15.0=hd211d86_0
+  - freeglut=3.8.0=hfcef157_0
+  - freetype=2.14.1=hfbffc0b_0
+  - fribidi=1.0.16=haf45083_0
+  - gast=0.7.0=pyhd3eb1b0_0
+  - gflags=2.2.2=hd77b12b_1
+  - giflib=5.2.2=h7edc060_0
+  - glog=0.5.0=hd77b12b_1
+  - google-pasta=0.2.0=pyhd3eb1b0_0
+  - graphite2=1.3.14=hd77b12b_1
+  - grpcio=1.74.1=py310h5c751cc_0
+  - h5py=3.15.1=py310he283ef2_1
+  - harfbuzz=12.3.0=h3ef6528_1
+  - hdf5=1.14.5=ha36df97_2
+  - icc_rt=2022.1.0=h6049295_2
+  - icu=73.1=h6c2663c_0
+  - idna=3.11=py310haa95532_0
+  - intel-openmp=2025.0.0=haa95532_1164
+  - ipykernel=7.2.0=pyh6dadd2b_1
+  - ipython=8.37.0=pyha7b4d00_0
+  - jedi=0.19.2=pyhd8ed1ab_1
+  - joblib=1.5.3=py310haa95532_0
+  - jpeg=9f=ha349fce_0
+  - jupyter_client=8.8.0=pyhcf101f3_0
+  - jupyter_core=5.9.1=pyh6dadd2b_0
+  - keras=3.11.2=py310h51baaa3_0
+  - krb5=1.21.3=hdf4eb48_0
+  - lcms2=2.17=h3732fa5_0
+  - lerc=4.0.0=h5da7b33_0
+  - libabseil=20250814.1=cxx17_hcd311fc_0
+  - libavif=1.3.0=h5bd13ec_0
+  - libbrotlicommon=1.2.0=h907acca_0
+  - libbrotlidec=1.2.0=h02c67a5_0
+  - libbrotlienc=1.2.0=h483e6b9_0
+  - libcurl=8.17.0=h6e672f4_1
+  - libdeflate=1.22=h5bf469e_0
+  - libexpat=2.7.4=hd7fb8db_0
+  - libffi=3.4.4=hd77b12b_1
+  - libglib=2.86.3=h9bccc14_0
+  - libgrpc=1.74.1=hde67744_0
+  - libhwloc=2.12.1=default_hfa10c62_1000
+  - libiconv=1.16=h2bbff1b_3
+  - libkrb5=1.22.1=hb237eb7_0
+  - libopenjpeg=2.5.4=h02ab6af_1
+  - libpng=1.6.54=ha15c746_0
+  - libprotobuf=6.33.0=h2a56892_1
+  - libre2-11=2025.11.05=ha6b10e7_0
+  - libsodium=1.0.20=hc70643c_0
+  - libssh2=1.11.1=h2addb87_0
+  - libthrift=0.22.0=ha2884a9_0
+  - libtiff=4.7.1=h3a18249_0
+  - libwebp-base=1.6.0=hbf3958f_0
+  - libxgboost=3.1.2=h585ebfc_0
+  - libxml2=2.13.9=h6201b9f_0
+  - libzlib=1.3.1=h02ab6af_0
+  - lz4-c=1.9.4=h2bbff1b_1
+  - m2w64-gcc-libgfortran=5.3.0=6
+  - m2w64-gcc-libs=5.3.0=7
+  - m2w64-gcc-libs-core=5.3.0=7
+  - m2w64-gmp=6.1.0=2
+  - m2w64-libwinpthread-git=5.0.0.4634.697f757=2
+  - markdown=3.10=py310haa95532_0
+  - markdown-it-py=2.2.0=py310haa95532_1
+  - markupsafe=3.0.2=py310h827c3e9_0
+  - matplotlib-inline=0.2.1=pyhd8ed1ab_0
+  - mdurl=0.1.2=py310haa95532_0
+  - mkl=2025.0.0=h5da7b33_930
+  - mkl-service=2.5.2=py310h0b37514_0
+  - mkl_fft=2.1.1=py310h300f80d_0
+  - mkl_random=1.3.0=py310ha5e6156_0
+  - ml_dtypes=0.5.4=py310h42c1672_0
+  - mpi=1.0=msmpi
+  - mpi4py=4.0.3=py310h02ab6af_1
+  - msmpi=10.1.1=had4844c_0
+  - msys2-conda-epoch=20160418=1
+  - namex=0.1.0=py310haa95532_0
+  - nest-asyncio=1.6.0=pyhd8ed1ab_1
+  - numpy-base=2.1.3=py310he4e2855_3
+  - openssl=3.6.1=hf411b9b_1
+  - opt_einsum=3.3.0=pyhd3eb1b0_1
+  - optree=0.18.0=py310h03f52e7_0
+  - orc=2.2.0=h79e1e1e_1
+  - packaging=25.0=py310haa95532_1
+  - paho-mqtt=2.1.0=pyhe01879c_1
+  - parso=0.8.6=pyhcf101f3_0
+  - pcre2=10.46=h5740b90_0
+  - pickleshare=0.7.5=pyhd8ed1ab_1004
+  - pillow=12.1.0=py310h6b7a805_0
+  - pip=26.0.1=pyhc872135_0
+  - pixman=0.46.4=h4043f72_0
+  - platformdirs=4.9.2=pyhcf101f3_0
+  - prompt-toolkit=3.0.52=pyha770c72_0
+  - protobuf=6.33.0=py310ha4c6e68_0
+  - psutil=7.2.2=py310h1637853_0
+  - pure_eval=0.2.3=pyhd8ed1ab_1
+  - py-xgboost=3.1.2=py310haa95532_0
+  - pyarrow=21.0.0=py310h42c1672_1
+  - pycparser=2.23=py310haa95532_0
+  - pygments=2.19.2=pyhd8ed1ab_0
+  - pysocks=1.7.1=py310haa95532_1
+  - python=3.10.19=h981015d_0
+  - python-dateutil=2.9.0.post0=pyhe01879c_2
+  - python-flatbuffers=24.3.25=py310haa95532_0
+  - python_abi=3.10=2_cp310
+  - pywin32=311=py310h282bd7d_1
+  - pyyaml=6.0.3=py310hb9a58be_0
+  - pyzmq=27.1.0=py310h535538e_0
+  - re2=2025.11.05=hc24cdf5_0
+  - requests=2.32.5=py310haa95532_1
+  - rich=14.2.0=py310haa95532_0
+  - scipy=1.15.3=py310h1bbe36f_1
+  - setuptools=80.10.2=py310haa95532_0
+  - six=1.17.0=pyhe01879c_1
+  - snappy=1.2.2=hab6b7b3_1
+  - sqlite=3.51.1=hda9a48d_0
+  - stack_data=0.6.3=pyhd8ed1ab_1
+  - tbb=2022.3.0=h90c84d6_0
+  - tbb-devel=2022.3.0=h90c84d6_0
+  - tensorboard=2.20.0=py310haa95532_0
+  - tensorboard-data-server=0.7.0=py310haa95532_1
+  - tensorflow=2.20.0=cpu_py310h6605a60_0
+  - tensorflow-base=2.20.0=cpu_py310hce87ebc_0
+  - termcolor=3.2.0=py310haa95532_0
+  - threadpoolctl=3.5.0=py310h4442805_1
+  - tk=8.6.15=hf199647_0
+  - tornado=6.5.4=py310h29418f3_0
+  - traitlets=5.14.3=pyhd8ed1ab_1
+  - typing-extensions=4.15.0=py310haa95532_0
+  - typing_extensions=4.15.0=py310haa95532_0
+  - ucrt=10.0.22621.0=haa95532_0
+  - urllib3=2.6.3=py310haa95532_0
+  - utf8proc=2.6.1=h2bbff1b_1
+  - vc=14.42=haa95532_5
+  - vc14_runtime=14.44.35208=h4927774_10
+  - vs2015_runtime=14.44.35208=ha6b5a95_10
+  - wcwidth=0.6.0=pyhd8ed1ab_0
+  - werkzeug=3.1.3=py310haa95532_0
+  - wheel=0.46.3=py310haa95532_0
+  - win_inet_pton=1.1.0=py310haa95532_1
+  - wrapt=2.0.1=py310h02ab6af_0
+  - xgboost=3.1.2=py310haa95532_0
+  - xz=5.6.4=h4754444_1
+  - yaml=0.2.5=he774522_0
+  - zeromq=4.3.5=h5bddc39_9
+  - zlib=1.3.1=h02ab6af_0
+  - zstd=1.5.7=h56299aa_0
+  - pip:
+      - numpy==1.24.4
+      - pandas==2.3.0
+      - pyocclient==0.6
+      - pytz==2025.2
+      - scikit-learn==1.6.1
+      - tzdata==2025.3
+

project_report.md

@@ -0,0 +1,551 @@
+# Project Report: Multimodal Driver State Analysis
+
+## 1) Project Scope
+
+This repository implements an end-to-end workflow for multimodal driver-state analysis in a simulator setup.  
+The system combines:
+- Facial Action Units (AUs)
+- Eye-tracking features (fixations, saccades, blinks, pupil behavior)
+
+Apart from this, several machine learning model architectures are presented and evaluated.
+
+Content:
+- Dataset generation
+- Exploratory data analysis
+- Model training experiments
+- Real-time inference with SQlite, systemd and MQTT
+- Repository file inventory
+- Additional nformation 
+
+
+## 2) Dataset generation
+
+### 2.1 Data Access, Filtering, and Data Conversion
+
+Main scripts:
+- `dataset_creation/create_parquet_files_from_owncloud.py`
+- `dataset_creation/parquet_file_creation.py`
+
+Purpose:
+- Download and/or access dataset files (either download first via ```EDA/owncloud_file_access.ipynb``` or all in one with ```dataset_creation/create_parquet_files_from_owncloud.py```
+- Keep relevant columns (FACE_AUs and eye-tracking raw values)
+- Filter invalid samples (e.g., invalid level segments): Make sure not to drop rows where NaN is necessary for later feature creation, therefore use subset argument in dropNa()! 
+- Export subject-level parquet files
+- Before running the scripts: be aware that the whole dataset contains 30 files with around 900 Mbytes each, provide enough storage and expect this to take a while.
+
+
+### 2.2 Feature Engineering (Offline)
+
+Main script:
+- `dataset_creation/combined_feature_creation.py`
+
+Behavior:
+- Builds fixed-size sliding windows over subject time series (window size and step size can be adjusted)
+- Uses prepared parquet files from 2.1
+- Aggregates AU statistics per window (e.g., `FACE_AUxx_mean`)
+- Computes eye-feature aggregates (fix/sacc/blink/pupil metrics)
+- Produces training-ready feature tables = dataset
+- Parameter ```MIN_DUR_BLINKS``` can be adjusted, although this value needs to make sense in combination with your sampling frequency
+- With low videostream rates, consider to reevaluate the meaningfulness of some eye-tracking features, especially the fixations
+- running the script requires a manual installation of [pygaze Analyser library](https://github.com/esdalmaijer/PyGazeAnalyser.git) from github
+
+### 2.3 Online Camera + Eye + AU Feature Extraction
+
+Main scripts:
+- `dataset_creation/camera_handling/camera_stream_AU_and_ET_new.py`
+- `dataset_creation/camera_handling/eyeFeature_new.py`
+- `dataset_creation/camera_handling/db_helper.py`
+
+Runtime behavior:
+- Captures webcam stream with OpenCV
+- Extracts gaze/iris-based signals via MediaPipe
+- Records overlapping windows (`VIDEO_DURATION=50s`, `START_INTERVAL=5s`, `FPS=25`)
+- Runs AU extraction (`py-feat`) from recorded video segments
+  - Explanation of the py-feat functionality is located in `dataset_creation/AU_creation/pyfeat_docu.ipynb`
+- Computes eye-feature summary from generated gaze parquet
+- Writes merged rows to SQLite table `feature_table`
+
+Operational note:
+- `DB_PATH` and other paths are currently code-configured and must be adapted per deployment.
+
+### 2.4 Two Approaches to Eye-Tracking Data Collection
+
+Eye-tracking can be implemented using two main approaches:
+
+Used Approach: Relative Iris Position
+- Tracks the position of the pupil within the eye region
+- The position is normalized relative to the eye itself
+- No reference to the screen or physical environment is required
+
+Not Used Approach: Screen Calibration
+- Requires the user to look at 9 predefined points on the screen
+- A mapping model is trained based on these points
+- Establishes a relationship between eye movement and screen coordinates
+
+Important Considerations (for both methods)
+- Keep the head as still as possible
+- Ensure consistent and even lighting conditions
+
+## 3) EDA
+The directory EDA provides several files to get insights into both the raw data from AdaBase and your own dataset.
+
+- `EDA.ipynb` - Main EDA notebook: recreates the plot from AdaBase documentation, lists all experiments and in general serves as a playground for you to get to know the files. 
+- `distribution_plots.ipynb` - This notebook aimes to visualize the data distributions for each experiment - the goal is the find out, whether the split of experiments into high and low cognitive load is clearer if some experiments are dropped.
+- `histogramms.ipynb` - Histogram analysis of low load vs high load per feature. Additionaly, scatter plots per feature are available.
+- `researchOnSubjectPerformance.ipynb` - This noteboooks aims to see how the performance values range for the 30 subjects. The code creates and saves a table in csv-format, which will later be used as the foundation of the performance based split in ```model_training/tools/performance_based_split```
+- `owncloud_file_access.ipynb` - Get access to the files via owncloud and safe them as .h5 files, in correspondence to the parquet file creation script 
+- `login.yaml` -Used to store URL and password to access files from owncloud, used in previous notebook
+- `calculate_replacement_values.ipynb` -Fallback / median computation notebook for deployment, creation of yaml syntax embedding
+
+General information: 
+- Due to their size, its absolutely recommended to download and save the dataset files once in the beginning
+- For better data understanding, read the [AdaBase publication](https://www.mdpi.com/1424-8220/23/1/340)
+
+
+## 4) Model Training
+
+Included model families:
+- CNN variants (different fusion strategies)
+- XGBoost
+- Isolation Forest*
+- OCSVM*
+- DeepSVDD*
+
+\* These training strategies are unsupervised, which means only low cognitive load samples are used for training. Validation then also considers high low samples.  
+
+
+Supporting utilities in ```model_training/tools```:
+- `scaler.py`: Functions to fit, transform, save and load either MinMaxScaler or StandardScaler, subject-wise and globally - for new subjects, a fallback scaler (using mean of all subjects scaling parameters) is used
+- `performance_split.py`: Provides a function to split a group of subjects based on their performance in the AdaBase experiments, based on the results created in `researchOnSubjectPerformance.ipynb`. To split into three groups for train, validation & test, call the function twice
+- `mad_outlier_removal.py`: Functions to fit and transform data with MAD outlier removal
+- `evaluation_tools.py`: Especially used for Isolation Forest, Functions for ROC curve as well as confusion matrix
+
+
+### 4.1 CNNs
+This section summarizes all CNN‑based supervised learning approaches implemented in the project.  
+All models operate on facial Action Unit (AU) features and, depending on the notebook, additional eye‑tracking features.  
+The notebooks differ in evaluation methodology, fusion strategy, and experimental intention.
+
+### 4.1.1 Baseline CNN (Notebook: *CNN_simple*)  
+The first notebook implements a simple 1D CNN to establish a baseline for AU‑only classification.  
+The model uses two convolutional layers, batch normalization, max pooling, and a regularized dense head.  
+A single subject‑exclusive train/validation/test split is used.
+
+The intention behind this notebook is to:  
+- Provide a baseline performance level  
+- Validate that AU features contain discriminative information  
+- Identify overfitting tendencies before moving to more rigorous evaluation  
+
+
+### 4.1.2 Cross‑Validated CNN (Notebook: *CNN_crossVal*)  
+This notebook introduces 5‑fold GroupKFold cross‑validation, ensuring subject‑exclusive folds.  
+The architecture is similar to the baseline but includes stronger regularization and a lower learning rate.
+
+The intention behind this notebook is to:  
+- Provide robust generalization estimates  
+- Reduce variance caused by single‑split evaluation  
+- Establish a cross‑validated AU‑only benchmark  
+
+
+### 4.1.3 Cross‑Validated CNN (Face AUs Only)  (Notebook: *CNN_crossVal_faceAUs*)  
+This notebook is a streamlined version of the previous one, removing unused eye‑tracking features and focusing exclusively on AUs.
+
+The intention behind this notebook is to:  
+- Provide a clean AU‑only benchmark  
+- Improve reproducibility and interpretability  
+- Prepare for multimodal comparisons  
+
+
+### 4.1.4 Cross‑Validated CNN with Early Fusion (AUs + Eye Features)  
+(Notebook: *CNN_crossVal_faceAUs_eyeFeatures*)  
+This notebook introduces early fusion, concatenating AU and eye‑tracking features into a single input vector.  
+The architecture remains identical to AU‑only models.
+
+The intention behind this notebook is to:  
+- Evaluate whether multimodal early fusion improves performance  
+- Establish a first multimodal baseline  
+- Analyze class‑specific behavior via confusion matrices  
+
+This notebook didn't lead to any useful results.
+
+### 4.1.5 Cross‑Validated CNN with Early Fusion (Refined Version)  (Notebook: *CNN_crossVal_EarlyFusion*)  
+This notebook refines the early‑fusion approach by removing samples with missing values and ensuring consistent multimodal input quality.
+
+The intention behind this notebook is to:  
+- Provide a clean and fully validated early‑fusion model  
+- Investigate multimodal complementarity under rigorous CV  
+- Improve interpretability through aggregated confusion matrices  
+
+
+### 4.1.6 Cross‑Validated CNN with Early Fusion and Subset Filtering  (Notebook: *CNN_crossVal_EarlyFusion_Filter*)  
+This notebook applies domain‑specific filtering to isolate a more homogeneous subset of cognitive states before training.
+
+The intention behind this notebook is to:  
+- Evaluate whether subset filtering improves multimodal learning  
+- Reduce dataset heterogeneity  
+- Provide a controlled multimodal benchmark  
+
+
+### 4.1.7 Hybrid‑Fusion CNN  (Notebook: *CNN_crossVal_HybridFusion*)  
+This notebook introduces a hybrid‑fusion architecture with two modality‑specific branches:  
+- A 1D CNN for AUs  
+- A dense MLP for eye‑tracking features  
+
+The branches are fused before classification.
+
+The intention behind this notebook is to:  
+- Allow each modality to learn specialized representations  
+- Evaluate whether hybrid fusion outperforms early fusion  
+- Provide a strong multimodal benchmark  
+
+
+### 4.1.8 Early‑Fusion CNN with Independent Test Evaluation  (Notebook: *CNN_crossVal_EarlyFusion_Test_Eval*)  
+This notebook introduces the first true held‑out test evaluation for an early‑fusion CNN.  
+A subject‑exclusive train/test split is created before cross‑validation.
+
+The intention behind this notebook is to:  
+- Provide a deployment‑realistic performance estimate  
+- Compare validation‑fold behavior with true test‑set behavior  
+- Visualize ROC and PR curves for threshold analysis  
+
+| Metric / Model | CNN_crossVal_EarlyFusion_Test_Eval |
+|----------------|-------------------------------------|
+| Test Accuracy |   0.913                                |
+| Test F1        |   0.927                                   |
+| Test AUC       |   0.967                                 |
+| Balanced Accuracy |   0.907                              |
+| Precision       |   0.918                                  |
+| Recall          |   0.937                                |
+
+#### Confusion Matrix
+![Konfusionsmatrix](results/Konfusionsmatrix_EarlyFusion.png)
+
+*Figure 4.1.8.1: Confusion matrix of the Early‑Fusion model.*
+
+#### ROC-Curve
+![ROC-Kurve](results/ROC_EarlyFusion.png)
+
+*Figure 4.1.8.2: ROC-Curve of the Early‑Fusion model.*
+
+### 4.1.9 Hybrid‑Fusion CNN with Independent Test Evaluation  (Notebook: *CNN_crossVal_HybridFusion_Test_Eval*)  
+This notebook extends hybrid fusion with a subject‑exclusive train/test split and full test‑set evaluation.
+
+The intention behind this notebook is to:  
+- Evaluate hybrid fusion under realistic deployment conditions  
+- Compare hybrid vs. early fusion on unseen subjects  
+- Provide full diagnostic plots (ROC, PR, confusion matrices)  
+
+| Metric / Model | CNN_crossVal_HybridFusion_Test_Eval |
+|----------------|--------------------------------------|
+| Test Accuracy |   0.950                                  |
+| Test F1        |   0.959                                |
+| Test AUC       |   0.983                                   |
+| Balanced Accuracy |    0.942                          |
+| Precision       |   0.933                                |
+| Recall          |   0.986                                 |
+
+#### Confusion Matrix
+![Konfusionsmatrix](results/Konfusionsmatrix_HybridFusion.png)
+
+*Figure 4.1.9.1: Confusion matrix of the Hybrid‑Fusion model.*
+
+#### ROC-Curve
+![ROC-Kurve](results/ROC_HybridFusion.png)
+
+*Figure 4.1.9.2: ROC-Curve of the Hybrid‑Fusion model.*
+
+### 4.1.10 Summary
+Across all nine notebooks, the project progresses from a simple AU‑only baseline to advanced multimodal hybrid‑fusion architectures with independent test evaluation.  
+
+The final experiments revealed that hybrid fusion provides a measurable performance advantage over early fusion. While both approaches achieve strong results, the hybrid‑fusion model reaches higher overall accuracy (95% vs. 91.3%) and substantially stronger recall (98.6% vs. 93.7%), indicating that it is more effective at correctly identifying high‑workload samples.
+Early fusion, however, shows slightly better precision, suggesting that it produces fewer false positives.
+
+Looking ahead, further improvements could likely be achieved through more extensive hyperparameter tuning, as the current results suggest that additional optimization headroom remains.
+
+### 4.2 XGBoost
+This documentation outlines the evolution of the XGBoost classification pipeline for cognitive workload detection. The project transitioned from a basic unimodal setup to a sophisticated, multi-stage hybrid system incorporating advanced statistical filtering and deep feature extraction. 
+During the model creation several methods were used to improve the model accuracy. During training, the biggest challenge was always the high overfitting of the model. Even in the last version with explicit regulation parameters the overall accuracy couldn't be improved more than the different methods before.
+The model overall was not that good, as the highest accuracy we could achieve was around 65%, which is a bit higher than Fraunhofer achieved in the ADABase-Paper.
+
+### 4.2.1 Classical XGBoost Baseline
+
+To establish a performance baseline, a classical Extreme Gradient Boosting (XGBoost) model was implemented. XGBoost was selected for its ability to handle non-linear relationships and its inherent regularization, which helps prevent overfitting in high-dimensional feature spaces like Facial Action Units. XGBoost was picked because of its usage in the ADABase Paper. Initially, the model utilized raw Action Unit sums with global normalization to determine the basic predictability of workload from facial muscle activity alone.
+
+| Metric / Model | Classical XGBoost |
+| --- | --- |
+| Accuracy | 0.581 |
+| AUC | 0.562 |
+| F1-Score | 0.652 |
+
+### 4.2.2 XGBoost with GroupKFold Validation
+
+To address the challenge of inter-subject variability, the validation strategy was upgraded to `GroupKFold`. In behavioral data, samples from the same subject are highly correlated. Standard cross-validation often leads to data leakage, where the model memorizes individual facial characteristics. By ensuring that a subject's data is never shared between the training and validation sets, this iteration provides a scientifically rigorous measure of how the model generalizes to entirely unseen individuals.
+
+| Metric / Model | XGBoost (GroupKFold) |
+| --- | --- |
+| Accuracy | 0.586 |
+| AUC | 0.573 |
+| F1-Score | 0.651 |
+
+### 4.2.3 Hybrid XGBoost with Autoencoder
+
+To improve feature quality, a hybrid approach was introduced by pre-training a deep Autoencoder. The encoder branch was used to compress 20 raw Action Units into a 5-dimensional latent space. This non-linear dimensionality reduction aims to capture muscle synergies and filter out noise that decision trees might struggle with. The XGBoost classifier was then trained on these machine-learned representations rather than raw inputs.
+
+| Metric / Model | XGBoost + Autoencoder |
+| --- | --- |
+| Accuracy | 0.589 |
+| AUC | 0.575 |
+| F1-Score | 0.650 |
+
+### 4.2.4 Robust XGBoost with MAD Outlier Removal
+
+Recognizing that physiological and AU data often contain sensor artifacts, a robust preprocessing layer was added using Median Absolute Deviation (MAD). Unlike standard deviation, MAD is resilient to extreme outliers. By calculating a Robust Z-score and filtering signals in the training set, the model learned from a "clean" representation of cognitive states, significantly improving the stability of the gradient boosting process.
+
+| Metric / Model | XGBoost + MAD |
+| --- | --- |
+| Accuracy | 0.641 |
+| AUC | 0.610 |
+| F1-Score | 0.733 |
+
+### 4.2.5 Combined Dataset of Action Units and EyeTracking
+
+This iteration involved training, which refined a robust pipeline on a new, expanded dataset. This dataset integrated both high-frequency facial action units and advanced eye-tracking metrics (pupillometry and fixations). 
+Since the recreation of the EyeTracking data in the lab was in doubt, only Action Units were used in the first XGBoost models. Now the model also implemented the EyeTracking data as features.
+By applying performance-based subject splitting, we ensured that the training and test sets were balanced not only by label but by the subjects' underlying skill levels, resulting in the most deployable version of the AI.
+
+| Metric / Model | Final Combined Model |
+| --- | --- |
+| Accuracy | 0.659 |
+| AUC | 0.621 |
+| F1-Score | 0.715 |
+
+### 4.2.6 Regularized XGBoost with Complexity Control
+
+Building upon the robust preprocessing of the previous steps, this iteration focuses on strict **complexity control** within the XGBoost architecture. To mitigate the 100% training accuracy observed in earlier unimodal tests—a clear indicator of overfitting—we introduced explicit **L1 (reg_alpha)** and **L2 (reg_lambda)** regularization parameters into the GridSearch space.
+
+By penalizing large weights and promoting feature sparsity, the model is forced to prioritize the most globally relevant Action Units. Furthermore, the tree depth was intentionally restricted (`max_depth`: 2-4), and an **Early Stopping** callback with a 30-round patience window was implemented. This ensures that training terminates at the point of optimal generalization, capturing the essential physiological trends of cognitive load while ignoring subject-specific noise.
+
+| Metric / Model | Regularized XGBoost |
+| --- | --- |
+| Accuracy | 0.665 |
+| AUC | 0.646 |
+| F1-Score | 0.727 |
+
+### 4.3 Isolation Forest
+To start with unsupervised learning techniques, `IsolationForest.ipynb`was created to research how well a simple ensemble classificator performs on the created dataset. 
+The notebook comes with one class grid search for hyperparameter tuning as well as a ROC curve that allows manual fine tuning. 
+Overall, our experiments have shown, that this approach is not sufficient, with the following results: 
+| Metric / Model | Isolation Forest | 
+|----------------|---------|
+| Best Balanced Accuracy |0.57|
+| Best AUC | 0.61|
+
+In detail, the classificator tends to classify the majority of samples as low load and is therefore not sufficient to be used for later deployment.
+
+
+### 4.4 One Class SVM with Autoencoder
+The training of an On Class SVM on the data from the dataset resulted in every sample was predicted as an anomaly. 
+In the next step, an autoencoder is pretrained to learn representation of the data. Afterwards, the encoder is used for preprocessing, which leads to OCSVM training on encoder output. 
+The training includes hyperparameter tuning through gridsearch cv. 
+Encoder output is visualized with print statements and plots that show the encoded data for both low and high load samples. 
+We see that the encoder struggles to represent the unseen high load samples differently. As a consequence, the One Class SVM also does not achieve sufficient performance. 
+| Metric / Model | One Class SVM | 
+|----------------|---------|
+| Best Balanced Accuracy |0.62|
+
+When the notebook is run completely, both the trained encoder and svm are saved for later use given the save paths are set correctly. 
+### 4.5 Deep SVDD
+Similar to the OCSVM training, an autoencoder is used to preprocess the data before the actual Deep SVDD training. Nevertheless, the usage is partialy different. The Dee SVDD uses a pretrained encoder to fine tune it by apllying a different loss function (which results from the theoretical concept behind Deep SVDD). This means that the encoder weights are still modified in the actual Deep SVDD training. 
+Also, this approach includes **hybrid fusion of modalities**. Instead of putting all features into the same input layer, the neural network is divided into two branches, that process action units and eye-tracking features separately.
+Then, after two Dense layers each, the branches are fusioned by concatenation. From there, another two Dense layers process the data. 
+The decoder is not exactly similar, as the split of the modalities happens are the very end. 
+To compute the total loss, loss from both modalities is combined by sum. Users are able to change loss weights. Training includes 2x2 phases, both autoencoder and later Deep SVDD are first trained with larger learning rate, then fine tuned with a smaller learning rate. 
+
+| Metric / Model | Deep SVDD | 
+|----------------|---------|
+| Best Balanced Accuracy |0.60|
+| Best AUC | 0.57|
+
+### 4.6 General information on unsupervised approaches
+As described above, the approachs didn't meet the requirements in terms of prediction performance. For all models, both MinMax-Scaling as well as Standard-Scaling was done. Also, both subjectwise and globally. Unfortunately, the differences were not that large, which may explain why preprocessing wasn't mentioned above. 
+
+Future research should always keep in mind while subject-wise scaling might be better for training, it makes deployment on new subjects more difficult. Our solution, as implemented in `model_training/tools/scaler.py` calculates a fallback scaler (using mean of all subjects scaling parameters). 
+
+## 5) Real-Time Prediction and Messaging
+
+Main script:
+- `predict_pipeline/predict_sample.py`
+
+Pipeline:
+- Loads runtime config (`predict_pipeline/config.yaml`)
+- Pulls latest row from SQLite
+- Replaces missing values using `fallback` map from config file - if more than 50% of values need to be replaced, the sample is dropped and "valid=False"
+- Optionally applies scaler (`.pkl`/`.joblib`) - set via config file
+- Loads model (`.keras`, `.pkl`, `.joblib`) and predicts
+- Publishes JSON payload to MQTT topic
+
+Expected payload form:
+```json
+{
+  "valid": true,    # false only if too many signals are invalid
+  "_id": 123,       # this is the sample ID from the database
+  "prediction": 0   # 0 for low load, 1 for high load
+}
+```
+
+### 5.1 Scheduled Prediction (Linux)
+
+Files:
+- `predict_pipeline/predict.service`
+- `predict_pipeline/predict.timer`
+
+Role:
+- Run inference repeatedly without manual execution
+- Timer/service configuration can be customized
+
+More information on how to use and interact with the system service and timer can be found in [predict_service_timer_documentation.md](/predict_pipeline/predict_service_timer_documentation.md)
+
+## 5.2 Runtime Configuration
+
+Primary config file:
+- `predict_pipeline/config.yaml`
+
+Sections:
+- `database`: SQLite location + table + sort key
+- `model`: model path
+- `scaler`: scaler usage + path
+- `mqtt`: broker and publish format
+- `sample.columns`: expected feature order
+- `fallback`: default values for NaN replacement
+
+Important:
+- The repository currently uses environment-specific absolute paths in some scripts/configs to ensure functionality on Ohm-UX driving simulator.
+
+
+## 5.3 Data and Feature Expectations
+
+Prediction expects SQLite rows containing:
+- `_Id`
+- `start_time` - this is not yet used for either predictions or messages
+- All configured model features (AUs + eye metrics)
+
+Common feature groups (similar to own dataset):
+- `FACE_AUxx_mean` columns
+- Fixation counters and duration statistics
+- Saccade count/amplitude/duration statistics
+- Blink count/duration statistics
+- Pupil mean and IPA
+
+## 5.4 Create database from scratch
+To (re-)create the custom database for deployment, use `fill_db.ipynb`. Enter the path to your dataset, drop unnecessary columns and insert a subset of data with tool functions from `tools/db_helpers`
+
+## 6) Installation and Dependencies
+Due to unsolvable dependency conflicts, several environemnts need to be used in the same time. 
+### 6.1 Environemnt for camera handling
+The setup of a virtual environment for the camera handling is difficult due to vary dependency conflicts.
+Therefore it is necessary to create the virtual environment with every package in the specific version and each package in the specific order.
+Furthermore the environment needs to be based on Python 3.10. The specific versions and order of the packages are described int the file: 
+`requirements.txt`
+
+
+### 6.2 Environment for predictions
+If you want to use the existing deployment on Ohm-UX driving simulator's jetson board, activate conda environment `p310_FS_TF`, a python 3.10 environment including tensorflow and all other packages required to run `predict_sample.py`
+Otherwise, as described in `readme.md: Setup`, you can use `prediction_env.yaml`, to create a new environment that fulfills the requirements.
+
+## 7) Repository File Inventory
+
+### Root
+
+- `.gitignore` - Git ignore rules
+- `readme.md` - minimal quickstart documentation
+- `project_report.md` - full technical documentation (this file)
+- `requirements.txt` - Python dependencies
+
+### Dataset Creation
+
+- `dataset_creation/parquet_file_creation.py` - local files to parquet conversion
+- `dataset_creation/create_parquet_files_from_owncloud.py` - ownCloud download + parquet conversion
+- `dataset_creation/combined_feature_creation.py` - sliding-window multimodal feature generation
+- `dataset_creation/maxDist.py` - helper/statistical utility script for eye-tracking feature creation 
+
+#### AU Creation
+- `dataset_creation/AU_creation/pyfeat_docu.ipynb` - py-feat exploratory notes
+
+#### Camera Handling
+- `dataset_creation/camera_handling/camera_stream_AU_and_ET_new.py` - current camera + AU + eye online pipeline
+- `dataset_creation/camera_handling/eyeFeature_new.py` - eye-feature extraction from gaze parquet
+- `dataset_creation/camera_handling/db_helper.py` - SQLite helper functions (camera pipeline)
+- `dataset_creation/camera_handling/camera_stream.py` - baseline camera streaming script
+- `dataset_creation/camera_handling/eyeFeature_kalibrierung.py` - eye-feature extraction with calibration
+- `dataset_creation/camera_handling/db_test.py` - DB test utility
+
+### EDA
+
+- `EDA/EDA.ipynb` - main EDA notebook
+- `EDA/distribution_plots.ipynb` - distribution visualization
+- `EDA/histogramms.ipynb` - histogram analysis
+- `EDA/researchOnSubjectPerformance.ipynb` - subject-level analysis
+- `EDA/owncloud_file_access.ipynb` - ownCloud exploration/access notebook
+- `EDA/calculate_replacement_values.ipynb` - fallback/median computation notebook
+- `EDA/login.yaml` - local auth/config artifact for EDA workflows
+
+### Model Training
+
+#### CNN
+- `model_training/CNN/CNN_simple.ipynb`
+- `model_training/CNN/CNN_crossVal.ipynb`
+- `model_training/CNN/CNN_crossVal_EarlyFusion.ipynb`
+- `model_training/CNN/CNN_crossVal_EarlyFusion_Filter.ipynb`
+- `model_training/CNN/CNN_crossVal_EarlyFusion_Test_Eval.ipynb`
+- `model_training/CNN/CNN_crossVal_faceAUs.ipynb`
+- `model_training/CNN/CNN_crossVal_faceAUs_eyeFeatures.ipynb`
+- `model_training/CNN/CNN_crossVal_HybridFusion.ipynb`
+- `model_training/CNN/CNN_crossVal_HybridFusion_Test_Eval.ipynb`
+- `model_training/CNN/deployment_pipeline.ipynb`
+
+#### XGBoost
+- `model_training/xgboost/xgboost.ipynb`
+- `model_training/xgboost/xgboost_groupfold.ipynb`
+- `model_training/xgboost/xgboost_new_dataset.ipynb`
+- `model_training/xgboost/xgboost_regulated.ipynb`
+- `model_training/xgboost/xgboost_with_AE.ipynb`
+- `model_training/xgboost/xgboost_with_MAD.ipynb`
+
+#### Isolation Forest
+- `model_training/IsolationForest/iforest_training.ipynb`
+
+#### OCSVM
+- `model_training/OCSVM/ocsvm_with_AE.ipynb`
+
+#### DeepSVDD
+- `model_training/DeepSVDD/deepSVDD.ipynb`
+
+#### MAD Outlier Removal
+- `model_training/MAD_outlier_removal/mad_outlier_removal.ipynb`
+- `model_training/MAD_outlier_removal/mad_outlier_removal_median.ipynb`
+
+#### Shared Training Tools
+- `model_training/tools/scaler.py`
+- `model_training/tools/performance_split.py`
+- `model_training/tools/mad_outlier_removal.py`
+- `model_training/tools/evaluation_tools.py`
+
+### Prediction Pipeline
+
+- `predict_pipeline/predict_sample.py` - runtime prediction + MQTT publish
+- `predict_pipeline/config.yaml` - runtime database/model/scaler/mqtt config
+- `predict_pipeline/fill_db.ipynb` - helper notebook for DB setup/testing
+- `predict_pipeline/predict.service` - systemd service unit
+- `predict_pipeline/predict.timer` - systemd timer unit
+- `predict_pipeline/predict_service_timer_documentation.md` - Linux service/timer guide
+
+### Generic Tools
+
+- `tools/db_helpers.py` - common SQLite utilities used to get newest sample for prediction
+
+## 8) Additional Information
+
+- Several paths are hardcoded on purpose to ensure compability with the jetsonboard at the OHM-UX driving simulator.
+- Camera and AU processing are resource-intensive; version pinning and hardware validation are recommended.
+- To access our dataset and other valuables, the `data-paulusjafahrsimulator-gpu` 
+ directory contains the raw data, performance results, parquet files, and the final dataset.

readme.md

@@ -1,67 +1,54 @@
 # Multimodal Driver State Analysis
 
-Ein umfassendes Framework zur Analyse von Fahrerverhalten durch kombinierte Feature-Extraktion aus Facial Action Units (AU) und Eye-Tracking Daten.
+Short overview: this repository contains the data, feature, training, and inference pipeline for multimodal driver-state analysis using facial AUs and eye-tracking signals.
 
-## 📋 Projektübersicht
+For full documentation, see [project_report.md](project_report.md).
 
-Dieses Projekt verarbeitet multimodale Sensordaten aus Fahrsimulator-Studien und extrahiert zeitbasierte Features für die Analyse von Fahrerzuständen. Die Pipeline kombiniert:
+## Quickstart
 
-- **Facial Action Units (AU)**: 20 Gesichtsaktionseinheiten zur Emotionserkennung
-- **Eye-Tracking**: Fixationen, Sakkaden, Blinks und Pupillenmetriken
-
-## 🎯 Features
-
-### Datenverarbeitung
-- **Sliding Window Aggregation**: 50-Sekunden-Fenster mit 5-Sekunden-Schrittweite
-- **Hierarchische Gruppierung**: Automatische Segmentierung nach STUDY/LEVEL/PHASE
-- **Robuste Fehlerbehandlung**: Graceful Degradation bei fehlenden Modalitäten
-
-### Extrahierte Features
-
-#### Facial Action Units (20 AUs)
-Für jede AU wird der Mittelwert pro Window berechnet:
-- AU01 (Inner Brow Raiser) bis AU43 (Eyes Closed)
-- Aggregation: `mean` über 50s Window
-
-#### Eye-Tracking Features
-**Fixationen:**
-- Anzahl nach Dauer-Kategorien (66-150ms, 300-500ms, >1000ms, >100ms)
-- Mittelwert und Median der Fixationsdauer
-
-**Sakkaden:**
-- Anzahl, mittlere Amplitude, mittlere/mediane Dauer
-
-**Blinks:**
-- Anzahl, mittlere/mediane Dauer
-
-**Pupille:**
-- Mittlere Pupillengröße
-- Index of Pupillary Activity (IPA) - Hochfrequenzkomponente (0.6-2.0 Hz)
-
-## 🏗️ Projektstruktur
-
-to be continued.
-
-## 🚀 Installation
-
-### Voraussetzungen
+### 1) Setup
+Activate the conda-repository "camera_stream_AU_ET_test". 
 ```bash
-Python 3.12
+    conda activate camera_stream_AU_ET_test
+```
+**Make sure, another environment that fulfills prediction_env.yaml is available**, matching with predict_pipeline/predict.service 
+See `predict_pipeline/predict_service_timer_documentation.md`
+to get an overview over all available conda environments on your device, use this command in anaconda prompt terminal: 
+```bash
+    conda info --envs
+```
+Optionally, create a new environment based on the yaml-file:
+```bash
+    conda env create -f prediction_env.yaml
+```
+Ohm-UX driving simulator jetson board only: The conda-environment `p310_FS_TF` is used for predictions. 
+
+### 2) Camera AU + Eye Pipeline (`camera_stream_AU_and_ET_new.py`)
+
+1. Open `dataset_creation/camera_handling/camera_stream_AU_and_ET_new.py` and adjust:
+- `DB_PATH`
+- `CAMERA_INDEX`
+- `OUTPUT_DIR` (optional)
+
+2. Start camera capture and feature extraction:
+```bash
+python dataset_creation/camera_handling/camera_stream_AU_and_ET_new.py
 ```
 
-### Dependencies
+3. Stop with `q` in the camera window.
+
+
+### 3) Predict Pipeline (`predict_pipeline/predict_sample.py`)
+
+1. Edit `predict_pipeline/config.yaml` and set:
+- `database.path`, `database.table`, `database.key`
+- `model.path`
+- `scaler.path` (if `use_scaling: true`)
+- MQTT settings under `mqtt`
+
+2. Run one prediction cycle:
 ```bash
-pip install -r requirements.txt
+python predict_pipeline/predict_sample.py
 ```
 
-**Wichtigste Pakete:**
-- `pandas`, `numpy` - Datenverarbeitung
-- `scipy` - Signalverarbeitung
-- `scikit-learn` - Feature-Skalierung & ML
-- `pygazeanalyser` - Eye-Tracking Analyse
-- `pyarrow` - Parquet I/O
-
-## 💻 Usage
-
-### 1. Feature-Extraktion
- to be continued
+3. Use  [predict_service_timer_documentation.md](/predict_pipeline/predict_service_timer_documentation.md) to see how to use the service and timer for automation. On Ohm-UX driving simulator's jetson board, the service runs in the background and starts automatically when the device is booting. 

requirements.txt

@@ -0,0 +1,27 @@
+# Core data + ML utilities
+numpy
+pandas
+scipy
+scikit-learn
+pyarrow
+joblib
+PyYAML
+matplotlib
+
+# Prediction pipeline
+paho-mqtt
+tensorflow
+
+# Camera / feature extraction stack
+# It is necessary to create an extra environment for the extraction pipeline, because of the different version needs
+Python==3.10
+numpy==1.24.4
+scipy==1.10.1
+opencv-python==4.7.0.72
+mediapipe==0.10.13
+py-feat
+
+# Data ingestion (ownCloud + HDF)
+pyocclient
+h5py
+tables