- added xgboost to classify AU and EyeTracking features

2026-01-11 17:21:07 +01:00
parent 10bc7c568a
commit fc6c593f6b
2 changed files with 884 additions and 0 deletions
@@ -0,0 +1,77 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "id": "e790b157",
+   "metadata": {},
+   "source": [
+    "Im folgenden wird auf die Daten das MAD Outlier removal angewendet."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "46bd036d",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import numpy as np\n",
+    "import pandas as pd\n",
+    "\n",
+    "def calculate_mad_params(df, columns):\n",
+    "    \"\"\"\n",
+    "    Calculate median and MAD parameters for each column.\n",
+    "    This should be run ONLY on the training data.\n",
+    "    \n",
+    "    Returns a dictionary: {col: (median, mad)}\n",
+    "    \"\"\"\n",
+    "    params = {}\n",
+    "    for col in columns:\n",
+    "        median = df[col].median()\n",
+    "        mad = np.median(np.abs(df[col] - median))\n",
+    "        params[col] = (median, mad)\n",
+    "    return params"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "e0691732",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def apply_mad_filter(df, params, threshold=3.5):\n",
+    "    \"\"\"\n",
+    "    Apply MAD-based outlier removal using precomputed parameters.\n",
+    "    Works on training, validation, and test data.\n",
+    "    \n",
+    "    df: DataFrame to filter\n",
+    "    params: dictionary {col: (median, mad)} from training data\n",
+    "    threshold: cutoff for robust Z-score\n",
+    "    \"\"\"\n",
+    "    df_clean = df.copy()\n",
+    "\n",
+    "    for col, (median, mad) in params.items():\n",
+    "        if mad == 0:\n",
+    "            continue  # no spread; nothing to remove for this column\n",
+    "\n",
+    "        robust_z = 0.6745 * (df_clean[col] - median) / mad\n",
+    "        outlier_mask = np.abs(robust_z) > threshold\n",
+    "\n",
+    "        # Remove values only in this specific column\n",
+    "        df_clean.loc[outlier_mask, col] = median\n",
+    "        print(df_clean.shape)\n",
+    "        \n",
+    "    print(df_clean.shape)\n",
+    "    return df_clean"
+   ]
+  }
+ ],
+ "metadata": {
+  "language_info": {
+   "name": "python"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}