{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "e790b157",
   "metadata": {},
   "source": [
    "Im folgenden wird auf die Daten das MAD Outlier removal angewendet."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "46bd036d",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "def calculate_mad_params(df, columns):\n",
    "    \"\"\"\n",
    "    Calculate median and MAD parameters for each column.\n",
    "    This should be run ONLY on the training data.\n",
    "    \n",
    "    Returns a dictionary: {col: (median, mad)}\n",
    "    \"\"\"\n",
    "    params = {}\n",
    "    for col in columns:\n",
    "        median = df[col].median()\n",
    "        mad = np.median(np.abs(df[col] - median))\n",
    "        params[col] = (median, mad)\n",
    "    return params"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e0691732",
   "metadata": {},
   "outputs": [],
   "source": [
    "def apply_mad_filter(df, params, threshold=3.5):\n",
    "    \"\"\"\n",
    "    Apply MAD-based outlier removal using precomputed parameters.\n",
    "    Works on training, validation, and test data.\n",
    "    \n",
    "    df: DataFrame to filter\n",
    "    params: dictionary {col: (median, mad)} from training data\n",
    "    threshold: cutoff for robust Z-score\n",
    "    \"\"\"\n",
    "    df_clean = df.copy()\n",
    "\n",
    "    for col, (median, mad) in params.items():\n",
    "        if mad == 0:\n",
    "            continue  # no spread; nothing to remove for this column\n",
    "\n",
    "        robust_z = 0.6745 * (df_clean[col] - median) / mad\n",
    "        outlier_mask = np.abs(robust_z) > threshold\n",
    "\n",
    "        # Remove values only in this specific column\n",
    "        df_clean.loc[outlier_mask, col] = median\n",
    "        print(df_clean.shape)\n",
    "        \n",
    "    print(df_clean.shape)\n",
    "    return df_clean"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
