kurzti88066/Fahrsimulator_MSY2526_AI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

outsourcing of scaler in iforest and deep svdd, removal of paths

Browse Source
This commit is contained in:
weigmi87303
2026-03-04 16:51:22 +01:00
parent 6cc38291df
commit 13bd76631f
2 changed files with 69 additions and 402 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model_training/IsolationForest/iforest_training.ipynb
+8 -132
View File
@@ -28,7 +28,7 @@
"sys.path.append(base_dir)\n",
"print(base_dir)\n",
"\n",
"from Fahrsimulator_MSY2526_AI.model_training.tools import evaluation_tools\n",
"from Fahrsimulator_MSY2526_AI.model_training.tools import evaluation_tools, scaler\n",
"from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
"from sklearn.ensemble import IsolationForest\n",
"from sklearn.model_selection import GridSearchCV, KFold\n",
@@ -52,7 +52,7 @@
"metadata": {},
"outputs": [],
"source": [
"data_path = Path(r\"/home/jovyan/data-paulusjafahrsimulator-gpu/new_datasets/50s_25Hz_dataset.parquet\")"
"data_path = Path(r\".parquet\") # TODO: set manually"
]
},
{
@@ -115,118 +115,6 @@
"print(f\"high all: {high_all.shape}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "47a0f44d",
"metadata": {},
"outputs": [],
"source": [
"def fit_normalizer(train_data, au_columns, method='standard', scope='global'):\n",
" \"\"\"\n",
" Fit normalization scalers on training data.\n",
" \n",
" Parameters:\n",
" -----------\n",
" train_data : pd.DataFrame\n",
" Training dataframe with AU columns and subjectID\n",
" au_columns : list\n",
" List of AU column names to normalize\n",
" method : str, default='standard'\n",
" Normalization method: 'standard' for StandardScaler or 'minmax' for MinMaxScaler\n",
" scope : str, default='global'\n",
" Normalization scope: 'subject' for per-subject or 'global' for across all subjects\n",
" \n",
" Returns:\n",
" --------\n",
" dict\n",
" Dictionary containing fitted scalers\n",
" \"\"\"\n",
" # Select scaler based on method\n",
" if method == 'standard':\n",
" Scaler = StandardScaler\n",
" elif method == 'minmax':\n",
" Scaler = MinMaxScaler\n",
" else:\n",
" raise ValueError(\"method must be 'standard' or 'minmax'\")\n",
" \n",
" scalers = {}\n",
" \n",
" if scope == 'subject':\n",
" # Fit one scaler per subject\n",
" for subject in train_data['subjectID'].unique():\n",
" subject_mask = train_data['subjectID'] == subject\n",
" scaler = Scaler()\n",
" scaler.fit(train_data.loc[subject_mask, au_columns])\n",
" scalers[subject] = scaler\n",
" \n",
" elif scope == 'global':\n",
" # Fit one scaler for all subjects\n",
" scaler = Scaler()\n",
" scaler.fit(train_data[au_columns])\n",
" scalers['global'] = scaler\n",
" \n",
" else:\n",
" raise ValueError(\"scope must be 'subject' or 'global'\")\n",
" \n",
" return {'scalers': scalers, 'method': method, 'scope': scope}"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "642d0017",
"metadata": {},
"outputs": [],
"source": [
"def apply_normalizer(data, au_columns, normalizer_dict):\n",
" \"\"\"\n",
" Apply fitted normalization scalers to data.\n",
" \n",
" Parameters:\n",
" -----------\n",
" data : pd.DataFrame\n",
" Dataframe with AU columns and subjectID\n",
" au_columns : list\n",
" List of AU column names to normalize\n",
" normalizer_dict : dict\n",
" Dictionary containing fitted scalers from fit_normalizer()\n",
" \n",
" Returns:\n",
" --------\n",
" pd.DataFrame\n",
" DataFrame with normalized AU columns\n",
" \"\"\"\n",
" normalized_data = data.copy()\n",
" scalers = normalizer_dict['scalers']\n",
" scope = normalizer_dict['scope']\n",
" \n",
" if scope == 'subject':\n",
" # Apply per-subject normalization\n",
" for subject in data['subjectID'].unique():\n",
" subject_mask = data['subjectID'] == subject\n",
" \n",
" # Use the subject's scaler if available, otherwise use a fitted scaler from training\n",
" if subject in scalers:\n",
" scaler = scalers[subject]\n",
" else:\n",
" # For new subjects not seen in training, use the first available scaler\n",
" # (This is a fallback - ideally all test subjects should be in training for subject-level normalization)\n",
" print(f\"Warning: Subject {subject} not found in training data. Using fallback scaler.\")\n",
" scaler = list(scalers.values())[0]\n",
" \n",
" normalized_data.loc[subject_mask, au_columns] = scaler.transform(\n",
" data.loc[subject_mask, au_columns]\n",
" )\n",
" \n",
" elif scope == 'global':\n",
" # Apply global normalization\n",
" scaler = scalers['global']\n",
" normalized_data[au_columns] = scaler.transform(data[au_columns])\n",
" \n",
" return normalized_data"
]
},
{
"cell_type": "markdown",
"id": "697b3cf7",
@@ -335,7 +223,7 @@
"outputs": [],
"source": [
"# Fit normalizer on training data\n",
"normalizer = fit_normalizer(train_data, cols, method='minmax', scope='global')\n",
"normalizer = scaler.fit_normalizer(train_data, cols, method='minmax', scope='global')\n",
"print(\"Normalizer fitted on training data\")"
]
},
@@ -347,11 +235,11 @@
"outputs": [],
"source": [
"# Apply normalization to all datasets\n",
"train_normalized = apply_normalizer(train_data, cols, normalizer)\n",
"val_normal_normalized = apply_normalizer(val_normal_data, cols, normalizer)\n",
"val_high_normalized = apply_normalizer(val_high_data, cols, normalizer)\n",
"test_normal_normalized = apply_normalizer(test_normal_data, cols, normalizer)\n",
"test_high_normalized = apply_normalizer(test_high_data, cols, normalizer)\n",
"train_normalized = scaler.apply_normalizer(train_data, cols, normalizer)\n",
"val_normal_normalized = scaler.apply_normalizer(val_normal_data, cols, normalizer)\n",
"val_high_normalized = scaler.apply_normalizer(val_high_data, cols, normalizer)\n",
"test_normal_normalized = scaler.apply_normalizer(test_normal_data, cols, normalizer)\n",
"test_high_normalized = scaler.apply_normalizer(test_high_data, cols, normalizer)\n",
"\n",
"print(\"Normalization applied to all datasets\")"
]
@@ -490,18 +378,6 @@
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.12.10"
}
},
"nbformat": 4,