kurzti88066/Fahrsimulator_MSY2526_AI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

minor fixes to new paths / dataset with all columns

Browse Source
This commit is contained in:
weigmi87303
2026-03-04 12:25:07 +01:00
parent 3d8c7c6639
commit af3f9d16b2
2 changed files with 89 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files
model_training/IsolationForest/iforest_training.ipynb
+33 -29
View File
@@ -28,7 +28,7 @@
"sys.path.append(base_dir)\n",
"print(base_dir)\n",
"\n",
"from tools import evaluation_tools\n",
"from Fahrsimulator_MSY2526_AI.model_training.tools import evaluation_tools\n",
"from sklearn.preprocessing import StandardScaler, MinMaxScaler\n",
"from sklearn.ensemble import IsolationForest\n",
"from sklearn.model_selection import GridSearchCV, KFold\n",
@@ -52,7 +52,7 @@
"metadata": {},
"outputs": [],
"source": [
"data_path = Path(r\"C:\\Users\\micha\\FAUbox\\WS2526_Fahrsimulator_MSY (Celina Korzer)\\AU_dataset\\output_windowed.parquet\")"
"data_path = Path(r\"/home/jovyan/data-paulusjafahrsimulator-gpu/new_datasets/50s_25Hz_dataset.parquet\")"
]
},
{
@@ -301,20 +301,26 @@
"metadata": {},
"outputs": [],
"source": [
"# Cell 2: Get AU columns and prepare datasets\n",
"# Get all column names that start with 'AU'\n",
"au_columns = [col for col in low_all.columns if col.startswith('AU')]\n",
"au_columns = [col for col in low_all.columns if \"face\" in col.lower()] \n",
"\n",
"eye_columns = [ \n",
" 'Fix_count_short_66_150','Fix_count_medium_300_500','Fix_count_long_gt_1000', \n",
" 'Fix_count_100','Fix_mean_duration','Fix_median_duration', \n",
" 'Sac_count','Sac_mean_amp','Sac_mean_dur','Sac_median_dur', \n",
" 'Blink_count','Blink_mean_dur','Blink_median_dur', \n",
" 'Pupil_mean','Pupil_IPA' \n",
"] \n",
"cols = au_columns +eye_columns\n",
"# Prepare training data (only normal/low data)\n",
"train_data = low_all[low_all['subjectID'].isin(train_subjects)][['subjectID'] + au_columns].copy()\n",
"train_data = low_all[low_all['subjectID'].isin(train_subjects)][['subjectID'] + cols].copy()\n",
"\n",
"# Prepare validation data (normal and anomaly)\n",
"val_normal_data = low_all[low_all['subjectID'].isin(val_subjects)][['subjectID'] + au_columns].copy()\n",
"val_high_data = high_all[high_all['subjectID'].isin(val_subjects)][['subjectID'] + au_columns].copy()\n",
"val_normal_data = low_all[low_all['subjectID'].isin(val_subjects)][['subjectID'] + cols].copy()\n",
"val_high_data = high_all[high_all['subjectID'].isin(val_subjects)][['subjectID'] + cols].copy()\n",
"\n",
"# Prepare test data (normal and anomaly)\n",
"test_normal_data = low_all[low_all['subjectID'].isin(test_subjects)][['subjectID'] + au_columns].copy()\n",
"test_high_data = high_all[high_all['subjectID'].isin(test_subjects)][['subjectID'] + au_columns].copy()\n",
"test_normal_data = low_all[low_all['subjectID'].isin(test_subjects)][['subjectID'] + cols].copy()\n",
"test_high_data = high_all[high_all['subjectID'].isin(test_subjects)][['subjectID'] + cols].copy()\n",
"\n",
"print(f\"Train samples: {len(train_data)}\")\n",
"print(f\"Val normal samples: {len(val_normal_data)}, Val high samples: {len(val_high_data)}\")\n",
@@ -328,8 +334,8 @@
"metadata": {},
"outputs": [],
"source": [
"# Cell 3: Fit normalizer on training data\n",
"normalizer = fit_normalizer(train_data, au_columns, method='minmax', scope='global')\n",
"# Fit normalizer on training data\n",
"normalizer = fit_normalizer(train_data, cols, method='minmax', scope='global')\n",
"print(\"Normalizer fitted on training data\")"
]
},
@@ -340,12 +346,12 @@
"metadata": {},
"outputs": [],
"source": [
"# Cell 4: Apply normalization to all datasets\n",
"train_normalized = apply_normalizer(train_data, au_columns, normalizer)\n",
"val_normal_normalized = apply_normalizer(val_normal_data, au_columns, normalizer)\n",
"val_high_normalized = apply_normalizer(val_high_data, au_columns, normalizer)\n",
"test_normal_normalized = apply_normalizer(test_normal_data, au_columns, normalizer)\n",
"test_high_normalized = apply_normalizer(test_high_data, au_columns, normalizer)\n",
"# Apply normalization to all datasets\n",
"train_normalized = apply_normalizer(train_data, cols, normalizer)\n",
"val_normal_normalized = apply_normalizer(val_normal_data, cols, normalizer)\n",
"val_high_normalized = apply_normalizer(val_high_data, cols, normalizer)\n",
"test_normal_normalized = apply_normalizer(test_normal_data, cols, normalizer)\n",
"test_high_normalized = apply_normalizer(test_high_data, cols, normalizer)\n",
"\n",
"print(\"Normalization applied to all datasets\")"
]
@@ -357,11 +363,9 @@
"metadata": {},
"outputs": [],
"source": [
"# Cell 5: Extract AU columns and create labels for grid search\n",
"# Extract only AU columns (drop subjectID)\n",
"X_train = train_normalized[au_columns].copy()\n",
"X_val_normal = val_normal_normalized[au_columns].copy()\n",
"X_val_high = val_high_normalized[au_columns].copy()\n",
"X_train = train_normalized[cols].copy()\n",
"X_val_normal = val_normal_normalized[cols].copy()\n",
"X_val_high = val_high_normalized[cols].copy()\n",
"\n",
"# Combine train and validation sets for grid search\n",
"X_grid_search = pd.concat([X_train, X_val_normal, X_val_high], ignore_index=True)\n",
@@ -416,7 +420,7 @@
"metadata": {},
"outputs": [],
"source": [
"# Cell 7: Train final model with best parameters on training data\n",
"# Train final model with best parameters on training data\n",
"final_model = IsolationForest(**best_params, random_state=42)\n",
"final_model.fit(X_train.values)\n",
"\n",
@@ -430,9 +434,9 @@
"metadata": {},
"outputs": [],
"source": [
"# Cell 8: Prepare independent test set\n",
"X_test_normal = test_normal_normalized[au_columns].copy()\n",
"X_test_high = test_high_normalized[au_columns].copy()\n",
"# Prepare independent test set\n",
"X_test_normal = test_normal_normalized[cols].copy()\n",
"X_test_high = test_high_normalized[cols].copy()\n",
"\n",
"# Combine test sets\n",
"X_test = pd.concat([X_test_normal, X_test_high], ignore_index=True)\n",
@@ -483,7 +487,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "base",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
@@ -497,7 +501,7 @@
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.5"
"version": "3.12.10"
}
},
"nbformat": 4,