- added description to use py-feat
- added script: -- to split a video into 2min clips -- to extract AUs from a video
This commit is contained in:
TimoKurz
parent
a1bdf470cc
commit
f5796a5cdd
58
dataset_creation/AU_creation/AU_creation_service.py
Normal file
58
dataset_creation/AU_creation/AU_creation_service.py
Normal file
@ -0,0 +1,58 @@
|
||||
from feat import Detector
|
||||
from feat.utils.io import get_test_data_path
|
||||
from moviepy.video.io.VideoFileClip import VideoFileClip
|
||||
import os
|
||||
|
||||
def extract_aus(path, model):
|
||||
detector = Detector(au_model=model)
|
||||
|
||||
video_prediction = detector.detect(
|
||||
path, data_type="video", skip_frames=24*5, face_detection_threshold=0.95 # alle 5 Sekunden einbeziehen - 24 Frames pro Sekunde
|
||||
)
|
||||
|
||||
return video_prediction.aus.sum()
|
||||
|
||||
def split_video(path, chunk_length=120):
|
||||
video = VideoFileClip(path)
|
||||
duration = int(video.duration)
|
||||
|
||||
subclips_dir = os.path.join(os.dirname(path), "subclips")
|
||||
os.makedirs(subclips_dir, exist_ok=True)
|
||||
paths = []
|
||||
|
||||
for start in range(0, duration, chunk_length):
|
||||
end = min(start + chunk_length, duration)
|
||||
|
||||
subclip = (
|
||||
video
|
||||
.subclip(start, end)
|
||||
.without_audio()
|
||||
.set_fps(video.fps)
|
||||
)
|
||||
|
||||
output_path = f"{subclips_dir}_part_{start//chunk_length + 1}.mp4"
|
||||
subclip.write_videofile(
|
||||
output_path,
|
||||
)
|
||||
paths.append(output_path)
|
||||
|
||||
return output_path
|
||||
|
||||
def start(path):
|
||||
results = []
|
||||
clips = split_video(path)
|
||||
|
||||
for clip in clips:
|
||||
results.append(extract_aus(clip, 'svm'))
|
||||
return results
|
||||
|
||||
if __name__ == "__main__":
|
||||
results = []
|
||||
clips = []
|
||||
test_video_path = "AU_creation/YTDown.com_YouTube_Was-ist-los-bei-7-vs-Wild_Media_Gtj9zu_WikU_001_1080p.mp4"
|
||||
clips = split_video(test_video_path)
|
||||
|
||||
for clippath in clips:
|
||||
results.append(extract_aus(clippath, 'svm'))
|
||||
|
||||
print(results)
|
||||
154
dataset_creation/AU_creation/pyfeat_docu.ipynb
Normal file
154
dataset_creation/AU_creation/pyfeat_docu.ipynb
Normal file
@ -0,0 +1,154 @@
|
||||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"id": "3b0c6c82",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Hier entsteht die Dokumentation, wie die Action Units erzeugt wurden.\n",
|
||||
"Daraus wird dann letztendlich ein Skript erstellt, welches automatisch AUs aus Videodateien erstellen soll.\n",
|
||||
"\n",
|
||||
"Py-Feat besitzt Dependencies, die ab Python 3.12 nicht mehr verfügbar sind.\n",
|
||||
"Dazu muss ein Kernel mit Python 3.11 erstellt werden.\n",
|
||||
"Folgendes Vorgehen:\n",
|
||||
"1. Seite des Jupyter Labs öffnen\n",
|
||||
"2. Terminal öffnen und folgende Befehle eingeben:\n",
|
||||
" conda create -n py311 python=3.11\n",
|
||||
" source ~/.bashrc\n",
|
||||
" conda activate py311\n",
|
||||
" conda install jupyter\n",
|
||||
" python -m ipykernel install --user --name=py311 --display-name \"Python 3.11\"\n",
|
||||
" pip install py-feat\n",
|
||||
" pip install \"moviepy<2.0\" (falls benötigt)\n",
|
||||
"3. den Kernel neustarten\n",
|
||||
"4. in VSC den Kernel neu hinzufügen und dann den Kernel mit dem Namen \"Python 3.11\" auswählen.\n",
|
||||
"\n",
|
||||
"Der Code unten zeigt eine beispielhafte Integration der py-feat Bibliothek.\n",
|
||||
"Die Klassifizierung zu 0,1 kommt durch die Wahl des AU-Modells zustande. Dabei wird SVM gewählt. (ADABase Paper)\n",
|
||||
"Gibt die Klassifizierung einen Gleitkommawert zwischen 0 & 1 aus, dann kommt XGB zum Einsatz. (REVELIO Paper)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c262899f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"%pip install git+https://github.com/cosanlab/py-feat.git"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "25d8d708",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from feat import Detector\n",
|
||||
"\n",
|
||||
"# detector = Detector(face_model='RetinaFace', facepose_model='Img2Pose', landmark_model='MobileFaceNet', au_model='svm')\n",
|
||||
"# detector = Detector(face_model='img2pose', landmark_model='mobilefacenet', au_model='xgb', emotion_model='resmasknet', facepose_model='img2pose', identity_model='facenet')\n",
|
||||
"detector = Detector(au_model='svm')\n",
|
||||
"# Detector(face_model='RetinaFace', facepose_model='Img2Pose', landmark_model='MobileFaceNet', au_model='svm')\n",
|
||||
"detector"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "c361665f",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from feat.utils.io import get_test_data_path\n",
|
||||
"from feat.plotting import imshow\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"# Helper to point to the test data folder\n",
|
||||
"test_data_dir = get_test_data_path()\n",
|
||||
"print(get_test_data_path())\n",
|
||||
"\n",
|
||||
"# Get the full path\n",
|
||||
"folder = r\"AU_creation\"\n",
|
||||
"paths = [os.path.join(folder, f) for f in os.listdir(folder)]\n",
|
||||
"\n",
|
||||
"# Plot it\n",
|
||||
"imshow(paths[0])"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "e1e0780c",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"single_face_prediction = detector.detect(paths, data_type=\"image\")\n",
|
||||
"\n",
|
||||
"type(single_face_prediction) # instace of a Fex class\n",
|
||||
"\n",
|
||||
"# Show results\n",
|
||||
"single_face_prediction.aus\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "5d844b17",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"single_face_prediction.emotions.plot()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "76771cb4",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"test_data_dir = get_test_data_path()\n",
|
||||
"test_video_path = os.path.join(test_data_dir, \"WolfgangLanger_Pexels.mp4\")\n",
|
||||
"\n",
|
||||
"video_prediction = detector.detect(\n",
|
||||
" test_video_path, data_type=\"video\", skip_frames=24*5, face_detection_threshold=0.95 # alle 5 Sekunden einbeziehen - 24 Frames pro Sekunde\n",
|
||||
")\n",
|
||||
"\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "d8a06268",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"au_counts = video_prediction.aus.sum()\n",
|
||||
"print(au_counts)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.11",
|
||||
"language": "python",
|
||||
"name": "py311"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.11.14"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 5
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user