559 lines
16 KiB
Python
559 lines
16 KiB
Python
import pickle
|
|
from pathlib import Path
|
|
|
|
import cv2
|
|
import mediapipe as mp
|
|
import numpy as np
|
|
from mediapipe.tasks.python import vision
|
|
from sklearn.ensemble import RandomForestClassifier
|
|
from sklearn.linear_model import LogisticRegression
|
|
from sklearn.neighbors import KNeighborsClassifier
|
|
from sklearn.preprocessing import LabelEncoder, StandardScaler
|
|
from sklearn.svm import SVC
|
|
|
|
|
|
WRIST = 0
|
|
|
|
THUMB_CMC = 1
|
|
THUMB_MCP = 2
|
|
THUMB_IP = 3
|
|
THUMB_TIP = 4
|
|
|
|
INDEX_MCP = 5
|
|
INDEX_PIP = 6
|
|
INDEX_DIP = 7
|
|
INDEX_TIP = 8
|
|
|
|
MIDDLE_MCP = 9
|
|
MIDDLE_PIP = 10
|
|
MIDDLE_DIP = 11
|
|
MIDDLE_TIP = 12
|
|
|
|
RING_MCP = 13
|
|
RING_PIP = 14
|
|
RING_DIP = 15
|
|
RING_TIP = 16
|
|
|
|
PINKY_MCP = 17
|
|
PINKY_PIP = 18
|
|
PINKY_DIP = 19
|
|
PINKY_TIP = 20
|
|
|
|
HAND_CONNECTIONS = [
|
|
(0, 1), (1, 2), (2, 3), (3, 4),
|
|
(0, 5), (5, 6), (6, 7), (7, 8),
|
|
(0, 9), (9, 10), (10, 11), (11, 12),
|
|
(0, 13), (13, 14), (14, 15), (15, 16),
|
|
(0, 17), (17, 18), (18, 19), (19, 20),
|
|
(5, 9), (9, 13), (13, 17),
|
|
]
|
|
|
|
DEFAULT_UNKNOWN_THRESHOLD = 0.7
|
|
DEFAULT_KNN_K = 3
|
|
DEFAULT_RF_TREES = 200
|
|
DEFAULT_NUM_HANDS = 2
|
|
|
|
|
|
def landmark_to_pixel(lm, width, height):
|
|
return int(lm.x * width), int(lm.y * height)
|
|
|
|
|
|
def vec2_from_landmarks(hand_landmarks):
|
|
return np.array([[lm.x, lm.y] for lm in hand_landmarks], dtype=np.float32)
|
|
|
|
|
|
def vec3_from_world_landmarks(world_landmarks):
|
|
return np.array([[lm.x, lm.y, lm.z] for lm in world_landmarks], dtype=np.float32)
|
|
|
|
|
|
def hand_scale_2d(points2d):
|
|
s = np.linalg.norm(points2d[MIDDLE_MCP] - points2d[WRIST])
|
|
return max(float(s), 1e-6)
|
|
|
|
|
|
def hand_scale_3d(points3d):
|
|
s = np.linalg.norm(points3d[MIDDLE_MCP] - points3d[WRIST])
|
|
return max(float(s), 1e-6)
|
|
|
|
|
|
def normalize_2d_features(hand_landmarks):
|
|
pts = vec2_from_landmarks(hand_landmarks)
|
|
origin = pts[WRIST].copy()
|
|
pts = pts - origin
|
|
scale = hand_scale_2d(pts + origin)
|
|
pts = pts / scale
|
|
return pts.flatten().astype(np.float32)
|
|
|
|
|
|
def normalize_3d_features(world_landmarks):
|
|
pts = vec3_from_world_landmarks(world_landmarks)
|
|
origin = pts[WRIST].copy()
|
|
pts = pts - origin
|
|
scale = hand_scale_3d(pts + origin)
|
|
pts = pts / scale
|
|
return pts.flatten().astype(np.float32)
|
|
|
|
|
|
def angle_between(v1, v2):
|
|
n1 = np.linalg.norm(v1)
|
|
n2 = np.linalg.norm(v2)
|
|
if n1 < 1e-6 or n2 < 1e-6:
|
|
return 0.0
|
|
c = np.dot(v1, v2) / (n1 * n2)
|
|
c = np.clip(c, -1.0, 1.0)
|
|
return float(np.arccos(c))
|
|
|
|
|
|
def joint_angle(points, a, b, c):
|
|
ba = points[a] - points[b]
|
|
bc = points[c] - points[b]
|
|
return angle_between(ba, bc)
|
|
|
|
|
|
def extract_angle_features(world_landmarks):
|
|
pts = vec3_from_world_landmarks(world_landmarks)
|
|
|
|
thumb_angle_1 = joint_angle(pts, THUMB_CMC, THUMB_MCP, THUMB_IP)
|
|
thumb_angle_2 = joint_angle(pts, THUMB_MCP, THUMB_IP, THUMB_TIP)
|
|
|
|
index_angle_1 = joint_angle(pts, INDEX_MCP, INDEX_PIP, INDEX_DIP)
|
|
index_angle_2 = joint_angle(pts, INDEX_PIP, INDEX_DIP, INDEX_TIP)
|
|
|
|
middle_angle_1 = joint_angle(pts, MIDDLE_MCP, MIDDLE_PIP, MIDDLE_DIP)
|
|
middle_angle_2 = joint_angle(pts, MIDDLE_PIP, MIDDLE_DIP, MIDDLE_TIP)
|
|
|
|
ring_angle_1 = joint_angle(pts, RING_MCP, RING_PIP, RING_DIP)
|
|
ring_angle_2 = joint_angle(pts, RING_PIP, RING_DIP, RING_TIP)
|
|
|
|
pinky_angle_1 = joint_angle(pts, PINKY_MCP, PINKY_PIP, PINKY_DIP)
|
|
pinky_angle_2 = joint_angle(pts, PINKY_PIP, PINKY_DIP, PINKY_TIP)
|
|
|
|
wrist_to_thumb = pts[THUMB_TIP] - pts[WRIST]
|
|
wrist_to_index = pts[INDEX_TIP] - pts[WRIST]
|
|
wrist_to_middle = pts[MIDDLE_TIP] - pts[WRIST]
|
|
wrist_to_ring = pts[RING_TIP] - pts[WRIST]
|
|
wrist_to_pinky = pts[PINKY_TIP] - pts[WRIST]
|
|
|
|
spread_thumb_index = angle_between(wrist_to_thumb, wrist_to_index)
|
|
spread_index_middle = angle_between(wrist_to_index, wrist_to_middle)
|
|
spread_middle_ring = angle_between(wrist_to_middle, wrist_to_ring)
|
|
spread_ring_pinky = angle_between(wrist_to_ring, wrist_to_pinky)
|
|
|
|
feats = np.array([
|
|
thumb_angle_1, thumb_angle_2,
|
|
index_angle_1, index_angle_2,
|
|
middle_angle_1, middle_angle_2,
|
|
ring_angle_1, ring_angle_2,
|
|
pinky_angle_1, pinky_angle_2,
|
|
spread_thumb_index,
|
|
spread_index_middle,
|
|
spread_middle_ring,
|
|
spread_ring_pinky,
|
|
], dtype=np.float32)
|
|
|
|
return feats
|
|
|
|
|
|
def extract_feature_family(hand_landmarks, world_landmarks, family_name):
|
|
family_name = family_name.upper()
|
|
|
|
if family_name == "2D":
|
|
return normalize_2d_features(hand_landmarks)
|
|
if family_name == "3D":
|
|
return normalize_3d_features(world_landmarks)
|
|
if family_name == "ANGLES":
|
|
return extract_angle_features(world_landmarks)
|
|
|
|
raise ValueError(f"Unknown feature family: {family_name}")
|
|
|
|
|
|
def load_pickle(path):
|
|
with open(path, "rb") as f:
|
|
return pickle.load(f)
|
|
|
|
|
|
def family_key_from_name(name):
|
|
name = name.upper()
|
|
if name == "2D":
|
|
return "features_2d"
|
|
if name == "3D":
|
|
return "features_3d"
|
|
if name == "ANGLES":
|
|
return "features_angles"
|
|
raise ValueError(f"Unknown family name: {name}")
|
|
|
|
|
|
def ranking_key_from_method(method):
|
|
method = method.lower()
|
|
if method == "f":
|
|
return "rank_f"
|
|
if method == "mi":
|
|
return "rank_mi"
|
|
if method == "fisher":
|
|
return "rank_fisher"
|
|
raise ValueError(f"Unknown ranking method: {method}")
|
|
|
|
|
|
def flatten_training_data(data, family_key):
|
|
X = []
|
|
y = []
|
|
|
|
for label in sorted(data.keys()):
|
|
for sample in data[label]:
|
|
X.append(np.asarray(sample[family_key], dtype=np.float32))
|
|
y.append(label)
|
|
|
|
X = np.vstack([x.reshape(1, -1) for x in X])
|
|
y = np.asarray(y)
|
|
return X, y
|
|
|
|
|
|
def build_classifier(classifier_type):
|
|
classifier_type = classifier_type.lower()
|
|
|
|
if classifier_type == "knn":
|
|
return KNeighborsClassifier(n_neighbors=DEFAULT_KNN_K)
|
|
|
|
if classifier_type == "svm":
|
|
return SVC(kernel="rbf", probability=True)
|
|
|
|
if classifier_type == "rf":
|
|
return RandomForestClassifier(
|
|
n_estimators=DEFAULT_RF_TREES,
|
|
random_state=42,
|
|
class_weight=None,
|
|
)
|
|
|
|
if classifier_type == "logreg":
|
|
return LogisticRegression(
|
|
max_iter=2000,
|
|
solver="lbfgs",
|
|
)
|
|
|
|
raise ValueError(f"Unknown classifier type: {classifier_type}")
|
|
|
|
|
|
def get_selected_feature_indices(analysis, family_key, ranking_method, num_features):
|
|
rank_key = ranking_key_from_method(ranking_method)
|
|
rank = analysis[family_key][rank_key]
|
|
return np.asarray(rank[:num_features], dtype=np.int32)
|
|
|
|
|
|
def train_model(data, analysis, feature_family, classifier_type, ranking_method, num_features):
|
|
family_key = family_key_from_name(feature_family)
|
|
|
|
X, y = flatten_training_data(data, family_key)
|
|
selected_idx = get_selected_feature_indices(analysis, family_key, ranking_method, num_features)
|
|
|
|
X_sel = X[:, selected_idx]
|
|
|
|
scaler = StandardScaler()
|
|
X_scaled = scaler.fit_transform(X_sel)
|
|
|
|
label_encoder = LabelEncoder()
|
|
y_enc = label_encoder.fit_transform(y)
|
|
|
|
clf = build_classifier(classifier_type)
|
|
clf.fit(X_scaled, y_enc)
|
|
|
|
return {
|
|
"classifier": clf,
|
|
"scaler": scaler,
|
|
"label_encoder": label_encoder,
|
|
"selected_idx": selected_idx,
|
|
"family_key": family_key,
|
|
}
|
|
|
|
|
|
def predict_with_confidence(model_bundle, feature_vector, unknown_threshold=DEFAULT_UNKNOWN_THRESHOLD):
|
|
selected_idx = model_bundle["selected_idx"]
|
|
clf = model_bundle["classifier"]
|
|
scaler = model_bundle["scaler"]
|
|
label_encoder = model_bundle["label_encoder"]
|
|
|
|
x = feature_vector[selected_idx].reshape(1, -1)
|
|
x = scaler.transform(x)
|
|
|
|
pred_id = clf.predict(x)[0]
|
|
pred_label = label_encoder.inverse_transform([pred_id])[0]
|
|
|
|
if hasattr(clf, "predict_proba"):
|
|
probs = clf.predict_proba(x)[0]
|
|
conf = float(np.max(probs))
|
|
else:
|
|
conf = 1.0
|
|
|
|
if conf < unknown_threshold:
|
|
return "UNKNOWN", conf
|
|
|
|
return pred_label, conf
|
|
|
|
|
|
def create_hand_landmarker(
|
|
model_path="hand_landmarker.task",
|
|
num_hands=DEFAULT_NUM_HANDS,
|
|
min_hand_detection_confidence=0.5,
|
|
min_hand_presence_confidence=0.5,
|
|
min_tracking_confidence=0.5,
|
|
):
|
|
BaseOptions = mp.tasks.BaseOptions
|
|
HandLandmarkerOptions = vision.HandLandmarkerOptions
|
|
VisionRunningMode = vision.RunningMode
|
|
|
|
options = HandLandmarkerOptions(
|
|
base_options=BaseOptions(model_asset_path=model_path),
|
|
running_mode=VisionRunningMode.IMAGE,
|
|
num_hands=num_hands,
|
|
min_hand_detection_confidence=min_hand_detection_confidence,
|
|
min_hand_presence_confidence=min_hand_presence_confidence,
|
|
min_tracking_confidence=min_tracking_confidence,
|
|
)
|
|
|
|
return vision.HandLandmarker.create_from_options(options)
|
|
|
|
|
|
def load_gesture_model_bundle(
|
|
data_pickle="gesture_samples.pkl",
|
|
analysis_pickle="gesture_feature_analysis.pkl",
|
|
feature_family="ANGLES",
|
|
classifier_type="knn",
|
|
ranking_method="fisher",
|
|
num_features=10,
|
|
unknown_threshold=DEFAULT_UNKNOWN_THRESHOLD,
|
|
hand_model_path="hand_landmarker.task",
|
|
):
|
|
if not Path(data_pickle).exists():
|
|
raise FileNotFoundError(f"Missing {data_pickle}")
|
|
if not Path(analysis_pickle).exists():
|
|
raise FileNotFoundError(f"Missing {analysis_pickle}")
|
|
|
|
data = load_pickle(data_pickle)
|
|
analysis = load_pickle(analysis_pickle)
|
|
|
|
model_bundle = train_model(
|
|
data=data,
|
|
analysis=analysis,
|
|
feature_family=feature_family,
|
|
classifier_type=classifier_type,
|
|
ranking_method=ranking_method,
|
|
num_features=num_features,
|
|
)
|
|
|
|
hand_landmarker = create_hand_landmarker(model_path=hand_model_path)
|
|
|
|
return {
|
|
"model_bundle": model_bundle,
|
|
"hand_landmarker": hand_landmarker,
|
|
"feature_family": feature_family,
|
|
"classifier_type": classifier_type,
|
|
"ranking_method": ranking_method,
|
|
"num_features": num_features,
|
|
"unknown_threshold": unknown_threshold,
|
|
}
|
|
|
|
|
|
def _put_text_with_background(frame, text, origin, font_scale=0.6, text_color=(255, 255, 255), bg_color=(0, 0, 0)):
|
|
x, y = origin
|
|
font = cv2.FONT_HERSHEY_SIMPLEX
|
|
thickness = 2
|
|
(text_w, text_h), baseline = cv2.getTextSize(text, font, font_scale, thickness)
|
|
top_left = (x, max(0, y - text_h - baseline - 6))
|
|
bottom_right = (x + text_w + 6, max(0, y + 4))
|
|
cv2.rectangle(frame, top_left, bottom_right, bg_color, -1)
|
|
cv2.putText(
|
|
frame,
|
|
text,
|
|
(x + 3, y - 3),
|
|
font,
|
|
font_scale,
|
|
text_color,
|
|
thickness,
|
|
cv2.LINE_AA,
|
|
)
|
|
|
|
|
|
def _ray_to_frame_bounds(start_point, direction_vector, frame_w, frame_h):
|
|
x0, y0 = start_point
|
|
dx, dy = direction_vector
|
|
|
|
norm = float(np.hypot(dx, dy))
|
|
if norm < 1e-6:
|
|
return None
|
|
|
|
dx /= norm
|
|
dy /= norm
|
|
|
|
candidates = []
|
|
|
|
if abs(dx) > 1e-6:
|
|
for x_bound in (0, frame_w - 1):
|
|
t = (x_bound - x0) / dx
|
|
if t <= 0:
|
|
continue
|
|
y = y0 + t * dy
|
|
if 0 <= y <= frame_h - 1:
|
|
candidates.append((t, (int(x_bound), int(round(y)))))
|
|
|
|
if abs(dy) > 1e-6:
|
|
for y_bound in (0, frame_h - 1):
|
|
t = (y_bound - y0) / dy
|
|
if t <= 0:
|
|
continue
|
|
x = x0 + t * dx
|
|
if 0 <= x <= frame_w - 1:
|
|
candidates.append((t, (int(round(x)), int(y_bound))))
|
|
|
|
if not candidates:
|
|
return None
|
|
|
|
candidates.sort(key=lambda item: item[0])
|
|
return candidates[0][1]
|
|
|
|
|
|
def draw_pointing_ray(frame, hand_landmarks, crop_x, crop_y, crop_w, crop_h, color=(255, 255, 0), thickness=3):
|
|
index_tip_x, index_tip_y = landmark_to_pixel(hand_landmarks[INDEX_TIP], crop_w, crop_h)
|
|
index_mcp_x, index_mcp_y = landmark_to_pixel(hand_landmarks[INDEX_MCP], crop_w, crop_h)
|
|
|
|
start_point = (index_tip_x + crop_x, index_tip_y + crop_y)
|
|
direction = (
|
|
(index_tip_x - index_mcp_x),
|
|
(index_tip_y - index_mcp_y),
|
|
)
|
|
|
|
end_point = _ray_to_frame_bounds(start_point, direction, frame.shape[1], frame.shape[0])
|
|
if end_point is None:
|
|
return
|
|
|
|
cv2.line(frame, start_point, end_point, color, thickness)
|
|
|
|
|
|
def draw_hand_on_frame(frame, hand_landmarks, crop_x, crop_y, crop_w, crop_h, label_text=""):
|
|
points = []
|
|
|
|
for a, b in HAND_CONNECTIONS:
|
|
x1, y1 = landmark_to_pixel(hand_landmarks[a], crop_w, crop_h)
|
|
x2, y2 = landmark_to_pixel(hand_landmarks[b], crop_w, crop_h)
|
|
x1 += crop_x
|
|
y1 += crop_y
|
|
x2 += crop_x
|
|
y2 += crop_y
|
|
cv2.line(frame, (x1, y1), (x2, y2), (0, 255, 0), 2)
|
|
|
|
for i, lm in enumerate(hand_landmarks):
|
|
x, y = landmark_to_pixel(lm, crop_w, crop_h)
|
|
x += crop_x
|
|
y += crop_y
|
|
points.append((x, y))
|
|
color = (0, 0, 255) if i in [THUMB_TIP, INDEX_TIP, MIDDLE_TIP, RING_TIP, PINKY_TIP] else (255, 255, 0)
|
|
radius = 5 if i in [THUMB_TIP, INDEX_TIP, MIDDLE_TIP, RING_TIP, PINKY_TIP] else 3
|
|
cv2.circle(frame, (x, y), radius, color, -1)
|
|
|
|
if label_text and points:
|
|
label_x = max(0, min(x for x, _ in points))
|
|
label_y = max(20, min(y for _, y in points) - 10)
|
|
_put_text_with_background(frame, label_text, (label_x, label_y), font_scale=0.55)
|
|
|
|
|
|
class GestureRecognizer:
|
|
def __init__(
|
|
self,
|
|
data_pickle="gesture_samples.pkl",
|
|
analysis_pickle="gesture_feature_analysis.pkl",
|
|
feature_family="ANGLES",
|
|
classifier_type="knn",
|
|
ranking_method="fisher",
|
|
num_features=10,
|
|
unknown_threshold=DEFAULT_UNKNOWN_THRESHOLD,
|
|
hand_model_path="hand_landmarker.task",
|
|
):
|
|
bundle = load_gesture_model_bundle(
|
|
data_pickle=data_pickle,
|
|
analysis_pickle=analysis_pickle,
|
|
feature_family=feature_family,
|
|
classifier_type=classifier_type,
|
|
ranking_method=ranking_method,
|
|
num_features=num_features,
|
|
unknown_threshold=unknown_threshold,
|
|
hand_model_path=hand_model_path,
|
|
)
|
|
self.model_bundle = bundle["model_bundle"]
|
|
self.hand_landmarker = bundle["hand_landmarker"]
|
|
self.feature_family = bundle["feature_family"]
|
|
self.classifier_type = bundle["classifier_type"]
|
|
self.ranking_method = bundle["ranking_method"]
|
|
self.num_features = bundle["num_features"]
|
|
self.unknown_threshold = bundle["unknown_threshold"]
|
|
|
|
def detect_gestures(self, crop_bgr):
|
|
crop_rgb = cv2.cvtColor(crop_bgr, cv2.COLOR_BGR2RGB)
|
|
mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=crop_rgb)
|
|
result = self.hand_landmarker.detect(mp_image)
|
|
|
|
detections = []
|
|
if not result.hand_landmarks or not result.hand_world_landmarks:
|
|
return detections
|
|
|
|
hand_count = min(len(result.hand_landmarks), len(result.hand_world_landmarks))
|
|
for hand_index in range(hand_count):
|
|
hand_landmarks = result.hand_landmarks[hand_index]
|
|
world_landmarks = result.hand_world_landmarks[hand_index]
|
|
|
|
feature_vector = extract_feature_family(
|
|
hand_landmarks=hand_landmarks,
|
|
world_landmarks=world_landmarks,
|
|
family_name=self.feature_family,
|
|
)
|
|
pred_label, conf = predict_with_confidence(
|
|
self.model_bundle,
|
|
feature_vector,
|
|
unknown_threshold=self.unknown_threshold,
|
|
)
|
|
|
|
handedness_text = ""
|
|
if result.handedness and len(result.handedness) > hand_index and len(result.handedness[hand_index]) > 0:
|
|
handedness_text = result.handedness[hand_index][0].category_name
|
|
|
|
detections.append(
|
|
{
|
|
"hand_landmarks": hand_landmarks,
|
|
"label": pred_label,
|
|
"confidence": conf,
|
|
"handedness": handedness_text,
|
|
}
|
|
)
|
|
|
|
return detections
|
|
|
|
def draw_gesture(self, frame, detection, crop_x, crop_y, crop_w, crop_h):
|
|
label = detection["label"]
|
|
confidence = detection["confidence"]
|
|
handedness_text = detection["handedness"]
|
|
|
|
if handedness_text:
|
|
label_text = f"{handedness_text}: {label} {confidence:.2f}"
|
|
else:
|
|
label_text = f"{label} {confidence:.2f}"
|
|
|
|
draw_hand_on_frame(
|
|
frame,
|
|
detection["hand_landmarks"],
|
|
crop_x,
|
|
crop_y,
|
|
crop_w,
|
|
crop_h,
|
|
label_text=label_text,
|
|
)
|
|
|
|
if label.upper() == "A":
|
|
draw_pointing_ray(
|
|
frame,
|
|
detection["hand_landmarks"],
|
|
crop_x,
|
|
crop_y,
|
|
crop_w,
|
|
crop_h,
|
|
)
|
|
|
|
def close(self):
|
|
self.hand_landmarker.close()
|