import pickle from pathlib import Path import cv2 import mediapipe as mp import numpy as np from mediapipe.tasks.python import vision from sklearn.ensemble import RandomForestClassifier from sklearn.linear_model import LogisticRegression from sklearn.neighbors import KNeighborsClassifier from sklearn.preprocessing import LabelEncoder, StandardScaler from sklearn.svm import SVC WRIST = 0 THUMB_CMC = 1 THUMB_MCP = 2 THUMB_IP = 3 THUMB_TIP = 4 INDEX_MCP = 5 INDEX_PIP = 6 INDEX_DIP = 7 INDEX_TIP = 8 MIDDLE_MCP = 9 MIDDLE_PIP = 10 MIDDLE_DIP = 11 MIDDLE_TIP = 12 RING_MCP = 13 RING_PIP = 14 RING_DIP = 15 RING_TIP = 16 PINKY_MCP = 17 PINKY_PIP = 18 PINKY_DIP = 19 PINKY_TIP = 20 HAND_CONNECTIONS = [ (0, 1), (1, 2), (2, 3), (3, 4), (0, 5), (5, 6), (6, 7), (7, 8), (0, 9), (9, 10), (10, 11), (11, 12), (0, 13), (13, 14), (14, 15), (15, 16), (0, 17), (17, 18), (18, 19), (19, 20), (5, 9), (9, 13), (13, 17), ] DEFAULT_UNKNOWN_THRESHOLD = 0.7 DEFAULT_KNN_K = 3 DEFAULT_RF_TREES = 200 DEFAULT_NUM_HANDS = 2 def landmark_to_pixel(lm, width, height): return int(lm.x * width), int(lm.y * height) def vec2_from_landmarks(hand_landmarks): return np.array([[lm.x, lm.y] for lm in hand_landmarks], dtype=np.float32) def vec3_from_world_landmarks(world_landmarks): return np.array([[lm.x, lm.y, lm.z] for lm in world_landmarks], dtype=np.float32) def hand_scale_2d(points2d): s = np.linalg.norm(points2d[MIDDLE_MCP] - points2d[WRIST]) return max(float(s), 1e-6) def hand_scale_3d(points3d): s = np.linalg.norm(points3d[MIDDLE_MCP] - points3d[WRIST]) return max(float(s), 1e-6) def normalize_2d_features(hand_landmarks): pts = vec2_from_landmarks(hand_landmarks) origin = pts[WRIST].copy() pts = pts - origin scale = hand_scale_2d(pts + origin) pts = pts / scale return pts.flatten().astype(np.float32) def normalize_3d_features(world_landmarks): pts = vec3_from_world_landmarks(world_landmarks) origin = pts[WRIST].copy() pts = pts - origin scale = hand_scale_3d(pts + origin) pts = pts / scale return pts.flatten().astype(np.float32) def angle_between(v1, v2): n1 = np.linalg.norm(v1) n2 = np.linalg.norm(v2) if n1 < 1e-6 or n2 < 1e-6: return 0.0 c = np.dot(v1, v2) / (n1 * n2) c = np.clip(c, -1.0, 1.0) return float(np.arccos(c)) def joint_angle(points, a, b, c): ba = points[a] - points[b] bc = points[c] - points[b] return angle_between(ba, bc) def extract_angle_features(world_landmarks): pts = vec3_from_world_landmarks(world_landmarks) thumb_angle_1 = joint_angle(pts, THUMB_CMC, THUMB_MCP, THUMB_IP) thumb_angle_2 = joint_angle(pts, THUMB_MCP, THUMB_IP, THUMB_TIP) index_angle_1 = joint_angle(pts, INDEX_MCP, INDEX_PIP, INDEX_DIP) index_angle_2 = joint_angle(pts, INDEX_PIP, INDEX_DIP, INDEX_TIP) middle_angle_1 = joint_angle(pts, MIDDLE_MCP, MIDDLE_PIP, MIDDLE_DIP) middle_angle_2 = joint_angle(pts, MIDDLE_PIP, MIDDLE_DIP, MIDDLE_TIP) ring_angle_1 = joint_angle(pts, RING_MCP, RING_PIP, RING_DIP) ring_angle_2 = joint_angle(pts, RING_PIP, RING_DIP, RING_TIP) pinky_angle_1 = joint_angle(pts, PINKY_MCP, PINKY_PIP, PINKY_DIP) pinky_angle_2 = joint_angle(pts, PINKY_PIP, PINKY_DIP, PINKY_TIP) wrist_to_thumb = pts[THUMB_TIP] - pts[WRIST] wrist_to_index = pts[INDEX_TIP] - pts[WRIST] wrist_to_middle = pts[MIDDLE_TIP] - pts[WRIST] wrist_to_ring = pts[RING_TIP] - pts[WRIST] wrist_to_pinky = pts[PINKY_TIP] - pts[WRIST] spread_thumb_index = angle_between(wrist_to_thumb, wrist_to_index) spread_index_middle = angle_between(wrist_to_index, wrist_to_middle) spread_middle_ring = angle_between(wrist_to_middle, wrist_to_ring) spread_ring_pinky = angle_between(wrist_to_ring, wrist_to_pinky) feats = np.array([ thumb_angle_1, thumb_angle_2, index_angle_1, index_angle_2, middle_angle_1, middle_angle_2, ring_angle_1, ring_angle_2, pinky_angle_1, pinky_angle_2, spread_thumb_index, spread_index_middle, spread_middle_ring, spread_ring_pinky, ], dtype=np.float32) return feats def extract_feature_family(hand_landmarks, world_landmarks, family_name): family_name = family_name.upper() if family_name == "2D": return normalize_2d_features(hand_landmarks) if family_name == "3D": return normalize_3d_features(world_landmarks) if family_name == "ANGLES": return extract_angle_features(world_landmarks) raise ValueError(f"Unknown feature family: {family_name}") def load_pickle(path): with open(path, "rb") as f: return pickle.load(f) def family_key_from_name(name): name = name.upper() if name == "2D": return "features_2d" if name == "3D": return "features_3d" if name == "ANGLES": return "features_angles" raise ValueError(f"Unknown family name: {name}") def ranking_key_from_method(method): method = method.lower() if method == "f": return "rank_f" if method == "mi": return "rank_mi" if method == "fisher": return "rank_fisher" raise ValueError(f"Unknown ranking method: {method}") def flatten_training_data(data, family_key): X = [] y = [] for label in sorted(data.keys()): for sample in data[label]: X.append(np.asarray(sample[family_key], dtype=np.float32)) y.append(label) X = np.vstack([x.reshape(1, -1) for x in X]) y = np.asarray(y) return X, y def build_classifier(classifier_type): classifier_type = classifier_type.lower() if classifier_type == "knn": return KNeighborsClassifier(n_neighbors=DEFAULT_KNN_K) if classifier_type == "svm": return SVC(kernel="rbf", probability=True) if classifier_type == "rf": return RandomForestClassifier( n_estimators=DEFAULT_RF_TREES, random_state=42, class_weight=None, ) if classifier_type == "logreg": return LogisticRegression( max_iter=2000, solver="lbfgs", ) raise ValueError(f"Unknown classifier type: {classifier_type}") def get_selected_feature_indices(analysis, family_key, ranking_method, num_features): rank_key = ranking_key_from_method(ranking_method) rank = analysis[family_key][rank_key] return np.asarray(rank[:num_features], dtype=np.int32) def train_model(data, analysis, feature_family, classifier_type, ranking_method, num_features): family_key = family_key_from_name(feature_family) X, y = flatten_training_data(data, family_key) selected_idx = get_selected_feature_indices(analysis, family_key, ranking_method, num_features) X_sel = X[:, selected_idx] scaler = StandardScaler() X_scaled = scaler.fit_transform(X_sel) label_encoder = LabelEncoder() y_enc = label_encoder.fit_transform(y) clf = build_classifier(classifier_type) clf.fit(X_scaled, y_enc) return { "classifier": clf, "scaler": scaler, "label_encoder": label_encoder, "selected_idx": selected_idx, "family_key": family_key, } def predict_with_confidence(model_bundle, feature_vector, unknown_threshold=DEFAULT_UNKNOWN_THRESHOLD): selected_idx = model_bundle["selected_idx"] clf = model_bundle["classifier"] scaler = model_bundle["scaler"] label_encoder = model_bundle["label_encoder"] x = feature_vector[selected_idx].reshape(1, -1) x = scaler.transform(x) pred_id = clf.predict(x)[0] pred_label = label_encoder.inverse_transform([pred_id])[0] if hasattr(clf, "predict_proba"): probs = clf.predict_proba(x)[0] conf = float(np.max(probs)) else: conf = 1.0 if conf < unknown_threshold: return "UNKNOWN", conf return pred_label, conf def create_hand_landmarker( model_path="hand_landmarker.task", num_hands=DEFAULT_NUM_HANDS, min_hand_detection_confidence=0.5, min_hand_presence_confidence=0.5, min_tracking_confidence=0.5, ): BaseOptions = mp.tasks.BaseOptions HandLandmarkerOptions = vision.HandLandmarkerOptions VisionRunningMode = vision.RunningMode options = HandLandmarkerOptions( base_options=BaseOptions(model_asset_path=model_path), running_mode=VisionRunningMode.IMAGE, num_hands=num_hands, min_hand_detection_confidence=min_hand_detection_confidence, min_hand_presence_confidence=min_hand_presence_confidence, min_tracking_confidence=min_tracking_confidence, ) return vision.HandLandmarker.create_from_options(options) def load_gesture_model_bundle( data_pickle="gesture_samples.pkl", analysis_pickle="gesture_feature_analysis.pkl", feature_family="ANGLES", classifier_type="knn", ranking_method="fisher", num_features=10, unknown_threshold=DEFAULT_UNKNOWN_THRESHOLD, hand_model_path="hand_landmarker.task", ): if not Path(data_pickle).exists(): raise FileNotFoundError(f"Missing {data_pickle}") if not Path(analysis_pickle).exists(): raise FileNotFoundError(f"Missing {analysis_pickle}") data = load_pickle(data_pickle) analysis = load_pickle(analysis_pickle) model_bundle = train_model( data=data, analysis=analysis, feature_family=feature_family, classifier_type=classifier_type, ranking_method=ranking_method, num_features=num_features, ) hand_landmarker = create_hand_landmarker(model_path=hand_model_path) return { "model_bundle": model_bundle, "hand_landmarker": hand_landmarker, "feature_family": feature_family, "classifier_type": classifier_type, "ranking_method": ranking_method, "num_features": num_features, "unknown_threshold": unknown_threshold, } def _put_text_with_background(frame, text, origin, font_scale=0.6, text_color=(255, 255, 255), bg_color=(0, 0, 0)): x, y = origin font = cv2.FONT_HERSHEY_SIMPLEX thickness = 2 (text_w, text_h), baseline = cv2.getTextSize(text, font, font_scale, thickness) top_left = (x, max(0, y - text_h - baseline - 6)) bottom_right = (x + text_w + 6, max(0, y + 4)) cv2.rectangle(frame, top_left, bottom_right, bg_color, -1) cv2.putText( frame, text, (x + 3, y - 3), font, font_scale, text_color, thickness, cv2.LINE_AA, ) def _ray_to_frame_bounds(start_point, direction_vector, frame_w, frame_h): x0, y0 = start_point dx, dy = direction_vector norm = float(np.hypot(dx, dy)) if norm < 1e-6: return None dx /= norm dy /= norm candidates = [] if abs(dx) > 1e-6: for x_bound in (0, frame_w - 1): t = (x_bound - x0) / dx if t <= 0: continue y = y0 + t * dy if 0 <= y <= frame_h - 1: candidates.append((t, (int(x_bound), int(round(y))))) if abs(dy) > 1e-6: for y_bound in (0, frame_h - 1): t = (y_bound - y0) / dy if t <= 0: continue x = x0 + t * dx if 0 <= x <= frame_w - 1: candidates.append((t, (int(round(x)), int(y_bound)))) if not candidates: return None candidates.sort(key=lambda item: item[0]) return candidates[0][1] def draw_pointing_ray(frame, hand_landmarks, crop_x, crop_y, crop_w, crop_h, color=(255, 255, 0), thickness=3): index_tip_x, index_tip_y = landmark_to_pixel(hand_landmarks[INDEX_TIP], crop_w, crop_h) index_mcp_x, index_mcp_y = landmark_to_pixel(hand_landmarks[INDEX_MCP], crop_w, crop_h) start_point = (index_tip_x + crop_x, index_tip_y + crop_y) direction = ( (index_tip_x - index_mcp_x), (index_tip_y - index_mcp_y), ) end_point = _ray_to_frame_bounds(start_point, direction, frame.shape[1], frame.shape[0]) if end_point is None: return cv2.line(frame, start_point, end_point, color, thickness) def draw_hand_on_frame(frame, hand_landmarks, crop_x, crop_y, crop_w, crop_h, label_text=""): points = [] for a, b in HAND_CONNECTIONS: x1, y1 = landmark_to_pixel(hand_landmarks[a], crop_w, crop_h) x2, y2 = landmark_to_pixel(hand_landmarks[b], crop_w, crop_h) x1 += crop_x y1 += crop_y x2 += crop_x y2 += crop_y cv2.line(frame, (x1, y1), (x2, y2), (0, 255, 0), 2) for i, lm in enumerate(hand_landmarks): x, y = landmark_to_pixel(lm, crop_w, crop_h) x += crop_x y += crop_y points.append((x, y)) color = (0, 0, 255) if i in [THUMB_TIP, INDEX_TIP, MIDDLE_TIP, RING_TIP, PINKY_TIP] else (255, 255, 0) radius = 5 if i in [THUMB_TIP, INDEX_TIP, MIDDLE_TIP, RING_TIP, PINKY_TIP] else 3 cv2.circle(frame, (x, y), radius, color, -1) if label_text and points: label_x = max(0, min(x for x, _ in points)) label_y = max(20, min(y for _, y in points) - 10) _put_text_with_background(frame, label_text, (label_x, label_y), font_scale=0.55) class GestureRecognizer: def __init__( self, data_pickle="gesture_samples.pkl", analysis_pickle="gesture_feature_analysis.pkl", feature_family="ANGLES", classifier_type="knn", ranking_method="fisher", num_features=10, unknown_threshold=DEFAULT_UNKNOWN_THRESHOLD, hand_model_path="hand_landmarker.task", ): bundle = load_gesture_model_bundle( data_pickle=data_pickle, analysis_pickle=analysis_pickle, feature_family=feature_family, classifier_type=classifier_type, ranking_method=ranking_method, num_features=num_features, unknown_threshold=unknown_threshold, hand_model_path=hand_model_path, ) self.model_bundle = bundle["model_bundle"] self.hand_landmarker = bundle["hand_landmarker"] self.feature_family = bundle["feature_family"] self.classifier_type = bundle["classifier_type"] self.ranking_method = bundle["ranking_method"] self.num_features = bundle["num_features"] self.unknown_threshold = bundle["unknown_threshold"] def detect_gestures(self, crop_bgr): crop_rgb = cv2.cvtColor(crop_bgr, cv2.COLOR_BGR2RGB) mp_image = mp.Image(image_format=mp.ImageFormat.SRGB, data=crop_rgb) result = self.hand_landmarker.detect(mp_image) detections = [] if not result.hand_landmarks or not result.hand_world_landmarks: return detections hand_count = min(len(result.hand_landmarks), len(result.hand_world_landmarks)) for hand_index in range(hand_count): hand_landmarks = result.hand_landmarks[hand_index] world_landmarks = result.hand_world_landmarks[hand_index] feature_vector = extract_feature_family( hand_landmarks=hand_landmarks, world_landmarks=world_landmarks, family_name=self.feature_family, ) pred_label, conf = predict_with_confidence( self.model_bundle, feature_vector, unknown_threshold=self.unknown_threshold, ) handedness_text = "" if result.handedness and len(result.handedness) > hand_index and len(result.handedness[hand_index]) > 0: handedness_text = result.handedness[hand_index][0].category_name detections.append( { "hand_landmarks": hand_landmarks, "label": pred_label, "confidence": conf, "handedness": handedness_text, } ) return detections def draw_gesture(self, frame, detection, crop_x, crop_y, crop_w, crop_h): label = detection["label"] confidence = detection["confidence"] handedness_text = detection["handedness"] if handedness_text: label_text = f"{handedness_text}: {label} {confidence:.2f}" else: label_text = f"{label} {confidence:.2f}" draw_hand_on_frame( frame, detection["hand_landmarks"], crop_x, crop_y, crop_w, crop_h, label_text=label_text, ) if label.upper() == "A": draw_pointing_ray( frame, detection["hand_landmarks"], crop_x, crop_y, crop_w, crop_h, ) def close(self): self.hand_landmarker.close()