bhattial100541/global_match_memory
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

1

Browse Source
This commit is contained in:
Kristoph Laemmerzahl 2025-12-11 11:22:35 +01:00
parent 953a0d294e
commit 622ebc9fb0
13 changed files with 1362 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
True_Finished_Game.py
View File

@ -51,7 +51,7 @@ BG_COLOR = (10, 42, 83)
BUTTON_FILL = (18, 122, 138)
BUTTON_BORDER = (255, 255, 255)
FPS = 30
SCREEN_WIDTH, SCREEN_HEIGHT = 900, 600
SCREEN_WIDTH, SCREEN_HEIGHT = 900, 600 #900,500
# --- Background / Logo ---
BASE_DIR = os.path.dirname(os.path.abspath(__file__))

2
calibration.json
View File

@ -1 +1 @@
[[24, 118], [630, 135], [627, 459], [36, 461]]
[[4, 8], [636, 6], [636, 471], [9, 477]]

4
gesture_input_osc.py
View File

@ -3,7 +3,7 @@ import mediapipe as mp
import numpy as np
import math, time, json
from pythonosc import udp_client
#Improve stability
# -------------------------------
# SETTINGS
# -------------------------------
@ -13,7 +13,7 @@ GESTURE_CAM_INDEX = 0 # Clap/Gesture Kamera
GAME_SCREEN_WIDTH = 900
GAME_SCREEN_HEIGHT = 600
STILL_REQUIRED = 1.0 # Sekunden die der Finger stabil sein muss
STILL_REQUIRED = 0.60 # Sekunden die der Finger stabil sein muss
MOVE_TOLERANCE = 25 # Bewegungsschwelle (Pixel)
client = udp_client.SimpleUDPClient("127.0.0.1", 5005)

186
gesture_input_osc_new_test.py Normal file
View File

@ -0,0 +1,186 @@
import cv2
import mediapipe as mp
import numpy as np
import math, json
from pythonosc import udp_client
# --------------------------------------------------
# SETTINGS (INTEGER ONLY) An PC_Leistung anpassen
# --------------------------------------------------
TOUCH_CAM_INDEX = 1
GESTURE_CAM_INDEX = 0 #<--------Index_Kamera
CAMERA_WIDTH = 480 #<------Resolution
CAMERA_HEIGHT = 320
CAMERA_FPS = 18 # INT FPS
MODEL_COMPLEXITY = 0 # 0=fast, 1=normal, 2=accurate
MOVE_TOLERANCE = 28 # Pixel
TOUCH_STILL_FRAMES = 18 # stabile Frames für Touch
TOUCH_COOLDOWN_FRAMES = 12 # Cooldown nach Touch
CLAP_DISTANCE_THRESHOLD = 110
CLAP_COOLDOWN_FRAMES = 32 #<----- Optional?
GAME_SCREEN_WIDTH = 900
GAME_SCREEN_HEIGHT = 600 #<----------------Screens_Controll
# --------------------------------------------------
client = udp_client.SimpleUDPClient("127.0.0.1", 5005)
# --------------------------------------------------
# GLOBAL STATES
# --------------------------------------------------
last_finger_pos = None
still_frames = 0
touch_cooldown = 0
clap_cooldown = 0
# --------------------------------------------------
# LOAD CALIBRATION
# --------------------------------------------------
try:
with open("calibration.json", "r") as f:
CALIB_POINTS = json.load(f)
print("📐 Kalibrierung geladen")
except:
CALIB_POINTS = None
print("⚠️ Keine Kalibrierung – Rohkoordinaten")
H = None
if CALIB_POINTS:
src = np.array(CALIB_POINTS, dtype=np.float32)
dst = np.array([
[0, 0],
[GAME_SCREEN_WIDTH, 0],
[GAME_SCREEN_WIDTH, GAME_SCREEN_HEIGHT],
[0, GAME_SCREEN_HEIGHT]
], dtype=np.float32)
H, _ = cv2.findHomography(src, dst)
def map_point(x, y):
if H is None:
return int(x), int(y)
p = np.array([[[x, y]]], dtype=np.float32)
m = cv2.perspectiveTransform(p, H)[0][0]
return int(m[0]), int(m[1])
# --------------------------------------------------
def run():
global last_finger_pos, still_frames
global touch_cooldown, clap_cooldown
mp_hands = mp.solutions.hands
hands_touch = mp_hands.Hands(
max_num_hands=1,
model_complexity=MODEL_COMPLEXITY,
min_detection_confidence=0.6
)
hands_gesture = mp_hands.Hands(
max_num_hands=2,
model_complexity=MODEL_COMPLEXITY,
min_detection_confidence=0.6
)
cam_touch = cv2.VideoCapture(TOUCH_CAM_INDEX)
cam_gest = cv2.VideoCapture(GESTURE_CAM_INDEX)
for cam in (cam_touch, cam_gest):
cam.set(cv2.CAP_PROP_FRAME_WIDTH, CAMERA_WIDTH)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT, CAMERA_HEIGHT)
cam.set(cv2.CAP_PROP_FPS, CAMERA_FPS)
frame_delay = int(1000 / CAMERA_FPS)
while True:
ok1, frame_touch = cam_touch.read()
ok2, frame_gest = cam_gest.read()
if not ok1 or not ok2:
break
frame_touch = cv2.flip(frame_touch, -1)
frame_gest = cv2.flip(frame_gest, 1)
# ---------------- TOUCH ----------------
rgb_t = cv2.cvtColor(frame_touch, cv2.COLOR_BGR2RGB)
res_t = hands_touch.process(rgb_t)
h, w, _ = frame_touch.shape
if res_t.multi_hand_landmarks:
lm = res_t.multi_hand_landmarks[0]
# Finger zeigt nach unten?
if lm.landmark[8].y > lm.landmark[5].y:
fx = int(lm.landmark[8].x * w)
fy = int(lm.landmark[8].y * h)
sx, sy = map_point(fx, fy)
current_pos = (fx, fy)
if last_finger_pos is None:
still_frames = 0
else:
dist = math.hypot(
current_pos[0] - last_finger_pos[0],
current_pos[1] - last_finger_pos[1]
)
if dist < MOVE_TOLERANCE:
still_frames += 1
if still_frames >= TOUCH_STILL_FRAMES and touch_cooldown == 0:
client.send_message("/touch", [sx, sy])
print(f"👉 TOUCH {sx},{sy}")
touch_cooldown = TOUCH_COOLDOWN_FRAMES
still_frames = 0
else:
still_frames = 0
last_finger_pos = current_pos
cv2.circle(frame_touch, (fx, fy), 8, (0, 255, 0), -1)
else:
last_finger_pos = None
still_frames = 0
if touch_cooldown > 0:
touch_cooldown -= 1
# ---------------- CLAP ----------------
rgb_g = cv2.cvtColor(frame_gest, cv2.COLOR_BGR2RGB)
res_g = hands_gesture.process(rgb_g)
gh, gw, _ = frame_gest.shape
if res_g.multi_hand_landmarks and len(res_g.multi_hand_landmarks) == 2:
h1, h2 = res_g.multi_hand_landmarks
x1 = np.mean([p.x for p in h1.landmark]) * gw
y1 = np.mean([p.y for p in h1.landmark]) * gh
x2 = np.mean([p.x for p in h2.landmark]) * gw
y2 = np.mean([p.y for p in h2.landmark]) * gh
dist = math.hypot(x2 - x1, y2 - y1)
if dist < CLAP_DISTANCE_THRESHOLD and clap_cooldown == 0:
client.send_message("/clap", 1)
print("👏 CLAP")
clap_cooldown = CLAP_COOLDOWN_FRAMES
if clap_cooldown > 0:
clap_cooldown -= 1
# ---------------- DISPLAY ----------------
cv2.imshow("Touch-Cam", frame_touch)
cv2.imshow("Gesture-Cam", frame_gest)
if cv2.waitKey(frame_delay) & 0xFF == 27:
break
cam_touch.release()
cam_gest.release()
cv2.destroyAllWindows()
# --------------------------------------------------
if __name__ == "__main__":
run()

199
gesture_input_osc_re1.py Normal file
View File

@ -0,0 +1,199 @@
import cv2
import mediapipe as mp
import numpy as np
import math, time, json
from pythonosc import udp_client
# -------------------------------
# SETTINGS
# -------------------------------
TOUCH_CAM_INDEX = 1
GESTURE_CAM_INDEX = 0
GAME_SCREEN_WIDTH = 900 #900
GAME_SCREEN_HEIGHT = 600 #600
STILL_REQUIRED = 1.0
MOVE_TOLERANCE = 25
# -------------------------------
# CAMERA / PERFORMANCE SETTINGS
# -------------------------------
CAMERA_FPS = 15
DISPLAY_WIDTH = 320 #1280
DISPLAY_HEIGHT = 240 #720
MODEL_COMPLEXITY = 0 # ✅ 0=fast | 1=balanced | 2=accurate
client = udp_client.SimpleUDPClient("127.0.0.1", 5005)
# -------------------------------
# GLOBAL STATES
# -------------------------------
last_finger_pos = None
finger_still_start = None
prev_touch_time = 0.0
prev_clap_time = 0.0
# -------------------------------
# CALIBRATION + HOMOGRAPHY
# -------------------------------
try:
with open("calibration.json", "r") as f:
CALIB_POINTS = json.load(f)
print("📐 Calibration loaded:", CALIB_POINTS)
except:
CALIB_POINTS = None
print("⚠️ No calibration found")
H = None
if CALIB_POINTS is not None:
src = np.array(CALIB_POINTS, dtype=np.float32)
dst = np.array([
[0, 0],
[GAME_SCREEN_WIDTH, 0],
[GAME_SCREEN_WIDTH, GAME_SCREEN_HEIGHT],
[0, GAME_SCREEN_HEIGHT]
], dtype=np.float32)
H, _ = cv2.findHomography(src, dst)
print("📐 Homography ready")
def map_point_homography(x, y):
if H is None:
return int(x), int(y)
p = np.array([[[x, y]]], dtype=np.float32)
mapped = cv2.perspectiveTransform(p, H)[0][0]
return int(mapped[0]), int(mapped[1])
# -----------------------------------------------------------------
def run_gesture_input():
global last_finger_pos, finger_still_start
global prev_touch_time, prev_clap_time
mp_hands = mp.solutions.hands
mp_draw = mp.solutions.drawing_utils
# ✅ model_complexity applied here
hands_touch = mp_hands.Hands(
max_num_hands=1,
model_complexity=MODEL_COMPLEXITY,
min_detection_confidence=0.6,
min_tracking_confidence=0.6
)
hands_gesture = mp_hands.Hands(
max_num_hands=2,
model_complexity=MODEL_COMPLEXITY,
min_detection_confidence=0.6,
min_tracking_confidence=0.6
)
cam_touch = cv2.VideoCapture(TOUCH_CAM_INDEX)
cam_gesture = cv2.VideoCapture(GESTURE_CAM_INDEX)
for cam in (cam_touch, cam_gesture):
cam.set(cv2.CAP_PROP_FRAME_WIDTH, DISPLAY_WIDTH)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT, DISPLAY_HEIGHT)
cam.set(cv2.CAP_PROP_FPS, CAMERA_FPS)
clap_cooldown = 1.5
frame_duration = 1.0 / CAMERA_FPS
last_frame_time = time.time()
while True:
ok1, frame_touch = cam_touch.read()
ok2, frame_gest = cam_gesture.read()
if not ok1 or not ok2:
break
frame_touch = cv2.flip(frame_touch, -1)
frame_gest = cv2.flip(frame_gest, 1)
# ---------------- TOUCH ----------------
res_t = hands_touch.process(cv2.cvtColor(frame_touch, cv2.COLOR_BGR2RGB))
th, tw, _ = frame_touch.shape
if res_t.multi_hand_landmarks:
lm = res_t.multi_hand_landmarks[0]
mp_draw.draw_landmarks(frame_touch, lm, mp_hands.HAND_CONNECTIONS)
if lm.landmark[8].y < lm.landmark[5].y:
last_finger_pos = None
finger_still_start = None
continue
fx = int(lm.landmark[8].x * tw)
fy = int(lm.landmark[8].y * th)
sx, sy = map_point_homography(fx, fy)
now = time.time()
curr = (fx, fy)
if last_finger_pos is None:
last_finger_pos = curr
finger_still_start = now
else:
dist = math.hypot(curr[0]-last_finger_pos[0],
curr[1]-last_finger_pos[1])
if dist < MOVE_TOLERANCE:
if finger_still_start and now-finger_still_start >= STILL_REQUIRED:
if now-prev_touch_time > 0.5:
client.send_message("/touch", [sx, sy])
print(f"👉 TOUCH {sx},{sy}")
prev_touch_time = now
finger_still_start = None
else:
finger_still_start = now
last_finger_pos = curr
cv2.circle(frame_touch, (fx, fy), 10, (0,255,0), -1)
else:
last_finger_pos = None
finger_still_start = None
# ---------------- CLAP ----------------
res_g = hands_gesture.process(cv2.cvtColor(frame_gest, cv2.COLOR_BGR2RGB))
gh, gw, _ = frame_gest.shape
if res_g.multi_hand_landmarks and len(res_g.multi_hand_landmarks) == 2:
h1, h2 = res_g.multi_hand_landmarks
x1 = np.mean([p.x for p in h1.landmark]) * gw
y1 = np.mean([p.y for p in h1.landmark]) * gh
x2 = np.mean([p.x for p in h2.landmark]) * gw
y2 = np.mean([p.y for p in h2.landmark]) * gh
dist = math.hypot(x2-x1, y2-y1)
if dist < 100 and time.time()-prev_clap_time > clap_cooldown:
prev_clap_time = time.time()
client.send_message("/clap", 1)
print("👏 CLAP")
cv2.putText(frame_touch,
f"FPS:{CAMERA_FPS} MC:{MODEL_COMPLEXITY}",
(10,30), cv2.FONT_HERSHEY_SIMPLEX,
0.8, (255,255,0), 2)
cv2.imshow("Touch-Cam", frame_touch)
cv2.imshow("Gesture-Cam", frame_gest)
# FPS limiter
sleep = frame_duration - (time.time()-last_frame_time)
if sleep > 0:
time.sleep(sleep)
last_frame_time = time.time()
if cv2.waitKey(1) & 0xFF == 27:
break
cam_touch.release()
cam_gesture.release()
cv2.destroyAllWindows()
if __name__ == "__main__":
run_gesture_input()

334
gesture_input_osc_re2.py Normal file
View File

@ -0,0 +1,334 @@
import cv2
import mediapipe as mp
import numpy as np
import math
import time
import json
import threading
from queue import Queue, Empty
from pythonosc import udp_client
# -------------------------------
# SETTINGS (anpassen für 16GB Laptop)
# -------------------------------
TOUCH_CAM_INDEX = 1
GESTURE_CAM_INDEX = 0
GAME_SCREEN_WIDTH = 900
GAME_SCREEN_HEIGHT = 600 #600
STILL_REQUIRED = 1.0 # Sekunden Finger still
MOVE_TOLERANCE = 25 # Pixel
# Kamera / Performance
MODEL_COMPLEXITY = 1 # 0 = schnell, 1 = balanced, 2 = genau
CAMERA_FPS = 15 #30 # Ziel-FPS (best effort)
DISPLAY_WIDTH = 360 #1280
DISPLAY_HEIGHT = 240 #720
# Robustheits-Parameter
CAM_RECONNECT_DELAY = 2.0 # Sekunden Wartezeit beim Reconnect-Versuch
MAX_FRAME_AGE = 0.5 # Sekunden: wie alt ein Frame maximal sein darf
CAM_BUFFER_SIZE = 1 # versucht Puffer zu kappen
CLAP_COOLDOWN = 1.5
client = udp_client.SimpleUDPClient("127.0.0.1", 5005)
# -------------------------------
# Kalibrierung + Homographie
# -------------------------------
try:
with open("calibration.json", "r") as f:
CALIB_POINTS = json.load(f)
print("📐 Kalibrierung geladen:", CALIB_POINTS)
except Exception:
CALIB_POINTS = None
print("⚠️ Keine Kalibrierung gefunden – benutze Rohkoordinaten!")
H = None
if CALIB_POINTS is not None:
try:
src = np.array(CALIB_POINTS, dtype=np.float32)
dst = np.array([
[0, 0],
[GAME_SCREEN_WIDTH, 0],
[GAME_SCREEN_WIDTH, GAME_SCREEN_HEIGHT],
[0, GAME_SCREEN_HEIGHT]
], dtype=np.float32)
H, _ = cv2.findHomography(src, dst)
print("📐 Homographie-Matrix berechnet!")
except Exception as e:
print("⚠️ Homographie fehlgeschlagen:", e)
H = None
def map_point_homography(x, y):
if H is None:
return int(x), int(y)
p = np.array([[[x, y]]], dtype=np.float32)
mapped = cv2.perspectiveTransform(p, H)[0][0]
return int(mapped[0]), int(mapped[1])
# -------------------------------
# Kamerathread (liest non-stop, hält nur das letzte Frame)
# -------------------------------
class CameraReader(threading.Thread):
def __init__(self, index, width, height, fps, name="Cam"):
super().__init__(daemon=True)
self.index = index
self.width = width
self.height = height
self.fps = fps
self.name = f"{name}-{index}"
self.cap = None
self.latest_frame = None
self.latest_ts = 0.0
self.lock = threading.Lock()
self.stop_event = threading.Event()
self.connected = False
def run(self):
backoff = 0.5
while not self.stop_event.is_set():
if not self.connected:
try:
self.cap = cv2.VideoCapture(self.index, cv2.CAP_DSHOW) if hasattr(cv2, 'CAP_DSHOW') else cv2.VideoCapture(self.index)
# Versuche Einstellungen (best effort)
self.cap.set(cv2.CAP_PROP_FRAME_WIDTH, self.width)
self.cap.set(cv2.CAP_PROP_FRAME_HEIGHT, self.height)
self.cap.set(cv2.CAP_PROP_FPS, self.fps)
try:
self.cap.set(cv2.CAP_PROP_BUFFERSIZE, CAM_BUFFER_SIZE)
except Exception:
pass
time.sleep(0.2) # kurz warten, damit Kamera initialisiert
if self.cap.isOpened():
self.connected = True
backoff = 0.5
print(f"▶ {self.name} verbunden (Index {self.index})")
else:
self.cap.release()
raise RuntimeError("Cannot open camera")
except Exception as e:
print(f"⚠️ {self.name} Verbindungsfehler: {e} — retry in {backoff:.1f}s")
time.sleep(backoff)
backoff = min(backoff * 2, CAM_RECONNECT_DELAY)
continue
# Falls verbunden: Frames lesen (non-blocking best effort)
try:
ok, frame = self.cap.read()
if not ok or frame is None:
# Kamera hat kurz Probleme → reconnect
print(f"⚠️ {self.name} read failed, reconnecting...")
self._reconnect()
continue
ts = time.time()
# optional: flip depending on camera usage. Keep raw here; main thread entscheidet
with self.lock:
self.latest_frame = frame
self.latest_ts = ts
# kein sleep hier — schneller Reader liefert aktuellstes Frame
# aber ein sehr kleines sleep reduziert CPU-Last der Reader-Threads
time.sleep(0.001)
except Exception as e:
print(f"⚠️ {self.name} Laufzeitfehler: {e}")
self._reconnect()
# cleanup
if self.cap and self.cap.isOpened():
self.cap.release()
print(f"■ {self.name} gestoppt")
def _reconnect(self):
try:
if self.cap and self.cap.isOpened():
self.cap.release()
except Exception:
pass
self.connected = False
time.sleep(0.5)
def read_latest(self):
with self.lock:
return self.latest_frame.copy() if self.latest_frame is not None else None, self.latest_ts
def stop(self):
self.stop_event.set()
# -------------------------------
# Hauptprogramm (MediaPipe im Main-Thread)
# -------------------------------
def run_gesture_input():
# Threads für Kameras starten
cam_touch = CameraReader(TOUCH_CAM_INDEX, DISPLAY_WIDTH, DISPLAY_HEIGHT, CAMERA_FPS, name="TouchCam")
cam_gest = CameraReader(GESTURE_CAM_INDEX, DISPLAY_WIDTH, DISPLAY_HEIGHT, CAMERA_FPS, name="GestCam")
cam_touch.start()
cam_gest.start()
# MediaPipe setup im Main-Thread
mp_hands = mp.solutions.hands
mp_draw = mp.solutions.drawing_utils
hands_touch = mp_hands.Hands(
max_num_hands=1,
model_complexity=MODEL_COMPLEXITY,
min_detection_confidence=0.6,
min_tracking_confidence=0.6
)
hands_gesture = mp_hands.Hands(
max_num_hands=2,
model_complexity=MODEL_COMPLEXITY,
min_detection_confidence=0.6,
min_tracking_confidence=0.6
)
last_finger_pos = None
finger_still_start = None
prev_touch_time = 0.0
prev_clap_time = 0.0
frame_duration = 1.0 / CAMERA_FPS
last_frame_time = time.time()
try:
while True:
loop_start = time.time()
# 1) Hol die aktuellsten Frames (wenn zu alt -> skippen)
frame_t, ts_t = cam_touch.read_latest()
frame_g, ts_g = cam_gest.read_latest()
now = time.time()
# Wenn kein Frame vorhanden, einfach kurz warten und weitermachen (nicht break)
if frame_t is None or (now - ts_t) > MAX_FRAME_AGE:
# kein gültiges Touch-Frame; zeige Hinweis oder skip
# wir machen einfach weiter (kein Freeze)
frame_t = None
if frame_g is None or (now - ts_g) > MAX_FRAME_AGE:
frame_g = None
# 2) Verarbeite Touch (falls vorhanden)
if frame_t is not None:
# Flip & convert
frame_touch = cv2.flip(frame_t, -1)
th, tw = frame_touch.shape[:2]
rgb_t = cv2.cvtColor(frame_touch, cv2.COLOR_BGR2RGB)
res_t = hands_touch.process(rgb_t)
if res_t.multi_hand_landmarks:
lm = res_t.multi_hand_landmarks[0]
mp_draw.draw_landmarks(frame_touch, lm, mp_hands.HAND_CONNECTIONS)
# Finger zeigt nach unten? (daumen-/finger-orientiert)
if lm.landmark[8].y < lm.landmark[5].y:
last_finger_pos = None
finger_still_start = None
else:
fx = int(lm.landmark[8].x * tw)
fy = int(lm.landmark[8].y * th)
sx, sy = map_point_homography(fx, fy)
current_pos = (fx, fy)
if last_finger_pos is None:
last_finger_pos = current_pos
finger_still_start = time.time()
else:
dist = math.hypot(current_pos[0] - last_finger_pos[0],
current_pos[1] - last_finger_pos[1])
if dist < MOVE_TOLERANCE:
if finger_still_start and (time.time() - finger_still_start) >= STILL_REQUIRED:
if time.time() - prev_touch_time > 0.5:
client.send_message("/touch", [sx, sy])
print(f"👉 TOUCH bei {sx},{sy}")
prev_touch_time = time.time()
finger_still_start = None
else:
finger_still_start = time.time()
last_finger_pos = current_pos
cv2.circle(frame_touch, (fx, fy), 10, (0, 255, 0), -1)
cv2.putText(frame_touch, f"{sx},{sy}", (fx+10, fy-10),
cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0,255,0), 2)
else:
last_finger_pos = None
finger_still_start = None
# Debug-Info
cv2.putText(frame_touch, f"FPS:{CAMERA_FPS} MC:{MODEL_COMPLEXITY}", (10, 30),
cv2.FONT_HERSHEY_SIMPLEX, 0.7, (255,255,0), 2)
else:
# Frame fehlt: zeige schwarzen Platzhalter
frame_touch = np.zeros((DISPLAY_HEIGHT, DISPLAY_WIDTH, 3), dtype=np.uint8)
cv2.putText(frame_touch, "No Touch Frame", (10, 30),
cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0,0,255), 2)
# 3) Verarbeite Gesture (Clap) falls vorhanden
if frame_g is not None:
frame_gest = cv2.flip(frame_g, 1)
gh, gw = frame_gest.shape[:2]
rgb_g = cv2.cvtColor(frame_gest, cv2.COLOR_BGR2RGB)
res_g = hands_gesture.process(rgb_g)
if res_g.multi_hand_landmarks and len(res_g.multi_hand_landmarks) == 2:
h1, h2 = res_g.multi_hand_landmarks
x1 = np.mean([p.x for p in h1.landmark]) * gw
y1 = np.mean([p.y for p in h1.landmark]) * gh
x2 = np.mean([p.x for p in h2.landmark]) * gw
y2 = np.mean([p.y for p in h2.landmark]) * gh
dist = math.hypot(x2 - x1, y2 - y1)
if dist < 100 and (time.time() - prev_clap_time) > CLAP_COOLDOWN:
prev_clap_time = time.time()
client.send_message("/clap", 1)
print("👏 SEND /clap")
cv2.putText(frame_gest, "👏", (int(gw/2)-20, 80),
cv2.FONT_HERSHEY_SIMPLEX, 2, (0,255,255), 3)
# Falls keine Hände: optionales Overlay entfernen / nichts tun
else:
frame_gest = np.zeros((DISPLAY_HEIGHT, DISPLAY_WIDTH, 3), dtype=np.uint8)
cv2.putText(frame_gest, "No Gesture Frame", (10, 30),
cv2.FONT_HERSHEY_SIMPLEX, 0.8, (0,0,255), 2)
# 4) Show (nicht blockierend)
cv2.imshow("Touch-Cam", frame_touch)
cv2.imshow("Gesture-Cam", frame_gest)
# 5) Input handling (sehr kurze wait, damit Fenster-Events gehandelt werden)
key = cv2.waitKey(1)
if key == 27:
print("⏹ ESC gedrückt - Beenden")
break
# 6) FPS-Limiter (best-effort, kein long sleep)
elapsed = time.time() - loop_start
remaining = frame_duration - elapsed
if remaining > 0:
# kleine sleep um CPU zu schonen; nicht lang blockieren
time.sleep(min(remaining, 0.01))
except KeyboardInterrupt:
print("■ KeyboardInterrupt erhalten")
finally:
# Sauber schließen
try:
hands_touch.close()
hands_gesture.close()
except Exception:
pass
cam_touch.stop()
cam_gest.stop()
# Threads können etwas Zeit brauchen
cam_touch.join(timeout=2.0)
cam_gest.join(timeout=2.0)
cv2.destroyAllWindows()
print("✔ Programm beendet")
if __name__ == "__main__":
run_gesture_input()

209
gesture_input_osc_test2.py Normal file
View File

@ -0,0 +1,209 @@
import cv2
import mediapipe as mp
import numpy as np
import math, json
from pythonosc import udp_client
# ==================================================
# CAMERA / PERFORMANCE (INTEGER)
# ==================================================
TOUCH_CAM_INDEX = 1
GESTURE_CAM_INDEX = 0
CAMERA_WIDTH = 900
CAMERA_HEIGHT = 500
CAMERA_FPS = 18 # laptop-stabil
MODEL_COMPLEXITY = 1 # MUST be 0 on laptops
# ==================================================
# TOUCH (INTEGER / FRAME BASED)
# ==================================================
MOVE_TOLERANCE = 28
TOUCH_STILL_FRAMES = 18 # ~1s @ 18 FPS
TOUCH_COOLDOWN_FRAMES = 12
# ==================================================
# CLAP (INTEGER)
# ==================================================
CLAP_DISTANCE_THRESHOLD = 110
CLAP_COOLDOWN_FRAMES = 32
# ==================================================
# DISPLAY SETTINGS (INTEGER)
# 0 = OFF, 1 = 320x240, 2 = 480x360, 3 = 640x480
# ==================================================
DISPLAY_TOUCH_RES = 1
DISPLAY_GESTURE_RES = 1
DISPLAY_RES_MAP = {
1: (320, 240),
2: (480, 360),
3: (640, 480)
}
# ==================================================
# GAME / HOMOGRAPHY
# ==================================================
GAME_SCREEN_WIDTH = 900
GAME_SCREEN_HEIGHT = 600
# ==================================================
client = udp_client.SimpleUDPClient("127.0.0.1", 5005)
# ==================================================
# GLOBAL STATES
# ==================================================
last_finger_pos = None
still_frames = 0
touch_cooldown = 0
clap_cooldown = 0
# ==================================================
# LOAD CALIBRATION
# ==================================================
try:
with open("calibration.json", "r") as f:
CALIB_POINTS = json.load(f)
print("📐 Calibration loaded")
except:
CALIB_POINTS = None
print("⚠️ No calibration found")
H = None
if CALIB_POINTS:
src = np.array(CALIB_POINTS, dtype=np.float32)
dst = np.array([
[0, 0],
[GAME_SCREEN_WIDTH, 0],
[GAME_SCREEN_WIDTH, GAME_SCREEN_HEIGHT],
[0, GAME_SCREEN_HEIGHT]
], dtype=np.float32)
H, _ = cv2.findHomography(src, dst)
def map_point(x, y):
if H is None:
return int(x), int(y)
p = np.array([[[x, y]]], dtype=np.float32)
m = cv2.perspectiveTransform(p, H)[0][0]
return int(m[0]), int(m[1])
# ==================================================
def run():
global last_finger_pos, still_frames
global touch_cooldown, clap_cooldown
mp_hands = mp.solutions.hands
hands_touch = mp_hands.Hands(
max_num_hands=1,
model_complexity=MODEL_COMPLEXITY,
min_detection_confidence=0.6
)
hands_gesture = mp_hands.Hands(
max_num_hands=2,
model_complexity=MODEL_COMPLEXITY,
min_detection_confidence=0.6
)
cam_touch = cv2.VideoCapture(TOUCH_CAM_INDEX)
cam_gest = cv2.VideoCapture(GESTURE_CAM_INDEX)
for cam in (cam_touch, cam_gest):
cam.set(cv2.CAP_PROP_FRAME_WIDTH, CAMERA_WIDTH)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT, CAMERA_HEIGHT)
cam.set(cv2.CAP_PROP_FPS, CAMERA_FPS)
frame_delay = int(1000 / CAMERA_FPS)
while True:
ok1, frame_touch = cam_touch.read()
ok2, frame_gest = cam_gest.read()
if not ok1 or not ok2:
break
frame_touch = cv2.flip(frame_touch, -1)
frame_gest = cv2.flip(frame_gest, 1)
# ================= TOUCH =================
rgb_t = cv2.cvtColor(frame_touch, cv2.COLOR_BGR2RGB)
res_t = hands_touch.process(rgb_t)
h, w, _ = frame_touch.shape
if res_t.multi_hand_landmarks:
lm = res_t.multi_hand_landmarks[0]
if lm.landmark[8].y > lm.landmark[5].y:
fx = int(lm.landmark[8].x * w)
fy = int(lm.landmark[8].y * h)
sx, sy = map_point(fx, fy)
current_pos = (fx, fy)
if last_finger_pos is not None:
dist = math.hypot(
current_pos[0] - last_finger_pos[0],
current_pos[1] - last_finger_pos[1]
)
if dist < MOVE_TOLERANCE:
still_frames += 1
if still_frames >= TOUCH_STILL_FRAMES and touch_cooldown == 0:
client.send_message("/touch", [sx, sy])
print(f"👉 TOUCH {sx},{sy}")
touch_cooldown = TOUCH_COOLDOWN_FRAMES
still_frames = 0
else:
still_frames = 0
else:
still_frames = 0
last_finger_pos = current_pos
cv2.circle(frame_touch, (fx, fy), 6, (0, 255, 0), -1)
else:
last_finger_pos = None
still_frames = 0
if touch_cooldown > 0:
touch_cooldown -= 1
# ================= CLAP =================
rgb_g = cv2.cvtColor(frame_gest, cv2.COLOR_BGR2RGB)
res_g = hands_gesture.process(rgb_g)
gh, gw, _ = frame_gest.shape
if res_g.multi_hand_landmarks and len(res_g.multi_hand_landmarks) == 2:
h1, h2 = res_g.multi_hand_landmarks
x1 = np.mean([p.x for p in h1.landmark]) * gw
y1 = np.mean([p.y for p in h1.landmark]) * gh
x2 = np.mean([p.x for p in h2.landmark]) * gw
y2 = np.mean([p.y for p in h2.landmark]) * gh
if math.hypot(x2 - x1, y2 - y1) < CLAP_DISTANCE_THRESHOLD and clap_cooldown == 0:
client.send_message("/clap", 1)
print("👏 CLAP")
clap_cooldown = CLAP_COOLDOWN_FRAMES
if clap_cooldown > 0:
clap_cooldown -= 1
# ================= DISPLAY =================
if DISPLAY_TOUCH_RES > 0:
dw, dh = DISPLAY_RES_MAP[DISPLAY_TOUCH_RES]
cv2.imshow("Touch-Cam", cv2.resize(frame_touch, (dw, dh)))
if DISPLAY_GESTURE_RES > 0:
dw, dh = DISPLAY_RES_MAP[DISPLAY_GESTURE_RES]
cv2.imshow("Gesture-Cam", cv2.resize(frame_gest, (dw, dh)))
if cv2.waitKey(frame_delay) & 0xFF == 27:
break
cam_touch.release()
cam_gest.release()
cv2.destroyAllWindows()
# ==================================================
if __name__ == "__main__":
run()

196
gesture_input_osc_test3.py Normal file
View File

@ -0,0 +1,196 @@
import cv2
import mediapipe as mp
import numpy as np
import math, time, json
from pythonosc import udp_client
# =====================================================
# =================== SETTINGS ========================
# =====================================================
# -------- Camera Index --------
TOUCH_CAM_INDEX = 1
GESTURE_CAM_INDEX = 0
# -------- Camera Capture Resolution / FPS --------
CAM_WIDTH = 1280
CAM_HEIGHT = 720
CAM_FPS = 30
# -------- Display Resolution (INTEGER) --------
DISPLAY_WIDTH = 480 #960
DISPLAY_HEIGHT = 270 #540
# -------- Screen Mapping --------
GAME_SCREEN_WIDTH = 900
GAME_SCREEN_HEIGHT = 600
# -------- MediaPipe Model Complexity --------
MODEL_COMPLEXITY_TOUCH = 1
MODEL_COMPLEXITY_GESTURE = 0
# -------- Touch Trigger --------
STILL_REQUIRED = 1.0
MOVE_TOLERANCE = 25
TOUCH_COOLDOWN = 0.5
# -------- Clap Trigger --------
CLAP_DISTANCE = 100
CLAP_COOLDOWN = 1
# -------- OSC --------
OSC_IP = "127.0.0.1"
OSC_PORT = 5005
# =====================================================
# ================= GLOBAL STATE ======================
# =====================================================
client = udp_client.SimpleUDPClient(OSC_IP, OSC_PORT)
last_finger_pos = None
finger_still_start = None
prev_touch_time = 0.0
prev_clap_time = 0.0
# =====================================================
# ============ CALIBRATION / HOMOGRAPHY ===============
# =====================================================
try:
with open("calibration.json", "r") as f:
CALIB_POINTS = json.load(f)
print("📐 Calibration loaded")
except:
CALIB_POINTS = None
print("⚠️ No calibration found")
H = None
if CALIB_POINTS is not None:
src = np.array(CALIB_POINTS, dtype=np.float32)
dst = np.array([
[0, 0],
[GAME_SCREEN_WIDTH, 0],
[GAME_SCREEN_WIDTH, GAME_SCREEN_HEIGHT],
[0, GAME_SCREEN_HEIGHT]
], dtype=np.float32)
H, _ = cv2.findHomography(src, dst)
def map_point_homography(x, y):
if H is None:
return int(x), int(y)
p = np.array([[[x, y]]], dtype=np.float32)
m = cv2.perspectiveTransform(p, H)[0][0]
return int(m[0]), int(m[1])
# =====================================================
# ===================== MAIN ==========================
# =====================================================
def run_gesture_input():
global last_finger_pos, finger_still_start
global prev_touch_time, prev_clap_time
mp_hands = mp.solutions.hands
mp_draw = mp.solutions.drawing_utils
hands_touch = mp_hands.Hands(
max_num_hands=1,
model_complexity=MODEL_COMPLEXITY_TOUCH,
min_detection_confidence=0.6,
min_tracking_confidence=0.6
)
hands_gesture = mp_hands.Hands(
max_num_hands=2,
model_complexity=MODEL_COMPLEXITY_GESTURE,
min_detection_confidence=0.6,
min_tracking_confidence=0.6
)
cam_touch = cv2.VideoCapture(TOUCH_CAM_INDEX)
cam_gesture = cv2.VideoCapture(GESTURE_CAM_INDEX)
for cam in (cam_touch, cam_gesture):
cam.set(cv2.CAP_PROP_FRAME_WIDTH, CAM_WIDTH)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT, CAM_HEIGHT)
cam.set(cv2.CAP_PROP_FPS, CAM_FPS)
while True:
ok1, frame_touch = cam_touch.read()
ok2, frame_gest = cam_gesture.read()
if not ok1 or not ok2:
break
frame_touch = cv2.flip(frame_touch, -1)
frame_gest = cv2.flip(frame_gest, 1)
# ---------------- TOUCH ----------------
rgb_t = cv2.cvtColor(frame_touch, cv2.COLOR_BGR2RGB)
res_t = hands_touch.process(rgb_t)
th, tw, _ = frame_touch.shape
if res_t.multi_hand_landmarks:
lm = res_t.multi_hand_landmarks[0]
mp_draw.draw_landmarks(frame_touch, lm, mp_hands.HAND_CONNECTIONS)
if lm.landmark[8].y >= lm.landmark[5].y:
fx = int(lm.landmark[8].x * tw)
fy = int(lm.landmark[8].y * th)
sx, sy = map_point_homography(fx, fy)
now = time.time()
cur = (fx, fy)
if last_finger_pos is None:
last_finger_pos = cur
finger_still_start = now
else:
dist = math.hypot(cur[0]-last_finger_pos[0], cur[1]-last_finger_pos[1])
if dist < MOVE_TOLERANCE:
if now - finger_still_start >= STILL_REQUIRED and now - prev_touch_time > TOUCH_COOLDOWN:
client.send_message("/touch", [sx, sy])
prev_touch_time = now
finger_still_start = now
else:
finger_still_start = now
last_finger_pos = cur
cv2.circle(frame_touch, (fx, fy), 10, (0,255,0), -1)
else:
last_finger_pos = None
# ---------------- CLAP ----------------
rgb_g = cv2.cvtColor(frame_gest, cv2.COLOR_BGR2RGB)
res_g = hands_gesture.process(rgb_g)
gh, gw, _ = frame_gest.shape
if res_g.multi_hand_landmarks and len(res_g.multi_hand_landmarks) == 2:
h1, h2 = res_g.multi_hand_landmarks
c1 = np.mean([[p.x*gw, p.y*gh] for p in h1.landmark], axis=0)
c2 = np.mean([[p.x*gw, p.y*gh] for p in h2.landmark], axis=0)
dist = np.linalg.norm(c2 - c1)
if dist < CLAP_DISTANCE and time.time() - prev_clap_time > CLAP_COOLDOWN:
prev_clap_time = time.time()
client.send_message("/clap", 1)
# ---------------- DISPLAY SCALING ----------------
disp_touch = cv2.resize(frame_touch, (DISPLAY_WIDTH, DISPLAY_HEIGHT))
disp_gest = cv2.resize(frame_gest, (DISPLAY_WIDTH, DISPLAY_HEIGHT))
cv2.imshow("Touch Camera", disp_touch)
cv2.imshow("Gesture Camera", disp_gest)
if cv2.waitKey(5) & 0xFF == 27:
break
cam_touch.release()
cam_gesture.release()
cv2.destroyAllWindows()
# =====================================================
if __name__ == "__main__":
run_gesture_input()

200
gesture_input_osc_tryfix.py Normal file
View File

@ -0,0 +1,200 @@
import cv2
import mediapipe as mp
import numpy as np
import math, time, json
from pythonosc import udp_client
# -------------------------------
# SETTINGS
# -------------------------------
TOUCH_CAM_INDEX = 0
GESTURE_CAM_INDEX = 1
GAME_SCREEN_WIDTH = 900
GAME_SCREEN_HEIGHT = 600
STILL_REQUIRED = 1.0
MOVE_TOLERANCE = 25
TARGET_FPS = 20
FRAME_TIME = 1.0 / TARGET_FPS
client = udp_client.SimpleUDPClient("127.0.0.1", 5005)
# Global states
last_finger_pos = None
finger_still_start = None
prev_touch_time = 0.0
prev_clap_time = 0.0
# -------------------------------------
# LOAD CALIBRATION + HOMOGRAPHY
# -------------------------------------
try:
with open("calibration.json", "r") as f:
CALIB_POINTS = json.load(f)
print("📐 Kalibrierung geladen:", CALIB_POINTS)
except:
CALIB_POINTS = None
print("⚠️ Keine Kalibrierung gefunden – benutze Rohkoordinaten!")
H = None
if CALIB_POINTS is not None:
src = np.array(CALIB_POINTS, dtype=np.float32)
dst = np.array([
[0, 0],
[GAME_SCREEN_WIDTH, 0],
[GAME_SCREEN_WIDTH, GAME_SCREEN_HEIGHT],
[0, GAME_SCREEN_HEIGHT]
], dtype=np.float32)
H, _ = cv2.findHomography(src, dst)
print("📐 Homographie-Matrix berechnet!")
def map_point_homography(x, y):
if H is None:
return int(x), int(y)
p = np.array([[[x, y]]], dtype=np.float32)
mapped = cv2.perspectiveTransform(p, H)[0][0]
return int(mapped[0]), int(mapped[1])
# -----------------------------------------------------------------
def run_gesture_input():
global last_finger_pos, finger_still_start
global prev_touch_time, prev_clap_time
mp_hands = mp.solutions.hands
mp_draw = mp.solutions.drawing_utils
# FASTER HAND MODELS
hands_touch = mp_hands.Hands(max_num_hands=1, min_detection_confidence=0.6,
model_complexity=0)
hands_gesture = mp_hands.Hands(max_num_hands=2, min_detection_confidence=0.6,
model_complexity=0)
# Cameras
cam_touch = cv2.VideoCapture(TOUCH_CAM_INDEX)
cam_gesture = cv2.VideoCapture(GESTURE_CAM_INDEX)
# Set to 640x480
for cam in (cam_touch, cam_gesture):
cam.set(cv2.CAP_PROP_FRAME_WIDTH, 640)
cam.set(cv2.CAP_PROP_FRAME_HEIGHT, 480)
if not cam_touch.isOpened():
print("❌ Touch-Kamera konnte NICHT geöffnet werden!")
if not cam_gesture.isOpened():
print("❌ Gesture-Kamera konnte NICICHT geöffnet werden!")
clap_cooldown = 1.5
while True:
loop_start = time.time()
ok1, frame_touch = cam_touch.read()
ok2, frame_gest = cam_gesture.read()
if not ok1 or not ok2:
print("❌ Eine Kamera liefert kein Bild.")
break
# Flip für Orientierung
frame_touch = cv2.flip(frame_touch, -1)
frame_gest = cv2.flip(frame_gest, 1)
# ---------------- TOUCH detection ----------------
rgb_t = cv2.cvtColor(frame_touch, cv2.COLOR_BGR2RGB)
res_t = hands_touch.process(rgb_t)
th, tw, _ = frame_touch.shape
if res_t.multi_hand_landmarks:
lm = res_t.multi_hand_landmarks[0]
# Finger zeigt nach unten?
if lm.landmark[8].y < lm.landmark[5].y:
last_finger_pos = None
finger_still_start = None
continue
fx = int(lm.landmark[8].x * tw)
fy = int(lm.landmark[8].y * th)
sx, sy = map_point_homography(fx, fy)
now = time.time()
current_pos = (fx, fy)
if last_finger_pos is None:
last_finger_pos = current_pos
finger_still_start = now
else:
dist = math.hypot(current_pos[0] - last_finger_pos[0],
current_pos[1] - last_finger_pos[1])
if dist < MOVE_TOLERANCE:
if finger_still_start is None:
finger_still_start = now
else:
still_time = now - finger_still_start
if still_time >= STILL_REQUIRED and (now - prev_touch_time) > 0.5:
client.send_message("/touch", [sx, sy])
print(f"👉 TOUCH bei {sx},{sy} nach {still_time:.2f}s")
prev_touch_time = now
finger_still_start = None
else:
finger_still_start = now
last_finger_pos = current_pos
cv2.circle(frame_touch, (fx, fy), 10, (0, 255, 0), -1)
cv2.putText(frame_touch, f"{sx},{sy}", (fx + 10, fy - 10),
cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0,255,0), 2)
else:
last_finger_pos = None
finger_still_start = None
# ---------------- Clap detection ----------------
rgb_g = cv2.cvtColor(frame_gest, cv2.COLOR_BGR2RGB)
res_g = hands_gesture.process(rgb_g)
gh, gw, _ = frame_gest.shape
if res_g.multi_hand_landmarks and len(res_g.multi_hand_landmarks) == 2:
h1, h2 = res_g.multi_hand_landmarks
x1 = np.mean([p.x for p in h1.landmark]) * gw
y1 = np.mean([p.y for p in h1.landmark]) * gh
x2 = np.mean([p.x for p in h2.landmark]) * gw
y2 = np.mean([p.y for p in h2.landmark]) * gh
dist = math.hypot(x2 - x1, y2 - y1)
if dist < 100 and (time.time() - prev_clap_time) > clap_cooldown:
prev_clap_time = time.time()
client.send_message("/clap", 1)
print("👏 SEND /clap")
cv2.putText(frame_gest, "👏", (int(gw/2)-20, 80),
cv2.FONT_HERSHEY_SIMPLEX, 2, (0,255,255), 3)
# Display
cv2.imshow("Touch-Cam", frame_touch)
cv2.imshow("Gesture-Cam", frame_gest)
# -------------- FPS LIMITER --------------
elapsed = time.time() - loop_start
sleep_time = FRAME_TIME - elapsed
if sleep_time > 0:
time.sleep(sleep_time)
if cv2.waitKey(1) & 0xFF == 27:
break
cam_touch.release()
cam_gesture.release()
cv2.destroyAllWindows()
if __name__ == "__main__":
run_gesture_input()

0
ideas/Extra_Ordner_für_Continents.txt Normal file
View File

15
presets_PC/Laptop.txt Normal file
View File

@ -0,0 +1,15 @@
# --------------------------------------------------
# WINDOWS 11 LAPTOP PRESET (16 GB RAM)
# --------------------------------------------------
CAMERA_WIDTH = 640
CAMERA_HEIGHT = 480
CAMERA_FPS = 18
MODEL_COMPLEXITY = 0
MOVE_TOLERANCE = 28
TOUCH_STILL_FRAMES = 18
TOUCH_COOLDOWN_FRAMES = 12
CLAP_DISTANCE_THRESHOLD = 110
CLAP_COOLDOWN_FRAMES = 32

18
presets_PC/old.txt Normal file
View File

@ -0,0 +1,18 @@
TOUCH_CAM_INDEX = 0
GESTURE_CAM_INDEX = 1 #<--------Index_Kamera
CAMERA_WIDTH = 640 #<------Resolution
CAMERA_HEIGHT = 480
CAMERA_FPS = 20 # INT FPS
MODEL_COMPLEXITY = 0 # 0=fast, 1=normal, 2=accurate
MOVE_TOLERANCE = 25 # Pixel
TOUCH_STILL_FRAMES = 20 # stabile Frames für Touch
TOUCH_COOLDOWN_FRAMES = 10 # Cooldown nach Touch
CLAP_DISTANCE_THRESHOLD = 100
CLAP_COOLDOWN_FRAMES = 30 #<----- Optional?
GAME_SCREEN_WIDTH = 900
GAME_SCREEN_HEIGHT = 600 #<----------------Screens_Controll

2
touch_area.py
View File

@ -3,7 +3,7 @@ from pythonosc import dispatcher, osc_server
import threading
#python test_touch_area.py
SCREEN_WIDTH = 900
SCREEN_HEIGHT = 500
SCREEN_HEIGHT = 600
# Letzter Touchpunkt
touch_pos = None