Overhaul lag reduction

app.py

@@ -9,9 +9,16 @@ import mediapipe as mp
 from fer import FER  # Facial emotion recognition
 
 # -----------------------------
-# Configuration: Adjust skip rate (lower = more frequent heavy updates)
+# Configuration
 # -----------------------------
-SKIP_RATE = 5
+# 1) Increase skip rate
+SKIP_RATE = 15
+
+# 2) Use GPU if available
+device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+# 3) Desired input size for faster inference
+DESIRED_SIZE = (640, 480)
 
 # -----------------------------
 # Global caches for overlay info and frame counters
@@ -34,16 +41,18 @@ face_detection = mp_face_detection.FaceDetection(min_detection_confidence=0.5)
 object_detection_model = models.detection.fasterrcnn_resnet50_fpn(
     weights=FasterRCNN_ResNet50_FPN_Weights.DEFAULT
 )
-object_detection_model.eval()
+object_detection_model.eval().to(device)  # Move model to GPU (if available)
+
 obj_transform = transforms.Compose([transforms.ToTensor()])
 
+# If the FER library supports GPU, it may pick it up automatically. 
+# Some versions allow device specification, e.g. FER(mtcnn=True, device=device).
 emotion_detector = FER(mtcnn=True)
 
 # -----------------------------
-# Fast Overlay Functions
+# Overlay Drawing Functions
 # -----------------------------
 def draw_posture_overlay(raw_frame, landmarks):
-    # Draw each landmark as a small circle
     for (x, y) in landmarks:
         cv2.circle(raw_frame, (x, y), 4, (0, 255, 0), -1)
     return raw_frame
@@ -55,27 +64,42 @@ def draw_boxes_overlay(raw_frame, boxes, color):
 
 # -----------------------------
 # Heavy (Synchronous) Detection Functions
-# These functions compute the overlay info on the current frame.
 # -----------------------------
 def compute_posture_overlay(image):
-    frame = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
-    h, w, _ = frame.shape
-    frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
-    pose_results = pose.process(frame_rgb)
+    # Convert to BGR for MediaPipe
+    frame_bgr = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
+    h, w, _ = frame_bgr.shape
+
+    # 2) Downscale before processing (optional for posture)
+    frame_bgr_small = cv2.resize(frame_bgr, DESIRED_SIZE)
+    small_h, small_w, _ = frame_bgr_small.shape
+
+    frame_rgb_small = cv2.cvtColor(frame_bgr_small, cv2.COLOR_BGR2RGB)
+    pose_results = pose.process(frame_rgb_small)
+
+    # Scale landmarks back up to original size if needed
     if pose_results.pose_landmarks:
         landmarks = []
         for lm in pose_results.pose_landmarks.landmark:
-            landmarks.append((int(lm.x * w), int(lm.y * h)))
+            # Rescale from the smaller frame to the original size
+            x = int(lm.x * small_w * (w / small_w))
+            y = int(lm.y * small_h * (h / small_h))
+            landmarks.append((x, y))
         text = "Posture detected"
     else:
         landmarks = []
         text = "No posture detected"
+
     return landmarks, text
 
 def compute_emotion_overlay(image):
-    frame = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
-    frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
-    emotions = emotion_detector.detect_emotions(frame_rgb)
+    # Convert to BGR
+    frame_bgr = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
+    # 2) Downscale
+    frame_bgr_small = cv2.resize(frame_bgr, DESIRED_SIZE)
+    frame_rgb_small = cv2.cvtColor(frame_bgr_small, cv2.COLOR_BGR2RGB)
+
+    emotions = emotion_detector.detect_emotions(frame_rgb_small)
     if emotions:
         top_emotion, score = max(emotions[0]["emotions"].items(), key=lambda x: x[1])
         text = f"{top_emotion} ({score:.2f})"
@@ -84,33 +108,48 @@ def compute_emotion_overlay(image):
     return text
 
 def compute_objects_overlay(image):
-    frame = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
-    frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
-    image_pil = Image.fromarray(frame_rgb)
-    img_tensor = obj_transform(image_pil)
+    frame_bgr = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
+    # 2) Downscale
+    frame_bgr_small = cv2.resize(frame_bgr, DESIRED_SIZE)
+    frame_rgb_small = cv2.cvtColor(frame_bgr_small, cv2.COLOR_BGR2RGB)
+
+    image_pil = Image.fromarray(frame_rgb_small)
+    img_tensor = obj_transform(image_pil).to(device)
+
     with torch.no_grad():
         detections = object_detection_model([img_tensor])[0]
+
     threshold = 0.8
     boxes = []
     for box, score in zip(detections["boxes"], detections["scores"]):
         if score > threshold:
+            # box is in the scaled-down coordinates; 
+            # you may want to scale them back to the original if needed
             boxes.append(tuple(box.int().cpu().numpy()))
+
     text = f"Detected {len(boxes)} object(s)" if boxes else "No objects detected"
     return boxes, text
 
 def compute_faces_overlay(image):
-    frame = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
-    h, w, _ = frame.shape
-    frame_rgb = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
-    face_results = face_detection.process(frame_rgb)
+    frame_bgr = cv2.cvtColor(np.array(image), cv2.COLOR_RGB2BGR)
+    h, w, _ = frame_bgr.shape
+    # 2) Downscale
+    frame_bgr_small = cv2.resize(frame_bgr, DESIRED_SIZE)
+    small_h, small_w, _ = frame_bgr_small.shape
+
+    frame_rgb_small = cv2.cvtColor(frame_bgr_small, cv2.COLOR_BGR2RGB)
+    face_results = face_detection.process(frame_rgb_small)
+
     boxes = []
     if face_results.detections:
         for detection in face_results.detections:
             bbox = detection.location_data.relative_bounding_box
-            x = int(bbox.xmin * w)
-            y = int(bbox.ymin * h)
-            box_w = int(bbox.width * w)
-            box_h = int(bbox.height * h)
+            x = int(bbox.xmin * small_w)
+            y = int(bbox.ymin * small_h)
+            box_w = int(bbox.width * small_w)
+            box_h = int(bbox.height * small_h)
+            # Scale bounding box coords back to original if you need full resolution
+            # E.g., x_original = int(x * (w / small_w)), etc.
             boxes.append((x, y, x + box_w, y + box_h))
         text = f"Detected {len(boxes)} face(s)"
     else:
@@ -118,62 +157,69 @@ def compute_faces_overlay(image):
     return boxes, text
 
 # -----------------------------
-# Main Analysis Functions (run every frame)
-# They update the cache every SKIP_RATE frames and always return a current frame with overlay.
+# Main Analysis Functions
 # -----------------------------
 def analyze_posture_current(image):
     global posture_cache
     posture_cache["counter"] += 1
-    current_frame = np.array(image)  # raw RGB frame (as numpy array)
-    # Update overlay info every SKIP_RATE frames
+    current_frame = np.array(image)
+
     if posture_cache["counter"] % SKIP_RATE == 0 or posture_cache["landmarks"] is None:
         landmarks, text = compute_posture_overlay(image)
         posture_cache["landmarks"] = landmarks
         posture_cache["text"] = text
-    # Draw cached landmarks on the current frame copy
+
     output = current_frame.copy()
     if posture_cache["landmarks"]:
         output = draw_posture_overlay(output, posture_cache["landmarks"])
+
     return output, f"Posture Analysis: {posture_cache['text']}"
 
 def analyze_emotion_current(image):
     global emotion_cache
     emotion_cache["counter"] += 1
     current_frame = np.array(image)
+
     if emotion_cache["counter"] % SKIP_RATE == 0 or emotion_cache["text"] is None:
         text = compute_emotion_overlay(image)
         emotion_cache["text"] = text
-    # For emotion, we don't overlay anything; just return the current frame.
+
     return current_frame, f"Emotion Analysis: {emotion_cache['text']}"
 
 def analyze_objects_current(image):
     global objects_cache
     objects_cache["counter"] += 1
     current_frame = np.array(image)
+
     if objects_cache["counter"] % SKIP_RATE == 0 or objects_cache["boxes"] is None:
         boxes, text = compute_objects_overlay(image)
         objects_cache["boxes"] = boxes
         objects_cache["text"] = text
+
     output = current_frame.copy()
     if objects_cache["boxes"]:
         output = draw_boxes_overlay(output, objects_cache["boxes"], (255, 255, 0))
+
     return output, f"Object Detection: {objects_cache['text']}"
 
 def analyze_faces_current(image):
     global faces_cache
     faces_cache["counter"] += 1
     current_frame = np.array(image)
+
     if faces_cache["counter"] % SKIP_RATE == 0 or faces_cache["boxes"] is None:
         boxes, text = compute_faces_overlay(image)
         faces_cache["boxes"] = boxes
         faces_cache["text"] = text
+
     output = current_frame.copy()
     if faces_cache["boxes"]:
         output = draw_boxes_overlay(output, faces_cache["boxes"], (0, 0, 255))
+
     return output, f"Face Detection: {faces_cache['text']}"
 
 # -----------------------------
-# Custom CSS for a High-Tech Look (White Font)
+# Custom CSS
 # -----------------------------
 custom_css = """
 @import url('https://fonts.googleapis.com/css2?family=Orbitron:wght@400;700&display=swap');
@@ -206,7 +252,7 @@ body {
 """
 
 # -----------------------------
-# Create Individual Interfaces for Each Analysis
+# Create Individual Interfaces
 # -----------------------------
 posture_interface = gr.Interface(
     fn=analyze_posture_current,
@@ -214,7 +260,7 @@ posture_interface = gr.Interface(
     outputs=[gr.Image(type="numpy", label="Annotated Output"), gr.Textbox(label="Posture Analysis")],
     title="Posture Analysis",
     description="Detects your posture using MediaPipe.",
-    live=True
+    live=True  # Keep only this interface live to avoid multiple heavy computations
 )
 
 emotion_interface = gr.Interface(
@@ -223,7 +269,7 @@ emotion_interface = gr.Interface(
     outputs=[gr.Image(type="numpy", label="Annotated Output"), gr.Textbox(label="Emotion Analysis")],
     title="Emotion Analysis",
     description="Detects facial emotions using FER.",
-    live=True
+    live=False  # Turn off streaming to reduce overhead
 )
 
 objects_interface = gr.Interface(
@@ -232,7 +278,7 @@ objects_interface = gr.Interface(
     outputs=[gr.Image(type="numpy", label="Annotated Output"), gr.Textbox(label="Object Detection")],
     title="Object Detection",
     description="Detects objects using a pretrained Faster R-CNN.",
-    live=True
+    live=False
 )
 
 faces_interface = gr.Interface(
@@ -241,11 +287,11 @@ faces_interface = gr.Interface(
     outputs=[gr.Image(type="numpy", label="Annotated Output"), gr.Textbox(label="Face Detection")],
     title="Face Detection",
     description="Detects faces using MediaPipe.",
-    live=True
+    live=False
 )
 
 # -----------------------------
-# Create a Tabbed Interface for All Analyses
+# Create a Tabbed Interface
 # -----------------------------
 tabbed_interface = gr.TabbedInterface(
     interface_list=[posture_interface, emotion_interface, objects_interface, faces_interface],
@@ -253,12 +299,15 @@ tabbed_interface = gr.TabbedInterface(
 )
 
 # -----------------------------
-# Wrap Everything in a Blocks Layout with Custom CSS
+# Wrap in a Blocks Layout
 # -----------------------------
 demo = gr.Blocks(css=custom_css)
 with demo:
     gr.Markdown("<h1 class='gradio-title'>Real-Time Multi-Analysis App</h1>")
-    gr.Markdown("<p class='gradio-description'>Experience a high-tech cinematic interface for real-time analysis of your posture, emotions, objects, and faces using your webcam.</p>")
+    gr.Markdown(
+        "<p class='gradio-description'>Experience a high-tech cinematic interface for real-time "
+        "analysis of your posture, emotions, objects, and faces using your webcam.</p>"
+    )
     tabbed_interface.render()
 
 if __name__ == "__main__":