AI safety check

cloud/app.py

@@ -1,4 +1,5 @@
 import os
+import asyncio
 import base64
 import uuid
 import datetime
@@ -9,6 +10,7 @@ from pymongo import MongoClient
 from APIs import generate, EdamamAPI
 from APIs.mqtt import send_command
 from microwaveCookPlanner import MicrowaveCookPlanner
+import safety_checker
 
 sys.path.insert(0, '..')
 try:
@@ -51,64 +53,73 @@ def hello_world():
 
 
 @app.route("/cooking-params", methods=["POST"])
-def cooking_params():
+async def cooking_params():
     data = request.get_json()
     
     if not data:
         return jsonify({"error": "Invalid or missing JSON payload"}), 400
 
-    # Extract user or device parameters (with fallback defaults)
+    # Extract user or device parameters
     height_cm = float(data.get("dish_height", 4.0))
-    initial_temp_c = float(data.get("ir_initial_temp", 20.0))  # e.g., 4.0 for fridge, -18.0 for freezer
-    microwave_wattage = int(data.get("microwave_wattage", 900))  # e.g., 900W
-    defrost_mode = bool(data.get("defrost_mode", False))  # True for defrost, False for cook/reheat
-    print("Parsed parameters - Height (cm):", height_cm, "Initial Temp (C):", initial_temp_c, "Microwave Wattage:", microwave_wattage, "Defrost Mode:", defrost_mode)
+    initial_temp_c = float(data.get("ir_initial_temp", 20.0))
+    microwave_wattage = int(data.get("microwave_wattage", 900))
+    defrost_mode = bool(data.get("defrost_mode", False))
 
-    # 1. Handle the Camera Image
+    # 1. Save Camera Image
     camera_image_b64 = data.get("camera_image")
-    filepath = None
-    
-    if camera_image_b64:
-        filename = f"dish_{uuid.uuid4().hex}.jpg"
-        filepath = os.path.join(CAMERA_IMAGE_DIR, filename)
-        
-        try:
-            with open(filepath, "wb") as f:
-                f.write(base64.b64decode(camera_image_b64))
-            
-            data["camera_image"] = filepath
-            
-        except Exception as e:
-            return jsonify({"error": f"Failed to save camera image: {str(e)}"}), 500
-    else:
+    if not camera_image_b64:
         return jsonify({"error": "Missing required field 'camera_image'"}), 400
 
-    # 2. Run the Cook Planning Engine
+    filename = f"dish_{uuid.uuid4().hex}.jpg"
+    filepath = os.path.join(CAMERA_IMAGE_DIR, filename)
+    
     try:
-        cook_plan = microwave_cook_planner.generate_plan(
+        with open(filepath, "wb") as f:
+            f.write(base64.b64decode(camera_image_b64))
+        data["camera_image"] = filepath
+    except Exception as e:
+        return jsonify({"error": f"Failed to save camera image: {str(e)}"}), 500
+
+    # 2. Run Vision Safety Check & Cook Planner Concurrently
+    try:
+        safety_task = asyncio.to_thread(safety_checker.check_dish_safety, filepath)
+        planner_task = asyncio.to_thread(
+            microwave_cook_planner.generate_plan,
             image_path=filepath,
             height_cm=height_cm,
             initial_temp_c=initial_temp_c,
             microwave_wattage=microwave_wattage,
             defrost_mode=defrost_mode
         )
-    except Exception as e:
-        return jsonify({"error": f"Failed to compute cooking plan: {str(e)}"}), 500
 
-    # 3. Attach cooking parameters to database record
+        # Execute both concurrently and await results
+        safety_result, cook_plan = await asyncio.gather(safety_task, planner_task)
+
+    except Exception as e:
+        return jsonify({"error": f"Task execution failed: {str(e)}"}), 500
+
+    # 3. Evaluate Safety Result
+    data["safety_check"] = safety_result
+    if not safety_result.get("is_safe", True):
+        print(f"[Safety Warning] Unsafe dish detected: {safety_result}")
+        return jsonify({
+            "error": "Safety hazard detected in microwave area",
+            "is_safe": False,
+            "warning": safety_result.get("warning_message", "Unsafe materials detected."),
+            "detected_hazards": safety_result.get("detected_hazards", [])
+        }), 200
+
+    # 4. Attach Cooking Plan & Save to MongoDB
     data["analysis_results"] = cook_plan
 
-    # 4. Save to MongoDB
     try:
         cooking_collection.insert_one(data)
         data.pop("_id", None)
     except Exception as e:
         return jsonify({"error": f"Database error: {str(e)}"}), 500
 
-    # 5. Return complete output
     return jsonify(cook_plan), 201
 
-
 @app.route("/telemetry", methods=["POST"])
 def telemetry():
     data = request.get_json()

cloud/requirements.txt

@@ -1,4 +1,4 @@
-Flask==3.0.2
+Flask[async]==3.0.2
 pymongo==4.6.1
 gunicorn==21.2.0
 opencv-python-headless

cloud/safety_checker.py

@@ -0,0 +1,42 @@
+import json
+from APIs.aichat import generate
+
+DISH_SAFETY_SCHEMA = {
+    "type": "object",
+    "properties": {
+        "is_safe": {
+            "type": "boolean",
+            "description": "True if no microwave hazards (metal, foil, sealed packaging) are present."
+        },
+        "warning_message": {
+            "type": "string",
+            "description": "Explanation of any hazard found, or an empty string if safe."
+        },
+        "detected_hazards": {
+            "type": "array",
+            "items": {
+                "type": "string"
+            },
+            "description": "List of specific hazard items detected."
+        }
+    },
+    "required": ["is_safe", "warning_message", "detected_hazards"]
+}
+
+def check_dish_safety(image_path: str) -> dict:
+    prompt = (
+        "Analyze this top-down photo of a dish prepared for microwave cooking. "
+        "Inspect the area for metal utensils, aluminum foil, metallic dish patterns, or unvented plastic wraps."
+    )
+
+    # Pass the schema directly to the generation call
+    response_raw = generate(
+        prompt=prompt,
+        images=[image_path],
+        output_format=DISH_SAFETY_SCHEMA  # Passed as Ollama's `format` parameter
+    )
+
+    if isinstance(response_raw, dict):
+        return response_raw
+
+    return json.loads(response_raw)