feat: publish workphone SDK deployment and API docs

sdk/agent/vision_helper.py

@@ -0,0 +1,86 @@
+"""
+截屏 + AI 视觉：每次操作先看屏再决定
+
+以实际截屏为主，AI 分析界面后输出下一步动作
+"""
+
+import base64
+import json
+import logging
+import os
+import re
+
+logger = logging.getLogger(__name__)
+
+def _gemini_url():
+    key = os.getenv("GEMINI_API_KEY", "AIzaSyCPARryq8o6MKptLoT4STAvCsRB7uZuOK8")
+    return f"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key={key}"
+
+
+def ask_vision(screenshot_bytes: bytes, goal: str, step_hint: str = "") -> dict:
+    """
+    截屏 + AI 看屏，返回下一步动作
+    
+    Returns:
+        {"action": "click", "x": 540, "y": 1200}
+        {"action": "input", "text": "吉米雨"}
+        {"action": "done"}
+        {"action": "back"}
+    """
+    try:
+        import httpx
+    except ImportError:
+        logger.warning("httpx 未安装，无法调用视觉 API")
+        return {"action": "error", "error": "httpx 未安装"}
+    
+    b64 = base64.b64encode(screenshot_bytes).decode("utf-8")
+    prompt = f"""这是微信手机界面截图。
+
+目标：{goal}
+
+{step_hint}
+
+请分析截图，输出下一步动作的 JSON（只返回 JSON，不要其他文字）：
+- 需要点击某处：{{"action":"click","x":中点x,"y":中点y}}
+- 需要输入文字：{{"action":"input","text":"要输入的文字"}}
+- 已完成目标：{{"action":"done"}}
+- 需要返回：{{"action":"back"}}
+- 无法继续：{{"action":"error","error":"原因"}}
+
+屏幕尺寸通常为 1080x2400，坐标在范围内。"""
+    
+    try:
+        with httpx.Client(timeout=30) as client:
+            resp = client.post(
+                _gemini_url(),
+                json={
+                    "contents": [{
+                        "parts": [
+                            {"inline_data": {"mime_type": "image/png", "data": b64}},
+                            {"text": prompt}
+                        ]
+                    }],
+                    "generationConfig": {
+                        "temperature": 0.1,
+                        "maxOutputTokens": 256,
+                    }
+                }
+            )
+        
+        if resp.status_code != 200:
+            logger.error(f"Gemini API 错误: {resp.status_code} {resp.text[:200]}")
+            return {"action": "error", "error": f"API {resp.status_code}"}
+        
+        data = resp.json()
+        text = data.get("candidates", [{}])[0].get("content", {}).get("parts", [{}])[0].get("text", "")
+        text = text.strip()
+        
+        # 提取 JSON
+        m = re.search(r'\{[^{}]*\}', text)
+        if m:
+            return json.loads(m.group())
+        return {"action": "error", "error": "无法解析 JSON"}
+    
+    except Exception as e:
+        logger.error(f"视觉调用失败: {e}")
+        return {"action": "error", "error": str(e)}