9118df9eee
Some checks failed
SDK CI / python-compile (push) Has been cancelled
- 新增 sdk/app/agent（Hook/Skills/Anti-ban 等）及 NAS/ADB 脚本 - 更新 Android 端、unified、PHP SDK、Docker Compose - Soul 文档迁移至 Soul调研/；移除资料目录内 APK - .gitignore 排除 sdk/tmp、sdk/logs、sdk/tmp_rom Co-authored-by: Cursor <cursoragent@cursor.com>
87 lines
2.7 KiB
Python
87 lines
2.7 KiB
Python
"""
|
||
截屏 + AI 视觉：每次操作先看屏再决定
|
||
|
||
以实际截屏为主，AI 分析界面后输出下一步动作
|
||
"""
|
||
|
||
import base64
|
||
import json
|
||
import logging
|
||
import os
|
||
import re
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
def _gemini_url():
|
||
key = os.getenv("GEMINI_API_KEY", "AIzaSyCPARryq8o6MKptLoT4STAvCsRB7uZuOK8")
|
||
return f"https://generativelanguage.googleapis.com/v1beta/models/gemini-2.0-flash:generateContent?key={key}"
|
||
|
||
|
||
def ask_vision(screenshot_bytes: bytes, goal: str, step_hint: str = "") -> dict:
|
||
"""
|
||
截屏 + AI 看屏，返回下一步动作
|
||
|
||
Returns:
|
||
{"action": "click", "x": 540, "y": 1200}
|
||
{"action": "input", "text": "吉米雨"}
|
||
{"action": "done"}
|
||
{"action": "back"}
|
||
"""
|
||
try:
|
||
import httpx
|
||
except ImportError:
|
||
logger.warning("httpx 未安装，无法调用视觉 API")
|
||
return {"action": "error", "error": "httpx 未安装"}
|
||
|
||
b64 = base64.b64encode(screenshot_bytes).decode("utf-8")
|
||
prompt = f"""这是微信手机界面截图。
|
||
|
||
目标：{goal}
|
||
|
||
{step_hint}
|
||
|
||
请分析截图，输出下一步动作的 JSON（只返回 JSON，不要其他文字）：
|
||
- 需要点击某处：{{"action":"click","x":中点x,"y":中点y}}
|
||
- 需要输入文字：{{"action":"input","text":"要输入的文字"}}
|
||
- 已完成目标：{{"action":"done"}}
|
||
- 需要返回：{{"action":"back"}}
|
||
- 无法继续：{{"action":"error","error":"原因"}}
|
||
|
||
屏幕尺寸通常为 1080x2400，坐标在范围内。"""
|
||
|
||
try:
|
||
with httpx.Client(timeout=30) as client:
|
||
resp = client.post(
|
||
_gemini_url(),
|
||
json={
|
||
"contents": [{
|
||
"parts": [
|
||
{"inline_data": {"mime_type": "image/png", "data": b64}},
|
||
{"text": prompt}
|
||
]
|
||
}],
|
||
"generationConfig": {
|
||
"temperature": 0.1,
|
||
"maxOutputTokens": 256,
|
||
}
|
||
}
|
||
)
|
||
|
||
if resp.status_code != 200:
|
||
logger.error(f"Gemini API 错误: {resp.status_code} {resp.text[:200]}")
|
||
return {"action": "error", "error": f"API {resp.status_code}"}
|
||
|
||
data = resp.json()
|
||
text = data.get("candidates", [{}])[0].get("content", {}).get("parts", [{}])[0].get("text", "")
|
||
text = text.strip()
|
||
|
||
# 提取 JSON
|
||
m = re.search(r'\{[^{}]*\}', text)
|
||
if m:
|
||
return json.loads(m.group())
|
||
return {"action": "error", "error": "无法解析 JSON"}
|
||
|
||
except Exception as e:
|
||
logger.error(f"视觉调用失败: {e}")
|
||
return {"action": "error", "error": str(e)}
|