""" 工作手机SDK v3.0 - AI 智能命令引擎 使用本地 Ollama 或 Gemini 免费 API 解析自然语言命令 支持: - Ollama 本地模型(免费,默认) - Google Gemini Flash(免费 API) - DeepSeek(付费备选) """ import json import logging import httpx import os from typing import List, Dict, Optional, Any logger = logging.getLogger(__name__) # ========== AI 后端配置 ========== # Ollama(本地免费,首选) OLLAMA_URL = os.getenv("OLLAMA_URL", "http://localhost:11434") OLLAMA_MODEL = os.getenv("OLLAMA_MODEL", "qwen2.5:1.5b") # Gemini(免费 API,备选) GEMINI_API_KEY = os.getenv("GEMINI_API_KEY", "") GEMINI_MODEL = "gemini-2.0-flash" # ========== 系统提示词 ========== SYSTEM_PROMPT = """你是安卓手机操作助手。用户说指令,你只输出JSON数组。 【重要】你只能用以下10种action,禁止发明新action: open_app, click, input, swipe, back, home, wait, screenshot, key_event, shell APP表:微信、QQ、抖音、小红书、快手、B站、淘宝、京东、拼多多、闲鱼、支付宝、美团、饿了么、高德地图、滴滴、设置、Chrome、Termux、飞书、钉钉、企业微信、相机、电话、短信、相册、文件管理、日历、地图 操作格式(只用这些): {"action":"open_app","app":"APP名"} {"action":"click","text":"按钮文字"} {"action":"input","text":"内容"} {"action":"swipe","direction":"up/down/left/right"} {"action":"back"} {"action":"home"} {"action":"wait","seconds":2} {"action":"screenshot"} {"action":"key_event","key":"enter/back/home/search/del/tab"} {"action":"shell","command":"adb shell命令"} 示例: Q: 打开微信 A: [{"action":"open_app","app":"微信"}] Q: 打开Chrome搜索今天天气 A: [{"action":"open_app","app":"Chrome"},{"action":"wait","seconds":2},{"action":"click","text":"搜索或输入网址"},{"action":"input","text":"今天天气"},{"action":"key_event","key":"enter"}] Q: 打开微信发消息给文件传输助手说你好 A: [{"action":"open_app","app":"微信"},{"action":"wait","seconds":2},{"action":"click","text":"搜索"},{"action":"input","text":"文件传输助手"},{"action":"click","text":"文件传输助手"},{"action":"wait","seconds":1},{"action":"input","text":"你好"},{"action":"click","text":"发送"}] Q: 打开抖音刷视频 A: [{"action":"open_app","app":"抖音"},{"action":"wait","seconds":3},{"action":"swipe","direction":"up"}] Q: 打开设置看WIFI A: [{"action":"open_app","app":"设置"},{"action":"wait","seconds":1},{"action":"click","text":"WLAN"}] Q: 打开支付宝扫一扫 A: [{"action":"open_app","app":"支付宝"},{"action":"wait","seconds":2},{"action":"click","text":"扫一扫"}] Q: 将系统语言设置成中文 A: [{"action":"open_app","app":"设置"},{"action":"wait","seconds":1},{"action":"click","text":"System"},{"action":"click","text":"Languages"},{"action":"click","text":"中文"}] Q: 修改系统语言 A: [{"action":"open_app","app":"设置"},{"action":"wait","seconds":1},{"action":"shell","command":"settings put system system_locales zh-CN"},{"action":"shell","command":"setprop persist.sys.locale zh-CN"}] Q: 点击设置里的某个选项 A: [{"action":"open_app","app":"设置"},{"action":"wait","seconds":1},{"action":"click","text":"选项文字"}] Q: 注册微信账号 A: [{"action":"open_app","app":"微信"},{"action":"wait","seconds":3},{"action":"click","text":"注册"},{"action":"wait","seconds":2}] 规则: 1. 只输出JSON数组,无其他文字 2. action只能是上面10种之一,绝对不要发明新的action名 3. 打开APP后加wait等待加载 4. 搜索操作用input+key_event enter 5. 修改系统设置优先用shell命令 6. 不确定就拆成open_app+click步骤""" class AICommandEngine: """AI 命令解析引擎""" def __init__(self): self._backend = None # 缓存已检测到的后端 self._available = None async def detect_backend(self, force=False) -> str: """检测可用的 AI 后端""" if self._backend and self._backend != "none" and not force: return self._backend # 1. Ollama(本地免费) try: async with httpx.AsyncClient(timeout=5.0) as c: r = await c.get(f"{OLLAMA_URL}/api/tags") if r.status_code == 200: models = [m["name"] for m in r.json().get("models", [])] if models: self._backend = "ollama" self._available = models logger.info(f"AI 后端: Ollama ({', '.join(models)})") return "ollama" else: logger.info("Ollama 在线但无模型(可能在下载中)") except Exception: pass # 2. Gemini 免费 if GEMINI_API_KEY: self._backend = "gemini" logger.info("AI 后端: Gemini Flash (免费)") return "gemini" self._backend = "none" logger.warning("无可用 AI 后端(Ollama 未运行/无模型,Gemini 未配置)") return "none" async def parse(self, user_input: str, context: Optional[Dict] = None) -> List[Dict[str, Any]]: """ 解析自然语言命令,返回操作列表。 Args: user_input: 用户输入的自然语言 context: 可选,当前屏幕上下文(当前APP等) Returns: 操作列表 [{"action": "...", ...}, ...] """ backend = await self.detect_backend() # 构建 prompt prompt = user_input if context: prompt = f"当前APP: {context.get('current_app', '未知')}\n用户指令: {user_input}" if backend == "ollama": return await self._call_ollama(prompt) elif backend == "gemini": return await self._call_gemini(prompt) else: return [] async def _call_ollama(self, prompt: str) -> List[Dict]: """调用本地 Ollama""" try: async with httpx.AsyncClient(timeout=60.0) as c: r = await c.post(f"{OLLAMA_URL}/api/chat", json={ "model": OLLAMA_MODEL, "messages": [ {"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": prompt}, ], "stream": False, "options": {"temperature": 0.3, "num_predict": 500}, }) if r.status_code != 200: logger.error(f"Ollama 错误: {r.status_code} {r.text[:200]}") return [] text = r.json().get("message", {}).get("content", "").strip() logger.info(f"Ollama 原始回复: {text[:200]}") return self._extract_actions(text) except Exception as e: logger.error(f"Ollama 调用失败: {type(e).__name__}: {e}") return [] async def _call_gemini(self, prompt: str) -> List[Dict]: """调用 Google Gemini 免费 API""" try: url = f"https://generativelanguage.googleapis.com/v1beta/models/{GEMINI_MODEL}:generateContent?key={GEMINI_API_KEY}" async with httpx.AsyncClient(timeout=30.0) as c: r = await c.post(url, json={ "contents": [{"parts": [{"text": f"{SYSTEM_PROMPT}\n\n用户: {prompt}\n返回:"}]}], "generationConfig": {"temperature": 0.3, "maxOutputTokens": 500}, }) if r.status_code != 200: logger.error(f"Gemini 错误: {r.status_code} {r.text[:200]}") return [] text = r.json()["candidates"][0]["content"]["parts"][0]["text"].strip() return self._extract_actions(text) except Exception as e: logger.error(f"Gemini 调用失败: {e}") return [] # 有效的 action 类型白名单 VALID_ACTIONS = { "open_app", "click", "input", "swipe", "back", "home", "wait", "screenshot", "key_event", "shell", "close_app", } # 无效 action 的智能修复映射 ACTION_FIX_MAP = { # 系统设置相关 "settings": "open_app", # settings -> 打开设置 "setlocale": "shell", # setlocale -> shell 命令 "set_language": "shell", # set_language -> shell 命令 "change_language": "shell", "system_setting": "open_app", "setting": "open_app", # 操作类别 "tap": "click", "press": "key_event", "type": "input", "write": "input", "scroll": "swipe", "scroll_up": "swipe", "scroll_down": "swipe", "launch": "open_app", "start": "open_app", "open": "open_app", "close": "close_app", "stop": "close_app", "kill": "close_app", "go_back": "back", "go_home": "home", "return": "back", "capture": "screenshot", "delay": "wait", "sleep": "wait", "pause": "wait", "send": "click", # send -> 点击发送按钮 "search": "click", # search -> 点击搜索 "register": "click", # register -> 点击注册 "login": "click", # login -> 点击登录 } def _fix_action(self, action: Dict) -> Dict: """修复无效的 action 类型,尽量保留用户意图""" act = action.get("action", "") if act in self.VALID_ACTIONS: return action # 已经有效 # 尝试映射修复 fixed_act = self.ACTION_FIX_MAP.get(act.lower()) if fixed_act: logger.info(f"AI action 修复: {act} -> {fixed_act}") new_action = dict(action) new_action["action"] = fixed_act # 根据修复后的类型补充参数 if fixed_act == "open_app" and "app" not in new_action: # settings/setting -> 打开设置 if "setting" in act.lower(): new_action["app"] = "设置" elif fixed_act == "shell": if "command" not in new_action: # setlocale/set_language -> 设置中文 if "locale" in act.lower() or "language" in act.lower(): new_action["command"] = "settings put system system_locales zh-CN" elif fixed_act == "click": if "text" not in new_action: # 用 action 名称本身作为按钮文字 text_map = { "send": "发送", "search": "搜索", "register": "注册", "login": "登录", } new_action["text"] = text_map.get(act.lower(), act) elif fixed_act == "swipe": if "direction" not in new_action: if "up" in act.lower(): new_action["direction"] = "up" elif "down" in act.lower(): new_action["direction"] = "down" else: new_action["direction"] = "up" return new_action # 最后尝试:含有关键词的智能推断 act_lower = act.lower() if any(kw in act_lower for kw in ["app", "open", "launch", "start"]): action["action"] = "open_app" if "app" not in action: action["app"] = "设置" return action if any(kw in act_lower for kw in ["click", "tap", "press", "button"]): action["action"] = "click" return action if any(kw in act_lower for kw in ["type", "input", "text", "write"]): action["action"] = "input" return action logger.warning(f"AI 生成了无法修复的操作: {act},跳过") return None # 无法修复 def _extract_actions(self, text: str) -> List[Dict]: """从 AI 返回的文本中提取 JSON 操作列表""" text = text.strip() # 去除 markdown 代码块 if "```json" in text: text = text.split("```json")[-1].split("```")[0].strip() elif "```" in text: text = text.split("```")[1].split("```")[0].strip() if text.count("```") >= 2 else text # 找到 JSON 数组 start = text.find("[") end = text.rfind("]") if start >= 0 and end > start: text = text[start:end + 1] try: data = json.loads(text) if isinstance(data, dict): data = [data] if isinstance(data, list): raw_actions = [d for d in data if isinstance(d, dict) and "action" in d] # 校验并修复每个 action fixed_actions = [] for a in raw_actions: fixed = self._fix_action(a) if fixed is not None: fixed_actions.append(fixed) return fixed_actions except json.JSONDecodeError: logger.warning(f"AI 返回无法解析: {text[:200]}") return [] async def get_status(self) -> Dict: """获取 AI 引擎状态(每次重新检测)""" backend = await self.detect_backend(force=True) return { "backend": backend, "model": OLLAMA_MODEL if backend == "ollama" else GEMINI_MODEL if backend == "gemini" else "无", "available": backend != "none", "models": self._available or [], } # 全局实例 ai_engine = AICommandEngine()