283 lines
9.1 KiB
Python
283 lines
9.1 KiB
Python
"""
|
||
语音控制技能 - 基于本地AI的语音命令执行
|
||
"""
|
||
|
||
import logging
|
||
import re
|
||
import sys
|
||
import os
|
||
from typing import Dict, Any, List
|
||
|
||
# 兼容独立运行和包导入
|
||
_agent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
if _agent_dir not in sys.path:
|
||
sys.path.insert(0, _agent_dir)
|
||
|
||
try:
|
||
from skills.base import BaseSkill
|
||
except ImportError:
|
||
from ..base import BaseSkill
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class VoiceControlSkill(BaseSkill):
|
||
"""语音控制技能"""
|
||
|
||
PACKAGE = "" # 通用技能，不绑定特定APP
|
||
NAME = "语音控制"
|
||
|
||
# 常用应用包名映射
|
||
APP_PACKAGES = {
|
||
"微信": "com.tencent.mm",
|
||
"抖音": "com.ss.android.ugc.aweme",
|
||
"支付宝": "com.eg.android.AlipayGphone",
|
||
"淘宝": "com.taobao.taobao",
|
||
"微博": "com.sina.weibo",
|
||
"qq": "com.tencent.mobileqq",
|
||
"QQ": "com.tencent.mobileqq",
|
||
"设置": "com.android.settings",
|
||
"相机": "com.android.camera",
|
||
"浏览器": "com.android.chrome",
|
||
"豆包": "com.bytedance.doubao",
|
||
"豆包AI": "com.bytedance.doubao",
|
||
"小红书": "com.xingin.xhs",
|
||
"闲鱼": "com.taobao.idlefish",
|
||
"bilibili": "tv.danmaku.bili",
|
||
"B站": "tv.danmaku.bili",
|
||
"知乎": "com.zhihu.android",
|
||
}
|
||
|
||
def parse_voice_command(self, text: str) -> Dict[str, Any]:
|
||
"""
|
||
解析语音命令，返回操作序列
|
||
|
||
Args:
|
||
text: 语音文本
|
||
|
||
Returns:
|
||
解析结果和操作序列
|
||
"""
|
||
cmd = text.lower().strip()
|
||
actions = []
|
||
|
||
# 1. 处理复合命令（用逗号、然后、再等分隔）
|
||
parts = re.split(r'[,，、]|然后|再|接着', cmd)
|
||
if len(parts) > 1:
|
||
for part in parts:
|
||
part_actions = self._parse_single_command(part.strip())
|
||
actions.extend(part_actions)
|
||
return {
|
||
"success": True,
|
||
"text": text,
|
||
"actions": actions,
|
||
"type": "compound"
|
||
}
|
||
|
||
# 2. 处理单个命令
|
||
actions = self._parse_single_command(cmd)
|
||
|
||
return {
|
||
"success": len(actions) > 0,
|
||
"text": text,
|
||
"actions": actions,
|
||
"type": "single"
|
||
}
|
||
|
||
def _parse_single_command(self, cmd: str) -> List[Dict[str, Any]]:
|
||
"""解析单个命令"""
|
||
actions = []
|
||
|
||
# 打开应用（可能带搜索）
|
||
for name, pkg in self.APP_PACKAGES.items():
|
||
patterns = [
|
||
f"打开{name}",
|
||
f"启动{name}",
|
||
f"打开{name.lower()}",
|
||
f"开{name}"
|
||
]
|
||
|
||
for pattern in patterns:
|
||
if pattern in cmd:
|
||
actions.append({
|
||
"action": "open_app",
|
||
"params": {"package": pkg, "name": name}
|
||
})
|
||
|
||
# 检查是否有搜索关键词
|
||
search_text = self._extract_search_text(cmd)
|
||
if search_text:
|
||
actions.append({
|
||
"action": "wait",
|
||
"params": {"seconds": 2}
|
||
})
|
||
actions.append({
|
||
"action": "search",
|
||
"params": {"keyword": search_text}
|
||
})
|
||
|
||
return actions
|
||
|
||
# 搜索命令
|
||
if "搜索" in cmd or "查找" in cmd or "找" in cmd:
|
||
search_text = self._extract_search_text(cmd)
|
||
if search_text:
|
||
actions.append({
|
||
"action": "search",
|
||
"params": {"keyword": search_text}
|
||
})
|
||
return actions
|
||
|
||
# 返回
|
||
if "返回" in cmd or "回去" in cmd or "后退" in cmd:
|
||
actions.append({"action": "back", "params": {}})
|
||
return actions
|
||
|
||
# 回到桌面
|
||
if "桌面" in cmd or "主页" in cmd or "home" in cmd:
|
||
actions.append({"action": "home", "params": {}})
|
||
return actions
|
||
|
||
# 滑动
|
||
if "向上滑" in cmd or "上滑" in cmd:
|
||
actions.append({
|
||
"action": "swipe",
|
||
"params": {"direction": "up", "scale": 0.5}
|
||
})
|
||
return actions
|
||
|
||
if "向下滑" in cmd or "下滑" in cmd:
|
||
actions.append({
|
||
"action": "swipe",
|
||
"params": {"direction": "down", "scale": 0.5}
|
||
})
|
||
return actions
|
||
|
||
# 截图
|
||
if "截图" in cmd or "截屏" in cmd:
|
||
actions.append({
|
||
"action": "screenshot",
|
||
"params": {}
|
||
})
|
||
return actions
|
||
|
||
return actions
|
||
|
||
def _extract_search_text(self, cmd: str) -> str:
|
||
"""提取搜索关键词"""
|
||
patterns = [
|
||
r"搜索(.+)",
|
||
r"查找(.+)",
|
||
r"找(.+)",
|
||
r"搜(.+)"
|
||
]
|
||
|
||
for pattern in patterns:
|
||
match = re.search(pattern, cmd)
|
||
if match:
|
||
text = match.group(1).strip()
|
||
# 移除标点
|
||
text = re.sub(r'[,，。！？、]', '', text).strip()
|
||
return text
|
||
|
||
return ""
|
||
|
||
def execute_voice_command(self, text: str) -> Dict[str, Any]:
|
||
"""
|
||
执行语音命令
|
||
|
||
Args:
|
||
text: 语音文本
|
||
|
||
Returns:
|
||
执行结果
|
||
"""
|
||
try:
|
||
# 解析命令
|
||
parsed = self.parse_voice_command(text)
|
||
|
||
if not parsed["success"]:
|
||
return {
|
||
"success": False,
|
||
"error": f"无法理解命令: {text}",
|
||
"text": text
|
||
}
|
||
|
||
# 执行操作序列
|
||
results = []
|
||
for action_data in parsed["actions"]:
|
||
action = action_data["action"]
|
||
params = action_data.get("params", {})
|
||
|
||
try:
|
||
if action == "open_app":
|
||
pkg = params["package"]
|
||
self.d.app_start(pkg)
|
||
self.sleep(2)
|
||
results.append({"action": action, "success": True})
|
||
|
||
elif action == "search":
|
||
keyword = params["keyword"]
|
||
result = self.search(keyword)
|
||
results.append(result)
|
||
|
||
elif action == "back":
|
||
self.back()
|
||
results.append({"action": action, "success": True})
|
||
|
||
elif action == "home":
|
||
self.home()
|
||
results.append({"action": action, "success": True})
|
||
|
||
elif action == "swipe":
|
||
direction = params.get("direction", "up")
|
||
scale = params.get("scale", 0.5)
|
||
self.swipe(direction, scale)
|
||
results.append({"action": action, "success": True})
|
||
|
||
elif action == "screenshot":
|
||
filepath = self.screenshot_to_file()
|
||
results.append({
|
||
"action": action,
|
||
"success": True,
|
||
"filepath": filepath
|
||
})
|
||
|
||
elif action == "wait":
|
||
seconds = params.get("seconds", 1)
|
||
self.sleep(seconds)
|
||
results.append({"action": action, "success": True})
|
||
|
||
else:
|
||
results.append({
|
||
"action": action,
|
||
"success": False,
|
||
"error": f"未知操作: {action}"
|
||
})
|
||
|
||
except Exception as e:
|
||
logger.error(f"执行操作失败 {action}: {e}")
|
||
results.append({
|
||
"action": action,
|
||
"success": False,
|
||
"error": str(e)
|
||
})
|
||
|
||
success_count = sum(1 for r in results if r.get("success", False))
|
||
|
||
return {
|
||
"success": success_count > 0,
|
||
"text": text,
|
||
"total_actions": len(results),
|
||
"success_count": success_count,
|
||
"results": results
|
||
}
|
||
|
||
except Exception as e:
|
||
logger.error(f"执行语音命令失败: {e}")
|
||
return {
|
||
"success": False,
|
||
"error": str(e),
|
||
"text": text
|
||
}
|