feat: publish workphone SDK deployment and API docs
This commit is contained in:
Manus AI
282
sdk/agent/skills/voice_control.py
Normal file
282
sdk/agent/skills/voice_control.py
Normal file
@@ -0,0 +1,282 @@
|
||||
"""
|
||||
语音控制技能 - 基于本地AI的语音命令执行
|
||||
"""
|
||||
|
||||
import logging
|
||||
import re
|
||||
import sys
|
||||
import os
|
||||
from typing import Dict, Any, List
|
||||
|
||||
# 兼容独立运行和包导入
|
||||
_agent_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
if _agent_dir not in sys.path:
|
||||
sys.path.insert(0, _agent_dir)
|
||||
|
||||
try:
|
||||
from skills.base import BaseSkill
|
||||
except ImportError:
|
||||
from ..base import BaseSkill
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class VoiceControlSkill(BaseSkill):
|
||||
"""语音控制技能"""
|
||||
|
||||
PACKAGE = "" # 通用技能，不绑定特定APP
|
||||
NAME = "语音控制"
|
||||
|
||||
# 常用应用包名映射
|
||||
APP_PACKAGES = {
|
||||
"微信": "com.tencent.mm",
|
||||
"抖音": "com.ss.android.ugc.aweme",
|
||||
"支付宝": "com.eg.android.AlipayGphone",
|
||||
"淘宝": "com.taobao.taobao",
|
||||
"微博": "com.sina.weibo",
|
||||
"qq": "com.tencent.mobileqq",
|
||||
"QQ": "com.tencent.mobileqq",
|
||||
"设置": "com.android.settings",
|
||||
"相机": "com.android.camera",
|
||||
"浏览器": "com.android.chrome",
|
||||
"豆包": "com.bytedance.doubao",
|
||||
"豆包AI": "com.bytedance.doubao",
|
||||
"小红书": "com.xingin.xhs",
|
||||
"闲鱼": "com.taobao.idlefish",
|
||||
"bilibili": "tv.danmaku.bili",
|
||||
"B站": "tv.danmaku.bili",
|
||||
"知乎": "com.zhihu.android",
|
||||
}
|
||||
|
||||
def parse_voice_command(self, text: str) -> Dict[str, Any]:
|
||||
"""
|
||||
解析语音命令，返回操作序列
|
||||
|
||||
Args:
|
||||
text: 语音文本
|
||||
|
||||
Returns:
|
||||
解析结果和操作序列
|
||||
"""
|
||||
cmd = text.lower().strip()
|
||||
actions = []
|
||||
|
||||
# 1. 处理复合命令（用逗号、然后、再等分隔）
|
||||
parts = re.split(r'[,，、]|然后|再|接着', cmd)
|
||||
if len(parts) > 1:
|
||||
for part in parts:
|
||||
part_actions = self._parse_single_command(part.strip())
|
||||
actions.extend(part_actions)
|
||||
return {
|
||||
"success": True,
|
||||
"text": text,
|
||||
"actions": actions,
|
||||
"type": "compound"
|
||||
}
|
||||
|
||||
# 2. 处理单个命令
|
||||
actions = self._parse_single_command(cmd)
|
||||
|
||||
return {
|
||||
"success": len(actions) > 0,
|
||||
"text": text,
|
||||
"actions": actions,
|
||||
"type": "single"
|
||||
}
|
||||
|
||||
def _parse_single_command(self, cmd: str) -> List[Dict[str, Any]]:
|
||||
"""解析单个命令"""
|
||||
actions = []
|
||||
|
||||
# 打开应用（可能带搜索）
|
||||
for name, pkg in self.APP_PACKAGES.items():
|
||||
patterns = [
|
||||
f"打开{name}",
|
||||
f"启动{name}",
|
||||
f"打开{name.lower()}",
|
||||
f"开{name}"
|
||||
]
|
||||
|
||||
for pattern in patterns:
|
||||
if pattern in cmd:
|
||||
actions.append({
|
||||
"action": "open_app",
|
||||
"params": {"package": pkg, "name": name}
|
||||
})
|
||||
|
||||
# 检查是否有搜索关键词
|
||||
search_text = self._extract_search_text(cmd)
|
||||
if search_text:
|
||||
actions.append({
|
||||
"action": "wait",
|
||||
"params": {"seconds": 2}
|
||||
})
|
||||
actions.append({
|
||||
"action": "search",
|
||||
"params": {"keyword": search_text}
|
||||
})
|
||||
|
||||
return actions
|
||||
|
||||
# 搜索命令
|
||||
if "搜索" in cmd or "查找" in cmd or "找" in cmd:
|
||||
search_text = self._extract_search_text(cmd)
|
||||
if search_text:
|
||||
actions.append({
|
||||
"action": "search",
|
||||
"params": {"keyword": search_text}
|
||||
})
|
||||
return actions
|
||||
|
||||
# 返回
|
||||
if "返回" in cmd or "回去" in cmd or "后退" in cmd:
|
||||
actions.append({"action": "back", "params": {}})
|
||||
return actions
|
||||
|
||||
# 回到桌面
|
||||
if "桌面" in cmd or "主页" in cmd or "home" in cmd:
|
||||
actions.append({"action": "home", "params": {}})
|
||||
return actions
|
||||
|
||||
# 滑动
|
||||
if "向上滑" in cmd or "上滑" in cmd:
|
||||
actions.append({
|
||||
"action": "swipe",
|
||||
"params": {"direction": "up", "scale": 0.5}
|
||||
})
|
||||
return actions
|
||||
|
||||
if "向下滑" in cmd or "下滑" in cmd:
|
||||
actions.append({
|
||||
"action": "swipe",
|
||||
"params": {"direction": "down", "scale": 0.5}
|
||||
})
|
||||
return actions
|
||||
|
||||
# 截图
|
||||
if "截图" in cmd or "截屏" in cmd:
|
||||
actions.append({
|
||||
"action": "screenshot",
|
||||
"params": {}
|
||||
})
|
||||
return actions
|
||||
|
||||
return actions
|
||||
|
||||
def _extract_search_text(self, cmd: str) -> str:
|
||||
"""提取搜索关键词"""
|
||||
patterns = [
|
||||
r"搜索(.+)",
|
||||
r"查找(.+)",
|
||||
r"找(.+)",
|
||||
r"搜(.+)"
|
||||
]
|
||||
|
||||
for pattern in patterns:
|
||||
match = re.search(pattern, cmd)
|
||||
if match:
|
||||
text = match.group(1).strip()
|
||||
# 移除标点
|
||||
text = re.sub(r'[,，。！？、]', '', text).strip()
|
||||
return text
|
||||
|
||||
return ""
|
||||
|
||||
def execute_voice_command(self, text: str) -> Dict[str, Any]:
|
||||
"""
|
||||
执行语音命令
|
||||
|
||||
Args:
|
||||
text: 语音文本
|
||||
|
||||
Returns:
|
||||
执行结果
|
||||
"""
|
||||
try:
|
||||
# 解析命令
|
||||
parsed = self.parse_voice_command(text)
|
||||
|
||||
if not parsed["success"]:
|
||||
return {
|
||||
"success": False,
|
||||
"error": f"无法理解命令: {text}",
|
||||
"text": text
|
||||
}
|
||||
|
||||
# 执行操作序列
|
||||
results = []
|
||||
for action_data in parsed["actions"]:
|
||||
action = action_data["action"]
|
||||
params = action_data.get("params", {})
|
||||
|
||||
try:
|
||||
if action == "open_app":
|
||||
pkg = params["package"]
|
||||
self.d.app_start(pkg)
|
||||
self.sleep(2)
|
||||
results.append({"action": action, "success": True})
|
||||
|
||||
elif action == "search":
|
||||
keyword = params["keyword"]
|
||||
result = self.search(keyword)
|
||||
results.append(result)
|
||||
|
||||
elif action == "back":
|
||||
self.back()
|
||||
results.append({"action": action, "success": True})
|
||||
|
||||
elif action == "home":
|
||||
self.home()
|
||||
results.append({"action": action, "success": True})
|
||||
|
||||
elif action == "swipe":
|
||||
direction = params.get("direction", "up")
|
||||
scale = params.get("scale", 0.5)
|
||||
self.swipe(direction, scale)
|
||||
results.append({"action": action, "success": True})
|
||||
|
||||
elif action == "screenshot":
|
||||
filepath = self.screenshot_to_file()
|
||||
results.append({
|
||||
"action": action,
|
||||
"success": True,
|
||||
"filepath": filepath
|
||||
})
|
||||
|
||||
elif action == "wait":
|
||||
seconds = params.get("seconds", 1)
|
||||
self.sleep(seconds)
|
||||
results.append({"action": action, "success": True})
|
||||
|
||||
else:
|
||||
results.append({
|
||||
"action": action,
|
||||
"success": False,
|
||||
"error": f"未知操作: {action}"
|
||||
})
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"执行操作失败 {action}: {e}")
|
||||
results.append({
|
||||
"action": action,
|
||||
"success": False,
|
||||
"error": str(e)
|
||||
})
|
||||
|
||||
success_count = sum(1 for r in results if r.get("success", False))
|
||||
|
||||
return {
|
||||
"success": success_count > 0,
|
||||
"text": text,
|
||||
"total_actions": len(results),
|
||||
"success_count": success_count,
|
||||
"results": results
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
logger.error(f"执行语音命令失败: {e}")
|
||||
return {
|
||||
"success": False,
|
||||
"error": str(e),
|
||||
"text": text
|
||||
}
|
||||
Reference in New Issue
Block a user