siyu-bank/workphone-sdk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: publish workphone SDK deployment and API docs

Browse Source
This commit is contained in:
Manus AI
2026-07-14 18:10:52 +08:00
commit 021d633cc1
534 changed files with 122391 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

414
sdk/agent/voice_agent.py Normal file
View File

@@ -0,0 +1,414 @@
#!/usr/bin/env python3
"""
工作手机Agent - 语音控制版
运行在手机上，支持语音命令和自主执行任务
"""
import asyncio
import json
import logging
import argparse
import base64
import time
from datetime import datetime
from typing import Optional, Dict, Any, List
try:
import websockets
import uiautomator2 as u2
except ImportError:
print("请安装依赖: pip install websockets uiautomator2")
exit(1)
# 尝试导入语音识别
try:
import speech_recognition as sr
VOICE_ENABLED = True
except ImportError:
VOICE_ENABLED = False
print("语音功能未启用，安装: pip install SpeechRecognition")
logging.basicConfig(level=logging.INFO, format='%(asctime)s [%(levelname)s] %(message)s')
logger = logging.getLogger(__name__)
class VoiceAgent:
"""
语音控制Agent
功能：
1. 手机本地运行，无需电脑
2. 语音命令控制
3. 自主执行任务队列
4. 连接远程服务器接收命令
"""
def __init__(
self,
device_id: str,
server_url: str = None,
voice_enabled: bool = True
):
self.device_id = device_id
self.server_url = server_url
self.voice_enabled = voice_enabled and VOICE_ENABLED
# 初始化uiautomator2
self.d = u2.connect()
self.d.implicitly_wait(10.0)
# 任务队列
self.task_queue: List[Dict] = []
self.running = False
# 语音识别器
if self.voice_enabled:
self.recognizer = sr.Recognizer()
# 预设命令
self.commands = {
"打开微信": self._open_wechat,
"打开豆包": self._open_doubao,
"打开抖音": self._open_douyin,
"打开设置": self._open_settings,
"截图": self._screenshot,
"返回": self._go_back,
"回到桌面": self._go_home,
"向上滑": self._swipe_up,
"向下滑": self._swipe_down,
}
logger.info(f"VoiceAgent初始化完成: {device_id}")
logger.info(f"语音控制: {'启用' if self.voice_enabled else '禁用'}")
# ========== 基础操作 ==========
def _open_wechat(self):
"""打开微信"""
self.d.app_start("com.tencent.mm")
return {"success": True, "message": "微信已打开"}
def _open_doubao(self):
"""打开豆包"""
self.d.app_start("com.larus.nova")
return {"success": True, "message": "豆包已打开"}
def _open_douyin(self):
"""打开抖音"""
self.d.app_start("com.ss.android.ugc.aweme")
return {"success": True, "message": "抖音已打开"}
def _open_settings(self):
"""打开设置"""
self.d.app_start("com.android.settings")
return {"success": True, "message": "设置已打开"}
def _screenshot(self):
"""截图"""
img = self.d.screenshot(format='raw')
# 保存到手机
path = f"/sdcard/screenshot_{int(time.time())}.png"
with open(path, 'wb') as f:
f.write(img)
return {"success": True, "message": f"截图已保存: {path}"}
def _go_back(self):
"""返回"""
self.d.press("back")
return {"success": True, "message": "已返回"}
def _go_home(self):
"""回到桌面"""
self.d.press("home")
return {"success": True, "message": "已回到桌面"}
def _swipe_up(self):
"""向上滑动"""
self.d.swipe_ext("up", scale=0.8)
return {"success": True, "message": "已向上滑动"}
def _swipe_down(self):
"""向下滑动"""
self.d.swipe_ext("down", scale=0.8)
return {"success": True, "message": "已向下滑动"}
# ========== 语音识别 ==========
def listen_voice(self, timeout: int = 5) -> Optional[str]:
"""监听语音命令"""
if not self.voice_enabled:
return None
try:
with sr.Microphone() as source:
logger.info("请说话...")
self.recognizer.adjust_for_ambient_noise(source, duration=0.5)
audio = self.recognizer.listen(source, timeout=timeout)
# 使用Google语音识别（中文）
text = self.recognizer.recognize_google(audio, language="zh-CN")
logger.info(f"识别到: {text}")
return text
except sr.WaitTimeoutError:
return None
except sr.UnknownValueError:
logger.warning("无法识别语音")
return None
except Exception as e:
logger.error(f"语音识别错误: {e}")
return None
def process_voice_command(self, text: str) -> Dict:
"""处理语音命令"""
text = text.strip()
# 精确匹配
if text in self.commands:
return self.commands[text]()
# 模糊匹配
for cmd, func in self.commands.items():
if cmd in text or text in cmd:
return func()
# 发送消息命令: "给XXX发消息说XXX"
if "发消息" in text or "发送" in text:
return self._parse_send_message(text)
# 打开APP命令: "打开XXX"
if text.startswith("打开"):
app_name = text[2:].strip()
return self._open_app_by_name(app_name)
return {"success": False, "message": f"未识别的命令: {text}"}
def _parse_send_message(self, text: str) -> Dict:
"""解析发消息命令"""
# 示例: "给张三发消息说你好"
import re
match = re.search(r'给(.+?)(发消息|发送)(.+)', text)
if match:
contact = match.group(1).strip()
message = match.group(3).replace("说", "").strip()
return self._send_wechat_message(contact, message)
return {"success": False, "message": "无法解析发消息命令"}
def _send_wechat_message(self, contact: str, message: str) -> Dict:
"""发送微信消息"""
try:
# 打开微信
self.d.app_start("com.tencent.mm")
time.sleep(2)
# 搜索联系人
if self.d(text="搜索").exists(timeout=3):
self.d(text="搜索").click()
time.sleep(0.5)
self.d.send_keys(contact)
time.sleep(1)
if self.d(text=contact).exists(timeout=3):
self.d(text=contact).click()
time.sleep(1)
# 发送消息
self.d.send_keys(message)
time.sleep(0.3)
if self.d(text="发送").exists(timeout=3):
self.d(text="发送").click()
return {"success": True, "message": f"已发送给{contact}: {message}"}
except Exception as e:
return {"success": False, "message": str(e)}
def _open_app_by_name(self, name: str) -> Dict:
"""根据名称打开APP"""
app_map = {
"微信": "com.tencent.mm",
"豆包": "com.larus.nova",
"抖音": "com.ss.android.ugc.aweme",
"小红书": "com.xingin.xhs",
"淘宝": "com.taobao.taobao",
"支付宝": "com.eg.android.AlipayGphone",
"设置": "com.android.settings",
"相机": "com.android.camera",
"浏览器": "com.android.chrome",
}
package = app_map.get(name)
if package:
self.d.app_start(package)
return {"success": True, "message": f"{name}已打开"}
return {"success": False, "message": f"未知APP: {name}"}
# ========== 任务队列 ==========
def add_task(self, task: Dict):
"""添加任务到队列"""
task["id"] = f"task_{int(time.time() * 1000)}"
task["status"] = "pending"
task["created_at"] = datetime.now().isoformat()
self.task_queue.append(task)
logger.info(f"添加任务: {task}")
return task["id"]
async def process_task_queue(self):
"""处理任务队列"""
while self.running:
if self.task_queue:
task = self.task_queue[0]
if task["status"] == "pending":
task["status"] = "running"
logger.info(f"执行任务: {task['id']}")
try:
result = await self._execute_task(task)
task["status"] = "completed"
task["result"] = result
except Exception as e:
task["status"] = "failed"
task["error"] = str(e)
self.task_queue.pop(0)
await asyncio.sleep(1)
async def _execute_task(self, task: Dict) -> Dict:
"""执行单个任务"""
task_type = task.get("type")
params = task.get("params", {})
if task_type == "voice_command":
return self.process_voice_command(params.get("text", ""))
elif task_type == "click":
self.d.click(params["x"], params["y"])
return {"success": True}
elif task_type == "send_message":
return self._send_wechat_message(
params.get("contact"),
params.get("message")
)
elif task_type == "open_app":
return self._open_app_by_name(params.get("name"))
return {"success": False, "message": f"未知任务类型: {task_type}"}
# ========== 远程连接 ==========
async def connect_server(self):
"""连接远程服务器"""
if not self.server_url:
return
while self.running:
try:
logger.info(f"连接服务器: {self.server_url}")
async with websockets.connect(self.server_url) as ws:
# 注册
await ws.send(json.dumps({
"type": "register",
"device_id": self.device_id,
"capabilities": ["voice", "ui_control"]
}))
async for message in ws:
data = json.loads(message)
await self._handle_server_message(data, ws)
except Exception as e:
logger.error(f"服务器连接错误: {e}")
await asyncio.sleep(5)
async def _handle_server_message(self, data: Dict, ws):
"""处理服务器消息"""
msg_type = data.get("type")
if msg_type == "execute":
# 添加到任务队列
task_id = self.add_task(data.get("task", {}))
await ws.send(json.dumps({
"type": "task_queued",
"task_id": task_id
}))
elif msg_type == "voice_command":
result = self.process_voice_command(data.get("text", ""))
await ws.send(json.dumps({
"type": "result",
"command_id": data.get("command_id"),
"result": result
}))
# ========== 主循环 ==========
async def voice_loop(self):
"""语音监听循环"""
if not self.voice_enabled:
return
logger.info("开始语音监听...")
while self.running:
text = self.listen_voice(timeout=3)
if text:
# 唤醒词检测
if "小助手" in text or "你好" in text:
logger.info("唤醒成功，等待命令...")
command = self.listen_voice(timeout=5)
if command:
result = self.process_voice_command(command)
logger.info(f"执行结果: {result}")
await asyncio.sleep(0.1)
async def start(self):
"""启动Agent"""
self.running = True
logger.info("VoiceAgent启动")
tasks = [
self.process_task_queue(),
]
if self.server_url:
tasks.append(self.connect_server())
if self.voice_enabled:
tasks.append(self.voice_loop())
await asyncio.gather(*tasks)
def stop(self):
"""停止Agent"""
self.running = False
logger.info("VoiceAgent停止")
def main():
parser = argparse.ArgumentParser(description='语音控制Agent')
parser.add_argument('--device-id', default='voice-agent-001', help='设备ID')
parser.add_argument('--server', help='远程服务器地址 (ws://xxx:8899/ws/device/xxx)')
parser.add_argument('--no-voice', action='store_true', help='禁用语音控制')
args = parser.parse_args()
agent = VoiceAgent(
device_id=args.device_id,
server_url=args.server,
voice_enabled=not args.no_voice
)
try:
asyncio.run(agent.start())
except KeyboardInterrupt:
agent.stop()
if __name__ == "__main__":
main()