287 lines
9.8 KiB
Python
287 lines
9.8 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
本地 Mongo（库 wz）→ 打包 → 宝塔 API 上传/解压 → 面板 API 在服务器本机执行 mongorestore 覆盖 wzdj。
|
||
|
||
依赖：本机 mongodump、tar（Windows 10+ 自带 bsdtar 或需装 Git tar）；pip: requests。
|
||
配置：与 master.py 相同（BT_PANEL_URL、BT_API_KEY、DEPLOY_PROJECT_PATH 等），另读 .env.development / .env.production 仅用于本机 dump。
|
||
|
||
服务器侧：使用「网站/API 目录」里的 .env（部署后名为 .env）中 MONGODB_URI 连接 127.0.0.1 Mongo，
|
||
故无需把数据库密码塞进 API 请求体。
|
||
|
||
用法（在 new/wz-api 下）:
|
||
python scripts/mongo_push_baota_api.py
|
||
python scripts/mongo_push_baota_api.py --skip-dump # 已有 build/mongo_dump_wz
|
||
"""
|
||
from __future__ import print_function
|
||
|
||
import argparse
|
||
import hashlib
|
||
import os
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
import time
|
||
|
||
try:
|
||
import requests
|
||
|
||
try:
|
||
import urllib3
|
||
|
||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||
except Exception:
|
||
pass
|
||
except ImportError:
|
||
requests = None
|
||
|
||
|
||
def _bt_signed_post(base_url, key, path, extra_data, timeout=120):
|
||
req_time = int(time.time())
|
||
sk_md5 = hashlib.md5(key.encode()).hexdigest()
|
||
req_token = hashlib.md5(("%s%s" % (req_time, sk_md5)).encode()).hexdigest()
|
||
data = {"request_time": req_time, "request_token": req_token}
|
||
data.update(extra_data or {})
|
||
return requests.post(base_url + path, data=data, timeout=timeout, verify=False)
|
||
|
||
|
||
def _bt_parse_json(r):
|
||
if r is None or r.status_code != 200:
|
||
return None
|
||
try:
|
||
return r.json()
|
||
except Exception:
|
||
t = (r.text or "").lstrip()
|
||
if t.startswith("{"):
|
||
try:
|
||
return __import__("json").loads(r.text)
|
||
except Exception:
|
||
return None
|
||
return None
|
||
|
||
|
||
def _read_uri_from_env_file(path, key="MONGODB_URI"):
|
||
if not os.path.isfile(path):
|
||
return ""
|
||
try:
|
||
with open(path, "r", encoding="utf-8", errors="replace") as f:
|
||
for line in f:
|
||
s = line.strip()
|
||
if s.startswith("#") or "=" not in s:
|
||
continue
|
||
k, v = s.split("=", 1)
|
||
if k.strip() == key:
|
||
return v.strip().strip('"').strip("'")
|
||
except Exception:
|
||
pass
|
||
return ""
|
||
|
||
|
||
def _import_master(root):
|
||
sys.path.insert(0, root)
|
||
import importlib.util
|
||
|
||
p = os.path.join(root, "master.py")
|
||
spec = importlib.util.spec_from_file_location("wz_api_master", p)
|
||
m = importlib.util.module_from_spec(spec)
|
||
spec.loader.exec_module(m)
|
||
return m
|
||
|
||
|
||
def _which(cmd):
|
||
return shutil.which(cmd) is not None
|
||
|
||
|
||
def _run_local(cmd, cwd=None):
|
||
print("[local]", " ".join(cmd))
|
||
r = subprocess.run(cmd, cwd=cwd, shell=False)
|
||
return r.returncode == 0
|
||
|
||
|
||
def _make_tgz_from_dump(dump_parent, local_db):
|
||
"""dump_parent 下须有子目录 local_db（如 wz）。"""
|
||
src = os.path.join(dump_parent, local_db)
|
||
if not os.path.isdir(src):
|
||
return None
|
||
fd, tgz = tempfile.mkstemp(prefix="wz_mongo_", suffix=".tar.gz")
|
||
os.close(fd)
|
||
try:
|
||
# GNU tar: tar czf tgz -C dump_parent local_db
|
||
if not _run_local(["tar", "-czf", tgz, "-C", dump_parent, local_db]):
|
||
return None
|
||
return tgz
|
||
except Exception:
|
||
try:
|
||
os.remove(tgz)
|
||
except Exception:
|
||
pass
|
||
return None
|
||
|
||
|
||
def bt_exec_shell(cfg, shell, cwd_path):
|
||
url = (cfg["bt_panel_url"] or "").rstrip("/")
|
||
key = cfg["bt_api_key"]
|
||
j = _bt_parse_json(
|
||
_bt_signed_post(
|
||
url,
|
||
key,
|
||
"/files?action=ExecShell",
|
||
{"path": cwd_path, "shell": shell},
|
||
timeout=60,
|
||
)
|
||
)
|
||
if not isinstance(j, dict) or j.get("status") is not True:
|
||
print("[失败] ExecShell 未接受: %s" % (j,))
|
||
return False
|
||
return True
|
||
|
||
|
||
def bt_get_exec_msg(cfg):
|
||
url = (cfg["bt_panel_url"] or "").rstrip("/")
|
||
key = cfg["bt_api_key"]
|
||
return _bt_parse_json(_bt_signed_post(url, key, "/files?action=GetExecShellMsg", {}, timeout=60))
|
||
|
||
|
||
def wait_exec_shell(cfg, timeout_sec=300, poll=2):
|
||
"""轮询 GetExecShellMsg，直到任务结束或超时。"""
|
||
deadline = time.time() + timeout_sec
|
||
last_msg = None
|
||
while time.time() < deadline:
|
||
j = bt_get_exec_msg(cfg)
|
||
last_msg = j
|
||
if isinstance(j, dict):
|
||
st = j.get("status")
|
||
if st is True:
|
||
print("[信息] 远程命令已结束（面板返回 status=True）")
|
||
if j.get("msg"):
|
||
print(str(j.get("msg"))[-4000:])
|
||
return True
|
||
time.sleep(poll)
|
||
print("[失败] 等待远程 Shell 超时（%ss）。最后一次面板返回: %s" % (timeout_sec, last_msg))
|
||
return False
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description="Mongo 覆盖到线上（宝塔 API + 服务端 mongorestore）")
|
||
parser.add_argument("--skip-dump", action="store_true", help="跳过本机 mongodump，使用已有目录")
|
||
parser.add_argument(
|
||
"--dump-dir",
|
||
default="",
|
||
help="mongodump 输出父目录（内含 wz 子目录），默认 build/mongo_dump_wz",
|
||
)
|
||
parser.add_argument("--local-db", default=os.environ.get("LOCAL_DB_NAME", "wz"))
|
||
parser.add_argument("--remote-db", default=os.environ.get("REMOTE_DB_NAME", "wzdj"))
|
||
parser.add_argument("--skip-bt-ping", action="store_true")
|
||
parser.add_argument("--exec-timeout", type=int, default=int(os.environ.get("BT_MONGO_EXEC_TIMEOUT", "300") or "300"))
|
||
args = parser.parse_args()
|
||
|
||
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
m = _import_master(root)
|
||
cfg = m.get_cfg()
|
||
|
||
if not requests:
|
||
print("[失败] pip install requests")
|
||
return 1
|
||
if not args.skip_bt_ping and not m.bt_panel_ping(cfg):
|
||
print("[失败] 宝塔 API 探活失败")
|
||
return 1
|
||
|
||
local_db = (args.local_db or "wz").strip()
|
||
remote_db = (args.remote_db or "wzdj").strip()
|
||
dump_dir = (args.dump_dir or "").strip() or os.path.join(root, "build", "mongo_dump_wz")
|
||
|
||
if not args.skip_dump:
|
||
if not _which("mongodump"):
|
||
print("[失败] 未找到 mongodump")
|
||
return 1
|
||
os.makedirs(dump_dir, exist_ok=True)
|
||
env_dev = os.path.join(root, ".env.development")
|
||
local_uri = _read_uri_from_env_file(env_dev) or "mongodb://127.0.0.1:27017/wz"
|
||
if not _run_local(["mongodump", "--uri=%s" % local_uri, "--db", local_db, "--out", dump_dir]):
|
||
print("[失败] mongodump 失败")
|
||
return 1
|
||
else:
|
||
print("[信息] --skip-dump，使用已有: %s" % dump_dir)
|
||
|
||
if not os.path.isdir(os.path.join(dump_dir, local_db)):
|
||
print("[失败] 缺少子目录 %s/%s" % (dump_dir, local_db))
|
||
return 1
|
||
|
||
tgz = _make_tgz_from_dump(dump_dir, local_db)
|
||
if not tgz:
|
||
print("[失败] 打包 tar.gz 失败（需要系统 tar 命令）")
|
||
return 1
|
||
|
||
project_path = cfg["project_path"].rstrip("/")
|
||
remote_work = project_path + "/wz_mongo_bt_push"
|
||
remote_tgz = remote_work + "/mongo_wz_dump.tar.gz"
|
||
# 解压后布局：remote_work/<local_db>/*.bson；用 -d <remote_db> 指向目标库（避免 nsFrom/nsTo 与目录扫描兼容问题）
|
||
remote_dump_parent = remote_work
|
||
remote_db_dir = remote_work + "/" + local_db
|
||
|
||
try:
|
||
print("[1/4] 上传 dump 压缩包 ...")
|
||
if not m.bt_upload_file_resumable(cfg, tgz, remote_work, "mongo_wz_dump.tar.gz"):
|
||
print("[失败] 上传失败")
|
||
return 1
|
||
|
||
print("[2/4] 远程解压 ...")
|
||
if not m.bt_unzip_remote(cfg, remote_tgz, remote_work + "/"):
|
||
print("[失败] 远程解压失败")
|
||
return 1
|
||
|
||
if m.bt_delete_file(cfg, remote_tgz):
|
||
print("[信息] 已删除远程压缩包，避免 mongorestore 误扫 tar.gz")
|
||
else:
|
||
print("[警告] 删除远程 tar 失败，mongorestore 可能跳过集合；请在面板手动删: %s" % remote_tgz)
|
||
|
||
pp = project_path.replace("'", "'\\''")
|
||
rd = remote_dump_parent.replace("'", "'\\''")
|
||
rdbdir = remote_db_dir.replace("'", "'\\''")
|
||
shell = (
|
||
"set -e;"
|
||
"MRS=$(command -v mongorestore 2>/dev/null || true);"
|
||
"if [ -z \"$MRS\" ] && [ -x /www/server/mongodb/bin/mongorestore ]; then MRS=/www/server/mongodb/bin/mongorestore; fi;"
|
||
'if [ -z "$MRS" ]; then echo "mongorestore not found"; exit 127; fi;'
|
||
+ "ENVF='"
|
||
+ pp
|
||
+ "/.env';"
|
||
+ 'if [ ! -f "$ENVF" ]; then echo "missing $ENVF"; exit 1; fi;'
|
||
+ "URI=$(grep '^MONGODB_URI=' \"$ENVF\" | head -1 | cut -d= -f2- | tr -d '\\r');"
|
||
+ "URI=$(printf '%s' \"$URI\" | sed \"s/^\\\"//;s/\\\"$//\");"
|
||
+ "rm -f '"
|
||
+ rd
|
||
+ "/mongo_wz_dump.tar.gz' || true;"
|
||
+ "if [ ! -d '"
|
||
+ rdbdir
|
||
+ "' ]; then echo missing_dump; ls -la '"
|
||
+ rd
|
||
+ "'; exit 2; fi;"
|
||
+ '"$MRS" --uri="$URI" --drop -d '
|
||
+ remote_db
|
||
+ " '"
|
||
+ rdbdir
|
||
+ "'"
|
||
)
|
||
|
||
print("[3/4] 请求面板在服务器执行 mongorestore（读取线上 .env 的 MONGODB_URI）...")
|
||
if not bt_exec_shell(cfg, shell, "/tmp"):
|
||
return 1
|
||
|
||
print("[4/4] 等待执行结果（最多 %ss）..." % args.exec_timeout)
|
||
ok = wait_exec_shell(cfg, timeout_sec=args.exec_timeout)
|
||
if ok:
|
||
print("[成功] 远程 mongorestore 已完成（请核对上方的 bson 日志）。工作目录: %s" % remote_work)
|
||
return 0
|
||
return 1
|
||
finally:
|
||
try:
|
||
os.remove(tgz)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|