253 lines
9.8 KiB
Python
253 lines
9.8 KiB
Python
#!/usr/bin/env python3
|
||
# -*- coding: utf-8 -*-
|
||
"""
|
||
本地 MongoDB → 宝塔 Mongo（默认目标库 wzdj）一遍式同步。
|
||
|
||
依赖：mongodump、mongorestore（MongoDB Database Tools），且在 PATH 中。
|
||
|
||
示例（PowerShell）：
|
||
$env:REMOTE_MONGODB_URI = "mongodb://wzdj:你的密码@服务器IP:27017/wzdj?authSource=wzdj"
|
||
python scripts/mongo_sync_baota.py
|
||
|
||
仅导出、不恢复（并保留目录便于宝塔「导入」打包）：
|
||
python scripts/mongo_sync_baota.py --dump-only --dump-out ./build/mongo_dump_wz
|
||
|
||
本机推送时若 .env.production 里 Mongo 写的是 127.0.0.1（进程在服务器本机连库），脚本会读取环境变量
|
||
DEPLOY_HOST（未设置则 43.139.27.93，与 master.py 默认一致）自动替换远端主机再 mongorestore；
|
||
可用 --no-fix-remote-host 关闭。
|
||
"""
|
||
|
||
from __future__ import print_function
|
||
|
||
import argparse
|
||
import os
|
||
import shutil
|
||
import subprocess
|
||
import sys
|
||
import tempfile
|
||
from urllib.parse import urlparse, urlunparse, unquote
|
||
|
||
# 与 master.py 默认 DEPLOY_HOST 一致：用于本机 push 时把 .env.production 里的 127.0.0.1 换成可访问的服务器
|
||
DEFAULT_DEPLOY_HOST = "43.139.27.93"
|
||
|
||
|
||
def _replace_uri_hostname(uri, new_host):
|
||
"""仅替换 hostname，保留 userinfo、port、path、query。"""
|
||
try:
|
||
u = urlparse(uri.replace("mongodb+srv://", "mongodb://", 1))
|
||
if not u.scheme:
|
||
return uri
|
||
netloc = ""
|
||
if u.username:
|
||
netloc = u.username
|
||
if u.password:
|
||
netloc += ":" + u.password
|
||
netloc += "@"
|
||
netloc += new_host
|
||
if u.port:
|
||
netloc += ":%s" % u.port
|
||
return urlunparse((u.scheme, netloc, u.path, u.params, u.query, u.fragment))
|
||
except Exception:
|
||
return uri
|
||
|
||
|
||
def _db_name_from_uri(uri):
|
||
"""从 mongodb://host:port/dbname 取 dbname（忽略 query）。"""
|
||
try:
|
||
u = urlparse(uri.replace("mongodb+srv://", "mongodb://", 1))
|
||
path = (u.path or "").lstrip("/")
|
||
if not path:
|
||
return ""
|
||
return path.split("/")[0] if "/" in path else path.split("?")[0]
|
||
except Exception:
|
||
return ""
|
||
|
||
|
||
def _mask_uri(uri):
|
||
if not uri:
|
||
return uri
|
||
try:
|
||
u = urlparse(uri)
|
||
if u.username:
|
||
host = u.hostname or ""
|
||
port = ":%s" % u.port if u.port else ""
|
||
path = u.path or ""
|
||
q = "?%s" % u.query if u.query else ""
|
||
return "%s://%s:***@%s%s%s%s" % (u.scheme, u.username, host, port, path, q)
|
||
except Exception:
|
||
pass
|
||
return "***"
|
||
|
||
|
||
def _which(cmd):
|
||
return shutil.which(cmd) is not None
|
||
|
||
|
||
def _run(cmd, cwd=None):
|
||
disp = []
|
||
for x in cmd:
|
||
if x.startswith("--uri="):
|
||
disp.append("--uri=%s" % _mask_uri(x[6:]))
|
||
else:
|
||
disp.append(x)
|
||
print("[exec]", " ".join(disp))
|
||
r = subprocess.run(cmd, cwd=cwd, shell=False)
|
||
return r.returncode == 0
|
||
|
||
|
||
def _read_uri_from_env_file(path, key="MONGODB_URI"):
|
||
if not os.path.isfile(path):
|
||
return ""
|
||
try:
|
||
with open(path, "r", encoding="utf-8", errors="replace") as f:
|
||
for line in f:
|
||
s = line.strip()
|
||
if s.startswith("#") or "=" not in s:
|
||
continue
|
||
k, v = s.split("=", 1)
|
||
if k.strip() == key:
|
||
return v.strip().strip('"').strip("'")
|
||
except Exception:
|
||
pass
|
||
return ""
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(description="mongodump local -> mongorestore Baota wzdj")
|
||
parser.add_argument("--dump-only", action="store_true", help="只 dump 到临时目录并打印路径，不 restore")
|
||
parser.add_argument("--local-uri", default=os.environ.get("LOCAL_MONGODB_URI", ""))
|
||
parser.add_argument("--remote-uri", default=os.environ.get("REMOTE_MONGODB_URI", ""))
|
||
parser.add_argument("--local-db", default=os.environ.get("LOCAL_DB_NAME", ""))
|
||
parser.add_argument("--remote-db", default=os.environ.get("REMOTE_DB_NAME", "wzdj"))
|
||
parser.add_argument("--env-local", default="", help="从该文件读 LOCAL MONGODB_URI（key MONGODB_URI）")
|
||
parser.add_argument("--env-remote", default="", help="从该文件读 REMOTE MONGODB_URI（key MONGODB_URI）")
|
||
parser.add_argument(
|
||
"--dump-out",
|
||
default=os.environ.get("MONGO_DUMP_OUT", ""),
|
||
help="mongodump --out 目录（将生成 子目录/%s）；设置后不删目录。空则使用临时目录" % "库名",
|
||
)
|
||
parser.add_argument(
|
||
"--no-fix-remote-host",
|
||
action="store_true",
|
||
help="不要用 DEPLOY_HOST 替换远端 URI 中的 127.0.0.1/localhost",
|
||
)
|
||
parser.add_argument(
|
||
"--drop",
|
||
action="store_true",
|
||
help="mongorestore 在导入每个集合前先 drop（实现覆盖线上同库数据，慎用）",
|
||
)
|
||
parser.add_argument(
|
||
"--from-dump",
|
||
default="",
|
||
help="跳过 mongodump，直接使用已有 dump 目录（mongodump --out 的父目录，内含子目录=源库名）",
|
||
)
|
||
args = parser.parse_args()
|
||
|
||
root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
local_uri = (args.local_uri or "").strip()
|
||
if not local_uri:
|
||
env_path = (args.env_local or os.path.join(root, ".env.development")).strip()
|
||
local_uri = _read_uri_from_env_file(env_path)
|
||
if not local_uri:
|
||
local_uri = "mongodb://127.0.0.1:27017/wz"
|
||
|
||
remote_uri = (args.remote_uri or "").strip()
|
||
if not remote_uri:
|
||
env_path = (args.env_remote or os.path.join(root, ".env.production")).strip()
|
||
remote_uri = _read_uri_from_env_file(env_path)
|
||
if remote_uri and not args.no_fix_remote_host:
|
||
try:
|
||
u = urlparse(remote_uri.replace("mongodb+srv://", "mongodb://", 1))
|
||
h = (u.hostname or "").lower()
|
||
if h in ("127.0.0.1", "localhost"):
|
||
deploy_host = (os.environ.get("DEPLOY_HOST") or "").strip() or DEFAULT_DEPLOY_HOST
|
||
remote_uri = _replace_uri_hostname(remote_uri, deploy_host)
|
||
print("[信息] 远端 URI 原为主机本机地址，已改为 DEPLOY_HOST=%s 便于本机 mongorestore" % deploy_host)
|
||
except Exception:
|
||
pass
|
||
elif "127.0.0.1" in remote_uri or "localhost" in remote_uri:
|
||
print("[警告] REMOTE_MONGODB_URI 指向本机。可调 REMOTE_MONGODB_URI 或使用默认自动替换（勿加 --no-fix-remote-host）。")
|
||
|
||
local_db = (args.local_db or "").strip() or _db_name_from_uri(local_uri) or "wz"
|
||
remote_db = (args.remote_db or "wzdj").strip()
|
||
|
||
from_dump = (args.from_dump or "").strip()
|
||
if from_dump and args.dump_only:
|
||
print("[失败] 不能同时使用 --from-dump 与 --dump-only")
|
||
return 1
|
||
|
||
if not _which("mongorestore"):
|
||
print("[失败] 未找到 mongorestore，请先安装 MongoDB Database Tools 并加入 PATH。")
|
||
return 1
|
||
if not from_dump and not _which("mongodump"):
|
||
print("[失败] 未找到 mongodump，请先安装 MongoDB Database Tools 并加入 PATH。")
|
||
return 1
|
||
|
||
persistent = (args.dump_out or "").strip()
|
||
tmp = None
|
||
if from_dump:
|
||
dump_parent = os.path.abspath(from_dump)
|
||
elif persistent:
|
||
os.makedirs(persistent, exist_ok=True)
|
||
dump_parent = os.path.abspath(persistent)
|
||
else:
|
||
tmp = tempfile.mkdtemp(prefix="wz_mongo_dump_")
|
||
dump_parent = os.path.join(tmp, "out")
|
||
os.makedirs(dump_parent, exist_ok=True)
|
||
|
||
try:
|
||
if not from_dump:
|
||
# mongodump --out dump_parent 会在 dump_parent/<db>/ 下生成 .bson
|
||
dump_cmd = ["mongodump", "--uri=%s" % local_uri, "--db", local_db, "--out", dump_parent]
|
||
if not _run(dump_cmd):
|
||
print("[失败] mongodump 退出非 0")
|
||
return 1
|
||
else:
|
||
print("[信息] 使用已有 dump（--from-dump），跳过 mongodump")
|
||
|
||
src_dir = os.path.join(dump_parent, local_db)
|
||
if not os.path.isdir(src_dir):
|
||
print("[失败] 未找到 dump 子目录: %s" % src_dir)
|
||
return 1
|
||
|
||
if not from_dump:
|
||
print("[信息] 源库: %s (%s)" % (_mask_uri(local_uri), local_db))
|
||
print("[信息] dump 目录（含子库 %s）: %s" % (local_db, dump_parent))
|
||
|
||
if args.dump_only:
|
||
print("[信息] --dump-only：跳过后续 mongorestore。宝塔导入可压缩「%s」整目录或按面板要求选择。" % local_db)
|
||
return 0
|
||
|
||
if not remote_uri.strip():
|
||
print("[失败] 未设置远端 URI。请设置环境变量 REMOTE_MONGODB_URI 或 --remote-uri（或 --env-remote 指向含线上 MONGODB_URI 的文件）。")
|
||
return 1
|
||
|
||
ns_from = "%s.*" % local_db
|
||
ns_to = "%s.*" % remote_db
|
||
restore_cmd = [
|
||
"mongorestore",
|
||
"--uri=%s" % remote_uri,
|
||
"--nsFrom=%s" % ns_from,
|
||
"--nsTo=%s" % ns_to,
|
||
]
|
||
if args.drop:
|
||
restore_cmd.append("--drop")
|
||
restore_cmd.append(dump_parent)
|
||
print("[信息] 目标库: %s (%s)" % (_mask_uri(remote_uri), remote_db))
|
||
if not _run(restore_cmd):
|
||
print("[失败] mongorestore 退出非 0。")
|
||
print(" 若报错为 connection refused / timeout：线上 Mongo 常仅监听 127.0.0.1，公网连不到。")
|
||
print(" 做法：本机开 SSH 隧道 ssh -N -L 27018:127.0.0.1:27017 你的用户@服务器IP")
|
||
print(" 再设 REMOTE_MONGODB_URI 为 mongodb://wzdj:***@127.0.0.1:27018/wzdj?authSource=wzdj 并加 --no-fix-remote-host")
|
||
print(" 或把 dump 拷到服务器后对 127.0.0.1 执行 mongorestore；见 scripts/MONGO_宝塔同步.md")
|
||
return 1
|
||
print("[成功] 已从 %s 恢复到 %s" % (local_db, remote_db))
|
||
return 0
|
||
finally:
|
||
if tmp:
|
||
shutil.rmtree(tmp, ignore_errors=True)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
sys.exit(main())
|