"use client" import { useState, useEffect } from "react" import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" import { Button } from "@/components/ui/button" import { Badge } from "@/components/ui/badge" import { Input } from "@/components/ui/input" import { Progress } from "@/components/ui/progress" import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs" import { Switch } from "@/components/ui/switch" import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle, } from "@/components/ui/dialog" import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue, } from "@/components/ui/select" import { Label } from "@/components/ui/label" import { Textarea } from "@/components/ui/textarea" import { Zap, Play, Pause, Settings, Plus, CheckCircle2, Loader2, RefreshCw, Database, Edit, Trash2, AlertTriangle, FileCheck, Layers, ArrowRight, Sparkles, } from "lucide-react" // 清洗规则接口 interface CleaningRule { id: string name: string description: string field: string ruleType: 'format' | 'dedup' | 'enrich' | 'validate' | 'transform' ruleConfig: string enabled: boolean isDefault: boolean } // 清洗任务接口 interface CleaningTask { id: string name: string description: string sourceDb: string sourceCollection: string targetDb: string targetCollection: string rules: string[] // 应用的规则ID status: 'idle' | 'running' | 'paused' | 'completed' progress: number stats: { totalRecords: number processedRecords: number cleanedRecords: number issuesFound: number issuesFixed: number } } // 默认清洗规则 const DEFAULT_RULES: CleaningRule[] = [ { id: 'rule_phone', name: '手机号格式化', description: '将手机号格式化为标准11位格式', field: 'phone', ruleType: 'format', ruleConfig: `规则说明: 1. 去除空格、横杠、括号 2. 去除+86、0086前缀 3. 验证1[3-9]开头的11位数字 4. 无效手机号标记为null 示例: +86 138-0000-8888 → 13800008888 086 13800008888 → 13800008888 12345678901 → null (无效)`, enabled: true, isDefault: true, }, { id: 'rule_name', name: '姓名脱敏', description: '姓名中间字符用*替代', field: 'name', ruleType: 'transform', ruleConfig: `规则说明: 1. 保留第一个字(姓氏) 2. 中间名字用*替代 3. 保留最后一个字(可选) 示例: 张三 → 张* 王小明 → 王*明 欧阳修 → 欧*修`, enabled: true, isDefault: true, }, { id: 'rule_province', name: '省份标准化', description: '将省份名称标准化', field: 'province', ruleType: 'format', ruleConfig: `规则说明: 1. 去除"省"、"市"、"自治区"后缀 2. 直辖市特殊处理 3. 自治区简称处理 示例: 福建省 → 福建 北京市 → 北京 内蒙古自治区 → 内蒙古 广西壮族自治区 → 广西`, enabled: true, isDefault: true, }, { id: 'rule_dedup_phone', name: '手机号去重', description: '根据手机号去除重复记录', field: 'phone', ruleType: 'dedup', ruleConfig: `规则说明: 1. 以phone字段作为去重键 2. 保留最新的记录(按updated_at) 3. 合并其他记录的标签 去重策略: - 主键: phone - 保留策略: 最新记录 - 合并字段: tags, data_sources`, enabled: true, isDefault: true, }, { id: 'rule_qq', name: 'QQ号格式化', description: 'QQ号格式化为纯数字', field: 'qq', ruleType: 'format', ruleConfig: `规则说明: 1. 去除空格 2. 提取纯数字 3. 验证5-11位数字 示例: QQ: 12345678 → 12345678 12345678qq → 12345678 1234 → null (位数不足)`, enabled: true, isDefault: true, }, { id: 'rule_validate_rfm', name: 'RFM评分验证', description: '验证RFM评分在有效范围内', field: 'rfm_score', ruleType: 'validate', ruleConfig: `规则说明: 1. RFM评分范围: 0-100 2. 超出范围的修正为边界值 3. 非数字值设为null 验证逻辑: - rfm_score < 0 → 0 - rfm_score > 100 → 100 - 非数字 → null`, enabled: true, isDefault: true, }, { id: 'rule_enrich_level', name: '用户等级计算', description: '根据RFM评分计算用户等级', field: 'user_level', ruleType: 'enrich', ruleConfig: `规则说明: 根据rfm_score计算user_level: 等级映射: - rfm_score >= 90 → S级(钻石) - rfm_score >= 80 → A级(黄金) - rfm_score >= 60 → B级(白银) - rfm_score >= 40 → C级(青铜) - rfm_score < 40 → D级(潜力)`, enabled: true, isDefault: true, }, ] // 预定义清洗任务 const DEFAULT_TASKS: CleaningTask[] = [ { id: 'clean_1', name: 'QQ数据清洗', description: '清洗QQ库的手机号和地区数据', sourceDb: 'KR_腾讯', sourceCollection: 'qq_phone', targetDb: 'KR', targetCollection: '用户估值', rules: ['rule_phone', 'rule_province', 'rule_dedup_phone'], status: 'completed', progress: 100, stats: { totalRecords: 705000000, processedRecords: 705000000, cleanedRecords: 680000000, issuesFound: 25000000, issuesFixed: 24500000, }, }, { id: 'clean_2', name: '存客宝数据清洗', description: '清洗CRM数据,标准化字段格式', sourceDb: 'cunkebao_v3', sourceCollection: 'users', targetDb: 'KR_存客宝', targetCollection: '用户资产统一视图', rules: ['rule_phone', 'rule_name', 'rule_validate_rfm', 'rule_enrich_level'], status: 'running', progress: 45, stats: { totalRecords: 216000, processedRecords: 97200, cleanedRecords: 95000, issuesFound: 2200, issuesFixed: 2100, }, }, ] export default function DataCleaningPage() { const [tasks, setTasks] = useState(DEFAULT_TASKS) const [rules, setRules] = useState(DEFAULT_RULES) const [activeTab, setActiveTab] = useState("tasks") const [selectedTask, setSelectedTask] = useState(null) const [selectedRule, setSelectedRule] = useState(null) const [showCreateDialog, setShowCreateDialog] = useState(false) const [showRuleDialog, setShowRuleDialog] = useState(false) const [editingRuleConfig, setEditingRuleConfig] = useState('') const formatNumber = (num: number): string => { if (num >= 1000000000) return `${(num / 1000000000).toFixed(2)}B` if (num >= 1000000) return `${(num / 1000000).toFixed(1)}M` if (num >= 1000) return `${(num / 1000).toFixed(0)}K` return num.toLocaleString() } const getRuleTypeBadge = (type: string) => { switch (type) { case 'format': return 格式化 case 'dedup': return 去重 case 'enrich': return 补全 case 'validate': return 验证 case 'transform': return 转换 default: return {type} } } const getStatusBadge = (status: string) => { switch (status) { case 'completed': return 已完成 case 'running': return 运行中 case 'paused': return 已暂停 default: return 待执行 } } // 模拟进度更新 useEffect(() => { const interval = setInterval(() => { setTasks(prev => prev.map(t => { if (t.status === 'running' && t.progress < 100) { const newProgress = Math.min(t.progress + Math.random() * 3, 100) return { ...t, progress: newProgress, status: newProgress >= 100 ? 'completed' as const : t.status, stats: { ...t.stats, processedRecords: Math.floor(t.stats.totalRecords * newProgress / 100), cleanedRecords: Math.floor(t.stats.totalRecords * newProgress / 100 * 0.98), issuesFixed: Math.floor(t.stats.issuesFound * newProgress / 100 * 0.95), } } } return t })) }, 3000) return () => clearInterval(interval) }, []) const toggleRule = (ruleId: string) => { setRules(rules.map(r => r.id === ruleId ? { ...r, enabled: !r.enabled } : r)) } const openRuleEditor = (rule: CleaningRule) => { setSelectedRule(rule) setEditingRuleConfig(rule.ruleConfig) setShowRuleDialog(true) } const saveRule = () => { if (selectedRule) { setRules(rules.map(r => r.id === selectedRule.id ? { ...r, ruleConfig: editingRuleConfig } : r)) setShowRuleDialog(false) } } const runTask = (taskId: string) => { setTasks(tasks.map(t => t.id === taskId ? { ...t, status: 'running' as const } : t)) } return (
{/* 顶部标题 */}

AI数据清洗

配置清洗规则,自动格式化、去重、补全数据

{/* 统计卡片 */}

清洗规则

{rules.filter(r => r.enabled).length}/{rules.length}

已清洗

{formatNumber(tasks.reduce((sum, t) => sum + t.stats.cleanedRecords, 0))}

发现问题

{formatNumber(tasks.reduce((sum, t) => sum + t.stats.issuesFound, 0))}

已修复

{formatNumber(tasks.reduce((sum, t) => sum + t.stats.issuesFixed, 0))}

{/* 标签页 */} 清洗任务 清洗规则 {/* 清洗任务 */} {tasks.map(task => (

{task.name}

{getStatusBadge(task.status)}

{task.description}

{task.sourceDb}.{task.sourceCollection} {task.targetDb}.{task.targetCollection}
{task.status === 'idle' && ( )}
{/* 进度条 */}
处理进度 {task.progress.toFixed(0)}%
{/* 应用的规则 */}
{task.rules.map(ruleId => { const rule = rules.find(r => r.id === ruleId) return rule ? ( {rule.name} ) : null })}
{/* 统计 */}
总记录: {formatNumber(task.stats.totalRecords)}
已处理: {formatNumber(task.stats.processedRecords)}
已清洗: {formatNumber(task.stats.cleanedRecords)}
发现问题: {formatNumber(task.stats.issuesFound)}
已修复: {formatNumber(task.stats.issuesFixed)}
))}
{/* 清洗规则 */} {rules.map(rule => (
toggleRule(rule.id)} />

{rule.name}

{getRuleTypeBadge(rule.ruleType)} {rule.isDefault && 默认规则}

{rule.description}

字段: {rule.field}

))}
{/* 编辑规则弹窗 */} 编辑清洗规则 {selectedRule?.name} {selectedRule && (
{getRuleTypeBadge(selectedRule.ruleType)}