Files
users/app/ai-agent/data-cleaning/page.tsx

644 lines
23 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"use client"
import { useState, useEffect } from "react"
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"
import { Button } from "@/components/ui/button"
import { Badge } from "@/components/ui/badge"
import { Input } from "@/components/ui/input"
import { Progress } from "@/components/ui/progress"
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"
import { Switch } from "@/components/ui/switch"
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog"
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select"
import { Label } from "@/components/ui/label"
import { Textarea } from "@/components/ui/textarea"
import {
Zap,
Play,
Pause,
Settings,
Plus,
CheckCircle2,
Loader2,
RefreshCw,
Database,
Edit,
Trash2,
AlertTriangle,
FileCheck,
Layers,
ArrowRight,
Sparkles,
} from "lucide-react"
// 清洗规则接口
interface CleaningRule {
id: string
name: string
description: string
field: string
ruleType: 'format' | 'dedup' | 'enrich' | 'validate' | 'transform'
ruleConfig: string
enabled: boolean
isDefault: boolean
}
// 清洗任务接口
interface CleaningTask {
id: string
name: string
description: string
sourceDb: string
sourceCollection: string
targetDb: string
targetCollection: string
rules: string[] // 应用的规则ID
status: 'idle' | 'running' | 'paused' | 'completed'
progress: number
stats: {
totalRecords: number
processedRecords: number
cleanedRecords: number
issuesFound: number
issuesFixed: number
}
}
// 默认清洗规则
const DEFAULT_RULES: CleaningRule[] = [
{
id: 'rule_phone',
name: '手机号格式化',
description: '将手机号格式化为标准11位格式',
field: 'phone',
ruleType: 'format',
ruleConfig: `规则说明:
1. 去除空格、横杠、括号
2. 去除+86、0086前缀
3. 验证1[3-9]开头的11位数字
4. 无效手机号标记为null
示例:
+86 138-0000-8888 → 13800008888
086 13800008888 → 13800008888
12345678901 → null (无效)`,
enabled: true,
isDefault: true,
},
{
id: 'rule_name',
name: '姓名脱敏',
description: '姓名中间字符用*替代',
field: 'name',
ruleType: 'transform',
ruleConfig: `规则说明:
1. 保留第一个字(姓氏)
2. 中间名字用*替代
3. 保留最后一个字(可选)
示例:
张三 → 张*
王小明 → 王*明
欧阳修 → 欧*修`,
enabled: true,
isDefault: true,
},
{
id: 'rule_province',
name: '省份标准化',
description: '将省份名称标准化',
field: 'province',
ruleType: 'format',
ruleConfig: `规则说明:
1. 去除"省"、"市"、"自治区"后缀
2. 直辖市特殊处理
3. 自治区简称处理
示例:
福建省 → 福建
北京市 → 北京
内蒙古自治区 → 内蒙古
广西壮族自治区 → 广西`,
enabled: true,
isDefault: true,
},
{
id: 'rule_dedup_phone',
name: '手机号去重',
description: '根据手机号去除重复记录',
field: 'phone',
ruleType: 'dedup',
ruleConfig: `规则说明:
1. 以phone字段作为去重键
2. 保留最新的记录按updated_at
3. 合并其他记录的标签
去重策略:
- 主键: phone
- 保留策略: 最新记录
- 合并字段: tags, data_sources`,
enabled: true,
isDefault: true,
},
{
id: 'rule_qq',
name: 'QQ号格式化',
description: 'QQ号格式化为纯数字',
field: 'qq',
ruleType: 'format',
ruleConfig: `规则说明:
1. 去除空格
2. 提取纯数字
3. 验证5-11位数字
示例:
QQ: 12345678 → 12345678
12345678qq → 12345678
1234 → null (位数不足)`,
enabled: true,
isDefault: true,
},
{
id: 'rule_validate_rfm',
name: 'RFM评分验证',
description: '验证RFM评分在有效范围内',
field: 'rfm_score',
ruleType: 'validate',
ruleConfig: `规则说明:
1. RFM评分范围: 0-100
2. 超出范围的修正为边界值
3. 非数字值设为null
验证逻辑:
- rfm_score < 0 → 0
- rfm_score > 100 → 100
- 非数字 → null`,
enabled: true,
isDefault: true,
},
{
id: 'rule_enrich_level',
name: '用户等级计算',
description: '根据RFM评分计算用户等级',
field: 'user_level',
ruleType: 'enrich',
ruleConfig: `规则说明:
根据rfm_score计算user_level:
等级映射:
- rfm_score >= 90 → S级(钻石)
- rfm_score >= 80 → A级(黄金)
- rfm_score >= 60 → B级(白银)
- rfm_score >= 40 → C级(青铜)
- rfm_score < 40 → D级(潜力)`,
enabled: true,
isDefault: true,
},
]
// 预定义清洗任务
const DEFAULT_TASKS: CleaningTask[] = [
{
id: 'clean_1',
name: 'QQ数据清洗',
description: '清洗QQ库的手机号和地区数据',
sourceDb: 'KR_腾讯',
sourceCollection: 'qq_phone',
targetDb: 'KR',
targetCollection: '用户估值',
rules: ['rule_phone', 'rule_province', 'rule_dedup_phone'],
status: 'completed',
progress: 100,
stats: {
totalRecords: 705000000,
processedRecords: 705000000,
cleanedRecords: 680000000,
issuesFound: 25000000,
issuesFixed: 24500000,
},
},
{
id: 'clean_2',
name: '存客宝数据清洗',
description: '清洗CRM数据标准化字段格式',
sourceDb: 'cunkebao_v3',
sourceCollection: 'users',
targetDb: 'KR_存客宝',
targetCollection: '用户资产统一视图',
rules: ['rule_phone', 'rule_name', 'rule_validate_rfm', 'rule_enrich_level'],
status: 'running',
progress: 45,
stats: {
totalRecords: 216000,
processedRecords: 97200,
cleanedRecords: 95000,
issuesFound: 2200,
issuesFixed: 2100,
},
},
]
export default function DataCleaningPage() {
const [tasks, setTasks] = useState<CleaningTask[]>(DEFAULT_TASKS)
const [rules, setRules] = useState<CleaningRule[]>(DEFAULT_RULES)
const [activeTab, setActiveTab] = useState("tasks")
const [selectedTask, setSelectedTask] = useState<CleaningTask | null>(null)
const [selectedRule, setSelectedRule] = useState<CleaningRule | null>(null)
const [showCreateDialog, setShowCreateDialog] = useState(false)
const [showRuleDialog, setShowRuleDialog] = useState(false)
const [editingRuleConfig, setEditingRuleConfig] = useState('')
const formatNumber = (num: number): string => {
if (num >= 1000000000) return `${(num / 1000000000).toFixed(2)}B`
if (num >= 1000000) return `${(num / 1000000).toFixed(1)}M`
if (num >= 1000) return `${(num / 1000).toFixed(0)}K`
return num.toLocaleString()
}
const getRuleTypeBadge = (type: string) => {
switch (type) {
case 'format': return <Badge className="bg-blue-100 text-blue-700"></Badge>
case 'dedup': return <Badge className="bg-purple-100 text-purple-700"></Badge>
case 'enrich': return <Badge className="bg-green-100 text-green-700"></Badge>
case 'validate': return <Badge className="bg-orange-100 text-orange-700"></Badge>
case 'transform': return <Badge className="bg-pink-100 text-pink-700"></Badge>
default: return <Badge>{type}</Badge>
}
}
const getStatusBadge = (status: string) => {
switch (status) {
case 'completed':
return <Badge className="bg-green-100 text-green-700"><CheckCircle2 className="h-3 w-3 mr-1" /></Badge>
case 'running':
return <Badge className="bg-blue-100 text-blue-700"><Loader2 className="h-3 w-3 mr-1 animate-spin" /></Badge>
case 'paused':
return <Badge className="bg-yellow-100 text-yellow-700"><Pause className="h-3 w-3 mr-1" /></Badge>
default:
return <Badge className="bg-gray-100 text-gray-700"></Badge>
}
}
// 模拟进度更新
useEffect(() => {
const interval = setInterval(() => {
setTasks(prev => prev.map(t => {
if (t.status === 'running' && t.progress < 100) {
const newProgress = Math.min(t.progress + Math.random() * 3, 100)
return {
...t,
progress: newProgress,
status: newProgress >= 100 ? 'completed' as const : t.status,
stats: {
...t.stats,
processedRecords: Math.floor(t.stats.totalRecords * newProgress / 100),
cleanedRecords: Math.floor(t.stats.totalRecords * newProgress / 100 * 0.98),
issuesFixed: Math.floor(t.stats.issuesFound * newProgress / 100 * 0.95),
}
}
}
return t
}))
}, 3000)
return () => clearInterval(interval)
}, [])
const toggleRule = (ruleId: string) => {
setRules(rules.map(r => r.id === ruleId ? { ...r, enabled: !r.enabled } : r))
}
const openRuleEditor = (rule: CleaningRule) => {
setSelectedRule(rule)
setEditingRuleConfig(rule.ruleConfig)
setShowRuleDialog(true)
}
const saveRule = () => {
if (selectedRule) {
setRules(rules.map(r => r.id === selectedRule.id ? { ...r, ruleConfig: editingRuleConfig } : r))
setShowRuleDialog(false)
}
}
const runTask = (taskId: string) => {
setTasks(tasks.map(t => t.id === taskId ? { ...t, status: 'running' as const } : t))
}
return (
<div className="min-h-screen bg-gradient-to-br from-slate-50 via-blue-50/30 to-purple-50/20">
<div className="p-6 space-y-6">
{/* 顶部标题 */}
<div className="flex items-center justify-between">
<div>
<h1 className="text-2xl font-bold text-gray-900">AI数据清洗</h1>
<p className="text-sm text-gray-500 mt-1"></p>
</div>
<div className="flex items-center gap-3">
<Button variant="outline">
<RefreshCw className="h-4 w-4 mr-2" />
</Button>
<Button onClick={() => setShowCreateDialog(true)}>
<Plus className="h-4 w-4 mr-2" />
</Button>
</div>
</div>
{/* 统计卡片 */}
<div className="grid grid-cols-4 gap-4">
<Card className="border-0 shadow-sm bg-white/80 backdrop-blur">
<CardContent className="p-4">
<div className="flex items-center justify-between">
<div>
<p className="text-sm text-gray-500"></p>
<p className="text-2xl font-bold">{rules.filter(r => r.enabled).length}/{rules.length}</p>
</div>
<div className="p-3 rounded-xl bg-blue-100">
<FileCheck className="h-6 w-6 text-blue-600" />
</div>
</div>
</CardContent>
</Card>
<Card className="border-0 shadow-sm bg-white/80 backdrop-blur">
<CardContent className="p-4">
<div className="flex items-center justify-between">
<div>
<p className="text-sm text-gray-500"></p>
<p className="text-2xl font-bold text-green-600">{formatNumber(tasks.reduce((sum, t) => sum + t.stats.cleanedRecords, 0))}</p>
</div>
<div className="p-3 rounded-xl bg-green-100">
<CheckCircle2 className="h-6 w-6 text-green-600" />
</div>
</div>
</CardContent>
</Card>
<Card className="border-0 shadow-sm bg-white/80 backdrop-blur">
<CardContent className="p-4">
<div className="flex items-center justify-between">
<div>
<p className="text-sm text-gray-500"></p>
<p className="text-2xl font-bold text-orange-600">{formatNumber(tasks.reduce((sum, t) => sum + t.stats.issuesFound, 0))}</p>
</div>
<div className="p-3 rounded-xl bg-orange-100">
<AlertTriangle className="h-6 w-6 text-orange-600" />
</div>
</div>
</CardContent>
</Card>
<Card className="border-0 shadow-sm bg-white/80 backdrop-blur">
<CardContent className="p-4">
<div className="flex items-center justify-between">
<div>
<p className="text-sm text-gray-500"></p>
<p className="text-2xl font-bold text-purple-600">{formatNumber(tasks.reduce((sum, t) => sum + t.stats.issuesFixed, 0))}</p>
</div>
<div className="p-3 rounded-xl bg-purple-100">
<Zap className="h-6 w-6 text-purple-600" />
</div>
</div>
</CardContent>
</Card>
</div>
{/* 标签页 */}
<Tabs value={activeTab} onValueChange={setActiveTab}>
<TabsList className="bg-white/80 backdrop-blur">
<TabsTrigger value="tasks"></TabsTrigger>
<TabsTrigger value="rules"></TabsTrigger>
</TabsList>
{/* 清洗任务 */}
<TabsContent value="tasks" className="mt-4 space-y-3">
{tasks.map(task => (
<Card key={task.id} className="border-0 shadow-sm bg-white/80 backdrop-blur">
<CardContent className="p-4">
<div className="flex items-start justify-between mb-3">
<div className="flex items-center gap-3">
<div className="p-2.5 rounded-xl bg-gradient-to-r from-blue-500 to-purple-500">
<Zap className="h-5 w-5 text-white" />
</div>
<div>
<div className="flex items-center gap-2">
<h3 className="font-semibold text-gray-900">{task.name}</h3>
{getStatusBadge(task.status)}
</div>
<p className="text-sm text-gray-500">{task.description}</p>
<div className="flex items-center gap-2 text-xs text-gray-400 mt-1 font-mono">
<span>{task.sourceDb}.{task.sourceCollection}</span>
<ArrowRight className="h-3 w-3" />
<span>{task.targetDb}.{task.targetCollection}</span>
</div>
</div>
</div>
<div className="flex items-center gap-2">
{task.status === 'idle' && (
<Button size="sm" onClick={() => runTask(task.id)}>
<Play className="h-4 w-4 mr-1" />
</Button>
)}
<Button variant="ghost" size="sm">
<Settings className="h-4 w-4" />
</Button>
</div>
</div>
{/* 进度条 */}
<div className="mb-3">
<div className="flex items-center justify-between text-sm mb-1">
<span className="text-gray-500"></span>
<span className="font-medium">{task.progress.toFixed(0)}%</span>
</div>
<Progress value={task.progress} className="h-2" />
</div>
{/* 应用的规则 */}
<div className="flex flex-wrap gap-1 mb-3">
{task.rules.map(ruleId => {
const rule = rules.find(r => r.id === ruleId)
return rule ? (
<Badge key={ruleId} variant="secondary" className="text-xs">
{rule.name}
</Badge>
) : null
})}
</div>
{/* 统计 */}
<div className="grid grid-cols-5 gap-4 text-sm pt-3 border-t">
<div>
<span className="text-gray-500">:</span>
<span className="ml-2 font-medium">{formatNumber(task.stats.totalRecords)}</span>
</div>
<div>
<span className="text-gray-500">:</span>
<span className="ml-2 font-medium text-blue-600">{formatNumber(task.stats.processedRecords)}</span>
</div>
<div>
<span className="text-gray-500">:</span>
<span className="ml-2 font-medium text-green-600">{formatNumber(task.stats.cleanedRecords)}</span>
</div>
<div>
<span className="text-gray-500">:</span>
<span className="ml-2 font-medium text-orange-600">{formatNumber(task.stats.issuesFound)}</span>
</div>
<div>
<span className="text-gray-500">:</span>
<span className="ml-2 font-medium text-purple-600">{formatNumber(task.stats.issuesFixed)}</span>
</div>
</div>
</CardContent>
</Card>
))}
</TabsContent>
{/* 清洗规则 */}
<TabsContent value="rules" className="mt-4 space-y-3">
{rules.map(rule => (
<Card key={rule.id} className="border-0 shadow-sm bg-white/80 backdrop-blur">
<CardContent className="p-4">
<div className="flex items-start justify-between">
<div className="flex items-center gap-3">
<Switch checked={rule.enabled} onCheckedChange={() => toggleRule(rule.id)} />
<div>
<div className="flex items-center gap-2">
<h3 className="font-semibold text-gray-900">{rule.name}</h3>
{getRuleTypeBadge(rule.ruleType)}
{rule.isDefault && <Badge variant="outline" className="text-xs"></Badge>}
</div>
<p className="text-sm text-gray-500">{rule.description}</p>
<p className="text-xs text-gray-400 mt-1">: <code className="bg-gray-100 px-1 rounded">{rule.field}</code></p>
</div>
</div>
<Button variant="outline" size="sm" onClick={() => openRuleEditor(rule)}>
<Edit className="h-4 w-4 mr-1" />
</Button>
</div>
</CardContent>
</Card>
))}
</TabsContent>
</Tabs>
{/* 编辑规则弹窗 */}
<Dialog open={showRuleDialog} onOpenChange={setShowRuleDialog}>
<DialogContent className="max-w-2xl">
<DialogHeader>
<DialogTitle></DialogTitle>
<DialogDescription>{selectedRule?.name}</DialogDescription>
</DialogHeader>
{selectedRule && (
<div className="space-y-4 py-4">
<div className="grid grid-cols-2 gap-4">
<div className="space-y-2">
<Label></Label>
<Input value={selectedRule.name} readOnly className="bg-gray-50" />
</div>
<div className="space-y-2">
<Label></Label>
<Input value={selectedRule.field} readOnly className="bg-gray-50 font-mono" />
</div>
</div>
<div className="space-y-2">
<Label></Label>
{getRuleTypeBadge(selectedRule.ruleType)}
</div>
<div className="space-y-2">
<Label></Label>
<Textarea
value={editingRuleConfig}
onChange={(e) => setEditingRuleConfig(e.target.value)}
rows={12}
className="font-mono text-sm"
/>
</div>
</div>
)}
<DialogFooter>
<Button variant="outline" onClick={() => setShowRuleDialog(false)}></Button>
<Button onClick={saveRule}>
<CheckCircle2 className="h-4 w-4 mr-2" />
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
{/* 新建任务弹窗 */}
<Dialog open={showCreateDialog} onOpenChange={setShowCreateDialog}>
<DialogContent className="max-w-lg">
<DialogHeader>
<DialogTitle></DialogTitle>
<DialogDescription></DialogDescription>
</DialogHeader>
<div className="space-y-4 py-4">
<div className="space-y-2">
<Label></Label>
<Input placeholder="例如:用户数据清洗" />
</div>
<div className="grid grid-cols-2 gap-4">
<div className="space-y-2">
<Label></Label>
<Select>
<SelectTrigger><SelectValue placeholder="选择数据库" /></SelectTrigger>
<SelectContent>
<SelectItem value="kr_tencent">KR_腾讯</SelectItem>
<SelectItem value="kr_weibo">KR_微博</SelectItem>
<SelectItem value="ckb">cunkebao_v3</SelectItem>
</SelectContent>
</Select>
</div>
<div className="space-y-2">
<Label></Label>
<Select>
<SelectTrigger><SelectValue placeholder="选择目标" /></SelectTrigger>
<SelectContent>
<SelectItem value="kr_valuation">KR.</SelectItem>
<SelectItem value="kr_ckb">KR_存客宝.</SelectItem>
</SelectContent>
</Select>
</div>
</div>
<div className="space-y-2">
<Label></Label>
<div className="space-y-2 max-h-[200px] overflow-y-auto">
{rules.map(rule => (
<div key={rule.id} className="flex items-center justify-between p-2 rounded-lg border">
<div className="flex items-center gap-2">
<Switch defaultChecked={rule.enabled} />
<span className="text-sm">{rule.name}</span>
</div>
{getRuleTypeBadge(rule.ruleType)}
</div>
))}
</div>
</div>
</div>
<DialogFooter>
<Button variant="outline" onClick={() => setShowCreateDialog(false)}></Button>
<Button onClick={() => setShowCreateDialog(false)}>
<Zap className="h-4 w-4 mr-2" />
</Button>
</DialogFooter>
</DialogContent>
</Dialog>
</div>
</div>
)
}