chore: 以本地为准，上传全部并替换 GitHub
This commit is contained in:
卡若
510
app/data-ingestion/cleaning/page.tsx
Normal file
510
app/data-ingestion/cleaning/page.tsx
Normal file
@@ -0,0 +1,510 @@
|
||||
"use client"
|
||||
|
||||
import { useState, useEffect } from "react"
|
||||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"
|
||||
import { Button } from "@/components/ui/button"
|
||||
import { Badge } from "@/components/ui/badge"
|
||||
import { Input } from "@/components/ui/input"
|
||||
import { Switch } from "@/components/ui/switch"
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from "@/components/ui/dialog"
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@/components/ui/select"
|
||||
import { Label } from "@/components/ui/label"
|
||||
import { Textarea } from "@/components/ui/textarea"
|
||||
import {
|
||||
Zap,
|
||||
Plus,
|
||||
Search,
|
||||
RefreshCw,
|
||||
Play,
|
||||
CheckCircle2,
|
||||
AlertTriangle,
|
||||
Settings,
|
||||
Trash2,
|
||||
Copy,
|
||||
Eye,
|
||||
Loader2,
|
||||
Filter,
|
||||
Shield,
|
||||
Sparkles,
|
||||
FileText,
|
||||
Database,
|
||||
} from "lucide-react"
|
||||
|
||||
// 清洗规则接口
|
||||
interface CleaningRule {
|
||||
id: string
|
||||
name: string
|
||||
description: string
|
||||
type: 'format' | 'dedup' | 'validate' | 'enrich' | 'mask'
|
||||
targetField: string
|
||||
config: {
|
||||
pattern?: string
|
||||
replacement?: string
|
||||
validation?: string
|
||||
enrichSource?: string
|
||||
maskType?: string
|
||||
}
|
||||
priority: number
|
||||
enabled: boolean
|
||||
createdAt: string
|
||||
lastRunAt?: string
|
||||
processedCount?: number
|
||||
}
|
||||
|
||||
// 规则类型配置
|
||||
const RULE_TYPES = {
|
||||
format: { label: '格式化', color: 'bg-blue-100 text-blue-700', icon: FileText, desc: '统一数据格式' },
|
||||
dedup: { label: '去重', color: 'bg-green-100 text-green-700', icon: Copy, desc: '删除重复数据' },
|
||||
validate: { label: '验证', color: 'bg-yellow-100 text-yellow-700', icon: Shield, desc: '校验数据有效性' },
|
||||
enrich: { label: '补全', color: 'bg-purple-100 text-purple-700', icon: Sparkles, desc: '补充缺失字段' },
|
||||
mask: { label: '脱敏', color: 'bg-red-100 text-red-700', icon: Eye, desc: '敏感数据脱敏' },
|
||||
}
|
||||
|
||||
export default function CleaningRulesPage() {
|
||||
const [rules, setRules] = useState<CleaningRule[]>([])
|
||||
const [loading, setLoading] = useState(true)
|
||||
const [executing, setExecuting] = useState<string | null>(null)
|
||||
const [showAddDialog, setShowAddDialog] = useState(false)
|
||||
const [searchQuery, setSearchQuery] = useState("")
|
||||
const [stats, setStats] = useState({
|
||||
totalRules: 0,
|
||||
enabledRules: 0,
|
||||
totalProcessed: 0
|
||||
})
|
||||
|
||||
// 新规则表单
|
||||
const [newRule, setNewRule] = useState({
|
||||
name: '',
|
||||
description: '',
|
||||
type: 'format' as const,
|
||||
targetField: 'phone',
|
||||
pattern: '',
|
||||
replacement: '',
|
||||
validation: ''
|
||||
})
|
||||
|
||||
// 加载规则
|
||||
useEffect(() => {
|
||||
loadRules()
|
||||
}, [])
|
||||
|
||||
const loadRules = async () => {
|
||||
setLoading(true)
|
||||
try {
|
||||
const [rulesRes, statsRes] = await Promise.all([
|
||||
fetch('/api/cleaning-rules'),
|
||||
fetch('/api/cleaning-rules?action=stats')
|
||||
])
|
||||
const rulesData = await rulesRes.json()
|
||||
const statsData = await statsRes.json()
|
||||
|
||||
if (rulesData.success) setRules(rulesData.rules)
|
||||
if (statsData.success) setStats(statsData.stats)
|
||||
} catch (error) {
|
||||
console.error('加载清洗规则失败:', error)
|
||||
} finally {
|
||||
setLoading(false)
|
||||
}
|
||||
}
|
||||
|
||||
// 执行单个规则
|
||||
const executeRule = async (ruleId: string) => {
|
||||
setExecuting(ruleId)
|
||||
try {
|
||||
const res = await fetch('/api/cleaning-rules', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ action: 'execute', ruleId, limit: 1000 })
|
||||
})
|
||||
const data = await res.json()
|
||||
if (data.success) {
|
||||
alert(`执行完成: 处理 ${data.result.processed} 条，清洗 ${data.result.cleaned} 条`)
|
||||
loadRules()
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('执行失败:', error)
|
||||
} finally {
|
||||
setExecuting(null)
|
||||
}
|
||||
}
|
||||
|
||||
// 执行所有规则
|
||||
const executeAllRules = async () => {
|
||||
setExecuting('all')
|
||||
try {
|
||||
const res = await fetch('/api/cleaning-rules', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ action: 'execute_all', limit: 100 })
|
||||
})
|
||||
const data = await res.json()
|
||||
if (data.success) {
|
||||
alert(`批量执行完成: 执行了 ${data.executed} 条规则`)
|
||||
loadRules()
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('批量执行失败:', error)
|
||||
} finally {
|
||||
setExecuting(null)
|
||||
}
|
||||
}
|
||||
|
||||
// 切换规则状态
|
||||
const toggleRule = async (ruleId: string) => {
|
||||
try {
|
||||
await fetch('/api/cleaning-rules', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ action: 'toggle', ruleId })
|
||||
})
|
||||
setRules(rules.map(r => r.id === ruleId ? { ...r, enabled: !r.enabled } : r))
|
||||
} catch (error) {
|
||||
console.error('切换失败:', error)
|
||||
}
|
||||
}
|
||||
|
||||
// 添加规则
|
||||
const handleAddRule = async () => {
|
||||
try {
|
||||
const res = await fetch('/api/cleaning-rules', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({
|
||||
action: 'create',
|
||||
rule: {
|
||||
name: newRule.name,
|
||||
description: newRule.description,
|
||||
type: newRule.type,
|
||||
targetField: newRule.targetField,
|
||||
config: {
|
||||
pattern: newRule.pattern,
|
||||
replacement: newRule.replacement,
|
||||
validation: newRule.validation
|
||||
}
|
||||
}
|
||||
})
|
||||
})
|
||||
const data = await res.json()
|
||||
if (data.success) {
|
||||
setShowAddDialog(false)
|
||||
loadRules()
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('添加失败:', error)
|
||||
}
|
||||
}
|
||||
|
||||
const formatNumber = (num: number): string => {
|
||||
if (num >= 1000000000) return `${(num / 1000000000).toFixed(2)}B`
|
||||
if (num >= 1000000) return `${(num / 1000000).toFixed(1)}M`
|
||||
if (num >= 1000) return `${(num / 1000).toFixed(0)}K`
|
||||
return num.toLocaleString()
|
||||
}
|
||||
|
||||
// 筛选规则
|
||||
const filteredRules = rules.filter(rule => {
|
||||
if (searchQuery) {
|
||||
const query = searchQuery.toLowerCase()
|
||||
return rule.name.toLowerCase().includes(query) ||
|
||||
rule.description.toLowerCase().includes(query) ||
|
||||
rule.targetField.toLowerCase().includes(query)
|
||||
}
|
||||
return true
|
||||
})
|
||||
|
||||
return (
|
||||
<div className="min-h-screen bg-gradient-to-br from-slate-50 via-blue-50/30 to-purple-50/20">
|
||||
<div className="p-6 space-y-6">
|
||||
{/* 顶部标题 */}
|
||||
<div className="flex flex-col md:flex-row md:items-center md:justify-between gap-4">
|
||||
<div>
|
||||
<h1 className="text-2xl font-bold text-gray-900">清洗规则</h1>
|
||||
<p className="text-sm text-gray-500 mt-1">
|
||||
配置数据清洗规则，实现数据格式化、去重、验证、补全和脱敏
|
||||
</p>
|
||||
</div>
|
||||
<div className="flex items-center gap-3">
|
||||
<Button
|
||||
variant="outline"
|
||||
onClick={executeAllRules}
|
||||
disabled={executing === 'all'}
|
||||
>
|
||||
{executing === 'all' ? (
|
||||
<Loader2 className="h-4 w-4 mr-2 animate-spin" />
|
||||
) : (
|
||||
<Play className="h-4 w-4 mr-2" />
|
||||
)}
|
||||
执行所有规则
|
||||
</Button>
|
||||
<Button onClick={() => setShowAddDialog(true)}>
|
||||
<Plus className="h-4 w-4 mr-2" />
|
||||
添加规则
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* 统计卡片 */}
|
||||
<div className="grid grid-cols-1 md:grid-cols-4 gap-4">
|
||||
<Card className="bg-white/80 backdrop-blur border-0 shadow-sm">
|
||||
<CardContent className="p-4">
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<p className="text-sm text-gray-500">规则总数</p>
|
||||
<p className="text-2xl font-bold text-gray-900">{stats.totalRules}</p>
|
||||
</div>
|
||||
<div className="w-12 h-12 rounded-xl bg-blue-100 flex items-center justify-center">
|
||||
<Filter className="h-6 w-6 text-blue-600" />
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
<Card className="bg-white/80 backdrop-blur border-0 shadow-sm">
|
||||
<CardContent className="p-4">
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<p className="text-sm text-gray-500">已启用</p>
|
||||
<p className="text-2xl font-bold text-green-600">{stats.enabledRules}</p>
|
||||
</div>
|
||||
<div className="w-12 h-12 rounded-xl bg-green-100 flex items-center justify-center">
|
||||
<CheckCircle2 className="h-6 w-6 text-green-600" />
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
<Card className="bg-white/80 backdrop-blur border-0 shadow-sm">
|
||||
<CardContent className="p-4">
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<p className="text-sm text-gray-500">已处理数据</p>
|
||||
<p className="text-2xl font-bold text-gray-900">{formatNumber(stats.totalProcessed)}</p>
|
||||
</div>
|
||||
<div className="w-12 h-12 rounded-xl bg-purple-100 flex items-center justify-center">
|
||||
<Database className="h-6 w-6 text-purple-600" />
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
<Card className="bg-white/80 backdrop-blur border-0 shadow-sm">
|
||||
<CardContent className="p-4">
|
||||
<div className="flex items-center justify-between">
|
||||
<div>
|
||||
<p className="text-sm text-gray-500">规则类型</p>
|
||||
<p className="text-2xl font-bold text-gray-900">5种</p>
|
||||
</div>
|
||||
<div className="w-12 h-12 rounded-xl bg-orange-100 flex items-center justify-center">
|
||||
<Zap className="h-6 w-6 text-orange-600" />
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
|
||||
{/* 搜索 */}
|
||||
<div className="relative max-w-md">
|
||||
<Search className="absolute left-3 top-1/2 -translate-y-1/2 h-4 w-4 text-gray-400" />
|
||||
<Input
|
||||
placeholder="搜索规则名称、描述..."
|
||||
value={searchQuery}
|
||||
onChange={(e) => setSearchQuery(e.target.value)}
|
||||
className="pl-9 bg-white"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{/* 规则列表 */}
|
||||
{loading ? (
|
||||
<div className="flex items-center justify-center py-20">
|
||||
<Loader2 className="h-8 w-8 animate-spin text-blue-500" />
|
||||
<span className="ml-3 text-gray-500">加载清洗规则...</span>
|
||||
</div>
|
||||
) : (
|
||||
<div className="space-y-4">
|
||||
{filteredRules.map((rule, index) => {
|
||||
const typeConfig = RULE_TYPES[rule.type]
|
||||
const TypeIcon = typeConfig.icon
|
||||
|
||||
return (
|
||||
<Card key={rule.id} className="bg-white/80 backdrop-blur border-0 shadow-sm">
|
||||
<CardContent className="p-5">
|
||||
<div className="flex items-start justify-between">
|
||||
<div className="flex items-start gap-4">
|
||||
<div className="flex items-center gap-3">
|
||||
<span className="text-lg font-bold text-gray-300 w-6">{index + 1}</span>
|
||||
<div className={`p-2.5 rounded-xl ${typeConfig.color.replace('text-', 'bg-').replace('-700', '-100')}`}>
|
||||
<TypeIcon className={`h-5 w-5 ${typeConfig.color.split(' ')[1]}`} />
|
||||
</div>
|
||||
</div>
|
||||
<div className="flex-1">
|
||||
<div className="flex items-center gap-2 mb-1">
|
||||
<h3 className="font-semibold text-gray-900">{rule.name}</h3>
|
||||
<Badge className={typeConfig.color}>{typeConfig.label}</Badge>
|
||||
{!rule.enabled && (
|
||||
<Badge variant="outline" className="text-gray-500">已禁用</Badge>
|
||||
)}
|
||||
</div>
|
||||
<p className="text-sm text-gray-600 mb-2">{rule.description}</p>
|
||||
<div className="flex items-center gap-4 text-xs text-gray-500">
|
||||
<span className="flex items-center gap-1">
|
||||
<Database className="h-3 w-3" />
|
||||
目标字段: <span className="font-mono text-purple-600">{rule.targetField}</span>
|
||||
</span>
|
||||
{rule.lastRunAt && (
|
||||
<span>上次执行: {rule.lastRunAt}</span>
|
||||
)}
|
||||
{rule.processedCount !== undefined && (
|
||||
<span>已处理: {formatNumber(rule.processedCount)} 条</span>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="flex items-center gap-3">
|
||||
<Switch
|
||||
checked={rule.enabled}
|
||||
onCheckedChange={() => toggleRule(rule.id)}
|
||||
/>
|
||||
<Button
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => executeRule(rule.id)}
|
||||
disabled={executing === rule.id || !rule.enabled}
|
||||
>
|
||||
{executing === rule.id ? (
|
||||
<Loader2 className="h-4 w-4 animate-spin" />
|
||||
) : (
|
||||
<Play className="h-4 w-4" />
|
||||
)}
|
||||
</Button>
|
||||
<Button variant="ghost" size="sm">
|
||||
<Settings className="h-4 w-4" />
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</CardContent>
|
||||
</Card>
|
||||
)
|
||||
})}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* 添加规则弹窗 */}
|
||||
<Dialog open={showAddDialog} onOpenChange={setShowAddDialog}>
|
||||
<DialogContent className="max-w-xl">
|
||||
<DialogHeader>
|
||||
<DialogTitle>添加清洗规则</DialogTitle>
|
||||
<DialogDescription>
|
||||
创建新的数据清洗规则，应用于数据接入过程
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<div className="space-y-4 py-4">
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div className="space-y-2">
|
||||
<Label>规则名称 *</Label>
|
||||
<Input
|
||||
placeholder="例如：手机号格式化"
|
||||
value={newRule.name}
|
||||
onChange={(e) => setNewRule({ ...newRule, name: e.target.value })}
|
||||
/>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label>规则类型 *</Label>
|
||||
<Select
|
||||
value={newRule.type}
|
||||
onValueChange={(v: any) => setNewRule({ ...newRule, type: v })}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{Object.entries(RULE_TYPES).map(([key, config]) => (
|
||||
<SelectItem key={key} value={key}>
|
||||
<div className="flex items-center gap-2">
|
||||
<config.icon className="h-4 w-4" />
|
||||
{config.label} - {config.desc}
|
||||
</div>
|
||||
</SelectItem>
|
||||
))}
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<Label>规则描述</Label>
|
||||
<Textarea
|
||||
placeholder="描述该规则的作用..."
|
||||
value={newRule.description}
|
||||
onChange={(e) => setNewRule({ ...newRule, description: e.target.value })}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-2 gap-4">
|
||||
<div className="space-y-2">
|
||||
<Label>目标字段 *</Label>
|
||||
<Select
|
||||
value={newRule.targetField}
|
||||
onValueChange={(v) => setNewRule({ ...newRule, targetField: v })}
|
||||
>
|
||||
<SelectTrigger>
|
||||
<SelectValue />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
<SelectItem value="phone">phone (手机号)</SelectItem>
|
||||
<SelectItem value="email">email (邮箱)</SelectItem>
|
||||
<SelectItem value="name">name (姓名)</SelectItem>
|
||||
<SelectItem value="idcard">idcard (身份证)</SelectItem>
|
||||
<SelectItem value="address">address (地址)</SelectItem>
|
||||
<SelectItem value="province">province (省份)</SelectItem>
|
||||
<SelectItem value="city">city (城市)</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label>匹配模式（正则）</Label>
|
||||
<Input
|
||||
placeholder="例如：^1[3-9]\d{9}$"
|
||||
value={newRule.pattern}
|
||||
onChange={(e) => setNewRule({ ...newRule, pattern: e.target.value })}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{newRule.type === 'format' && (
|
||||
<div className="space-y-2">
|
||||
<Label>替换为</Label>
|
||||
<Input
|
||||
placeholder="替换后的格式"
|
||||
value={newRule.replacement}
|
||||
onChange={(e) => setNewRule({ ...newRule, replacement: e.target.value })}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<DialogFooter>
|
||||
<Button variant="outline" onClick={() => setShowAddDialog(false)}>
|
||||
取消
|
||||
</Button>
|
||||
<Button onClick={handleAddRule}>
|
||||
创建规则
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
</div>
|
||||
</div>
|
||||
)
|
||||
}
|
||||
Reference in New Issue
Block a user