b17b488f8e
Reorganize navigation and module structure based on new requirements. Co-authored-by: null <4804959+fnvtk@users.noreply.github.com>
349 lines
14 KiB
TypeScript
349 lines
14 KiB
TypeScript
"use client"
|
||
|
||
import { useState } from "react"
|
||
import { Zap, Plus, Play, CheckCircle, Clock, AlertTriangle, FileText, Settings, Code, ArrowRight } from "lucide-react"
|
||
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"
|
||
import { Button } from "@/components/ui/button"
|
||
import { Badge } from "@/components/ui/badge"
|
||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"
|
||
import { CreateCleaningRuleDialog } from "@/components/dialogs/create-cleaning-rule-dialog"
|
||
import { CleaningRuleSettingsDialog } from "@/components/dialogs/cleaning-rule-settings-dialog"
|
||
import { CleaningRuleLogsDialog } from "@/components/dialogs/cleaning-rule-logs-dialog"
|
||
|
||
interface CleaningRule {
|
||
id: string
|
||
name: string
|
||
description: string
|
||
type: "dedup" | "format" | "mapping" | "validate" | "transform"
|
||
sourceTable: string
|
||
targetTable: string
|
||
status: "active" | "inactive" | "testing"
|
||
lastRun: string
|
||
processedRecords: number
|
||
errorRate: number
|
||
}
|
||
|
||
const MOCK_RULES: CleaningRule[] = [
|
||
{
|
||
id: "1",
|
||
name: "用户手机号标准化",
|
||
description: "统一手机号格式，去除+86前缀和空格",
|
||
type: "format",
|
||
sourceTable: "raw_users",
|
||
targetTable: "clean_users",
|
||
status: "active",
|
||
lastRun: "2025-12-12 14:30:00",
|
||
processedRecords: 12895634,
|
||
errorRate: 0.02,
|
||
},
|
||
{
|
||
id: "2",
|
||
name: "交易记录去重",
|
||
description: "基于交易ID和时间戳去除重复记录",
|
||
type: "dedup",
|
||
sourceTable: "raw_transactions",
|
||
targetTable: "clean_transactions",
|
||
status: "active",
|
||
lastRun: "2025-12-12 14:25:00",
|
||
processedRecords: 89234567,
|
||
errorRate: 0.01,
|
||
},
|
||
{
|
||
id: "3",
|
||
name: "城市编码映射",
|
||
description: "将城市名称映射为标准行政区划代码",
|
||
type: "mapping",
|
||
sourceTable: "raw_users",
|
||
targetTable: "clean_users",
|
||
status: "active",
|
||
lastRun: "2025-12-12 14:20:00",
|
||
processedRecords: 45678901,
|
||
errorRate: 0.05,
|
||
},
|
||
{
|
||
id: "4",
|
||
name: "金额字段校验",
|
||
description: "校验金额字段非负且在合理范围内",
|
||
type: "validate",
|
||
sourceTable: "raw_transactions",
|
||
targetTable: "clean_transactions",
|
||
status: "testing",
|
||
lastRun: "2025-12-12 10:00:00",
|
||
processedRecords: 1000000,
|
||
errorRate: 0.12,
|
||
},
|
||
{
|
||
id: "5",
|
||
name: "时间戳转换",
|
||
description: "将各种时间格式统一转换为UTC时间戳",
|
||
type: "transform",
|
||
sourceTable: "raw_events",
|
||
targetTable: "clean_events",
|
||
status: "active",
|
||
lastRun: "2025-12-12 14:35:00",
|
||
processedRecords: 256789012,
|
||
errorRate: 0.001,
|
||
},
|
||
]
|
||
|
||
const TYPE_CONFIG = {
|
||
dedup: { label: "去重", color: "bg-blue-100 text-blue-700" },
|
||
format: { label: "格式化", color: "bg-green-100 text-green-700" },
|
||
mapping: { label: "映射", color: "bg-purple-100 text-purple-700" },
|
||
validate: { label: "校验", color: "bg-yellow-100 text-yellow-700" },
|
||
transform: { label: "转换", color: "bg-orange-100 text-orange-700" },
|
||
}
|
||
|
||
const PRESETS = [
|
||
{ name: "去除首尾空格", code: "TRIM(column)" },
|
||
{ name: "转大写", code: "UPPER(column)" },
|
||
{ name: "转小写", code: "LOWER(column)" },
|
||
{ name: "日期格式化", code: "DATE_FORMAT(column, '%Y-%m-%d')" },
|
||
{ name: "空值填充", code: "COALESCE(column, default_value)" },
|
||
{ name: "正则替换", code: "REGEXP_REPLACE(column, pattern, replacement)" },
|
||
]
|
||
|
||
export default function DataCleaningPage() {
|
||
const [rules, setRules] = useState<CleaningRule[]>(MOCK_RULES)
|
||
const [selectedTab, setSelectedTab] = useState("rules")
|
||
const [showCreateDialog, setShowCreateDialog] = useState(false)
|
||
const [showSettingsDialog, setShowSettingsDialog] = useState(false)
|
||
const [showLogsDialog, setShowLogsDialog] = useState(false)
|
||
const [selectedRule, setSelectedRule] = useState<CleaningRule | null>(null)
|
||
|
||
const stats = {
|
||
totalRules: rules.length,
|
||
activeRules: rules.filter((r) => r.status === "active").length,
|
||
totalProcessed: rules.reduce((sum, r) => sum + r.processedRecords, 0),
|
||
avgErrorRate: (rules.reduce((sum, r) => sum + r.errorRate, 0) / rules.length).toFixed(3),
|
||
}
|
||
|
||
const formatNumber = (num: number): string => {
|
||
if (num >= 1000000000) return `${(num / 1000000000).toFixed(2)}B`
|
||
if (num >= 1000000) return `${(num / 1000000).toFixed(1)}M`
|
||
if (num >= 1000) return `${(num / 1000).toFixed(1)}K`
|
||
return num.toString()
|
||
}
|
||
|
||
const handleSettings = (rule: CleaningRule) => {
|
||
setSelectedRule(rule)
|
||
setShowSettingsDialog(true)
|
||
}
|
||
|
||
const handleViewLogs = (rule: CleaningRule) => {
|
||
setSelectedRule(rule)
|
||
setShowLogsDialog(true)
|
||
}
|
||
|
||
return (
|
||
<div className="space-y-6 p-6">
|
||
{/* Header */}
|
||
<div className="flex flex-col md:flex-row md:items-center justify-between gap-4">
|
||
<div>
|
||
<h1 className="text-2xl font-bold text-gray-900">数据清洗</h1>
|
||
<p className="text-gray-500 mt-1">配置清洗规则，保障数据质量</p>
|
||
</div>
|
||
<Button
|
||
className="bg-gradient-to-r from-blue-500 to-purple-500 text-white"
|
||
onClick={() => setShowCreateDialog(true)}
|
||
>
|
||
<Plus className="w-4 h-4 mr-2" />
|
||
新增清洗规则
|
||
</Button>
|
||
</div>
|
||
|
||
{/* Stats */}
|
||
<div className="grid grid-cols-2 lg:grid-cols-4 gap-4">
|
||
<Card className="border-none shadow-sm bg-white/60 backdrop-blur-sm">
|
||
<CardContent className="p-4">
|
||
<div className="flex items-center gap-2 text-gray-500 text-sm mb-1">
|
||
<FileText className="w-4 h-4" />
|
||
清洗规则
|
||
</div>
|
||
<div className="text-2xl font-bold text-gray-900">{stats.totalRules}</div>
|
||
</CardContent>
|
||
</Card>
|
||
<Card className="border-none shadow-sm bg-white/60 backdrop-blur-sm">
|
||
<CardContent className="p-4">
|
||
<div className="flex items-center gap-2 text-gray-500 text-sm mb-1">
|
||
<CheckCircle className="w-4 h-4 text-green-500" />
|
||
启用中
|
||
</div>
|
||
<div className="text-2xl font-bold text-green-600">{stats.activeRules}</div>
|
||
</CardContent>
|
||
</Card>
|
||
<Card className="border-none shadow-sm bg-white/60 backdrop-blur-sm">
|
||
<CardContent className="p-4">
|
||
<div className="flex items-center gap-2 text-gray-500 text-sm mb-1">
|
||
<Zap className="w-4 h-4" />
|
||
已处理记录
|
||
</div>
|
||
<div className="text-2xl font-bold text-gray-900">{formatNumber(stats.totalProcessed)}</div>
|
||
</CardContent>
|
||
</Card>
|
||
<Card className="border-none shadow-sm bg-white/60 backdrop-blur-sm">
|
||
<CardContent className="p-4">
|
||
<div className="flex items-center gap-2 text-gray-500 text-sm mb-1">
|
||
<AlertTriangle className="w-4 h-4 text-yellow-500" />
|
||
平均错误率
|
||
</div>
|
||
<div className="text-2xl font-bold text-gray-900">{stats.avgErrorRate}%</div>
|
||
</CardContent>
|
||
</Card>
|
||
</div>
|
||
|
||
<Tabs value={selectedTab} onValueChange={setSelectedTab}>
|
||
<TabsList className="bg-white/60 backdrop-blur-sm">
|
||
<TabsTrigger value="rules">清洗规则</TabsTrigger>
|
||
<TabsTrigger value="presets">预置函数</TabsTrigger>
|
||
<TabsTrigger value="sql">SQL编辑器</TabsTrigger>
|
||
</TabsList>
|
||
|
||
<TabsContent value="rules" className="mt-4">
|
||
<div className="space-y-4">
|
||
{rules.map((rule) => (
|
||
<Card key={rule.id} className="border-none shadow-sm bg-white/60 backdrop-blur-sm">
|
||
<CardContent className="p-5">
|
||
<div className="flex items-start justify-between">
|
||
<div className="flex-1">
|
||
<div className="flex items-center gap-3 mb-2">
|
||
<h3 className="font-semibold text-gray-900">{rule.name}</h3>
|
||
<Badge className={TYPE_CONFIG[rule.type].color}>{TYPE_CONFIG[rule.type].label}</Badge>
|
||
<Badge
|
||
variant="outline"
|
||
className={
|
||
rule.status === "active"
|
||
? "border-green-500 text-green-600"
|
||
: rule.status === "testing"
|
||
? "border-yellow-500 text-yellow-600"
|
||
: "border-gray-500 text-gray-600"
|
||
}
|
||
>
|
||
{rule.status === "active" ? "启用" : rule.status === "testing" ? "测试中" : "停用"}
|
||
</Badge>
|
||
</div>
|
||
<p className="text-sm text-gray-500 mb-3">{rule.description}</p>
|
||
<div className="flex items-center gap-6 text-sm">
|
||
<div className="flex items-center gap-2">
|
||
<span className="text-gray-500">数据流向:</span>
|
||
<code className="bg-gray-100 px-2 py-0.5 rounded text-xs">{rule.sourceTable}</code>
|
||
<ArrowRight className="w-3 h-3 text-gray-400" />
|
||
<code className="bg-gray-100 px-2 py-0.5 rounded text-xs">{rule.targetTable}</code>
|
||
</div>
|
||
<div>
|
||
<span className="text-gray-500">处理量:</span>
|
||
<span className="ml-1 font-medium">{formatNumber(rule.processedRecords)}</span>
|
||
</div>
|
||
<div>
|
||
<span className="text-gray-500">错误率:</span>
|
||
<span
|
||
className={`ml-1 font-medium ${rule.errorRate > 0.1 ? "text-red-600" : "text-green-600"}`}
|
||
>
|
||
{rule.errorRate}%
|
||
</span>
|
||
</div>
|
||
</div>
|
||
</div>
|
||
<div className="flex items-center gap-2">
|
||
<Button variant="outline" size="sm">
|
||
<Play className="w-4 h-4 mr-1" />
|
||
执行
|
||
</Button>
|
||
<Button variant="ghost" size="sm" onClick={() => handleSettings(rule)}>
|
||
<Settings className="w-4 h-4" />
|
||
</Button>
|
||
</div>
|
||
</div>
|
||
<div className="mt-4 pt-4 border-t border-gray-100 flex items-center justify-between text-xs text-gray-500">
|
||
<span className="flex items-center gap-1">
|
||
<Clock className="w-3 h-3" />
|
||
最后执行: {rule.lastRun}
|
||
</span>
|
||
<Button
|
||
variant="link"
|
||
size="sm"
|
||
className="h-auto p-0 text-blue-500"
|
||
onClick={() => handleViewLogs(rule)}
|
||
>
|
||
查看执行日志
|
||
</Button>
|
||
</div>
|
||
</CardContent>
|
||
</Card>
|
||
))}
|
||
</div>
|
||
</TabsContent>
|
||
|
||
<TabsContent value="presets" className="mt-4">
|
||
<Card className="border-none shadow-sm bg-white/60 backdrop-blur-sm">
|
||
<CardHeader>
|
||
<CardTitle className="text-lg">预置清洗函数</CardTitle>
|
||
</CardHeader>
|
||
<CardContent>
|
||
<div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-3 gap-4">
|
||
{PRESETS.map((preset, i) => (
|
||
<div key={i} className="p-4 rounded-xl bg-gray-50 hover:bg-gray-100 transition-colors cursor-pointer">
|
||
<h4 className="font-medium text-gray-900 mb-2">{preset.name}</h4>
|
||
<code className="text-xs text-gray-600 bg-white px-2 py-1 rounded block">{preset.code}</code>
|
||
</div>
|
||
))}
|
||
</div>
|
||
</CardContent>
|
||
</Card>
|
||
</TabsContent>
|
||
|
||
<TabsContent value="sql" className="mt-4">
|
||
<Card className="border-none shadow-sm bg-white/60 backdrop-blur-sm">
|
||
<CardHeader>
|
||
<CardTitle className="text-lg flex items-center gap-2">
|
||
<Code className="w-5 h-5" />
|
||
SQL清洗规则编辑器
|
||
</CardTitle>
|
||
</CardHeader>
|
||
<CardContent>
|
||
<div className="bg-gray-900 rounded-xl p-4 font-mono text-sm text-green-400 min-h-[200px]">
|
||
<pre>{`-- 示例：用户数据清洗
|
||
SELECT
|
||
user_id,
|
||
TRIM(LOWER(email)) as email,
|
||
REGEXP_REPLACE(phone, '[^0-9]', '') as phone,
|
||
COALESCE(city, '未知') as city,
|
||
DATE_FORMAT(created_at, '%Y-%m-%d') as created_date
|
||
FROM raw_users
|
||
WHERE email IS NOT NULL
|
||
AND LENGTH(phone) >= 11;`}</pre>
|
||
</div>
|
||
<div className="flex justify-end gap-2 mt-4">
|
||
<Button variant="outline">测试执行</Button>
|
||
<Button>保存规则</Button>
|
||
</div>
|
||
</CardContent>
|
||
</Card>
|
||
</TabsContent>
|
||
</Tabs>
|
||
|
||
{/* Create Cleaning Rule Dialog */}
|
||
<CreateCleaningRuleDialog open={showCreateDialog} onOpenChange={setShowCreateDialog} />
|
||
|
||
<CleaningRuleSettingsDialog
|
||
open={showSettingsDialog}
|
||
onOpenChange={setShowSettingsDialog}
|
||
rule={
|
||
selectedRule
|
||
? {
|
||
id: selectedRule.id,
|
||
name: selectedRule.name,
|
||
description: selectedRule.description,
|
||
type: selectedRule.type,
|
||
sourceTable: selectedRule.sourceTable,
|
||
targetTable: selectedRule.targetTable,
|
||
status: selectedRule.status,
|
||
}
|
||
: undefined
|
||
}
|
||
/>
|
||
<CleaningRuleLogsDialog open={showLogsDialog} onOpenChange={setShowLogsDialog} ruleName={selectedRule?.name} />
|
||
</div>
|
||
)
|
||
}
|