"use client" import { useState } from "react" import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card" import { Button } from "@/components/ui/button" import { Badge } from "@/components/ui/badge" import { Input } from "@/components/ui/input" import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs" import { Dialog, DialogContent, DialogDescription, DialogFooter, DialogHeader, DialogTitle, } from "@/components/ui/dialog" import { Label } from "@/components/ui/label" import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from "@/components/ui/select" import { Textarea } from "@/components/ui/textarea" import { Slider } from "@/components/ui/slider" import { ArrowLeft, Sparkles, CheckCircle, AlertTriangle, Play, Settings, Eye, RefreshCw, Database, Zap, Plus, Brain, MessageSquare, Shield, History, Check, X, Pause, FileText, } from "lucide-react" import Link from "next/link" const availableDataSources = [ { id: "1", name: "存客宝-用户主库", type: "mysql", records: 128956342, tables: ["users", "orders", "profiles"] }, { id: "2", name: "触客宝-行为数据", type: "kafka", records: 89234567, tables: ["events", "sessions"] }, { id: "3", name: "数智员工-账号API", type: "restapi", records: 45678901, tables: ["accounts", "activities"] }, { id: "7", name: "微信消息-Webhook", type: "webhook", records: 34567890, tables: ["messages", "contacts"] }, ] const aiCleaningRules = [ { id: 1, name: "用户数据智能清洗", description: "自动识别并修正用户数据中的格式错误、缺失值和异常值", dataSource: "存客宝-用户主库", targetTable: "users", prompt: "分析用户数据表,识别以下问题并提供修正建议:1) 手机号格式不规范 2) 邮箱格式错误 3) 姓名包含特殊字符 4) 地址信息不完整 5) 日期格式不统一。对于每个问题,给出具体的修正规则。", model: "GPT-4", status: "running", confidenceThreshold: 0.8, autoApproveThreshold: 0.95, processed: 125800, pending: 3400, approved: 120000, rejected: 2400, lastRun: "10分钟前", accuracy: 96.2, }, { id: 2, name: "交易流水异常检测", description: "识别交易数据中的异常记录和潜在错误", dataSource: "存客宝-用户主库", targetTable: "orders", prompt: "检查交易流水数据,识别以下异常:1) 金额为负或超出合理范围 2) 时间戳异常 3) 重复交易记录 4) 用户ID与订单不匹配 5) 状态字段值异常。标记问题记录并建议处理方式。", model: "GPT-4", status: "running", confidenceThreshold: 0.7, autoApproveThreshold: 0.9, processed: 89000, pending: 1200, approved: 85000, rejected: 2800, lastRun: "30分钟前", accuracy: 94.5, }, { id: 3, name: "聊天记录语义清洗", description: "清洗聊天记录中的敏感信息和无效内容", dataSource: "微信消息-Webhook", targetTable: "messages", prompt: "处理聊天记录数据:1) 识别并脱敏个人敏感信息(身份证、银行卡号等)2) 过滤垃圾信息和广告 3) 标准化时间格式 4) 提取并分类用户意图 5) 标记需要人工处理的复杂内容。", model: "Claude-3", status: "paused", confidenceThreshold: 0.6, autoApproveThreshold: 0.85, processed: 456000, pending: 15000, approved: 420000, rejected: 21000, lastRun: "2小时前", accuracy: 91.8, }, ] const pendingCleaningSuggestions = [ { id: 1, ruleId: 1, ruleName: "用户数据智能清洗", field: "phone", originalValue: "86-138-1234-5678", suggestedValue: "13812345678", confidence: 0.95, reason: "去除国际区号前缀和分隔符,保留纯数字手机号", affectedRows: 2350, createdAt: "2025-12-12 14:30", }, { id: 2, ruleId: 1, ruleName: "用户数据智能清洗", field: "email", originalValue: "user@example,com", suggestedValue: "user@example.com", confidence: 0.88, reason: "修正邮箱格式,将逗号替换为点号", affectedRows: 156, createdAt: "2025-12-12 14:28", }, { id: 3, ruleId: 2, ruleName: "交易流水异常检测", field: "amount", originalValue: "-500.00", suggestedValue: "500.00", confidence: 0.72, reason: "检测到负数金额,建议修正为正数(可能是退款记录录入错误)", affectedRows: 45, createdAt: "2025-12-12 14:25", }, { id: 4, ruleId: 3, ruleName: "聊天记录语义清洗", field: "content", originalValue: "我的身份证号是310101199001011234", suggestedValue: "我的身份证号是310***********1234", confidence: 0.92, reason: "检测到身份证号,进行脱敏处理保护用户隐私", affectedRows: 890, createdAt: "2025-12-12 14:22", }, ] // 清洗历史 const cleaningHistory = [ { id: 1, rule: "用户数据智能清洗", action: "approved", operator: "系统自动", count: 5000, time: "2025-12-12 14:00" }, { id: 2, rule: "交易流水异常检测", action: "approved", operator: "张管理员", count: 120, time: "2025-12-12 13:45" }, { id: 3, rule: "聊天记录语义清洗", action: "rejected", operator: "李管理员", count: 35, time: "2025-12-12 13:30", reason: "置信度过低", }, ] export default function AIDataCleaningPage() { const [activeTab, setActiveTab] = useState("rules") const [createRuleDialogOpen, setCreateRuleDialogOpen] = useState(false) const [settingsDialogOpen, setSettingsDialogOpen] = useState(false) const [previewDialogOpen, setPreviewDialogOpen] = useState(false) const [selectedRule, setSelectedRule] = useState<(typeof aiCleaningRules)[0] | null>(null) const [selectedSuggestion, setSelectedSuggestion] = useState<(typeof pendingCleaningSuggestions)[0] | null>(null) const [selectedSuggestions, setSelectedSuggestions] = useState([]) const [confidenceFilter, setConfidenceFilter] = useState([0.5]) const [newRule, setNewRule] = useState({ name: "", dataSource: "", targetTable: "", prompt: "", model: "gpt-4", confidenceThreshold: 0.7, autoApproveThreshold: 0.9, }) const totalPending = aiCleaningRules.reduce((sum, r) => sum + r.pending, 0) const totalApproved = aiCleaningRules.reduce((sum, r) => sum + r.approved, 0) const totalProcessed = aiCleaningRules.reduce((sum, r) => sum + r.processed, 0) const filteredSuggestions = pendingCleaningSuggestions.filter((s) => s.confidence >= confidenceFilter[0]) // 获取选中数据源的表列表 const getTablesForDataSource = (dsName: string) => { const ds = availableDataSources.find((d) => d.name === dsName) return ds?.tables || [] } return (
{/* 页面标题 */}

AI数据清洗

基于大模型智能识别数据质量问题,自动生成清洗规则,支持审核机制

{/* 统计概览 */}

总处理记录

{(totalProcessed / 1000).toFixed(0)}K

清洗成功率

{((totalApproved / totalProcessed) * 100).toFixed(1)}%

待审核

{totalPending.toLocaleString()}

运行中规则

{aiCleaningRules.filter((r) => r.status === "running").length} / {aiCleaningRules.length}

平均置信度

0.89

AI清洗规则 审核中心 {totalPending > 0 && {totalPending}} 操作历史 {/* AI清洗规则列表 */}
AI清洗规则配置

数据源来自数据治理模块,无需单独上传

{aiCleaningRules.map((rule) => (

{rule.name}

{rule.model} {rule.status === "running" ? ( 运行中 ) : ( 已暂停 )}

{rule.description}

{rule.status === "running" ? ( ) : ( )}
{/* 数据源和目标表 */}
数据源

{rule.dataSource}

目标表: {rule.targetTable}

置信度阈值

审核: {">="} {rule.confidenceThreshold} | 自动: {">="} {rule.autoApproveThreshold}

{/* 提示词 */}
AI提示词

{rule.prompt}

{/* 统计数据 */}

{rule.accuracy}%

准确率

{(rule.processed / 1000).toFixed(0)}K

已处理

{rule.pending.toLocaleString()}

待审核

{(rule.approved / 1000).toFixed(0)}K

已通过

{rule.rejected.toLocaleString()}

已拒绝

上次运行: {rule.lastRun}
))}
{/* 审核中心 */}
清洗建议审核

AI生成的数据修正建议,需人工确认后执行

置信度:
{">="} {confidenceFilter[0]}
{selectedSuggestions.length > 0 && (
已选 {selectedSuggestions.length} 项
)}
{filteredSuggestions.map((suggestion) => (
{ if (e.target.checked) { setSelectedSuggestions([...selectedSuggestions, suggestion.id]) } else { setSelectedSuggestions(selectedSuggestions.filter((id) => id !== suggestion.id)) } }} />
{suggestion.ruleName} 字段: {suggestion.field} = 0.9 ? "bg-emerald-100 text-emerald-700" : suggestion.confidence >= 0.7 ? "bg-amber-100 text-amber-700" : "bg-red-100 text-red-700" }`} > 置信度: {(suggestion.confidence * 100).toFixed(0)}% 影响 {suggestion.affectedRows} 行

原始值

{suggestion.originalValue}

建议值

{suggestion.suggestedValue}
AI分析理由

{suggestion.reason}

创建时间: {suggestion.createdAt}

))}
{/* 操作历史 */} 清洗操作历史
{cleaningHistory.map((item) => (
{item.rule} {item.action === "approved" ? "通过" : "拒绝"} {item.count} 条记录 {item.reason && 原因: {item.reason}}
{item.operator} {item.time}
))}
{/* 创建AI规则弹窗 */} 创建AI清洗规则 配置AI模型、选择数据源和目标表,定义清洗提示词
setNewRule({ ...newRule, name: e.target.value })} />