Files
users/app/data-ingestion/ai-engine/page.tsx

553 lines
23 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"use client"
import { useState, useEffect } from "react"
import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"
import { Button } from "@/components/ui/button"
import { Badge } from "@/components/ui/badge"
import { Progress } from "@/components/ui/progress"
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@/components/ui/tabs"
import {
Dialog,
DialogContent,
DialogDescription,
DialogFooter,
DialogHeader,
DialogTitle,
} from "@/components/ui/dialog"
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@/components/ui/select"
import { Label } from "@/components/ui/label"
import { Textarea } from "@/components/ui/textarea"
import {
Brain,
Zap,
Database,
ArrowRight,
CheckCircle2,
Loader2,
Play,
Settings,
RefreshCw,
Sparkles,
Target,
Tags,
FileText,
AlertCircle,
} from "lucide-react"
// AI标签引擎任务接口
interface EngineTask {
id: string
name: string
description: string
sourceDb: string
sourceCollection: string
status: 'idle' | 'analyzing' | 'tagging' | 'completed' | 'error'
progress: number
steps: {
name: string
status: 'pending' | 'running' | 'completed' | 'error'
result?: string
}[]
stats: {
totalRecords: number
analyzedRecords: number
taggedRecords: number
newTags: number
}
}
// 字段映射规则
interface FieldMapping {
sourceField: string
targetField: string
transformRule: string
aiEnhance: boolean
}
// 动态生成引擎任务从API加载
const generateDefaultTasks = (sources: any[]): EngineTask[] => {
return sources.slice(0, 5).map((src, i) => ({
id: `engine_${i + 1}`,
name: `${src.name || src.id}数据拆解`,
description: `解析${src.database || src.name}数据,提取用户标签`,
sourceDb: src.database || src.name,
sourceCollection: src.collection || '用户资产统一视图',
status: i === 0 ? 'completed' as const : 'idle' as const,
progress: i === 0 ? 100 : 0,
steps: [
{ name: '数据源连接', status: i === 0 ? 'completed' as const : 'pending' as const, result: i === 0 ? `${(src.recordCount / 1000000).toFixed(0)}M条记录` : undefined },
{ name: '字段分析', status: i === 0 ? 'completed' as const : 'pending' as const, result: i === 0 ? '识别字段完成' : undefined },
{ name: 'AI拆解', status: i === 0 ? 'completed' as const : 'pending' as const, result: i === 0 ? '提取关键字段' : undefined },
{ name: '标签生成', status: i === 0 ? 'completed' as const : 'pending' as const, result: i === 0 ? '生成用户标签' : undefined },
{ name: '数据写入', status: i === 0 ? 'completed' as const : 'pending' as const, result: i === 0 ? '写入用户估值表' : undefined },
],
stats: {
totalRecords: src.recordCount || 0,
analyzedRecords: i === 0 ? src.recordCount || 0 : 0,
taggedRecords: i === 0 ? Math.floor((src.recordCount || 0) * 0.95) : 0,
newTags: i === 0 ? 5 : 0,
}
}))
}
// 默认任务(京东示例)
const FALLBACK_TASKS: EngineTask[] = [
{
id: 'engine_1',
name: '腾讯QQ数据拆解',
description: '解析QQ库数据提取手机号关联并生成用户标签',
sourceDb: 'KR_腾讯',
sourceCollection: 'QQ+手机',
status: 'completed',
progress: 100,
steps: [
{ name: '数据源连接', status: 'completed', result: '705M条记录' },
{ name: '字段分析', status: 'completed', result: '识别12个字段' },
{ name: 'AI拆解', status: 'completed', result: '提取手机号、QQ号、地区' },
{ name: '标签生成', status: 'completed', result: '生成3类标签' },
{ name: '数据写入', status: 'completed', result: '写入用户估值表' },
],
stats: {
totalRecords: 705000000,
analyzedRecords: 705000000,
taggedRecords: 680000000,
newTags: 3,
}
},
{
id: 'engine_2',
name: '京东用户数据拆解',
description: '解析京东电商数据提取消费行为生成RFM标签',
sourceDb: 'KR_京东',
sourceCollection: 'users',
status: 'analyzing',
progress: 45,
steps: [
{ name: '数据源连接', status: 'completed', result: '142M条记录' },
{ name: '字段分析', status: 'completed', result: '识别28个字段' },
{ name: 'AI拆解', status: 'running', result: '分析消费行为...' },
{ name: '标签生成', status: 'pending' },
{ name: '数据写入', status: 'pending' },
],
stats: {
totalRecords: 142000000,
analyzedRecords: 63900000,
taggedRecords: 0,
newTags: 0,
}
},
{
id: 'engine_3',
name: '存客宝CRM数据拆解',
description: '解析存客宝私域数据,提取客户画像',
sourceDb: 'cunkebao_v3',
sourceCollection: 'users',
status: 'idle',
progress: 0,
steps: [
{ name: '数据源连接', status: 'pending' },
{ name: '字段分析', status: 'pending' },
{ name: 'AI拆解', status: 'pending' },
{ name: '标签生成', status: 'pending' },
{ name: '数据写入', status: 'pending' },
],
stats: {
totalRecords: 216000,
analyzedRecords: 0,
taggedRecords: 0,
newTags: 0,
}
},
]
// 标准字段映射规则
const STANDARD_FIELD_MAPPINGS: FieldMapping[] = [
{ sourceField: 'phone', targetField: 'phone', transformRule: '格式化为11位', aiEnhance: true },
{ sourceField: 'qq', targetField: 'qq', transformRule: '提取纯数字', aiEnhance: false },
{ sourceField: 'uid', targetField: 'weibo_uid', transformRule: '微博UID映射', aiEnhance: false },
{ sourceField: 'name', targetField: 'name', transformRule: '姓名脱敏', aiEnhance: true },
{ sourceField: 'province', targetField: 'province', transformRule: '省份标准化', aiEnhance: true },
{ sourceField: 'city', targetField: 'city', transformRule: '城市标准化', aiEnhance: true },
{ sourceField: 'gender', targetField: 'gender', transformRule: '性别标准化', aiEnhance: false },
{ sourceField: 'amount', targetField: 'total_spend', transformRule: '金额汇总', aiEnhance: false },
{ sourceField: 'last_active', targetField: 'recency', transformRule: 'R值计算', aiEnhance: true },
{ sourceField: 'buy_count', targetField: 'frequency', transformRule: 'F值计算', aiEnhance: true },
{ sourceField: 'total_amount', targetField: 'monetary', transformRule: 'M值计算', aiEnhance: true },
]
export default function AIEnginePage() {
const [tasks, setTasks] = useState<EngineTask[]>(FALLBACK_TASKS)
const [selectedTask, setSelectedTask] = useState<EngineTask | null>(null)
const [showConfigDialog, setShowConfigDialog] = useState(false)
const [activeTab, setActiveTab] = useState("tasks")
const [loading, setLoading] = useState(true)
// 从API加载真实数据源
useEffect(() => {
setLoading(true)
fetch('/api/data-sources')
.then(res => res.json())
.then(data => {
if (data.success && data.sources) {
// 只取MongoDB数据源
const mongoSources = data.sources.filter((s: any) => s.type === 'mongodb' && s.recordCount > 0)
if (mongoSources.length > 0) {
setTasks(generateDefaultTasks(mongoSources))
}
}
})
.catch(console.error)
.finally(() => setLoading(false))
}, [])
const formatNumber = (num: number): string => {
if (num >= 1000000000) return `${(num / 1000000000).toFixed(2)}B`
if (num >= 1000000) return `${(num / 1000000).toFixed(1)}M`
if (num >= 1000) return `${(num / 1000).toFixed(0)}K`
return num.toLocaleString()
}
const getStatusBadge = (status: string) => {
switch (status) {
case 'completed':
return <Badge className="bg-green-100 text-green-700"><CheckCircle2 className="h-3 w-3 mr-1" /></Badge>
case 'analyzing':
case 'tagging':
return <Badge className="bg-blue-100 text-blue-700"><Loader2 className="h-3 w-3 mr-1 animate-spin" /></Badge>
case 'error':
return <Badge className="bg-red-100 text-red-700"><AlertCircle className="h-3 w-3 mr-1" /></Badge>
default:
return <Badge className="bg-gray-100 text-gray-700"></Badge>
}
}
const getStepIcon = (status: string) => {
switch (status) {
case 'completed':
return <CheckCircle2 className="h-4 w-4 text-green-500" />
case 'running':
return <Loader2 className="h-4 w-4 text-blue-500 animate-spin" />
case 'error':
return <AlertCircle className="h-4 w-4 text-red-500" />
default:
return <div className="h-4 w-4 rounded-full border-2 border-gray-300" />
}
}
const runTask = (taskId: string) => {
setTasks(tasks.map(t => t.id === taskId ? {
...t,
status: 'analyzing' as const,
steps: t.steps.map((s, i) => i === 0 ? { ...s, status: 'running' as const } : s)
} : t))
}
// 模拟进度更新
useEffect(() => {
const interval = setInterval(() => {
setTasks(prev => prev.map(t => {
if (t.status === 'analyzing' || t.status === 'tagging') {
const newProgress = Math.min(t.progress + Math.random() * 5, 100)
const currentStepIndex = t.steps.findIndex(s => s.status === 'running')
let newSteps = [...t.steps]
if (newProgress > (currentStepIndex + 1) * 20 && currentStepIndex < t.steps.length - 1) {
newSteps[currentStepIndex] = { ...newSteps[currentStepIndex], status: 'completed' }
newSteps[currentStepIndex + 1] = { ...newSteps[currentStepIndex + 1], status: 'running' }
}
return {
...t,
progress: newProgress,
status: newProgress >= 100 ? 'completed' as const : t.status,
steps: newProgress >= 100 ? t.steps.map(s => ({ ...s, status: 'completed' as const })) : newSteps,
stats: {
...t.stats,
analyzedRecords: Math.floor(t.stats.totalRecords * newProgress / 100),
taggedRecords: newProgress >= 80 ? Math.floor(t.stats.totalRecords * (newProgress - 20) / 100) : 0,
newTags: newProgress >= 60 ? 5 : 0,
}
}
}
return t
}))
}, 2000)
return () => clearInterval(interval)
}, [])
return (
<div className="min-h-screen bg-gradient-to-br from-slate-50 via-blue-50/30 to-purple-50/20">
<div className="p-6 space-y-6">
{/* 顶部标题 */}
<div className="flex items-center justify-between">
<div>
<h1 className="text-2xl font-bold text-gray-900">AI标签引擎</h1>
<p className="text-sm text-gray-500 mt-1"></p>
</div>
<div className="flex items-center gap-3">
<Button variant="outline">
<RefreshCw className="h-4 w-4 mr-2" />
</Button>
<Button onClick={() => setShowConfigDialog(true)}>
<Brain className="h-4 w-4 mr-2" />
</Button>
</div>
</div>
{/* 引擎说明卡片 */}
<Card className="border-0 shadow-sm bg-gradient-to-r from-purple-50 to-blue-50">
<CardContent className="p-4">
<div className="flex items-start gap-4">
<div className="p-3 rounded-xl bg-gradient-to-r from-purple-500 to-blue-500">
<Brain className="h-6 w-6 text-white" />
</div>
<div className="flex-1">
<h3 className="font-semibold text-gray-900 mb-1">AI标签引擎工作流程</h3>
<div className="flex items-center gap-2 text-sm text-gray-600">
<Badge variant="outline">1. </Badge>
<ArrowRight className="h-4 w-4 text-gray-400" />
<Badge variant="outline">2. </Badge>
<ArrowRight className="h-4 w-4 text-gray-400" />
<Badge variant="outline">3. AI拆解</Badge>
<ArrowRight className="h-4 w-4 text-gray-400" />
<Badge variant="outline">4. </Badge>
<ArrowRight className="h-4 w-4 text-gray-400" />
<Badge variant="outline">5. </Badge>
</div>
<p className="text-xs text-gray-500 mt-2">
RFM评分和用户标签
</p>
</div>
</div>
</CardContent>
</Card>
{/* 标签页 */}
<Tabs value={activeTab} onValueChange={setActiveTab}>
<TabsList className="bg-white/80">
<TabsTrigger value="tasks"></TabsTrigger>
<TabsTrigger value="rules"></TabsTrigger>
<TabsTrigger value="logs"></TabsTrigger>
</TabsList>
{/* 拆解任务列表 */}
<TabsContent value="tasks" className="mt-4 space-y-3">
{tasks.map(task => (
<Card key={task.id} className="border-0 shadow-sm bg-white/80">
<CardContent className="p-4">
<div className="flex items-start justify-between mb-4">
<div className="flex items-center gap-3">
<div className="p-2.5 rounded-xl bg-purple-100">
<Brain className="h-5 w-5 text-purple-600" />
</div>
<div>
<div className="flex items-center gap-2">
<h3 className="font-semibold text-gray-900">{task.name}</h3>
{getStatusBadge(task.status)}
</div>
<p className="text-sm text-gray-500">{task.description}</p>
<div className="text-xs text-gray-400 mt-1 font-mono">
{task.sourceDb}.{task.sourceCollection}
</div>
</div>
</div>
<div className="flex items-center gap-2">
{task.status === 'idle' && (
<Button size="sm" onClick={() => runTask(task.id)}>
<Play className="h-4 w-4 mr-1" />
</Button>
)}
<Button size="sm" variant="ghost" onClick={() => setSelectedTask(task)}>
<Settings className="h-4 w-4" />
</Button>
</div>
</div>
{/* 进度条 */}
<div className="mb-4">
<div className="flex items-center justify-between text-sm mb-1">
<span className="text-gray-500"></span>
<span className="font-medium">{task.progress.toFixed(0)}%</span>
</div>
<Progress value={task.progress} className="h-2" />
</div>
{/* 步骤流程 */}
<div className="flex items-center justify-between mb-4">
{task.steps.map((step, i) => (
<div key={i} className="flex items-center">
<div className="flex flex-col items-center">
{getStepIcon(step.status)}
<span className="text-xs text-gray-500 mt-1">{step.name}</span>
{step.result && (
<span className="text-xs text-green-600">{step.result}</span>
)}
</div>
{i < task.steps.length - 1 && (
<div className={`w-16 h-0.5 mx-2 ${step.status === 'completed' ? 'bg-green-300' : 'bg-gray-200'}`} />
)}
</div>
))}
</div>
{/* 统计信息 */}
<div className="grid grid-cols-4 gap-4 text-sm pt-3 border-t">
<div>
<span className="text-gray-500">:</span>
<span className="ml-2 font-medium">{formatNumber(task.stats.totalRecords)}</span>
</div>
<div>
<span className="text-gray-500">:</span>
<span className="ml-2 font-medium text-blue-600">{formatNumber(task.stats.analyzedRecords)}</span>
</div>
<div>
<span className="text-gray-500">:</span>
<span className="ml-2 font-medium text-green-600">{formatNumber(task.stats.taggedRecords)}</span>
</div>
<div>
<span className="text-gray-500">:</span>
<span className="ml-2 font-medium text-purple-600">{task.stats.newTags}</span>
</div>
</div>
</CardContent>
</Card>
))}
</TabsContent>
{/* 字段映射规则 */}
<TabsContent value="rules" className="mt-4">
<Card className="border-0 shadow-sm bg-white/80">
<CardHeader>
<CardTitle className="text-sm font-medium"></CardTitle>
</CardHeader>
<CardContent>
<div className="space-y-2">
{STANDARD_FIELD_MAPPINGS.map((mapping, i) => (
<div key={i} className="flex items-center justify-between p-3 rounded-lg bg-gray-50 hover:bg-gray-100">
<div className="flex items-center gap-4">
<Badge variant="outline" className="font-mono">{mapping.sourceField}</Badge>
<ArrowRight className="h-4 w-4 text-gray-400" />
<Badge className="bg-purple-100 text-purple-700 font-mono">{mapping.targetField}</Badge>
</div>
<div className="flex items-center gap-3">
<span className="text-sm text-gray-500">{mapping.transformRule}</span>
{mapping.aiEnhance && (
<Badge className="bg-blue-100 text-blue-700">
<Sparkles className="h-3 w-3 mr-1" />AI增强
</Badge>
)}
</div>
</div>
))}
</div>
</CardContent>
</Card>
</TabsContent>
{/* 执行日志 */}
<TabsContent value="logs" className="mt-4">
<Card className="border-0 shadow-sm bg-white/80">
<CardHeader>
<CardTitle className="text-sm font-medium"></CardTitle>
</CardHeader>
<CardContent>
<div className="space-y-2 font-mono text-xs">
<div className="p-2 rounded bg-gray-900 text-green-400">
[2024-01-30 14:32:15] INFO: AI标签引擎启动
</div>
<div className="p-2 rounded bg-gray-900 text-green-400">
[2024-01-30 14:32:16] INFO: 连接数据源 KR_腾讯.qq_phone
</div>
<div className="p-2 rounded bg-gray-900 text-green-400">
[2024-01-30 14:32:18] INFO: 发现 705,000,000
</div>
<div className="p-2 rounded bg-gray-900 text-yellow-400">
[2024-01-30 14:32:20] WARN: 字段 'phone' AI格式化
</div>
<div className="p-2 rounded bg-gray-900 text-green-400">
[2024-01-30 14:35:45] INFO: AI拆解完成提取字段: phone, qq, province
</div>
<div className="p-2 rounded bg-gray-900 text-green-400">
[2024-01-30 14:40:12] INFO: 生成标签: , , QQ等级
</div>
<div className="p-2 rounded bg-gray-900 text-green-400">
[2024-01-30 14:45:30] INFO: 数据写入 KR.
</div>
</div>
</CardContent>
</Card>
</TabsContent>
</Tabs>
{/* 新建任务弹窗 */}
<Dialog open={showConfigDialog} onOpenChange={setShowConfigDialog}>
<DialogContent className="max-w-lg">
<DialogHeader>
<DialogTitle>AI拆解任务</DialogTitle>
<DialogDescription></DialogDescription>
</DialogHeader>
<div className="space-y-4 py-4">
<div className="grid grid-cols-2 gap-4">
<div className="space-y-2">
<Label></Label>
<Select>
<SelectTrigger>
<SelectValue placeholder="选择数据库" />
</SelectTrigger>
<SelectContent>
<SelectItem value="kr_tencent">KR_腾讯</SelectItem>
<SelectItem value="kr_weibo">KR_微博</SelectItem>
<SelectItem value="kr_jd">KR_京东</SelectItem>
<SelectItem value="ckb">cunkebao_v3</SelectItem>
</SelectContent>
</Select>
</div>
<div className="space-y-2">
<Label></Label>
<Select>
<SelectTrigger>
<SelectValue placeholder="选择目标" />
</SelectTrigger>
<SelectContent>
<SelectItem value="valuation">KR.</SelectItem>
<SelectItem value="ckb_view">KR_存客宝.</SelectItem>
</SelectContent>
</Select>
</div>
</div>
<div className="space-y-2">
<Label>AI模型</Label>
<Select defaultValue="qwen">
<SelectTrigger>
<SelectValue />
</SelectTrigger>
<SelectContent>
<SelectItem value="qwen">Qwen2.5 ()</SelectItem>
<SelectItem value="deepseek">DeepSeek-V3</SelectItem>
</SelectContent>
</Select>
</div>
<div className="space-y-2">
<Label></Label>
<Textarea placeholder="描述需要拆解的字段和转换规则..." rows={3} />
</div>
</div>
<DialogFooter>
<Button variant="outline" onClick={() => setShowConfigDialog(false)}></Button>
<Button onClick={() => setShowConfigDialog(false)}></Button>
</DialogFooter>
</DialogContent>
</Dialog>
</div>
</div>
)
}