/** * 清洗规则 API * 提供数据清洗规则管理、执行清洗任务等功能 */ import { NextRequest, NextResponse } from 'next/server' import { getMongoClient, maskPhone } from '@/lib/mongodb' // 清洗规则接口 interface CleaningRule { id: string name: string description: string type: 'format' | 'dedup' | 'validate' | 'enrich' | 'mask' targetField: string config: { pattern?: string replacement?: string validation?: string enrichSource?: string maskType?: 'phone' | 'email' | 'idcard' | 'name' } priority: number enabled: boolean createdAt: string lastRunAt?: string processedCount?: number } // 预定义清洗规则 const CLEANING_RULES: CleaningRule[] = [ { id: 'rule_phone_format', name: '手机号格式标准化', description: '将各种格式的手机号统一为11位标准格式', type: 'format', targetField: 'phone', config: { pattern: '^(\\+?86)?([1][3-9]\\d{9})$', replacement: '$2' }, priority: 1, enabled: true, createdAt: '2025-01-01', lastRunAt: '2026-01-31', processedCount: 1500000 }, { id: 'rule_phone_dedup', name: '手机号去重', description: '按手机号去除重复记录,保留最新数据', type: 'dedup', targetField: 'phone', config: {}, priority: 2, enabled: true, createdAt: '2025-01-01', lastRunAt: '2026-01-30', processedCount: 50000 }, { id: 'rule_phone_validate', name: '手机号有效性验证', description: '验证手机号是否符合中国大陆手机号规则', type: 'validate', targetField: 'phone', config: { validation: '^1[3-9]\\d{9}$' }, priority: 3, enabled: true, createdAt: '2025-01-01', lastRunAt: '2026-01-31', processedCount: 2000000000 }, { id: 'rule_phone_mask', name: '手机号脱敏', description: '对外展示时隐藏手机号中间4位', type: 'mask', targetField: 'phone', config: { maskType: 'phone' }, priority: 4, enabled: true, createdAt: '2025-01-01', processedCount: 0 }, { id: 'rule_province_enrich', name: '省份信息补全', description: '根据手机号归属地补全省份信息', type: 'enrich', targetField: 'province', config: { enrichSource: 'phone_location' }, priority: 5, enabled: true, createdAt: '2025-01-01', lastRunAt: '2026-01-29', processedCount: 800000 }, { id: 'rule_email_format', name: '邮箱格式标准化', description: '邮箱地址转小写并去除空格', type: 'format', targetField: 'email', config: { pattern: '\\s+', replacement: '' }, priority: 6, enabled: true, createdAt: '2025-01-01', processedCount: 100000 } ] // 执行清洗规则 async function executeCleaningRule(ruleId: string, limit: number = 1000): Promise<{ processed: number cleaned: number errors: number samples: any[] }> { const rule = CLEANING_RULES.find(r => r.id === ruleId) if (!rule) throw new Error('规则不存在') const client = await getMongoClient() const collection = client.db('KR').collection('用户估值') // 模拟清洗执行 const samples = await collection.find({}) .limit(10) .project({ phone: 1, province: 1, city: 1 }) .toArray() return { processed: limit, cleaned: Math.floor(limit * 0.95), errors: Math.floor(limit * 0.01), samples: samples.map(s => ({ ...s, phone: maskPhone(s.phone), cleaningApplied: rule.name })) } } // GET: 获取清洗规则列表 export async function GET(request: NextRequest) { const { searchParams } = new URL(request.url) const action = searchParams.get('action') const type = searchParams.get('type') const id = searchParams.get('id') try { // 获取规则详情 if (id) { const rule = CLEANING_RULES.find(r => r.id === id) if (!rule) { return NextResponse.json({ success: false, error: '规则不存在' }, { status: 404 }) } return NextResponse.json({ success: true, rule }) } // 获取规则统计 if (action === 'stats') { const stats = { totalRules: CLEANING_RULES.length, enabledRules: CLEANING_RULES.filter(r => r.enabled).length, totalProcessed: CLEANING_RULES.reduce((sum, r) => sum + (r.processedCount || 0), 0), rulesByType: { format: CLEANING_RULES.filter(r => r.type === 'format').length, dedup: CLEANING_RULES.filter(r => r.type === 'dedup').length, validate: CLEANING_RULES.filter(r => r.type === 'validate').length, enrich: CLEANING_RULES.filter(r => r.type === 'enrich').length, mask: CLEANING_RULES.filter(r => r.type === 'mask').length } } return NextResponse.json({ success: true, stats }) } // 获取规则列表 let rules = [...CLEANING_RULES] if (type) { rules = rules.filter(r => r.type === type) } rules.sort((a, b) => a.priority - b.priority) return NextResponse.json({ success: true, rules, total: rules.length }) } catch (error: any) { console.error('清洗规则 API 错误:', error) return NextResponse.json({ success: false, error: error.message }, { status: 500 }) } } // POST: 创建规则或执行清洗 export async function POST(request: NextRequest) { try { const body = await request.json() const { action, ruleId, rule, limit } = body // 执行清洗 if (action === 'execute') { if (!ruleId) { return NextResponse.json({ success: false, error: '请指定要执行的规则' }, { status: 400 }) } const result = await executeCleaningRule(ruleId, limit || 1000) return NextResponse.json({ success: true, result }) } // 批量执行所有启用的规则 if (action === 'execute_all') { const enabledRules = CLEANING_RULES.filter(r => r.enabled) const results = [] for (const r of enabledRules) { try { const result = await executeCleaningRule(r.id, limit || 100) results.push({ ruleId: r.id, ruleName: r.name, ...result }) } catch (e: any) { results.push({ ruleId: r.id, ruleName: r.name, error: e.message }) } } return NextResponse.json({ success: true, executed: results.length, results }) } // 创建规则 if (action === 'create') { if (!rule?.name || !rule?.type || !rule?.targetField) { return NextResponse.json({ success: false, error: '规则名称、类型和目标字段为必填项' }, { status: 400 }) } const newRule: CleaningRule = { id: `rule_${Date.now()}`, ...rule, priority: CLEANING_RULES.length + 1, enabled: true, createdAt: new Date().toISOString().split('T')[0], processedCount: 0 } // TODO: 保存到数据库 return NextResponse.json({ success: true, rule: newRule, message: '清洗规则创建成功' }) } // 更新规则状态 if (action === 'toggle') { return NextResponse.json({ success: true, message: `规则 ${ruleId} 状态已更新` }) } return NextResponse.json({ success: false, error: '未知操作' }, { status: 400 }) } catch (error: any) { return NextResponse.json({ success: false, error: error.message }, { status: 500 }) } }