feat: refactor data asset center for enhanced search and analytics

Refactor homepage for focused search and data display; streamline data platform; enhance user and tag management; focus AI assistant on data analysis and report generation. Co-authored-by: null <4804959+fnvtk@users.noreply.github.com>
2025-07-25 06:42:34 +00:00
parent ecd8a48863
commit 4eed69520c
40 changed files with 6853 additions and 6111 deletions
--- a/lib/data-dictionary.ts
+++ b/lib/data-dictionary.ts
@@ -0,0 +1,228 @@
+// 数据字典和映射规则管理
+// 根据需求文档中的要求，创建动态维护的数据字典
+
+export interface DataField {
+  name: string
+  type: "string" | "number" | "boolean" | "date" | "json" | "array"
+  description: string
+  isRequired: boolean
+  isPII: boolean // 个人身份信息标识
+  isIdentityKey: boolean // 身份识别关键字段
+  isAIFeature: boolean // 用于AI分析的特征字段
+  isDistributionKey: boolean // 用于分销/返点计算的关联字段
+}
+
+export interface SourceMapping {
+  sourceSystem: string
+  sourceField: string
+  targetField: string
+  transformRule?: string
+  validationRule?: string
+}
+
+export interface DataDictionary {
+  coreFields: Record<string, DataField>
+  unifiedTags: string[]
+  unifiedAttributes: Record<string, DataField>
+  sourceMappings: Record<string, SourceMapping[]>
+}
+
+// 核心数据字典定义
+export const DATA_DICTIONARY: DataDictionary = {
+  coreFields: {
+    userId: {
+      name: "userId",
+      type: "string",
+      description: "全局唯一用户ID",
+      isRequired: true,
+      isPII: false,
+      isIdentityKey: true,
+      isAIFeature: false,
+      isDistributionKey: true,
+    },
+    username: {
+      name: "username",
+      type: "string",
+      description: "用户名",
+      isRequired: false,
+      isPII: true,
+      isIdentityKey: true,
+      isAIFeature: true,
+      isDistributionKey: false,
+    },
+    email: {
+      name: "email",
+      type: "string",
+      description: "邮箱地址",
+      isRequired: false,
+      isPII: true,
+      isIdentityKey: true,
+      isAIFeature: false,
+      isDistributionKey: false,
+    },
+    phone: {
+      name: "phone",
+      type: "string",
+      description: "手机号码",
+      isRequired: false,
+      isPII: true,
+      isIdentityKey: true,
+      isAIFeature: false,
+      isDistributionKey: false,
+    },
+    fullName: {
+      name: "fullName",
+      type: "string",
+      description: "真实姓名",
+      isRequired: false,
+      isPII: true,
+      isIdentityKey: true,
+      isAIFeature: true,
+      isDistributionKey: false,
+    },
+    gender: {
+      name: "gender",
+      type: "string",
+      description: "性别",
+      isRequired: false,
+      isPII: false,
+      isIdentityKey: false,
+      isAIFeature: true,
+      isDistributionKey: false,
+    },
+    birthDate: {
+      name: "birthDate",
+      type: "date",
+      description: "出生日期",
+      isRequired: false,
+      isPII: true,
+      isIdentityKey: false,
+      isAIFeature: true,
+      isDistributionKey: false,
+    },
+    city: {
+      name: "city",
+      type: "string",
+      description: "所在城市",
+      isRequired: false,
+      isPII: false,
+      isIdentityKey: false,
+      isAIFeature: true,
+      isDistributionKey: false,
+    },
+    province: {
+      name: "province",
+      type: "string",
+      description: "所在省份",
+      isRequired: false,
+      isPII: false,
+      isIdentityKey: false,
+      isAIFeature: true,
+      isDistributionKey: false,
+    },
+  },
+  unifiedTags: [
+    "高价值客户",
+    "新用户",
+    "活跃用户",
+    "流失风险",
+    "科技爱好者",
+    "内容创作者",
+    "90后",
+    "00后",
+    "北京地区",
+    "上海地区",
+    "游戏爱好者",
+    "旅游达人",
+    "美食家",
+  ],
+  unifiedAttributes: {
+    totalSpend: {
+      name: "totalSpend",
+      type: "number",
+      description: "总消费金额",
+      isRequired: false,
+      isPII: false,
+      isIdentityKey: false,
+      isAIFeature: true,
+      isDistributionKey: true,
+    },
+    lastActiveDays: {
+      name: "lastActiveDays",
+      type: "number",
+      description: "最后活跃天数",
+      isRequired: false,
+      isPII: false,
+      isIdentityKey: false,
+      isAIFeature: true,
+      isDistributionKey: false,
+    },
+    contentPreference: {
+      name: "contentPreference",
+      type: "array",
+      description: "内容偏好",
+      isRequired: false,
+      isPII: false,
+      isIdentityKey: false,
+      isAIFeature: true,
+      isDistributionKey: false,
+    },
+    distributionLevel: {
+      name: "distributionLevel",
+      type: "number",
+      description: "分销层级",
+      isRequired: false,
+      isPII: false,
+      isIdentityKey: false,
+      isAIFeature: false,
+      isDistributionKey: true,
+    },
+  },
+  sourceMappings: {
+    douyin: [
+      { sourceSystem: "douyin", sourceField: "openid", targetField: "userId", transformRule: "prefix_dy_" },
+      { sourceSystem: "douyin", sourceField: "nickname", targetField: "username" },
+      { sourceSystem: "douyin", sourceField: "avatar", targetField: "avatarUrl" },
+    ],
+    xiaohongshu: [
+      { sourceSystem: "xiaohongshu", sourceField: "openid", targetField: "userId", transformRule: "prefix_xhs_" },
+      { sourceSystem: "xiaohongshu", sourceField: "nickname", targetField: "username" },
+    ],
+    cunkebao_form: [
+      { sourceSystem: "cunkebao_form", sourceField: "name", targetField: "fullName" },
+      { sourceSystem: "cunkebao_form", sourceField: "phone", targetField: "phone" },
+      { sourceSystem: "cunkebao_form", sourceField: "email", targetField: "email" },
+    ],
+    touchkebao_call: [
+      { sourceSystem: "touchkebao_call", sourceField: "phone_number", targetField: "phone" },
+      { sourceSystem: "touchkebao_call", sourceField: "call_time", targetField: "lastContactTime" },
+    ],
+  },
+}
+
+// 获取字段映射规则
+export function getFieldMapping(sourceSystem: string, sourceField: string): SourceMapping | undefined {
+  const mappings = DATA_DICTIONARY.sourceMappings[sourceSystem]
+  return mappings?.find((mapping) => mapping.sourceField === sourceField)
+}
+
+// 获取身份识别关键字段
+export function getIdentityKeyFields(): string[] {
+  return Object.entries(DATA_DICTIONARY.coreFields)
+    .filter(([_, field]) => field.isIdentityKey)
+    .map(([name, _]) => name)
+}
+
+// 获取AI特征字段
+export function getAIFeatureFields(): string[] {
+  return Object.entries(DATA_DICTIONARY.coreFields)
+    .filter(([_, field]) => field.isAIFeature)
+    .map(([name, _]) => name)
+}
+
+// 获取分销关联字段
+export function getDistributionKeyFields(): string[] {
+  return Object.entries(DATA_DICTIONARY.coreFields)
+    .filter(([_, field]) => field.isDistributionKey)
+    .map(([name, _]) => name)
+}
--- a/lib/mindsdb-connector.ts
+++ b/lib/mindsdb-connector.ts
@@ -0,0 +1,433 @@
+// MindsDB连接器 - 实现AI增强的数据查询和分析
+import { Client } from "mindsdb-js-sdk"
+
+export interface MindsDBConfig {
+  host: string
+  port: number
+  username: string
+  password: string
+  database?: string
+}
+
+export interface AIQueryRequest {
+  query: string
+  model?: string
+  parameters?: Record<string, any>
+  useCache?: boolean
+}
+
+export interface SearchRequest {
+  keyword: string
+  type: "user" | "traffic" | "all"
+  filters?: Record<string, any>
+  limit?: number
+  offset?: number
+}
+
+export interface VersionInfo {
+  version: string
+  timestamp: string
+  changes: string[]
+  author: string
+}
+
+export class MindsDBConnector {
+  private client: Client
+  private connected = false
+  private cache: Map<string, any> = new Map()
+
+  constructor(private config: MindsDBConfig) {
+    this.client = new Client({
+      host: config.host,
+      port: config.port,
+      username: config.username,
+      password: config.password,
+    })
+  }
+
+  // 连接到MindsDB
+  async connect(): Promise<void> {
+    try {
+      await this.client.connect()
+      this.connected = true
+      console.log("MindsDB连接成功")
+    } catch (error) {
+      console.error("MindsDB连接失败:", error)
+      throw error
+    }
+  }
+
+  // 断开连接
+  async disconnect(): Promise<void> {
+    if (this.connected) {
+      await this.client.disconnect()
+      this.connected = false
+    }
+  }
+
+  // AI增强查询 - 使用自然语言查询数据
+  async aiQuery(request: AIQueryRequest): Promise<any> {
+    if (!this.connected) {
+      await this.connect()
+    }
+
+    const cacheKey = `ai_query_${JSON.stringify(request)}`
+
+    // 检查缓存
+    if (request.useCache && this.cache.has(cacheKey)) {
+      return this.cache.get(cacheKey)
+    }
+
+    try {
+      // 使用MindsDB的AI模型进行查询
+      const query = `
+        SELECT * FROM mindsdb.${request.model || "gpt4"} 
+        WHERE text = '${request.query}'
+      `
+
+      const result = await this.client.query(query)
+
+      // 缓存结果
+      if (request.useCache) {
+        this.cache.set(cacheKey, result, 300000) // 5分钟缓存
+      }
+
+      return result
+    } catch (error) {
+      console.error("AI查询失败:", error)
+      throw error
+    }
+  }
+
+  // 智能搜索 - 支持用户数据和流量关键词的快速搜索
+  async intelligentSearch(request: SearchRequest): Promise<any> {
+    if (!this.connected) {
+      await this.connect()
+    }
+
+    const cacheKey = `search_${JSON.stringify(request)}`
+
+    // 检查缓存
+    if (this.cache.has(cacheKey)) {
+      return this.cache.get(cacheKey)
+    }
+
+    try {
+      let searchQuery = ""
+
+      switch (request.type) {
+        case "user":
+          searchQuery = this.buildUserSearchQuery(request)
+          break
+        case "traffic":
+          searchQuery = this.buildTrafficSearchQuery(request)
+          break
+        case "all":
+          searchQuery = this.buildUnifiedSearchQuery(request)
+          break
+      }
+
+      const result = await this.client.query(searchQuery)
+
+      // 缓存结果
+      this.cache.set(cacheKey, result, 60000) // 1分钟缓存
+
+      return result
+    } catch (error) {
+      console.error("智能搜索失败:", error)
+      throw error
+    }
+  }
+
+  // 构建用户搜索查询
+  private buildUserSearchQuery(request: SearchRequest): string {
+    const { keyword, filters, limit = 100, offset = 0 } = request
+
+    let query = `
+      SELECT 
+        u.user_id,
+        u.username,
+        u.phone,
+        u.email,
+        u.tags,
+        u.rfm_score,
+        u.last_active,
+        u.created_at,
+        MATCH(u.username, u.phone, u.email, u.tags) AGAINST('${keyword}' IN NATURAL LANGUAGE MODE) as relevance_score
+      FROM users u
+      WHERE MATCH(u.username, u.phone, u.email, u.tags) AGAINST('${keyword}' IN NATURAL LANGUAGE MODE)
+    `
+
+    // 添加过滤条件
+    if (filters) {
+      Object.entries(filters).forEach(([key, value]) => {
+        query += ` AND u.${key} = '${value}'`
+      })
+    }
+
+    query += ` ORDER BY relevance_score DESC, u.last_active DESC`
+    query += ` LIMIT ${limit} OFFSET ${offset}`
+
+    return query
+  }
+
+  // 构建流量关键词搜索查询
+  private buildTrafficSearchQuery(request: SearchRequest): string {
+    const { keyword, filters, limit = 100, offset = 0 } = request
+
+    let query = `
+      SELECT 
+        t.keyword_id,
+        t.keyword,
+        t.category,
+        t.search_volume,
+        t.competition,
+        t.cpc,
+        t.trend_data,
+        t.last_updated,
+        MATCH(t.keyword, t.category) AGAINST('${keyword}' IN NATURAL LANGUAGE MODE) as relevance_score
+      FROM traffic_keywords t
+      WHERE MATCH(t.keyword, t.category) AGAINST('${keyword}' IN NATURAL LANGUAGE MODE)
+    `
+
+    // 添加过滤条件
+    if (filters) {
+      Object.entries(filters).forEach(([key, value]) => {
+        query += ` AND t.${key} = '${value}'`
+      })
+    }
+
+    query += ` ORDER BY relevance_score DESC, t.search_volume DESC`
+    query += ` LIMIT ${limit} OFFSET ${offset}`
+
+    return query
+  }
+
+  // 构建统一搜索查询
+  private buildUnifiedSearchQuery(request: SearchRequest): string {
+    const { keyword, limit = 100, offset = 0 } = request
+
+    return `
+      (
+        SELECT 
+          'user' as type,
+          user_id as id,
+          username as title,
+          CONCAT(phone, ' | ', email) as description,
+          tags,
+          last_active as updated_at,
+          MATCH(username, phone, email, tags) AGAINST('${keyword}' IN NATURAL LANGUAGE MODE) as relevance_score
+        FROM users
+        WHERE MATCH(username, phone, email, tags) AGAINST('${keyword}' IN NATURAL LANGUAGE MODE)
+      )
+      UNION ALL
+      (
+        SELECT 
+          'traffic' as type,
+          keyword_id as id,
+          keyword as title,
+          CONCAT('搜索量: ', search_volume, ' | 竞争度: ', competition) as description,
+          category as tags,
+          last_updated as updated_at,
+          MATCH(keyword, category) AGAINST('${keyword}' IN NATURAL LANGUAGE MODE) as relevance_score
+        FROM traffic_keywords
+        WHERE MATCH(keyword, category) AGAINST('${keyword}' IN NATURAL LANGUAGE MODE)
+      )
+      ORDER BY relevance_score DESC
+      LIMIT ${limit} OFFSET ${offset}
+    `
+  }
+
+  // 用户数据分析 - 使用AI进行用户行为分析
+  async analyzeUserBehavior(userId: string): Promise<any> {
+    if (!this.connected) {
+      await this.connect()
+    }
+
+    try {
+      const query = `
+        SELECT 
+          prediction,
+          confidence,
+          explanation
+        FROM mindsdb.user_behavior_predictor
+        WHERE user_id = '${userId}'
+      `
+
+      return await this.client.query(query)
+    } catch (error) {
+      console.error("用户行为分析失败:", error)
+      throw error
+    }
+  }
+
+  // 流量预测 - 使用AI预测流量趋势
+  async predictTrafficTrends(keyword: string, timeframe = "30d"): Promise<any> {
+    if (!this.connected) {
+      await this.connect()
+    }
+
+    try {
+      const query = `
+        SELECT 
+          predicted_volume,
+          trend_direction,
+          confidence_interval,
+          factors
+        FROM mindsdb.traffic_predictor
+        WHERE keyword = '${keyword}' AND timeframe = '${timeframe}'
+      `
+
+      return await this.client.query(query)
+    } catch (error) {
+      console.error("流量预测失败:", error)
+      throw error
+    }
+  }
+
+  // 版本管理 - 创建数据版本
+  async createVersion(data: any, author: string, changes: string[]): Promise<VersionInfo> {
+    const version = `v${Date.now()}`
+    const timestamp = new Date().toISOString()
+
+    const versionInfo: VersionInfo = {
+      version,
+      timestamp,
+      changes,
+      author,
+    }
+
+    try {
+      // 存储版本信息
+      const query = `
+        INSERT INTO data_versions (version, timestamp, data_snapshot, changes, author)
+        VALUES ('${version}', '${timestamp}', '${JSON.stringify(data)}', '${JSON.stringify(changes)}', '${author}')
+      `
+
+      await this.client.query(query)
+      return versionInfo
+    } catch (error) {
+      console.error("创建版本失败:", error)
+      throw error
+    }
+  }
+
+  // 获取版本历史
+  async getVersionHistory(limit = 50): Promise<VersionInfo[]> {
+    if (!this.connected) {
+      await this.connect()
+    }
+
+    try {
+      const query = `
+        SELECT version, timestamp, changes, author
+        FROM data_versions
+        ORDER BY timestamp DESC
+        LIMIT ${limit}
+      `
+
+      const result = await this.client.query(query)
+      return result.rows || []
+    } catch (error) {
+      console.error("获取版本历史失败:", error)
+      throw error
+    }
+  }
+
+  // 恢复到指定版本
+  async restoreVersion(version: string): Promise<any> {
+    if (!this.connected) {
+      await this.connect()
+    }
+
+    try {
+      const query = `
+        SELECT data_snapshot
+        FROM data_versions
+        WHERE version = '${version}'
+      `
+
+      const result = await this.client.query(query)
+      if (result.rows && result.rows.length > 0) {
+        return JSON.parse(result.rows[0].data_snapshot)
+      }
+
+      throw new Error(`版本 ${version} 不存在`)
+    } catch (error) {
+      console.error("恢复版本失败:", error)
+      throw error
+    }
+  }
+
+  // 实时数据同步
+  async syncRealTimeData(source: string, data: any): Promise<void> {
+    if (!this.connected) {
+      await this.connect()
+    }
+
+    try {
+      const query = `
+        INSERT INTO real_time_data (source, data, timestamp)
+        VALUES ('${source}', '${JSON.stringify(data)}', NOW())
+        ON DUPLICATE KEY UPDATE
+        data = '${JSON.stringify(data)}',
+        timestamp = NOW()
+      `
+
+      await this.client.query(query)
+    } catch (error) {
+      console.error("实时数据同步失败:", error)
+      throw error
+    }
+  }
+
+  // 清理缓存
+  clearCache(): void {
+    this.cache.clear()
+  }
+
+  // 获取系统状态
+  async getSystemStatus(): Promise<any> {
+    if (!this.connected) {
+      await this.connect()
+    }
+
+    try {
+      const queries = [
+        "SELECT COUNT(*) as user_count FROM users",
+        "SELECT COUNT(*) as keyword_count FROM traffic_keywords",
+        "SELECT COUNT(*) as version_count FROM data_versions",
+        "SELECT AVG(response_time) as avg_response_time FROM query_logs WHERE created_at > DATE_SUB(NOW(), INTERVAL 1 HOUR)",
+      ]
+
+      const results = await Promise.all(queries.map((query) => this.client.query(query)))
+
+      return {
+        userCount: results[0].rows[0].user_count,
+        keywordCount: results[1].rows[0].keyword_count,
+        versionCount: results[2].rows[0].version_count,
+        avgResponseTime: results[3].rows[0].avg_response_time || 0,
+        cacheSize: this.cache.size,
+        connected: this.connected,
+      }
+    } catch (error) {
+      console.error("获取系统状态失败:", error)
+      throw error
+    }
+  }
+}
+
+// 单例模式
+let mindsDBInstance: MindsDBConnector | null = null
+
+export function getMindsDBConnector(config?: MindsDBConfig): MindsDBConnector {
+  if (!mindsDBInstance && config) {
+    mindsDBInstance = new MindsDBConnector(config)
+  }
+
+  if (!mindsDBInstance) {
+    throw new Error("MindsDB连接器未初始化，请提供配置信息")
+  }
+
+  return mindsDBInstance
+}