小红书同步每日数据

database/migrations/add_share_collect_to_user_day_statistics.sql

@@ -0,0 +1,9 @@
+-- Add share_count and collect_count to user_day_statistics
+-- Date: 2026-01-28
+
+USE media_manager;
+
+ALTER TABLE user_day_statistics
+  ADD COLUMN share_count INT DEFAULT 0 COMMENT 'share_count' AFTER like_count,
+  ADD COLUMN collect_count INT DEFAULT 0 COMMENT 'collect_count' AFTER share_count;
+

database/migrations/restore_user_day_statistics_full.sql

@@ -0,0 +1,157 @@
+-- Restore full schema for user_day_statistics (idempotent-ish)
+-- Date: 2026-01-28
+--
+-- Notes:
+-- - This script avoids TypeORM "synchronize" surprises by ensuring columns exist.
+-- - MySQL 8.0 does NOT support ADD COLUMN IF NOT EXISTS reliably across all syntaxes,
+--   so we use INFORMATION_SCHEMA checks + dynamic SQL.
+-- - It will try to add an FK to platform_accounts(id). If there are orphan rows, it will NOT add FK.
+
+USE media_manager;
+
+SET @db = DATABASE();
+SET @tbl = 'user_day_statistics';
+
+-- Helper: add column if missing
+-- play_count
+SET @col = 'play_count';
+SET @sql = (
+  SELECT IF(
+    (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA=@db AND TABLE_NAME=@tbl AND COLUMN_NAME=@col) > 0,
+    'SELECT 1',
+    'ALTER TABLE user_day_statistics ADD COLUMN play_count INT NOT NULL DEFAULT 0 AFTER works_count'
+  )
+);
+PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+-- comment_count
+SET @col = 'comment_count';
+SET @sql = (
+  SELECT IF(
+    (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA=@db AND TABLE_NAME=@tbl AND COLUMN_NAME=@col) > 0,
+    'SELECT 1',
+    'ALTER TABLE user_day_statistics ADD COLUMN comment_count INT NOT NULL DEFAULT 0 AFTER play_count'
+  )
+);
+PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+-- fans_increase
+SET @col = 'fans_increase';
+SET @sql = (
+  SELECT IF(
+    (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA=@db AND TABLE_NAME=@tbl AND COLUMN_NAME=@col) > 0,
+    'SELECT 1',
+    'ALTER TABLE user_day_statistics ADD COLUMN fans_increase INT NOT NULL DEFAULT 0 AFTER comment_count'
+  )
+);
+PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+-- like_count
+SET @col = 'like_count';
+SET @sql = (
+  SELECT IF(
+    (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA=@db AND TABLE_NAME=@tbl AND COLUMN_NAME=@col) > 0,
+    'SELECT 1',
+    'ALTER TABLE user_day_statistics ADD COLUMN like_count INT NOT NULL DEFAULT 0 AFTER fans_increase'
+  )
+);
+PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+-- share_count
+SET @col = 'share_count';
+SET @sql = (
+  SELECT IF(
+    (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA=@db AND TABLE_NAME=@tbl AND COLUMN_NAME=@col) > 0,
+    'SELECT 1',
+    'ALTER TABLE user_day_statistics ADD COLUMN share_count INT NOT NULL DEFAULT 0 AFTER like_count'
+  )
+);
+PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+-- collect_count
+SET @col = 'collect_count';
+SET @sql = (
+  SELECT IF(
+    (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA=@db AND TABLE_NAME=@tbl AND COLUMN_NAME=@col) > 0,
+    'SELECT 1',
+    'ALTER TABLE user_day_statistics ADD COLUMN collect_count INT NOT NULL DEFAULT 0 AFTER share_count'
+  )
+);
+PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+-- cover_click_rate (string)
+SET @col = 'cover_click_rate';
+SET @sql = (
+  SELECT IF(
+    (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA=@db AND TABLE_NAME=@tbl AND COLUMN_NAME=@col) > 0,
+    'ALTER TABLE user_day_statistics MODIFY COLUMN cover_click_rate VARCHAR(50) NOT NULL DEFAULT ''0''',
+    'ALTER TABLE user_day_statistics ADD COLUMN cover_click_rate VARCHAR(50) NOT NULL DEFAULT ''0'' AFTER collect_count'
+  )
+);
+PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+-- avg_watch_duration (string)
+SET @col = 'avg_watch_duration';
+SET @sql = (
+  SELECT IF(
+    (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA=@db AND TABLE_NAME=@tbl AND COLUMN_NAME=@col) > 0,
+    'ALTER TABLE user_day_statistics MODIFY COLUMN avg_watch_duration VARCHAR(50) NOT NULL DEFAULT ''0''',
+    'ALTER TABLE user_day_statistics ADD COLUMN avg_watch_duration VARCHAR(50) NOT NULL DEFAULT ''0'' AFTER cover_click_rate'
+  )
+);
+PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+-- total_watch_duration (string)
+SET @col = 'total_watch_duration';
+SET @sql = (
+  SELECT IF(
+    (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA=@db AND TABLE_NAME=@tbl AND COLUMN_NAME=@col) > 0,
+    'ALTER TABLE user_day_statistics MODIFY COLUMN total_watch_duration VARCHAR(50) NOT NULL DEFAULT ''0''',
+    'ALTER TABLE user_day_statistics ADD COLUMN total_watch_duration VARCHAR(50) NOT NULL DEFAULT ''0'' AFTER avg_watch_duration'
+  )
+);
+PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+-- completion_rate (string)
+SET @col = 'completion_rate';
+SET @sql = (
+  SELECT IF(
+    (SELECT COUNT(*) FROM INFORMATION_SCHEMA.COLUMNS WHERE TABLE_SCHEMA=@db AND TABLE_NAME=@tbl AND COLUMN_NAME=@col) > 0,
+    'ALTER TABLE user_day_statistics MODIFY COLUMN completion_rate VARCHAR(50) NOT NULL DEFAULT ''0''',
+    'ALTER TABLE user_day_statistics ADD COLUMN completion_rate VARCHAR(50) NOT NULL DEFAULT ''0'' AFTER total_watch_duration'
+  )
+);
+PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+-- Index on record_date
+SET @idx = 'idx_record_date';
+SET @sql = (
+  SELECT IF(
+    (SELECT COUNT(*) FROM INFORMATION_SCHEMA.STATISTICS WHERE TABLE_SCHEMA=@db AND TABLE_NAME=@tbl AND INDEX_NAME=@idx) > 0,
+    'SELECT 1',
+    'ALTER TABLE user_day_statistics ADD INDEX idx_record_date (record_date)'
+  )
+);
+PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+
+-- Foreign key to platform_accounts(id) if possible (no orphans)
+SET @fk = 'fk_user_day_statistics_account';
+SET @orphans = (
+  SELECT COUNT(*)
+  FROM user_day_statistics uds
+  LEFT JOIN platform_accounts pa ON pa.id = uds.account_id
+  WHERE pa.id IS NULL
+);
+
+SET @sql = (
+  SELECT IF(
+    (SELECT COUNT(*) FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=@db AND TABLE_NAME=@tbl AND CONSTRAINT_NAME=@fk) > 0,
+    'SELECT 1',
+    IF(@orphans > 0,
+      'SELECT ''Skip adding FK fk_user_day_statistics_account: orphan rows exist'' AS warning',
+      'ALTER TABLE user_day_statistics ADD CONSTRAINT fk_user_day_statistics_account FOREIGN KEY (account_id) REFERENCES platform_accounts(id) ON DELETE CASCADE'
+    )
+  )
+);
+PREPARE stmt FROM @sql; EXECUTE stmt; DEALLOCATE PREPARE stmt;
+

database/schema.sql

@@ -212,6 +212,8 @@ CREATE TABLE IF NOT EXISTS user_day_statistics (
     comment_count INT DEFAULT 0 COMMENT '评论数',
     fans_increase INT DEFAULT 0 COMMENT '涨粉数',
     like_count INT DEFAULT 0 COMMENT '点赞数',
+    share_count INT DEFAULT 0 COMMENT '分享数',
+    collect_count INT DEFAULT 0 COMMENT '收藏数',
     cover_click_rate VARCHAR(50) DEFAULT '0' COMMENT '封面点击率',
     avg_watch_duration VARCHAR(50) DEFAULT '0' COMMENT '平均观看时长（秒）',
     total_watch_duration VARCHAR(50) DEFAULT '0' COMMENT '观看总时长（秒）',

server/src/models/entities/UserDayStatistics.ts

@@ -30,6 +30,12 @@ export class UserDayStatistics {
   @Column({ name: 'like_count', type: 'int', default: 0, comment: '点赞数' })
   likeCount!: number;
 
+  @Column({ name: 'share_count', type: 'int', default: 0, comment: '分享数' })
+  shareCount!: number;
+
+  @Column({ name: 'collect_count', type: 'int', default: 0, comment: '收藏数' })
+  collectCount!: number;
+
   @Column({ name: 'cover_click_rate', type: 'varchar', length: 50, default: '0', comment: '封面点击率' })
   coverClickRate!: string;
 

server/src/models/index.ts

@@ -21,7 +21,8 @@ export const AppDataSource = new DataSource({
   username: config.database.username,
   password: config.database.password,
   database: config.database.database,
-  synchronize: config.env === 'development', // 开发环境自动同步
+  // 永远关闭 synchronize：避免自动改表/删列（统一走 migrations）
+  synchronize: false,
   logging: config.env === 'development',
   entities: [
     User,

server/src/services/UserDayStatisticsService.ts

@@ -9,6 +9,8 @@ export interface UserDayStatisticsItem {
   commentCount?: number;
   fansIncrease?: number;
   likeCount?: number;
+  shareCount?: number;
+  collectCount?: number;
   coverClickRate?: string;
   avgWatchDuration?: string;
   totalWatchDuration?: string;
@@ -51,6 +53,8 @@ export class UserDayStatisticsService {
         commentCount: item.commentCount ?? existing.commentCount,
         fansIncrease: item.fansIncrease ?? existing.fansIncrease,
         likeCount: item.likeCount ?? existing.likeCount,
+        shareCount: item.shareCount ?? existing.shareCount,
+        collectCount: item.collectCount ?? existing.collectCount,
         coverClickRate: item.coverClickRate ?? existing.coverClickRate ?? '0',
         avgWatchDuration: item.avgWatchDuration ?? existing.avgWatchDuration ?? '0',
         totalWatchDuration: item.totalWatchDuration ?? existing.totalWatchDuration ?? '0',
@@ -69,6 +73,8 @@ export class UserDayStatisticsService {
         commentCount: item.commentCount ?? 0,
         fansIncrease: item.fansIncrease ?? 0,
         likeCount: item.likeCount ?? 0,
+        shareCount: item.shareCount ?? 0,
+        collectCount: item.collectCount ?? 0,
         coverClickRate: item.coverClickRate ?? '0',
         avgWatchDuration: item.avgWatchDuration ?? '0',
         totalWatchDuration: item.totalWatchDuration ?? '0',
@@ -104,6 +110,8 @@ export class UserDayStatisticsService {
         commentCount: patch.commentCount ?? existing.commentCount,
         fansIncrease: patch.fansIncrease ?? existing.fansIncrease,
         likeCount: patch.likeCount ?? existing.likeCount,
+        shareCount: patch.shareCount ?? existing.shareCount,
+        collectCount: patch.collectCount ?? existing.collectCount,
         coverClickRate: patch.coverClickRate ?? existing.coverClickRate ?? '0',
         avgWatchDuration: patch.avgWatchDuration ?? existing.avgWatchDuration ?? '0',
         totalWatchDuration: patch.totalWatchDuration ?? existing.totalWatchDuration ?? '0',
@@ -121,6 +129,8 @@ export class UserDayStatisticsService {
       commentCount: patch.commentCount ?? 0,
       fansIncrease: patch.fansIncrease ?? 0,
       likeCount: patch.likeCount ?? 0,
+      shareCount: patch.shareCount ?? 0,
+      collectCount: patch.collectCount ?? 0,
       coverClickRate: patch.coverClickRate ?? '0',
       avgWatchDuration: patch.avgWatchDuration ?? '0',
       totalWatchDuration: patch.totalWatchDuration ?? '0',

server/src/services/XiaohongshuAccountOverviewImportService.ts

@@ -28,11 +28,18 @@ type PlaywrightCookie = {
 
 type MetricKind =
   | 'playCount'
+  | 'likeCount'
+  | 'commentCount'
+  | 'shareCount'
+  | 'collectCount'
+  | 'fansIncrease'
   | 'coverClickRate'
   | 'avgWatchDuration'
   | 'totalWatchDuration'
   | 'completionRate';
 
+type ExportMode = 'watch' | 'interaction' | 'fans';
+
 function ensureDir(p: string) {
   return fs.mkdir(p, { recursive: true });
 }
@@ -87,12 +94,21 @@ function parseChineseNumberLike(input: unknown): number | null {
 
 function detectMetricKind(sheetName: string): MetricKind | null {
   const n = sheetName.trim();
-  // 小红书导出的子表命名可能是「观看趋势」或「观看数趋势」
+  // 观看数据：子表命名可能是「观看趋势」或「观看数趋势」
   if (n.includes('观看趋势') || n.includes('观看数')) return 'playCount';
   if (n.includes('封面点击率')) return 'coverClickRate';
   if (n.includes('平均观看时长')) return 'avgWatchDuration';
   if (n.includes('观看总时长')) return 'totalWatchDuration';
   if (n.includes('完播率')) return 'completionRate';
+
+  // 互动数据
+  if (n.includes('点赞') && n.includes('趋势')) return 'likeCount';
+  if (n.includes('评论') && n.includes('趋势')) return 'commentCount';
+  if (n.includes('分享') && n.includes('趋势')) return 'shareCount';
+  if (n.includes('收藏') && n.includes('趋势')) return 'collectCount';
+
+  // 涨粉数据（只取净涨粉趋势）
+  if (n.includes('净涨粉') && n.includes('趋势')) return 'fansIncrease';
   return null;
 }
 
@@ -172,15 +188,31 @@ async function createBrowserForAccount(proxy: ProxyConfig | null): Promise<{ bro
   return { browser, shouldClose: false };
 }
 
-function parseXhsExcel(filePath: string): Map<string, { recordDate: Date } & Record<string, any>> {
+function parseXhsExcel(
+  filePath: string,
+  mode: ExportMode
+): Map<string, { recordDate: Date } & Record<string, any>> {
   const wb = XLSX.readFile(filePath);
   const result = new Map<string, { recordDate: Date } & Record<string, any>>();
 
-  logger.info(`[XHS Import] Excel loaded. file=${path.basename(filePath)} sheets=${wb.SheetNames.join(' | ')}`);
+  logger.info(
+    `[XHS Import] Excel loaded. mode=${mode} file=${path.basename(filePath)} sheets=${wb.SheetNames.join(' | ')}`
+  );
 
   for (const sheetName of wb.SheetNames) {
     const kind = detectMetricKind(sheetName);
     if (!kind) continue;
+
+    // 按导出类型过滤不相关子表，避免误写字段
+    if (
+      (mode === 'watch' &&
+        !['playCount', 'coverClickRate', 'avgWatchDuration', 'totalWatchDuration', 'completionRate'].includes(kind)) ||
+      (mode === 'interaction' && !['likeCount', 'commentCount', 'shareCount', 'collectCount'].includes(kind)) ||
+      (mode === 'fans' && kind !== 'fansIncrease')
+    ) {
+      continue;
+    }
+
     const sheet = wb.Sheets[sheetName];
     const rows = XLSX.utils.sheet_to_json<Record<string, any>>(sheet, { defval: '' });
 
@@ -200,9 +232,16 @@ function parseXhsExcel(filePath: string): Map<string, { recordDate: Date } & Rec
       if (!result.has(key)) result.set(key, { recordDate: d });
       const obj = result.get(key)!;
 
-      if (kind === 'playCount') {
+      if (kind === 'playCount' || kind === 'likeCount' || kind === 'commentCount' || kind === 'shareCount' || kind === 'collectCount' || kind === 'fansIncrease') {
         const n = parseChineseNumberLike(valueVal);
-        if (typeof n === 'number') obj.playCount = n;
+        if (typeof n === 'number') {
+          if (kind === 'playCount') obj.playCount = n;
+          if (kind === 'likeCount') obj.likeCount = n;
+          if (kind === 'commentCount') obj.commentCount = n;
+          if (kind === 'shareCount') obj.shareCount = n;
+          if (kind === 'collectCount') obj.collectCount = n;
+          if (kind === 'fansIncrease') obj.fansIncrease = n; // 允许负数
+        }
       } else {
         const s = String(valueVal ?? '').trim();
         if (kind === 'coverClickRate') obj.coverClickRate = s || '0';
@@ -347,62 +386,56 @@ export class XiaohongshuAccountOverviewImportService {
         throw new Error('小红书数据看板暂无访问权限/申请中，已标记 expired 并通知用户');
       }
 
-      // 尽量按用户描述进入：数据看板 -> 账号概览 -> 笔记数据 -> 观看数据 -> 近30日
-      // 页面结构可能会变，这里用“文本定位 + 容错”策略
+      // 统一入口：账号概览 -> 笔记数据
       await page.getByText('账号概览', { exact: true }).first().click().catch(() => undefined);
       await page.getByText('笔记数据', { exact: true }).first().click();
-      await page.getByText('观看数据', { exact: true }).first().click();
-
-      // 选择近30日：先点开时间范围，再点“近30日”
-      await page.getByText(/近\d+日/).first().click().catch(() => undefined);
-      await page.getByText('近30日', { exact: true }).click();
-
-      // 等待数据刷新完成（避免导出到全 0）
-      // 以页面上“观看数”卡片出现非 0 数字作为信号（页面文本会包含类似 8,077 / 4.8万）
-      await page
-        .waitForFunction(() => {
-          const t = document.body?.innerText || '';
-          if (!t.includes('观看数')) return false;
-          // 匹配“观看数”后出现非 0 的数值（允许逗号/万/亿）
-          return /观看数[\s\S]{0,50}([1-9]\d{0,2}(,\d{3})+|[1-9]\d*|[1-9]\d*(\.\d+)?\s*[万亿])/.test(t);
-        }, { timeout: 30_000 })
-        .catch(() => {
-          logger.warn('[XHS Import] Wait for non-zero watch count timed out. Continue export anyway.');
-        });
-
-      // 导出数据
-      const [download] = await Promise.all([
-        page.waitForEvent('download', { timeout: 60_000 }),
-        page.getByText('导出数据', { exact: true }).first().click(),
-      ]);
-
-      const filename = `${account.id}_${Date.now()}_${download.suggestedFilename()}`;
-      const filePath = path.join(this.downloadDir, filename);
-      await download.saveAs(filePath);
-
-      // 解析并入库
-      const perDay = parseXhsExcel(filePath);
-      let inserted = 0;
-      let updated = 0;
-
-      // 每天一条：accountId + date
-      for (const v of perDay.values()) {
-        const { recordDate, ...patch } = v;
-        const r = await this.userDayStatisticsService.saveStatisticsForDate(account.id, recordDate, patch);
-        inserted += r.inserted;
-        updated += r.updated;
-      }
-
-      // 删除 Excel（默认删除；设置 KEEP_XHS_XLSX=1 可保留用于排查）
-      if (process.env.KEEP_XHS_XLSX === '1') {
-        logger.warn(`[XHS Import] KEEP_XHS_XLSX=1, keep file: ${filePath}`);
-      } else {
-        await fs.unlink(filePath).catch(() => undefined);
-      }
 
-      logger.info(
-        `[XHS Import] Account done. accountId=${account.id} days=${perDay.size} inserted=${inserted} updated=${updated}`
-      );
+      const exportAndImport = async (tabText: '观看数据' | '互动数据' | '涨粉数据', mode: ExportMode) => {
+        await page.getByText(tabText, { exact: true }).first().click();
+        await page.getByText(/近\d+日/).first().click().catch(() => undefined);
+        await page.getByText('近30日', { exact: true }).click();
+        await page.waitForTimeout(1200);
+
+        const [download] = await Promise.all([
+          page.waitForEvent('download', { timeout: 60_000 }),
+          page.getByText('导出数据', { exact: true }).first().click(),
+        ]);
+
+        const filename = `${account.id}_${Date.now()}_${download.suggestedFilename()}`;
+        const filePath = path.join(this.downloadDir, filename);
+        await download.saveAs(filePath);
+
+        const perDay = parseXhsExcel(filePath, mode);
+        let inserted = 0;
+        let updated = 0;
+        for (const v of perDay.values()) {
+          const { recordDate, ...patch } = v;
+          const r = await this.userDayStatisticsService.saveStatisticsForDate(account.id, recordDate, patch);
+          inserted += r.inserted;
+          updated += r.updated;
+        }
+
+        if (process.env.KEEP_XHS_XLSX === '1') {
+          logger.warn(`[XHS Import] KEEP_XHS_XLSX=1, keep file: ${filePath}`);
+        } else {
+          await fs.unlink(filePath).catch(() => undefined);
+        }
+
+        logger.info(
+          `[XHS Import] ${tabText} imported. accountId=${account.id} days=${perDay.size} inserted=${inserted} updated=${updated}`
+        );
+      };
+
+      // 1) 观看数据：播放数 + 点击率/时长/完播率
+      await exportAndImport('观看数据', 'watch');
+
+      // 2) 互动数据：点赞/评论/收藏/分享
+      await exportAndImport('互动数据', 'interaction');
+
+      // 3) 涨粉数据：只取“净涨粉趋势”（解析器已过滤）
+      await exportAndImport('涨粉数据', 'fans');
+
+      logger.info(`[XHS Import] Account all tabs done. accountId=${account.id}`);
 
       await context.close();
     } finally {