Ethanfly 16 часов назад
Родитель
Сommit
b5d2f2064e

BIN
server/python/weixin_private_msg_17318.png


+ 83 - 0
server/src/scripts/test-data-sync.ts

@@ -0,0 +1,83 @@
+#!/usr/bin/env tsx
+/**
+ * 手动测试四个平台的数据同步任务
+ * 用于测试账号刷新重试机制
+ * 
+ * 运行: cd server && pnpm exec tsx src/scripts/test-data-sync.ts
+ */
+import { initDatabase } from '../models/index.js';
+import { XiaohongshuAccountOverviewImportService } from '../services/XiaohongshuAccountOverviewImportService.js';
+import { DouyinAccountOverviewImportService } from '../services/DouyinAccountOverviewImportService.js';
+import { BaijiahaoContentOverviewImportService } from '../services/BaijiahaoContentOverviewImportService.js';
+import { WeixinVideoDataCenterImportService } from '../services/WeixinVideoDataCenterImportService.js';
+import { logger } from '../utils/logger.js';
+
+async function main() {
+  logger.info('========================================');
+  logger.info('Starting manual data sync test...');
+  logger.info('========================================');
+
+  // 初始化数据库连接
+  await initDatabase();
+  logger.info('Database initialized');
+
+  const args = process.argv.slice(2);
+  const platform = args[0]?.toLowerCase();
+
+  if (!platform || !['xhs', 'dy', 'bj', 'wx', 'all'].includes(platform)) {
+    console.log('\nUsage: pnpm exec tsx src/scripts/test-data-sync.ts <platform>');
+    console.log('\nPlatforms:');
+    console.log('  xhs  - 小红书 (Xiaohongshu)');
+    console.log('  dy   - 抖音 (Douyin)');
+    console.log('  bj   - 百家号 (Baijiahao)');
+    console.log('  wx   - 视频号 (Weixin Video)');
+    console.log('  all  - 所有平台');
+    console.log('\nExample: pnpm exec tsx src/scripts/test-data-sync.ts xhs');
+    process.exit(1);
+  }
+
+  try {
+    if (platform === 'xhs' || platform === 'all') {
+      logger.info('\n========================================');
+      logger.info('Testing Xiaohongshu (小红书) data sync...');
+      logger.info('========================================');
+      await XiaohongshuAccountOverviewImportService.runDailyImport();
+      logger.info('✓ Xiaohongshu sync completed');
+    }
+
+    if (platform === 'dy' || platform === 'all') {
+      logger.info('\n========================================');
+      logger.info('Testing Douyin (抖音) data sync...');
+      logger.info('========================================');
+      await DouyinAccountOverviewImportService.runDailyImport();
+      logger.info('✓ Douyin sync completed');
+    }
+
+    if (platform === 'bj' || platform === 'all') {
+      logger.info('\n========================================');
+      logger.info('Testing Baijiahao (百家号) data sync...');
+      logger.info('========================================');
+      await BaijiahaoContentOverviewImportService.runDailyImport();
+      logger.info('✓ Baijiahao sync completed');
+    }
+
+    if (platform === 'wx' || platform === 'all') {
+      logger.info('\n========================================');
+      logger.info('Testing Weixin Video (视频号) data sync...');
+      logger.info('========================================');
+      await WeixinVideoDataCenterImportService.runDailyImport();
+      logger.info('✓ Weixin Video sync completed');
+    }
+
+    logger.info('\n========================================');
+    logger.info('All tests completed successfully!');
+    logger.info('========================================');
+  } catch (error) {
+    logger.error('Test failed:', error);
+    process.exit(1);
+  }
+
+  process.exit(0);
+}
+
+main();

+ 35 - 2
server/src/services/BaijiahaoContentOverviewImportService.ts

@@ -6,6 +6,7 @@ import { AppDataSource, PlatformAccount } from '../models/index.js';
 import { BrowserManager } from '../automation/browser.js';
 import { logger } from '../utils/logger.js';
 import { UserDayStatisticsService } from './UserDayStatisticsService.js';
+import { AccountService } from './AccountService.js';
 import type { ProxyConfig } from '@media-manager/shared';
 import { WS_EVENTS } from '@media-manager/shared';
 import { wsManager } from '../websocket/index.js';
@@ -483,7 +484,7 @@ export class BaijiahaoContentOverviewImportService {
   /**
    * 单账号:导出 Excel → 解析 → 入库 → 删除文件
    */
-  async importAccountLast30Days(account: PlatformAccount): Promise<void> {
+  async importAccountLast30Days(account: PlatformAccount, isRetry = false): Promise<void> {
     const cookies = parseCookiesFromAccount(account.cookieData);
     if (!cookies.length) {
       throw new Error('cookieData 为空或无法解析');
@@ -513,7 +514,39 @@ export class BaijiahaoContentOverviewImportService {
       await page.waitForTimeout(1500);
 
       if (page.url().includes('passport') || page.url().includes('login')) {
-        throw new Error('未登录/需要重新登录(跳转到登录页)');
+        // 第一次检测到登录失效时,尝试刷新账号
+        if (!isRetry) {
+          logger.info(`[BJ Import] Login expired detected for account ${account.id}, attempting to refresh...`);
+          await context.close();
+          if (shouldClose) await browser.close();
+          
+          try {
+            const accountService = new AccountService();
+            const refreshResult = await accountService.refreshAccount(account.userId, account.id);
+            
+            if (refreshResult.needReLogin) {
+              // 刷新后仍需要重新登录,走原先的失效流程
+              logger.warn(`[BJ Import] Account ${account.id} refresh failed, still needs re-login`);
+              throw new Error('未登录/需要重新登录(跳转到登录页)');
+            }
+            
+            // 刷新成功,重新获取账号信息并重试导入
+            logger.info(`[BJ Import] Account ${account.id} refreshed successfully, retrying import...`);
+            const refreshedAccount = await this.accountRepository.findOne({ where: { id: account.id } });
+            if (!refreshedAccount) {
+              throw new Error('账号刷新后未找到');
+            }
+            
+            // 递归调用,标记为重试
+            return await this.importAccountLast30Days(refreshedAccount, true);
+          } catch (refreshError) {
+            logger.error(`[BJ Import] Account ${account.id} refresh failed:`, refreshError);
+            throw new Error('未登录/需要重新登录(跳转到登录页)');
+          }
+        } else {
+          // 已经是重试了,不再尝试刷新
+          throw new Error('未登录/需要重新登录(跳转到登录页)');
+        }
       }
 
       const bodyText = (await page.textContent('body').catch(() => '')) || '';

+ 84 - 9
server/src/services/DouyinAccountOverviewImportService.ts

@@ -6,6 +6,7 @@ import { AppDataSource, PlatformAccount } from '../models/index.js';
 import { BrowserManager } from '../automation/browser.js';
 import { logger } from '../utils/logger.js';
 import { UserDayStatisticsService } from './UserDayStatisticsService.js';
+import { AccountService } from './AccountService.js';
 import type { ProxyConfig } from '@media-manager/shared';
 import { WS_EVENTS } from '@media-manager/shared';
 import { wsManager } from '../websocket/index.js';
@@ -340,7 +341,7 @@ export class DouyinAccountOverviewImportService {
   /**
    * 单账号:导出 Excel → 解析 → 入库 → 删除文件
    */
-  async importAccountLast30Days(account: PlatformAccount): Promise<void> {
+  async importAccountLast30Days(account: PlatformAccount, isRetry = false): Promise<void> {
     const cookies = parseCookiesFromAccount(account.cookieData);
     if (!cookies.length) {
       throw new Error('cookieData 为空或无法解析');
@@ -365,13 +366,46 @@ export class DouyinAccountOverviewImportService {
       }
 
       const page = await context.newPage();
+      logger.info(`[DY Import] accountId=${account.id} goto data-center...`);
       await page.goto('https://creator.douyin.com/creator-micro/data-center/operation', {
         waitUntil: 'domcontentloaded',
       });
       await page.waitForTimeout(1500);
 
       if (page.url().includes('login')) {
-        throw new Error('未登录/需要重新登录(跳转到 login)');
+        // 第一次检测到登录失效时,尝试刷新账号
+        if (!isRetry) {
+          logger.info(`[DY Import] Login expired detected for account ${account.id}, attempting to refresh...`);
+          await context.close();
+          if (shouldClose) await browser.close();
+          
+          try {
+            const accountService = new AccountService();
+            const refreshResult = await accountService.refreshAccount(account.userId, account.id);
+            
+            if (refreshResult.needReLogin) {
+              // 刷新后仍需要重新登录,走原先的失效流程
+              logger.warn(`[DY Import] Account ${account.id} refresh failed, still needs re-login`);
+              throw new Error('未登录/需要重新登录(跳转到 login)');
+            }
+            
+            // 刷新成功,重新获取账号信息并重试导入
+            logger.info(`[DY Import] Account ${account.id} refreshed successfully, retrying import...`);
+            const refreshedAccount = await this.accountRepository.findOne({ where: { id: account.id } });
+            if (!refreshedAccount) {
+              throw new Error('账号刷新后未找到');
+            }
+            
+            // 递归调用,标记为重试
+            return await this.importAccountLast30Days(refreshedAccount, true);
+          } catch (refreshError) {
+            logger.error(`[DY Import] Account ${account.id} refresh failed:`, refreshError);
+            throw new Error('未登录/需要重新登录(跳转到 login)');
+          }
+        } else {
+          // 已经是重试了,不再尝试刷新
+          throw new Error('未登录/需要重新登录(跳转到 login)');
+        }
       }
 
       // 检测“暂无访问权限 / 权限申请中 / 暂无数据”提示:标记账号 expired + 推送提示
@@ -395,14 +429,38 @@ export class DouyinAccountOverviewImportService {
         throw new Error('抖音数据看板暂无访问权限/申请中,已标记 expired 并通知用户');
       }
 
-      // 统一入口:数据中心 -> 账号总览 -> 短视频
-      await page.getByText('数据中心', { exact: false }).first().click().catch(() => undefined);
-      await page.getByText('账号总览', { exact: true }).first().click().catch(() => undefined);
-      await page.getByText('短视频', { exact: true }).first().click();
+      // 已直达账号总览页(data-center/operation),无需再点「数据中心/账号总览」,直接点「短视频」和「近30天」
+      await page.waitForTimeout(500);
+      logger.info(`[DY Import] accountId=${account.id} on 账号总览页, click 短视频 tab (#semiTabaweme)...`);
+      const shortVideoById = page.locator('#semiTabaweme');
+      if ((await shortVideoById.count().catch(() => 0)) > 0) {
+        await shortVideoById.first().click();
+      } else {
+        const shortVideoCandidates = ['短视频', '短视频数据'];
+        let shortVideoClicked = false;
+        for (const text of shortVideoCandidates) {
+          const loc = page.getByText(text, { exact: false }).first();
+          if ((await loc.count().catch(() => 0)) > 0) {
+            await loc.click().catch(() => undefined);
+            shortVideoClicked = true;
+            break;
+          }
+        }
+        if (!shortVideoClicked) {
+          throw new Error('页面上未找到「短视频」入口,请确认抖音创作者后台是否改版');
+        }
+      }
 
-      // 切换“近30天”
-      await page.getByText(/近\d+天?/).first().click().catch(() => undefined);
-      await page.getByText('近30天', { exact: true }).click();
+      // 切换“近30天”(优先用 ID #addon-aoc08fi,兜底文案)
+      await page.waitForTimeout(500);
+      logger.info(`[DY Import] accountId=${account.id} click 近30天 (#addon-aoc08fi)...`);
+      const last30DaysById = page.locator('#addon-aoc08fi');
+      if ((await last30DaysById.count().catch(() => 0)) > 0) {
+        await last30DaysById.first().click();
+      } else {
+        await page.getByText(/近\d+天?/).first().click().catch(() => undefined);
+        await page.getByText('近30天', { exact: true }).click();
+      }
       await page.waitForTimeout(1200);
 
       // 逐个指标导出(排除:主页访问 / 取关粉丝)
@@ -421,6 +479,7 @@ export class DouyinAccountOverviewImportService {
       let totalInserted = 0;
       let totalUpdated = 0;
       let mergedDays = new Map<string, { recordDate: Date } & Record<string, any>>();
+      const savedExcelPaths: string[] = [];
 
       const clickMetric = async (metric: { name: string; candidates: string[] }) => {
         // 先精确匹配,失败后用包含匹配(适配 UI 文案差异)
@@ -447,6 +506,7 @@ export class DouyinAccountOverviewImportService {
       };
 
       for (const metric of metricsToExport) {
+        logger.info(`[DY Import] accountId=${account.id} exporting metric: ${metric.name}...`);
         await clickMetric(metric);
 
         const [download] = await Promise.all([
@@ -457,6 +517,12 @@ export class DouyinAccountOverviewImportService {
         const filename = `${account.id}_${Date.now()}_${download.suggestedFilename()}`;
         const filePath = path.join(this.downloadDir, filename);
         await download.saveAs(filePath);
+        // 保留 Excel 不删除,便于核对数据;路径打日志方便查看
+        const absolutePath = path.resolve(filePath);
+        savedExcelPaths.push(absolutePath);
+        logger.info(
+          `[DY Import] Excel saved (${metric.name}): ${absolutePath}`
+        );
 
         try {
           const perDay = parseDouyinExcel(filePath);
@@ -470,6 +536,7 @@ export class DouyinAccountOverviewImportService {
             `[DY Import] metric exported & parsed. accountId=${account.id} metric=${metric.name} file=${path.basename(filePath)} days=${perDay.size}`
           );
         } finally {
+          // 默认导入后删除 Excel,避免磁盘堆积;仅在显式 KEEP_DY_XLSX=true 时保留(用于调试)
           if (process.env.KEEP_DY_XLSX === 'true') {
             logger.warn(`[DY Import] KEEP_DY_XLSX=true, keep file: ${filePath}`);
           } else {
@@ -478,6 +545,14 @@ export class DouyinAccountOverviewImportService {
         }
       }
 
+      // 汇总:本账号导出的 7 个 Excel 已解析
+      logger.info(
+        `[DY Import] accountId=${account.id} 共 ${savedExcelPaths.length} 个 Excel 已解析`
+      );
+      if (savedExcelPaths.length !== 7) {
+        logger.warn(`[DY Import] accountId=${account.id} 预期 7 个 Excel,实际 ${savedExcelPaths.length} 个`);
+      }
+
       // 合并完成后统一入库(避免同一天多次 update)
       for (const v of mergedDays.values()) {
         const { recordDate, ...patch } = v;

+ 35 - 2
server/src/services/WeixinVideoDataCenterImportService.ts

@@ -6,6 +6,7 @@ import { AppDataSource, PlatformAccount } from '../models/index.js';
 import { BrowserManager } from '../automation/browser.js';
 import { logger } from '../utils/logger.js';
 import { UserDayStatisticsService } from './UserDayStatisticsService.js';
+import { AccountService } from './AccountService.js';
 import type { ProxyConfig } from '@media-manager/shared';
 import { WS_EVENTS } from '@media-manager/shared';
 import { wsManager } from '../websocket/index.js';
@@ -420,7 +421,7 @@ export class WeixinVideoDataCenterImportService {
     logger.info('[WX Import] Done.');
   }
 
-  async importAccountLast30Days(account: PlatformAccount): Promise<void> {
+  async importAccountLast30Days(account: PlatformAccount, isRetry = false): Promise<void> {
     const cookies = parseCookiesFromAccount(account.cookieData);
     if (!cookies.length) throw new Error('cookieData 为空或无法解析');
 
@@ -447,7 +448,39 @@ export class WeixinVideoDataCenterImportService {
       await page.waitForTimeout(1500);
 
       if (page.url().includes('login') || page.url().includes('passport')) {
-        throw new Error('未登录/需要重新登录(跳转到登录页)');
+        // 第一次检测到登录失效时,尝试刷新账号
+        if (!isRetry) {
+          logger.info(`[WX Import] Login expired detected for account ${account.id}, attempting to refresh...`);
+          await context.close();
+          if (shouldClose) await browser.close();
+          
+          try {
+            const accountService = new AccountService();
+            const refreshResult = await accountService.refreshAccount(account.userId, account.id);
+            
+            if (refreshResult.needReLogin) {
+              // 刷新后仍需要重新登录,走原先的失效流程
+              logger.warn(`[WX Import] Account ${account.id} refresh failed, still needs re-login`);
+              throw new Error('未登录/需要重新登录(跳转到登录页)');
+            }
+            
+            // 刷新成功,重新获取账号信息并重试导入
+            logger.info(`[WX Import] Account ${account.id} refreshed successfully, retrying import...`);
+            const refreshedAccount = await this.accountRepository.findOne({ where: { id: account.id } });
+            if (!refreshedAccount) {
+              throw new Error('账号刷新后未找到');
+            }
+            
+            // 递归调用,标记为重试
+            return await this.importAccountLast30Days(refreshedAccount, true);
+          } catch (refreshError) {
+            logger.error(`[WX Import] Account ${account.id} refresh failed:`, refreshError);
+            throw new Error('未登录/需要重新登录(跳转到登录页)');
+          }
+        } else {
+          // 已经是重试了,不再尝试刷新
+          throw new Error('未登录/需要重新登录(跳转到登录页)');
+        }
       }
 
       // 进入 数据中心

+ 35 - 2
server/src/services/XiaohongshuAccountOverviewImportService.ts

@@ -6,6 +6,7 @@ import { AppDataSource, PlatformAccount } from '../models/index.js';
 import { BrowserManager } from '../automation/browser.js';
 import { logger } from '../utils/logger.js';
 import { UserDayStatisticsService } from './UserDayStatisticsService.js';
+import { AccountService } from './AccountService.js';
 import type { ProxyConfig } from '@media-manager/shared';
 import { WS_EVENTS } from '@media-manager/shared';
 import { wsManager } from '../websocket/index.js';
@@ -349,7 +350,7 @@ export class XiaohongshuAccountOverviewImportService {
   /**
    * 单账号:导出 Excel → 解析 → 入库 → 删除文件
    */
-  async importAccountLast30Days(account: PlatformAccount): Promise<void> {
+  async importAccountLast30Days(account: PlatformAccount, isRetry = false): Promise<void> {
     const cookies = parseCookiesFromAccount(account.cookieData);
     if (!cookies.length) {
       throw new Error('cookieData 为空或无法解析');
@@ -378,7 +379,39 @@ export class XiaohongshuAccountOverviewImportService {
       await page.waitForTimeout(1500);
 
       if (page.url().includes('login')) {
-        throw new Error('未登录/需要重新登录(跳转到 login)');
+        // 第一次检测到登录失效时,尝试刷新账号
+        if (!isRetry) {
+          logger.info(`[XHS Import] Login expired detected for account ${account.id}, attempting to refresh...`);
+          await context.close();
+          if (shouldClose) await browser.close();
+          
+          try {
+            const accountService = new AccountService();
+            const refreshResult = await accountService.refreshAccount(account.userId, account.id);
+            
+            if (refreshResult.needReLogin) {
+              // 刷新后仍需要重新登录,走原先的失效流程
+              logger.warn(`[XHS Import] Account ${account.id} refresh failed, still needs re-login`);
+              throw new Error('未登录/需要重新登录(跳转到 login)');
+            }
+            
+            // 刷新成功,重新获取账号信息并重试导入
+            logger.info(`[XHS Import] Account ${account.id} refreshed successfully, retrying import...`);
+            const refreshedAccount = await this.accountRepository.findOne({ where: { id: account.id } });
+            if (!refreshedAccount) {
+              throw new Error('账号刷新后未找到');
+            }
+            
+            // 递归调用,标记为重试
+            return await this.importAccountLast30Days(refreshedAccount, true);
+          } catch (refreshError) {
+            logger.error(`[XHS Import] Account ${account.id} refresh failed:`, refreshError);
+            throw new Error('未登录/需要重新登录(跳转到 login)');
+          }
+        } else {
+          // 已经是重试了,不再尝试刷新
+          throw new Error('未登录/需要重新登录(跳转到 login)');
+        }
       }
 
       // 检测“暂无访问权限 / 权限申请中”提示:仅推送提示,不修改账号状态(避免误判或用户不想自动变更)