multi-platform-media-manage/server/src/services/DouyinAccountOverviewImportService.ts

import fs from 'node:fs/promises';
import path from 'node:path';
import type { Browser } from 'playwright';
import { AppDataSource, PlatformAccount } from '../models/index.js';
import { BrowserManager } from '../automation/browser.js';
import { logger } from '../utils/logger.js';
import { UserDayStatisticsService } from './UserDayStatisticsService.js';
import { AccountService } from './AccountService.js';
import type { ProxyConfig } from '@media-manager/shared';
import { WS_EVENTS } from '@media-manager/shared';
import { wsManager } from '../websocket/index.js';
import { launchBrowser } from '../automation/browserProvider.js';

type PlaywrightCookie = {
  name: string;
  value: string;
  domain?: string;
  path?: string;
  url?: string;
  expires?: number;
  httpOnly?: boolean;
  secure?: boolean;
  sameSite?: 'Lax' | 'None' | 'Strict';
};

function ensureDir(p: string) {
  return fs.mkdir(p, { recursive: true });
}

function normalizeDateText(input: unknown): Date | null {
  if (!input) return null;
  if (input instanceof Date && !Number.isNaN(input.getTime())) {
    const d = new Date(input);
    d.setHours(0, 0, 0, 0);
    return d;
  }
  const s = String(input).trim();
  if (!s) return null;

  // 2026-01-27 / 2026/01/27
  const m1 = s.match(/(\d{4})\D(\d{1,2})\D(\d{1,2})/);
  if (m1) {
    const yyyy = Number(m1[1]);
    const mm = Number(m1[2]);
    const dd = Number(m1[3]);
    if (!yyyy || !mm || !dd) return null;
    const d = new Date(yyyy, mm - 1, dd);
    d.setHours(0, 0, 0, 0);
    return d;
  }

  // 01-27（兜底：用当前年份）
  const m2 = s.match(/^(\d{1,2})[-/](\d{1,2})$/);
  if (m2) {
    const yyyy = new Date().getFullYear();
    const mm = Number(m2[1]);
    const dd = Number(m2[2]);
    const d = new Date(yyyy, mm - 1, dd);
    d.setHours(0, 0, 0, 0);
    return d;
  }

  return null;
}

function toRatePercentStringFromValue(val: unknown): string | undefined {
  const n = typeof val === 'number' ? val : Number(val);
  if (!Number.isFinite(n)) return undefined;
  if (n === 0) return '0';
  const scaled = n * 100;
  const rounded = Math.round(scaled * 100) / 100;
  const s = rounded.toFixed(2).replace(/\.00$/, '').replace(/(\.\d)0$/, '$1');
  return `${s}%`;
}

function parseCookiesFromAccount(cookieData: string | null): PlaywrightCookie[] {
  if (!cookieData) return [];
  const raw = cookieData.trim();
  if (!raw) return [];

  // 1) JSON array（最常见：浏览器插件导出/前端保存）
  if (raw.startsWith('[') || raw.startsWith('{')) {
    try {
      const parsed = JSON.parse(raw);
      const arr = Array.isArray(parsed) ? parsed : (parsed?.cookies ? parsed.cookies : []);
      if (!Array.isArray(arr)) return [];
      return arr
        .map((c: any) => {
          const name = String(c?.name ?? '').trim();
          const value = String(c?.value ?? '').trim();
          if (!name) return null;
          const domain = c?.domain ? String(c.domain) : undefined;
          const pathVal = c?.path ? String(c.path) : '/';
          const url = !domain ? 'https://creator.douyin.com' : undefined;
          const sameSiteRaw = c?.sameSite;
          const sameSite =
            sameSiteRaw === 'Lax' || sameSiteRaw === 'None' || sameSiteRaw === 'Strict'
              ? sameSiteRaw
              : undefined;

          return {
            name,
            value,
            domain,
            path: pathVal,
            url,
            expires: typeof c?.expires === 'number' ? c.expires : undefined,
            httpOnly: typeof c?.httpOnly === 'boolean' ? c.httpOnly : undefined,
            secure: typeof c?.secure === 'boolean' ? c.secure : undefined,
            sameSite,
          } satisfies PlaywrightCookie;
        })
        .filter(Boolean) as PlaywrightCookie[];
    } catch {
      // fallthrough
    }
  }

  // 2) "a=b; c=d" 拼接格式
  const pairs = raw.split(';').map((p) => p.trim()).filter(Boolean);
  const cookies: PlaywrightCookie[] = [];
  for (const p of pairs) {
    const idx = p.indexOf('=');
    if (idx <= 0) continue;
    const name = p.slice(0, idx).trim();
    const value = p.slice(idx + 1).trim();
    if (!name) continue;
    cookies.push({ name, value, url: 'https://creator.douyin.com' });
  }
  return cookies;
}

async function createBrowserForAccount(proxy: ProxyConfig | null): Promise<{ browser: Browser; shouldClose: boolean }> {
  // 静默同步：默认一律 headless，不弹窗
  // 只有在“引导登录/验证”时（DY_STORAGE_STATE_BOOTSTRAP=1 且 DY_IMPORT_HEADLESS=0）才允许 headful
  const allowHeadfulForBootstrap = process.env.DY_STORAGE_STATE_BOOTSTRAP === '1' && process.env.DY_IMPORT_HEADLESS === '0';
  const headless = !allowHeadfulForBootstrap;
  if (proxy?.enabled) {
    const server = `${proxy.type}://${proxy.host}:${proxy.port}`;
    const browser = await launchBrowser({
      headless,
      proxy: {
        server,
        username: proxy.username,
        password: proxy.password,
      },
      args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu', '--window-size=1920,1080'],
    });
    return { browser, shouldClose: true };
  }
  const browser = await BrowserManager.getBrowser({ headless });
  return { browser, shouldClose: false };
}

type DashboardMetricTrendPoint = {
  date_time?: string; // YYYYMMDD
  value?: number;
  douyin_value?: number;
  xigua_value?: number;
  yumme_value?: number;
  change_rate?: number;
};

type DashboardMetricItem = {
  english_metric_name?: string;
  metric_name?: string;
  metric_value?: number;
  trends?: DashboardMetricTrendPoint[];
};

type DashboardResponse = {
  status_code?: number;
  status_msg?: string;
  metrics?: DashboardMetricItem[];
};

function parseYmdCompactToDate(ymd: unknown): Date | null {
  const s = String(ymd || '').trim();
  if (!/^\d{8}$/.test(s)) return null;
  const yyyy = Number(s.slice(0, 4));
  const mm = Number(s.slice(4, 6));
  const dd = Number(s.slice(6, 8));
  if (!yyyy || !mm || !dd) return null;
  const d = new Date(yyyy, mm - 1, dd);
  d.setHours(0, 0, 0, 0);
  return d;
}

function pickTrendValue(pt: DashboardMetricTrendPoint): number | undefined {
  // 优先使用聚合 value；若不存在则兜底 douyin_value
  const v =
    typeof pt?.value === 'number'
      ? pt.value
      : typeof pt?.douyin_value === 'number'
        ? pt.douyin_value
        : undefined;
  if (!Number.isFinite(v as number)) return undefined;
  return v as number;
}

export class DouyinAccountOverviewImportService {
  private accountRepository = AppDataSource.getRepository(PlatformAccount);
  private userDayStatisticsService = new UserDayStatisticsService();

  private stateDir = path.resolve(process.cwd(), 'tmp', 'douyin-storage-state');

  private getStatePath(accountId: number) {
    return path.join(this.stateDir, `${accountId}.json`);
  }

  private async ensureStorageState(account: PlatformAccount, cookies: PlaywrightCookie[]): Promise<string | null> {
    const statePath = this.getStatePath(account.id);
    try {
      await fs.access(statePath);
      return statePath;
    } catch {
      // no state
    }

    // 需要你在弹出的浏览器里完成一次登录/验证，然后脚本会自动保存 storageState
    // 启用方式：DY_IMPORT_HEADLESS=0 且 DY_STORAGE_STATE_BOOTSTRAP=1
    if (!(process.env.DY_IMPORT_HEADLESS === '0' && process.env.DY_STORAGE_STATE_BOOTSTRAP === '1')) {
      return null;
    }

    await ensureDir(this.stateDir);
    logger.warn(
      `[DY Import] No storageState for accountId=${account.id}. Bootstrapping... 请在弹出的浏览器中完成登录/验证。`
    );

    const { browser, shouldClose } = await createBrowserForAccount(account.proxyConfig);
    try {
      const context = await browser.newContext({
        viewport: { width: 1920, height: 1080 },
        locale: 'zh-CN',
        timezoneId: 'Asia/Shanghai',
      });
      await context.addCookies(cookies as any);
      const page = await context.newPage();
      await page.goto('https://creator.douyin.com/creator-micro/data-center/operation', {
        waitUntil: 'domcontentloaded',
      });

      // 最长等 5 分钟：让你手动完成登录/滑块/短信等
      await page
        .waitForFunction(() => {
          const t = document.body?.innerText || '';
          return t.includes('数据中心') || t.includes('账号总览') || t.includes('短视频');
        }, { timeout: 5 * 60_000 })
        .catch(() => undefined);

      await context.storageState({ path: statePath });
      logger.info(`[DY Import] storageState saved: ${statePath}`);
      await context.close();
      return statePath;
    } finally {
      if (shouldClose) await browser.close().catch(() => undefined);
    }
  }

  /**
   * 统一入口：定时任务与添加账号均调用此方法，执行“账号总览-短视频-数据表现-近30天”
   */
  static async runDailyImport(): Promise<void> {
    const svc = new DouyinAccountOverviewImportService();
    await svc.runDailyImportForAllDouyinAccounts();
  }

  /**
   * 单账号入口：仅为指定抖音账号执行近30天账号总览导入（用于账号从失效恢复为 active 时补数）
   */
  static async runDailyImportForAccount(accountId: number): Promise<void> {
    const svc = new DouyinAccountOverviewImportService();
    const account = await svc.accountRepository.findOne({
      where: { id: accountId, platform: 'douyin' as any },
    });
    if (!account) {
      throw new Error(`未找到抖音账号 id=${accountId}`);
    }
    await svc.importAccountLast30Days(account);
  }

  /**
   * 为所有抖音账号导出“账号总览-短视频-数据表现-近30天”并导入 user_day_statistics
   */
  async runDailyImportForAllDouyinAccounts(): Promise<void> {
    const accounts = await this.accountRepository.find({
      where: { platform: 'douyin' as any },
    });

    logger.info(`[DY Import] Start. total_accounts=${accounts.length}`);

    for (const account of accounts) {
      try {
        await this.importAccountLast30Days(account);
      } catch (e) {
        logger.error(
          `[DY Import] Account failed. accountId=${account.id} name=${account.accountName || ''}`,
          e
        );
      }
    }

    logger.info('[DY Import] Done.');
  }

  /**
   * 单账号：导出 Excel → 解析 → 入库 → 删除文件
   */
  async importAccountLast30Days(account: PlatformAccount, isRetry = false): Promise<void> {
    const cookies = parseCookiesFromAccount(account.cookieData);
    if (!cookies.length) {
      throw new Error('cookieData 为空或无法解析');
    }

    const { browser, shouldClose } = await createBrowserForAccount(account.proxyConfig);
    try {
      const statePath = await this.ensureStorageState(account, cookies);
      const context = await browser.newContext({
        acceptDownloads: true,
        viewport: { width: 1920, height: 1080 },
        locale: 'zh-CN',
        timezoneId: 'Asia/Shanghai',
        userAgent:
          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
        ...(statePath ? { storageState: statePath } : {}),
      });
      context.setDefaultTimeout(60_000);
      // 如果没 state，就退回 cookie-only（可能导出为 0）
      if (!statePath) {
        await context.addCookies(cookies as any);
      }

      const page = await context.newPage();
      logger.info(`[DY Import] accountId=${account.id} goto data-center...`);
      await page.goto('https://creator.douyin.com/creator-micro/data-center/operation', {
        waitUntil: 'domcontentloaded',
      });
      await page.waitForTimeout(1500);

      if (page.url().includes('login')) {
        // 第一次检测到登录失效时，尝试刷新账号
        if (!isRetry) {
          logger.info(`[DY Import] Login expired detected for account ${account.id}, attempting to refresh...`);
          await context.close();
          if (shouldClose) await browser.close();
          
          try {
            const accountService = new AccountService();
            const refreshResult = await accountService.refreshAccount(account.userId, account.id);
            
            if (refreshResult.needReLogin) {
              // 刷新后仍需要重新登录，走原先的失效流程
              logger.warn(`[DY Import] Account ${account.id} refresh failed, still needs re-login`);
              throw new Error('未登录/需要重新登录（跳转到 login）');
            }
            
            // 刷新成功，重新获取账号信息并重试导入
            logger.info(`[DY Import] Account ${account.id} refreshed successfully, retrying import...`);
            const refreshedAccount = await this.accountRepository.findOne({ where: { id: account.id } });
            if (!refreshedAccount) {
              throw new Error('账号刷新后未找到');
            }
            
            // 递归调用，标记为重试
            return await this.importAccountLast30Days(refreshedAccount, true);
          } catch (refreshError) {
            logger.error(`[DY Import] Account ${account.id} refresh failed:`, refreshError);
            throw new Error('未登录/需要重新登录（跳转到 login）');
          }
        } else {
          // 已经是重试了，不再尝试刷新
          throw new Error('未登录/需要重新登录（跳转到 login）');
        }
      }

      // 检测“暂无访问权限 / 权限申请中 / 暂无数据”提示：标记账号 expired + 推送提示
      const bodyText = (await page.textContent('body').catch(() => '')) || '';
      if (
        bodyText.includes('暂无访问权限') ||
        bodyText.includes('权限申请中') ||
        bodyText.includes('暂无数据权限') ||
        bodyText.includes('暂无数据，请稍后再试')
      ) {
        await this.accountRepository.update(account.id, { status: 'expired' as any });
        wsManager.sendToUser(account.userId, WS_EVENTS.ACCOUNT_UPDATED, {
          account: { id: account.id, status: 'expired', platform: 'douyin' },
        });
        wsManager.sendToUser(account.userId, WS_EVENTS.SYSTEM_MESSAGE, {
          level: 'warning',
          message: `抖音账号「${account.accountName || account.accountId || account.id}」暂无数据看板访问权限，请到抖音创作者中心申请数据权限（通过后一般次日生效）。`,
          platform: 'douyin',
          accountId: account.id,
        });
        throw new Error('抖音数据看板暂无访问权限/申请中，已标记 expired 并通知用户');
      }

      // 已直达账号总览页（data-center/operation），无需再点「数据中心/账号总览」，直接点「短视频」和「近30天」
      await page.waitForTimeout(500);
      logger.info(`[DY Import] accountId=${account.id} on 账号总览页, click 短视频 tab (#semiTabaweme)...`);
      const shortVideoById = page.locator('#semiTabaweme');
      if ((await shortVideoById.count().catch(() => 0)) > 0) {
        await shortVideoById.first().click();
      } else {
        const shortVideoCandidates = ['短视频', '短视频数据'];
        let shortVideoClicked = false;
        for (const text of shortVideoCandidates) {
          const loc = page.getByText(text, { exact: false }).first();
          if ((await loc.count().catch(() => 0)) > 0) {
            await loc.click().catch(() => undefined);
            shortVideoClicked = true;
            break;
          }
        }
        if (!shortVideoClicked) {
          throw new Error('页面上未找到「短视频」入口，请确认抖音创作者后台是否改版');
        }
      }

      // 切换“近30天”（优先用 ID #addon-aoc08fi，兜底文案）
      await page.waitForTimeout(500);
      logger.info(`[DY Import] accountId=${account.id} click 近30天 (#addon-aoc08fi)...`);
      const last30DaysById = page.locator('#addon-aoc08fi');
      if ((await last30DaysById.count().catch(() => 0)) > 0) {
        await last30DaysById.first().click();
      } else {
        await page.getByText(/近\d+天?/).first().click().catch(() => undefined);
        await page.getByText('近30天', { exact: true }).click();
      }
      await page.waitForTimeout(1200);

      let totalInserted = 0;
      let totalUpdated = 0;
      const apiUrl = 'https://creator.douyin.com/janus/douyin/creator/data/overview/dashboard';

      logger.info(`[DY Import] accountId=${account.id} fetch dashboard (POST recent_days=30)...`);

      // 优先监听页面自身是否会发起该请求；若没有，则在页面上下文里手动 fetch（浏览器自动带 cookie）
      const responsePromise = page
        .waitForResponse(
          (res) => res.request().method() === 'POST' && res.url().includes('/janus/douyin/creator/data/overview/dashboard'),
          { timeout: 8000 }
        )
        .catch(() => null);

      const evalPromise = page.evaluate(async (url) => {
        try {
          const r = await fetch(url, {
            method: 'POST',
            credentials: 'include',
            headers: {
              accept: 'application/json, text/plain, */*',
              'content-type': 'application/json',
            },
            body: JSON.stringify({ recent_days: 30 }),
          });
          const json = await r.json().catch(() => null);
          return json;
        } catch (e: any) {
          return { status_code: -1, status_msg: String(e?.message || e) };
        }
      }, apiUrl);

      const [res, evalJson] = await Promise.all([responsePromise, evalPromise]);
      const body = ((res ? await res.json().catch(() => null) : null) ?? evalJson ?? null) as DashboardResponse | null;

      if (!body || typeof body !== 'object') {
        throw new Error('overview/dashboard 响应不是 JSON');
      }
      if (Number(body.status_code) !== 0) {
        throw new Error(`overview/dashboard 返回非成功: code=${body.status_code} msg=${body.status_msg || ''}`);
      }

      const metrics = Array.isArray(body.metrics) ? body.metrics : [];
      if (!metrics.length) {
        logger.warn(`[DY Import] dashboard metrics empty. accountId=${account.id}`);
      }

      const mergedDays = new Map<string, { recordDate: Date } & Record<string, any>>();

      const setDay = (d: Date) => {
        const key = `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`;
        if (!mergedDays.has(key)) mergedDays.set(key, { recordDate: d });
        return mergedDays.get(key)!;
      };

      for (const m of metrics) {
        const en = String(m?.english_metric_name || '').trim();
        const trends = Array.isArray(m?.trends) ? m.trends : [];
        if (!en || !trends.length) continue;

        for (const pt of trends) {
          const d = parseYmdCompactToDate(pt?.date_time);
          if (!d) continue;
          const obj = setDay(d);
          const v = pickTrendValue(pt);
          if (v === undefined) continue;

          // 显式排除：主页访问 / 取关粉丝（库里没有对应字段）
          if (en === 'homepage_view_cnt' || en === 'cancel_fans_cnt') continue;

          if (en === 'total_fans_cnt') (obj as any).fansCount = Math.round(v);
          else if (en === 'play_cnt') (obj as any).playCount = Math.round(v);
          else if (en === 'digg_cnt') (obj as any).likeCount = Math.round(v);
          else if (en === 'comment_cnt') (obj as any).commentCount = Math.round(v);
          else if (en === 'share_count') (obj as any).shareCount = Math.round(v);
          else if (en === 'net_fans_cnt') (obj as any).fansIncrease = Math.round(v);
          else if (en === 'cover_click_ratio') {
            const s = toRatePercentStringFromValue(v);
            if (s != null) (obj as any).coverClickRate = s;
          }
        }
      }

      // 合并完成后统一入库（避免同一天多次 update）
      for (const v of mergedDays.values()) {
        const { recordDate, ...patch } = v;
        const r = await this.userDayStatisticsService.saveStatisticsForDate(account.id, recordDate, patch);
        totalInserted += r.inserted;
        totalUpdated += r.updated;
      }

      logger.info(
        `[DY Import] short-video imported. accountId=${account.id} days=${mergedDays.size} inserted=${totalInserted} updated=${totalUpdated}`
      );

      await context.close();
    } finally {
      if (shouldClose) {
        await browser.close().catch(() => undefined);
      }
    }
  }
}