import fs from 'node:fs/promises'; import path from 'node:path'; import type { Browser } from 'playwright'; import { AppDataSource, PlatformAccount } from '../models/index.js'; import { BrowserManager } from '../automation/browser.js'; import { logger } from '../utils/logger.js'; import { UserDayStatisticsService } from './UserDayStatisticsService.js'; import { AccountService } from './AccountService.js'; import type { ProxyConfig } from '@media-manager/shared'; import { WS_EVENTS } from '@media-manager/shared'; import { wsManager } from '../websocket/index.js'; import { launchBrowser } from '../automation/browserProvider.js'; type PlaywrightCookie = { name: string; value: string; domain?: string; path?: string; url?: string; expires?: number; httpOnly?: boolean; secure?: boolean; sameSite?: 'Lax' | 'None' | 'Strict'; }; function ensureDir(p: string) { return fs.mkdir(p, { recursive: true }); } function normalizeDateText(input: unknown): Date | null { if (!input) return null; if (input instanceof Date && !Number.isNaN(input.getTime())) { const d = new Date(input); d.setHours(0, 0, 0, 0); return d; } const s = String(input).trim(); if (!s) return null; // 2026-01-27 / 2026/01/27 const m1 = s.match(/(\d{4})\D(\d{1,2})\D(\d{1,2})/); if (m1) { const yyyy = Number(m1[1]); const mm = Number(m1[2]); const dd = Number(m1[3]); if (!yyyy || !mm || !dd) return null; const d = new Date(yyyy, mm - 1, dd); d.setHours(0, 0, 0, 0); return d; } // 01-27(兜底:用当前年份) const m2 = s.match(/^(\d{1,2})[-/](\d{1,2})$/); if (m2) { const yyyy = new Date().getFullYear(); const mm = Number(m2[1]); const dd = Number(m2[2]); const d = new Date(yyyy, mm - 1, dd); d.setHours(0, 0, 0, 0); return d; } return null; } function toRatePercentStringFromValue(val: unknown): string | undefined { const n = typeof val === 'number' ? val : Number(val); if (!Number.isFinite(n)) return undefined; if (n === 0) return '0'; const scaled = n * 100; const rounded = Math.round(scaled * 100) / 100; const s = rounded.toFixed(2).replace(/\.00$/, '').replace(/(\.\d)0$/, '$1'); return `${s}%`; } function parseCookiesFromAccount(cookieData: string | null): PlaywrightCookie[] { if (!cookieData) return []; const raw = cookieData.trim(); if (!raw) return []; // 1) JSON array(最常见:浏览器插件导出/前端保存) if (raw.startsWith('[') || raw.startsWith('{')) { try { const parsed = JSON.parse(raw); const arr = Array.isArray(parsed) ? parsed : (parsed?.cookies ? parsed.cookies : []); if (!Array.isArray(arr)) return []; return arr .map((c: any) => { const name = String(c?.name ?? '').trim(); const value = String(c?.value ?? '').trim(); if (!name) return null; const domain = c?.domain ? String(c.domain) : undefined; const pathVal = c?.path ? String(c.path) : '/'; const url = !domain ? 'https://creator.douyin.com' : undefined; const sameSiteRaw = c?.sameSite; const sameSite = sameSiteRaw === 'Lax' || sameSiteRaw === 'None' || sameSiteRaw === 'Strict' ? sameSiteRaw : undefined; return { name, value, domain, path: pathVal, url, expires: typeof c?.expires === 'number' ? c.expires : undefined, httpOnly: typeof c?.httpOnly === 'boolean' ? c.httpOnly : undefined, secure: typeof c?.secure === 'boolean' ? c.secure : undefined, sameSite, } satisfies PlaywrightCookie; }) .filter(Boolean) as PlaywrightCookie[]; } catch { // fallthrough } } // 2) "a=b; c=d" 拼接格式 const pairs = raw.split(';').map((p) => p.trim()).filter(Boolean); const cookies: PlaywrightCookie[] = []; for (const p of pairs) { const idx = p.indexOf('='); if (idx <= 0) continue; const name = p.slice(0, idx).trim(); const value = p.slice(idx + 1).trim(); if (!name) continue; cookies.push({ name, value, url: 'https://creator.douyin.com' }); } return cookies; } async function createBrowserForAccount(proxy: ProxyConfig | null): Promise<{ browser: Browser; shouldClose: boolean }> { // 静默同步:默认一律 headless,不弹窗 // 只有在“引导登录/验证”时(DY_STORAGE_STATE_BOOTSTRAP=1 且 DY_IMPORT_HEADLESS=0)才允许 headful const allowHeadfulForBootstrap = process.env.DY_STORAGE_STATE_BOOTSTRAP === '1' && process.env.DY_IMPORT_HEADLESS === '0'; const headless = !allowHeadfulForBootstrap; if (proxy?.enabled) { const server = `${proxy.type}://${proxy.host}:${proxy.port}`; const browser = await launchBrowser({ headless, proxy: { server, username: proxy.username, password: proxy.password, }, args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu', '--window-size=1920,1080'], }); return { browser, shouldClose: true }; } const browser = await BrowserManager.getBrowser({ headless }); return { browser, shouldClose: false }; } type DashboardMetricTrendPoint = { date_time?: string; // YYYYMMDD value?: number; douyin_value?: number; xigua_value?: number; yumme_value?: number; change_rate?: number; }; type DashboardMetricItem = { english_metric_name?: string; metric_name?: string; metric_value?: number; trends?: DashboardMetricTrendPoint[]; }; type DashboardResponse = { status_code?: number; status_msg?: string; metrics?: DashboardMetricItem[]; }; function parseYmdCompactToDate(ymd: unknown): Date | null { const s = String(ymd || '').trim(); if (!/^\d{8}$/.test(s)) return null; const yyyy = Number(s.slice(0, 4)); const mm = Number(s.slice(4, 6)); const dd = Number(s.slice(6, 8)); if (!yyyy || !mm || !dd) return null; const d = new Date(yyyy, mm - 1, dd); d.setHours(0, 0, 0, 0); return d; } function pickTrendValue(pt: DashboardMetricTrendPoint): number | undefined { // 优先使用聚合 value;若不存在则兜底 douyin_value const v = typeof pt?.value === 'number' ? pt.value : typeof pt?.douyin_value === 'number' ? pt.douyin_value : undefined; if (!Number.isFinite(v as number)) return undefined; return v as number; } export class DouyinAccountOverviewImportService { private accountRepository = AppDataSource.getRepository(PlatformAccount); private userDayStatisticsService = new UserDayStatisticsService(); private stateDir = path.resolve(process.cwd(), 'tmp', 'douyin-storage-state'); private getStatePath(accountId: number) { return path.join(this.stateDir, `${accountId}.json`); } private async ensureStorageState(account: PlatformAccount, cookies: PlaywrightCookie[]): Promise { const statePath = this.getStatePath(account.id); try { await fs.access(statePath); return statePath; } catch { // no state } // 需要你在弹出的浏览器里完成一次登录/验证,然后脚本会自动保存 storageState // 启用方式:DY_IMPORT_HEADLESS=0 且 DY_STORAGE_STATE_BOOTSTRAP=1 if (!(process.env.DY_IMPORT_HEADLESS === '0' && process.env.DY_STORAGE_STATE_BOOTSTRAP === '1')) { return null; } await ensureDir(this.stateDir); logger.warn( `[DY Import] No storageState for accountId=${account.id}. Bootstrapping... 请在弹出的浏览器中完成登录/验证。` ); const { browser, shouldClose } = await createBrowserForAccount(account.proxyConfig); try { const context = await browser.newContext({ viewport: { width: 1920, height: 1080 }, locale: 'zh-CN', timezoneId: 'Asia/Shanghai', }); await context.addCookies(cookies as any); const page = await context.newPage(); await page.goto('https://creator.douyin.com/creator-micro/data-center/operation', { waitUntil: 'domcontentloaded', }); // 最长等 5 分钟:让你手动完成登录/滑块/短信等 await page .waitForFunction(() => { const t = document.body?.innerText || ''; return t.includes('数据中心') || t.includes('账号总览') || t.includes('短视频'); }, { timeout: 5 * 60_000 }) .catch(() => undefined); await context.storageState({ path: statePath }); logger.info(`[DY Import] storageState saved: ${statePath}`); await context.close(); return statePath; } finally { if (shouldClose) await browser.close().catch(() => undefined); } } /** * 统一入口:定时任务与添加账号均调用此方法,执行“账号总览-短视频-数据表现-近30天” */ static async runDailyImport(): Promise { const svc = new DouyinAccountOverviewImportService(); await svc.runDailyImportForAllDouyinAccounts(); } /** * 单账号入口:仅为指定抖音账号执行近30天账号总览导入(用于账号从失效恢复为 active 时补数) */ static async runDailyImportForAccount(accountId: number): Promise { const svc = new DouyinAccountOverviewImportService(); const account = await svc.accountRepository.findOne({ where: { id: accountId, platform: 'douyin' as any }, }); if (!account) { throw new Error(`未找到抖音账号 id=${accountId}`); } await svc.importAccountLast30Days(account); } /** * 为所有抖音账号导出“账号总览-短视频-数据表现-近30天”并导入 user_day_statistics */ async runDailyImportForAllDouyinAccounts(): Promise { const accounts = await this.accountRepository.find({ where: { platform: 'douyin' as any }, }); logger.info(`[DY Import] Start. total_accounts=${accounts.length}`); for (const account of accounts) { try { await this.importAccountLast30Days(account); } catch (e) { logger.error( `[DY Import] Account failed. accountId=${account.id} name=${account.accountName || ''}`, e ); } } logger.info('[DY Import] Done.'); } /** * 单账号:导出 Excel → 解析 → 入库 → 删除文件 */ async importAccountLast30Days(account: PlatformAccount, isRetry = false): Promise { const cookies = parseCookiesFromAccount(account.cookieData); if (!cookies.length) { throw new Error('cookieData 为空或无法解析'); } const { browser, shouldClose } = await createBrowserForAccount(account.proxyConfig); try { const statePath = await this.ensureStorageState(account, cookies); const context = await browser.newContext({ acceptDownloads: true, viewport: { width: 1920, height: 1080 }, locale: 'zh-CN', timezoneId: 'Asia/Shanghai', userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', ...(statePath ? { storageState: statePath } : {}), }); context.setDefaultTimeout(60_000); // 如果没 state,就退回 cookie-only(可能导出为 0) if (!statePath) { await context.addCookies(cookies as any); } const page = await context.newPage(); logger.info(`[DY Import] accountId=${account.id} goto data-center...`); await page.goto('https://creator.douyin.com/creator-micro/data-center/operation', { waitUntil: 'domcontentloaded', }); await page.waitForTimeout(1500); if (page.url().includes('login')) { // 第一次检测到登录失效时,尝试刷新账号 if (!isRetry) { logger.info(`[DY Import] Login expired detected for account ${account.id}, attempting to refresh...`); await context.close(); if (shouldClose) await browser.close(); try { const accountService = new AccountService(); const refreshResult = await accountService.refreshAccount(account.userId, account.id); if (refreshResult.needReLogin) { // 刷新后仍需要重新登录,走原先的失效流程 logger.warn(`[DY Import] Account ${account.id} refresh failed, still needs re-login`); throw new Error('未登录/需要重新登录(跳转到 login)'); } // 刷新成功,重新获取账号信息并重试导入 logger.info(`[DY Import] Account ${account.id} refreshed successfully, retrying import...`); const refreshedAccount = await this.accountRepository.findOne({ where: { id: account.id } }); if (!refreshedAccount) { throw new Error('账号刷新后未找到'); } // 递归调用,标记为重试 return await this.importAccountLast30Days(refreshedAccount, true); } catch (refreshError) { logger.error(`[DY Import] Account ${account.id} refresh failed:`, refreshError); throw new Error('未登录/需要重新登录(跳转到 login)'); } } else { // 已经是重试了,不再尝试刷新 throw new Error('未登录/需要重新登录(跳转到 login)'); } } // 检测“暂无访问权限 / 权限申请中 / 暂无数据”提示:标记账号 expired + 推送提示 const bodyText = (await page.textContent('body').catch(() => '')) || ''; if ( bodyText.includes('暂无访问权限') || bodyText.includes('权限申请中') || bodyText.includes('暂无数据权限') || bodyText.includes('暂无数据,请稍后再试') ) { await this.accountRepository.update(account.id, { status: 'expired' as any }); wsManager.sendToUser(account.userId, WS_EVENTS.ACCOUNT_UPDATED, { account: { id: account.id, status: 'expired', platform: 'douyin' }, }); wsManager.sendToUser(account.userId, WS_EVENTS.SYSTEM_MESSAGE, { level: 'warning', message: `抖音账号「${account.accountName || account.accountId || account.id}」暂无数据看板访问权限,请到抖音创作者中心申请数据权限(通过后一般次日生效)。`, platform: 'douyin', accountId: account.id, }); throw new Error('抖音数据看板暂无访问权限/申请中,已标记 expired 并通知用户'); } // 已直达账号总览页(data-center/operation),无需再点「数据中心/账号总览」,直接点「短视频」和「近30天」 await page.waitForTimeout(500); logger.info(`[DY Import] accountId=${account.id} on 账号总览页, click 短视频 tab (#semiTabaweme)...`); const shortVideoById = page.locator('#semiTabaweme'); if ((await shortVideoById.count().catch(() => 0)) > 0) { await shortVideoById.first().click(); } else { const shortVideoCandidates = ['短视频', '短视频数据']; let shortVideoClicked = false; for (const text of shortVideoCandidates) { const loc = page.getByText(text, { exact: false }).first(); if ((await loc.count().catch(() => 0)) > 0) { await loc.click().catch(() => undefined); shortVideoClicked = true; break; } } if (!shortVideoClicked) { throw new Error('页面上未找到「短视频」入口,请确认抖音创作者后台是否改版'); } } // 切换“近30天”(优先用 ID #addon-aoc08fi,兜底文案) await page.waitForTimeout(500); logger.info(`[DY Import] accountId=${account.id} click 近30天 (#addon-aoc08fi)...`); const last30DaysById = page.locator('#addon-aoc08fi'); if ((await last30DaysById.count().catch(() => 0)) > 0) { await last30DaysById.first().click(); } else { await page.getByText(/近\d+天?/).first().click().catch(() => undefined); await page.getByText('近30天', { exact: true }).click(); } await page.waitForTimeout(1200); let totalInserted = 0; let totalUpdated = 0; const apiUrl = 'https://creator.douyin.com/janus/douyin/creator/data/overview/dashboard'; logger.info(`[DY Import] accountId=${account.id} fetch dashboard (POST recent_days=30)...`); // 优先监听页面自身是否会发起该请求;若没有,则在页面上下文里手动 fetch(浏览器自动带 cookie) const responsePromise = page .waitForResponse( (res) => res.request().method() === 'POST' && res.url().includes('/janus/douyin/creator/data/overview/dashboard'), { timeout: 8000 } ) .catch(() => null); const evalPromise = page.evaluate(async (url) => { try { const r = await fetch(url, { method: 'POST', credentials: 'include', headers: { accept: 'application/json, text/plain, */*', 'content-type': 'application/json', }, body: JSON.stringify({ recent_days: 30 }), }); const json = await r.json().catch(() => null); return json; } catch (e: any) { return { status_code: -1, status_msg: String(e?.message || e) }; } }, apiUrl); const [res, evalJson] = await Promise.all([responsePromise, evalPromise]); const body = ((res ? await res.json().catch(() => null) : null) ?? evalJson ?? null) as DashboardResponse | null; if (!body || typeof body !== 'object') { throw new Error('overview/dashboard 响应不是 JSON'); } if (Number(body.status_code) !== 0) { throw new Error(`overview/dashboard 返回非成功: code=${body.status_code} msg=${body.status_msg || ''}`); } const metrics = Array.isArray(body.metrics) ? body.metrics : []; if (!metrics.length) { logger.warn(`[DY Import] dashboard metrics empty. accountId=${account.id}`); } const mergedDays = new Map>(); const setDay = (d: Date) => { const key = `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`; if (!mergedDays.has(key)) mergedDays.set(key, { recordDate: d }); return mergedDays.get(key)!; }; for (const m of metrics) { const en = String(m?.english_metric_name || '').trim(); const trends = Array.isArray(m?.trends) ? m.trends : []; if (!en || !trends.length) continue; for (const pt of trends) { const d = parseYmdCompactToDate(pt?.date_time); if (!d) continue; const obj = setDay(d); const v = pickTrendValue(pt); if (v === undefined) continue; // 显式排除:主页访问 / 取关粉丝(库里没有对应字段) if (en === 'homepage_view_cnt' || en === 'cancel_fans_cnt') continue; if (en === 'total_fans_cnt') (obj as any).fansCount = Math.round(v); else if (en === 'play_cnt') (obj as any).playCount = Math.round(v); else if (en === 'digg_cnt') (obj as any).likeCount = Math.round(v); else if (en === 'comment_cnt') (obj as any).commentCount = Math.round(v); else if (en === 'share_count') (obj as any).shareCount = Math.round(v); else if (en === 'net_fans_cnt') (obj as any).fansIncrease = Math.round(v); else if (en === 'cover_click_ratio') { const s = toRatePercentStringFromValue(v); if (s != null) (obj as any).coverClickRate = s; } } } // 合并完成后统一入库(避免同一天多次 update) for (const v of mergedDays.values()) { const { recordDate, ...patch } = v; const r = await this.userDayStatisticsService.saveStatisticsForDate(account.id, recordDate, patch); totalInserted += r.inserted; totalUpdated += r.updated; } logger.info( `[DY Import] short-video imported. accountId=${account.id} days=${mergedDays.size} inserted=${totalInserted} updated=${totalUpdated}` ); await context.close(); } finally { if (shouldClose) { await browser.close().catch(() => undefined); } } } }