/// import { chromium, type Browser, type BrowserContext, type Page } from 'playwright'; import { AppDataSource, PlatformAccount, Work } from '../models/index.js'; import { logger } from '../utils/logger.js'; import { WorkDayStatisticsService } from './WorkDayStatisticsService.js'; import { AccountService } from './AccountService.js'; import { BrowserManager } from '../automation/browser.js'; import type { ProxyConfig } from '@media-manager/shared'; import { CookieManager } from '../automation/cookie.js'; import { WS_EVENTS } from '@media-manager/shared'; import { wsManager } from '../websocket/index.js'; import type { PlatformType } from '@media-manager/shared'; /** 抖音 metrics_trend 返回 user not match / 未登录时抛出,用于触发「先刷新账号、再决定是否账号失效」 */ export class DouyinLoginExpiredError extends Error { constructor(message = 'DOUYIN_LOGIN_EXPIRED') { super(message); this.name = 'DouyinLoginExpiredError'; } } type PlaywrightCookie = { name: string; value: string; domain?: string; path?: string; url?: string; expires?: number; httpOnly?: boolean; secure?: boolean; sameSite?: 'Lax' | 'None' | 'Strict'; }; type TrendPoint = { date_time?: string; value?: string | number }; type MetricsTrendResponse = { status_code: number; status_msg?: string; trend_map?: Record< string, Record >; }; interface DailyWorkStatPatch { workId: number; recordDate: Date; playCount?: number; likeCount?: number; commentCount?: number; shareCount?: number; collectCount?: number; fansIncrease?: number; completionRate?: string; twoSecondExitRate?: string; } function tryDecryptCookieData(cookieData: string | null): string | null { if (!cookieData) return null; const raw = cookieData.trim(); if (!raw) return null; try { return CookieManager.decrypt(raw); } catch { return raw; } } function parseCookiesFromAccount(cookieData: string | null): PlaywrightCookie[] { const rawOrDecrypted = tryDecryptCookieData(cookieData); if (!rawOrDecrypted) return []; const raw = rawOrDecrypted.trim(); if (!raw) return []; // 1) JSON array(最常见) if (raw.startsWith('[') || raw.startsWith('{')) { try { const parsed = JSON.parse(raw); const arr = Array.isArray(parsed) ? parsed : (parsed?.cookies ? parsed.cookies : []); if (!Array.isArray(arr)) return []; return arr .map((c: any) => { const name = String(c?.name ?? '').trim(); const value = String(c?.value ?? '').trim(); if (!name) return null; const domain = c?.domain ? String(c.domain) : undefined; const pathVal = c?.path ? String(c.path) : '/'; const url = !domain ? 'https://creator.douyin.com' : undefined; const sameSiteRaw = c?.sameSite; const sameSite = sameSiteRaw === 'Lax' || sameSiteRaw === 'None' || sameSiteRaw === 'Strict' ? sameSiteRaw : undefined; return { name, value, domain, path: pathVal, url, expires: typeof c?.expires === 'number' ? c.expires : undefined, httpOnly: typeof c?.httpOnly === 'boolean' ? c.httpOnly : undefined, secure: typeof c?.secure === 'boolean' ? c.secure : undefined, sameSite, } satisfies PlaywrightCookie; }) .filter(Boolean) as PlaywrightCookie[]; } catch { // fallthrough } } // 2) "a=b; c=d" 拼接格式 const pairs = raw.split(';').map((p) => p.trim()).filter(Boolean); const cookies: PlaywrightCookie[] = []; for (const p of pairs) { const idx = p.indexOf('='); if (idx <= 0) continue; const name = p.slice(0, idx).trim(); const value = p.slice(idx + 1).trim(); if (!name) continue; cookies.push({ name, value, url: 'https://creator.douyin.com' }); } return cookies; } async function createBrowserForAccount(proxy: ProxyConfig | null): Promise<{ browser: Browser; shouldClose: boolean }> { const headless = true; if (proxy?.enabled) { const server = `${proxy.type}://${proxy.host}:${proxy.port}`; const browser = await chromium.launch({ headless, proxy: { server, username: proxy.username, password: proxy.password, }, args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu', '--window-size=1920,1080'], }); return { browser, shouldClose: true }; } const browser = await BrowserManager.getBrowser({ headless }); return { browser, shouldClose: false }; } function parseChinaDateFromDateTimeString(dateTime: unknown): Date | null { if (!dateTime) return null; const s = String(dateTime).trim(); if (s.length < 10) return null; const ymd = s.slice(0, 10); // YYYY-MM-DD const m = ymd.match(/^(\d{4})-(\d{2})-(\d{2})$/); if (!m) return null; const yyyy = Number(m[1]); const mm = Number(m[2]); const dd = Number(m[3]); if (!yyyy || !mm || !dd) return null; const d = new Date(yyyy, mm - 1, dd, 0, 0, 0, 0); return d; } function toNumber(val: unknown, defaultValue = 0): number { if (typeof val === 'number') return Number.isFinite(val) ? val : defaultValue; if (typeof val === 'string') { const n = Number(val); return Number.isFinite(n) ? n : defaultValue; } return defaultValue; } function toInt(val: unknown, defaultValue = 0): number { const n = toNumber(val, NaN); if (!Number.isFinite(n)) return defaultValue; return Math.round(n); } function normalizePercentString(val: unknown): string | undefined { const n = toNumber(val, NaN); if (!Number.isFinite(n)) return undefined; if (n === 0) return '0'; // 去掉多余的 0:48.730000 -> 48.73 const s = n.toString(); return `${s}%`; } function isDouyinLoginExpiredByApi(body: any): boolean { const code = Number(body?.status_code); const msg = String(body?.status_msg || ''); if (code === 20001) return true; // user not match if (msg.includes('user not match')) return true; if (msg.includes('登录') && msg.includes('失效')) return true; return false; } class DouyinMetricsTrendClient { private capturedHeaders: Record | null = null; private buildTrendUrl(itemId: string, metric: string): string { const base = 'https://creator.douyin.com/janus/douyin/creator/data/item_analysis/metrics_trend'; const params = new URLSearchParams({ aid: '2906', app_name: 'aweme_creator_platform', device_platform: 'web', // referer/user_agent 等埋点参数保留,尽量贴近浏览器请求 referer: 'https://creator.douyin.com/creator-micro/data-center/content', user_agent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36', cookie_enabled: 'true', screen_width: '1920', screen_height: '1080', browser_language: 'zh-CN', browser_platform: 'Win32', browser_name: 'Mozilla', browser_version: '5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36', browser_online: 'true', timezone_name: 'Asia/Shanghai', item_id: itemId, trend_type: '1', time_unit: '1', metrics_group: '0,1,3', metrics: metric, }); return `${base}?${params.toString()}`; } private filterCapturedHeaders(h: Record): Record { const out: Record = {}; const allowList = new Set([ 'accept', 'accept-language', 'agw-js-conv', 'user-agent', 'x-secsdk-csrf-token', ]); for (const [k, v] of Object.entries(h || {})) { const key = k.toLowerCase(); if (key === 'cookie') continue; if (key === 'host') continue; if (key === 'authority') continue; if (key === 'content-length') continue; if (key === 'referer') continue; // 每次按作品详情页动态设置 if (allowList.has(key)) out[key] = v; // 保留所有 x- 前缀的 header(有些风控字段会放在 x- 里) else if (key.startsWith('x-')) out[key] = v; } return out; } private async captureHeadersFromRealRequest(page: Page, metricLabel: string, itemId: string, metric: string): Promise { // 通过点击 UI 触发一次真实请求,抓取其 headers(尤其是 x-secsdk-csrf-token) const apiPattern = /\/janus\/douyin\/creator\/data\/item_analysis\/metrics_trend/i; const wait = page.waitForResponse((res) => { if (res.request().method() !== 'GET') return false; const u = res.url(); return apiPattern.test(u) && u.includes(`item_id=${itemId}`) && u.includes(`metrics=${metric}`); }, { timeout: 25_000 }); // 指标卡片(如:点赞量/播放量/评论量...) // 允许用 "|" 提供多个候选文案(适配 UI 文案差异) const labels = metricLabel .split('|') .map((s) => s.trim()) .filter(Boolean); let clicked = false; for (const label of labels.length ? labels : [metricLabel]) { const loc = page.getByText(label, { exact: false }).first(); const cnt = await loc.count().catch(() => 0); if (!cnt) continue; await loc.waitFor({ state: 'visible', timeout: 10_000 }).catch(() => undefined); await loc.click().catch(() => undefined); clicked = true; break; } if (!clicked) { // 不强制失败:有些情况下 UI 不可点击,但直连请求仍可能成功 logger.warn(`[DY WorkStats] Could not click metric label on page for header capture. label=${metricLabel}`); } const res = await wait; const headers = res.request().headers(); this.capturedHeaders = this.filterCapturedHeaders(headers); logger.info(`[DY WorkStats] Captured request headers for metrics_trend. keys=${Object.keys(this.capturedHeaders).join(',')}`); } async fetchTrend( ctx: BrowserContext, page: Page, itemId: string, metric: string, metricLabelForFallback: string, refererUrl: string ): Promise { const url = this.buildTrendUrl(itemId, metric); const headers: Record = { accept: 'application/json, text/plain, */*', 'accept-language': 'zh-CN,zh;q=0.9', referer: refererUrl, 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36', ...(this.capturedHeaders || {}), }; // 1) 先尝试直接请求(最快) try { const res = await ctx.request.get(url, { headers, timeout: 25_000, }); const json = (await res.json().catch(() => null)) as MetricsTrendResponse | null; if (json && typeof json === 'object') return json; } catch { // fallthrough } // 2) 如果直连失败,抓一次真实请求 header 后重试 if (!this.capturedHeaders) { await this.captureHeadersFromRealRequest(page, metricLabelForFallback, itemId, metric).catch(() => undefined); } const headers2: Record = { accept: 'application/json, text/plain, */*', 'accept-language': 'zh-CN,zh;q=0.9', referer: refererUrl, 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36', ...(this.capturedHeaders || {}), }; const res2 = await ctx.request.get(url, { headers: headers2, timeout: 25_000, }); const json2 = (await res2.json().catch(() => null)) as MetricsTrendResponse | null; if (!json2) throw new Error('metrics_trend 响应不是 JSON'); return json2; } } export class DouyinWorkStatisticsImportService { private accountRepository = AppDataSource.getRepository(PlatformAccount); private workRepository = AppDataSource.getRepository(Work); private workDayStatisticsService = new WorkDayStatisticsService(); static async runDailyImport(): Promise { const svc = new DouyinWorkStatisticsImportService(); await svc.runDailyImportForAllDouyinAccounts(); } async runDailyImportForAllDouyinAccounts(): Promise { const accounts = await this.accountRepository.find({ where: { platform: 'douyin' as any }, }); logger.info(`[DY WorkStats] Start import for ${accounts.length} accounts`); for (const account of accounts) { try { await this.importAccountWorksStatistics(account); } catch (e) { logger.error( `[DY WorkStats] Account failed. accountId=${account.id} name=${account.accountName || ''}`, e ); // 单账号失败仅记录日志,不中断循环,其他账号照常同步 } } logger.info('[DY WorkStats] All accounts done'); } /** * 按账号同步作品日统计。检测到 cookie 失效时:先尝试同步/刷新账号一次;刷新仍失效则标记账号 expired。 * @param isRetry 是否为「刷新账号后的重试」,避免无限递归 */ private async importAccountWorksStatistics(account: PlatformAccount, isRetry = false): Promise { const cookies = parseCookiesFromAccount(account.cookieData); if (!cookies.length) { logger.warn(`[DY WorkStats] accountId=${account.id} cookieData 为空或无法解析,跳过`); return; } const works = await this.workRepository.find({ where: { accountId: account.id, platform: 'douyin' as any, }, }); if (!works.length) { logger.info(`[DY WorkStats] accountId=${account.id} 没有作品,跳过`); return; } const { browser, shouldClose } = await createBrowserForAccount(account.proxyConfig); let context: BrowserContext | null = null; let closedDueToLoginExpired = false; try { context = await browser.newContext({ viewport: { width: 1920, height: 1080 }, locale: 'zh-CN', timezoneId: 'Asia/Shanghai', userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36', }); await context.addCookies(cookies as any); context.setDefaultTimeout(60_000); if (!context) { throw new Error('BrowserContext 初始化失败'); } const ctx = context; const page = await context.newPage(); const client = new DouyinMetricsTrendClient(); let totalInserted = 0; let totalUpdated = 0; for (const work of works) { const itemId = (work.platformVideoId || '').trim(); if (!itemId) continue; const detailUrl = `https://creator.douyin.com/creator-micro/work-management/work-detail/${encodeURIComponent( itemId )}?enter_from=item_data`; try { await page.goto(detailUrl, { waitUntil: 'domcontentloaded' }).catch(() => undefined); await page.waitForTimeout(1200); if (page.url().includes('login') || page.url().includes('passport')) { throw new DouyinLoginExpiredError('work-detail 页面跳转登录,cookie 可能失效'); } // metrics -> 入库字段映射 const metricsPlan: Array<{ metric: string; label: string; apply: (patch: DailyWorkStatPatch, v: unknown) => void; }> = [ { metric: 'view_count', label: '播放量', apply: (p, v) => (p.playCount = toInt(v, 0)) }, { metric: 'like_count', label: '点赞量', apply: (p, v) => (p.likeCount = toInt(v, 0)) }, { metric: 'comment_count', label: '评论量', apply: (p, v) => (p.commentCount = toInt(v, 0)) }, { metric: 'share_count', label: '分享量', apply: (p, v) => (p.shareCount = toInt(v, 0)) }, { metric: 'favorite_count', label: '收藏量|收藏数', apply: (p, v) => (p.collectCount = toInt(v, 0)) }, { metric: 'subscribe_count', label: '涨粉量|涨粉数', apply: (p, v) => (p.fansIncrease = toInt(v, 0)) }, { metric: 'completion_rate', label: '完播率', apply: (p, v) => { const s = normalizePercentString(v); if (s != null) p.completionRate = s; }, }, { metric: 'bounce_rate_2s', label: '2s退出率|2s跳出率|2s跳出', apply: (p, v) => { const s = normalizePercentString(v); if (s != null) p.twoSecondExitRate = s; }, }, ]; const dayMap = new Map(); for (const m of metricsPlan) { const body = await client.fetchTrend(ctx, page, itemId, m.metric, m.label, detailUrl); if (!body || typeof body !== 'object') continue; if (isDouyinLoginExpiredByApi(body)) { throw new DouyinLoginExpiredError(body.status_msg || 'metrics_trend: user not match'); } if (Number(body.status_code) !== 0) { logger.warn( `[DY WorkStats] metrics_trend 非成功返回. accountId=${account.id} workId=${work.id} itemId=${itemId} metric=${m.metric} code=${body.status_code} msg=${body.status_msg || ''}` ); continue; } const trendMap = body.trend_map || {}; const metricMap = (trendMap as any)[m.metric] as Record | undefined; if (!metricMap) continue; // 优先取 group "0"(一般为“总计/全部”),否则兜底合并全部 group const points = Array.isArray(metricMap['0']) ? metricMap['0'] : Object.values(metricMap).flatMap((arr) => (Array.isArray(arr) ? arr : [])); for (const pt of points) { const d = parseChinaDateFromDateTimeString(pt?.date_time); if (!d) continue; const key = d.getTime(); let entry = dayMap.get(key); if (!entry) { entry = { workId: work.id, recordDate: d }; dayMap.set(key, entry); } m.apply(entry, pt?.value); } } const patches = Array.from(dayMap.values()).sort( (a, b) => a.recordDate.getTime() - b.recordDate.getTime() ); if (!patches.length) continue; const result = await this.workDayStatisticsService.saveStatisticsForDateBatch( patches.map((p) => ({ workId: p.workId, recordDate: p.recordDate, playCount: p.playCount, likeCount: p.likeCount, commentCount: p.commentCount, shareCount: p.shareCount, collectCount: p.collectCount, fansIncrease: p.fansIncrease, completionRate: p.completionRate, twoSecondExitRate: p.twoSecondExitRate, })) ); totalInserted += result.inserted; totalUpdated += result.updated; } catch (e) { if (e instanceof DouyinLoginExpiredError) { closedDueToLoginExpired = true; if (context) { await context.close().catch(() => undefined); context = null; } if (shouldClose) { await browser.close().catch(() => undefined); } // cookie 过期处理:先刷新一次账号,再决定是否标记 expired if (!isRetry) { logger.info(`[DY WorkStats] accountId=${account.id} 登录失效,尝试同步账号后重试...`); try { const accountService = new AccountService(); const refreshResult = await accountService.refreshAccount(account.userId, account.id); if (refreshResult.needReLogin) { await this.markAccountExpired(account, 'cookie 过期,需要重新登录'); return; } const refreshed = await this.accountRepository.findOne({ where: { id: account.id } }); if (refreshed) { logger.info(`[DY WorkStats] accountId=${account.id} 同步账号成功,重新拉取作品数据`); return this.importAccountWorksStatistics(refreshed, true); } } catch (refreshErr) { logger.error(`[DY WorkStats] accountId=${account.id} 同步账号失败`, refreshErr); await this.markAccountExpired(account, '同步账号失败,已标记过期'); return; } } else { await this.markAccountExpired(account, '同步后仍失效,已标记过期'); return; } } logger.error( `[DY WorkStats] Failed to import work stats. accountId=${account.id} workId=${work.id} itemId=${itemId}`, e ); } } logger.info( `[DY WorkStats] accountId=${account.id} completed. inserted=${totalInserted}, updated=${totalUpdated}` ); } finally { if (!closedDueToLoginExpired) { if (context) { await context.close().catch(() => undefined); } if (shouldClose) { await browser.close().catch(() => undefined); } } } } private async markAccountExpired(account: PlatformAccount, reason: string): Promise { await this.accountRepository.update(account.id, { status: 'expired' as any }); wsManager.sendToUser(account.userId, WS_EVENTS.ACCOUNT_UPDATED, { account: { id: account.id, status: 'expired', platform: 'douyin' as PlatformType }, }); wsManager.sendToUser(account.userId, WS_EVENTS.SYSTEM_MESSAGE, { level: 'warning', message: `抖音账号「${account.accountName || account.accountId || account.id}」登录已失效:${reason}`, platform: 'douyin', accountId: account.id, }); } }