///
import { chromium, type Browser, type BrowserContext, type Page } from 'playwright';
import { AppDataSource, PlatformAccount, Work } from '../models/index.js';
import { logger } from '../utils/logger.js';
import { WorkDayStatisticsService } from './WorkDayStatisticsService.js';
import { AccountService } from './AccountService.js';
import { BrowserManager } from '../automation/browser.js';
import type { ProxyConfig } from '@media-manager/shared';
import { CookieManager } from '../automation/cookie.js';
import { WS_EVENTS } from '@media-manager/shared';
import { wsManager } from '../websocket/index.js';
import type { PlatformType } from '@media-manager/shared';
/** 抖音 metrics_trend 返回 user not match / 未登录时抛出,用于触发「先刷新账号、再决定是否账号失效」 */
export class DouyinLoginExpiredError extends Error {
constructor(message = 'DOUYIN_LOGIN_EXPIRED') {
super(message);
this.name = 'DouyinLoginExpiredError';
}
}
type PlaywrightCookie = {
name: string;
value: string;
domain?: string;
path?: string;
url?: string;
expires?: number;
httpOnly?: boolean;
secure?: boolean;
sameSite?: 'Lax' | 'None' | 'Strict';
};
type TrendPoint = { date_time?: string; value?: string | number };
type MetricsTrendResponse = {
status_code: number;
status_msg?: string;
trend_map?: Record<
string,
Record
>;
};
interface DailyWorkStatPatch {
workId: number;
recordDate: Date;
playCount?: number;
likeCount?: number;
commentCount?: number;
shareCount?: number;
collectCount?: number;
fansIncrease?: number;
completionRate?: string;
twoSecondExitRate?: string;
}
function tryDecryptCookieData(cookieData: string | null): string | null {
if (!cookieData) return null;
const raw = cookieData.trim();
if (!raw) return null;
try {
return CookieManager.decrypt(raw);
} catch {
return raw;
}
}
function parseCookiesFromAccount(cookieData: string | null): PlaywrightCookie[] {
const rawOrDecrypted = tryDecryptCookieData(cookieData);
if (!rawOrDecrypted) return [];
const raw = rawOrDecrypted.trim();
if (!raw) return [];
// 1) JSON array(最常见)
if (raw.startsWith('[') || raw.startsWith('{')) {
try {
const parsed = JSON.parse(raw);
const arr = Array.isArray(parsed) ? parsed : (parsed?.cookies ? parsed.cookies : []);
if (!Array.isArray(arr)) return [];
return arr
.map((c: any) => {
const name = String(c?.name ?? '').trim();
const value = String(c?.value ?? '').trim();
if (!name) return null;
const domain = c?.domain ? String(c.domain) : undefined;
const pathVal = c?.path ? String(c.path) : '/';
const url = !domain ? 'https://creator.douyin.com' : undefined;
const sameSiteRaw = c?.sameSite;
const sameSite =
sameSiteRaw === 'Lax' || sameSiteRaw === 'None' || sameSiteRaw === 'Strict'
? sameSiteRaw
: undefined;
return {
name,
value,
domain,
path: pathVal,
url,
expires: typeof c?.expires === 'number' ? c.expires : undefined,
httpOnly: typeof c?.httpOnly === 'boolean' ? c.httpOnly : undefined,
secure: typeof c?.secure === 'boolean' ? c.secure : undefined,
sameSite,
} satisfies PlaywrightCookie;
})
.filter(Boolean) as PlaywrightCookie[];
} catch {
// fallthrough
}
}
// 2) "a=b; c=d" 拼接格式
const pairs = raw.split(';').map((p) => p.trim()).filter(Boolean);
const cookies: PlaywrightCookie[] = [];
for (const p of pairs) {
const idx = p.indexOf('=');
if (idx <= 0) continue;
const name = p.slice(0, idx).trim();
const value = p.slice(idx + 1).trim();
if (!name) continue;
cookies.push({ name, value, url: 'https://creator.douyin.com' });
}
return cookies;
}
async function createBrowserForAccount(proxy: ProxyConfig | null): Promise<{ browser: Browser; shouldClose: boolean }> {
const headless = true;
if (proxy?.enabled) {
const server = `${proxy.type}://${proxy.host}:${proxy.port}`;
const browser = await chromium.launch({
headless,
proxy: {
server,
username: proxy.username,
password: proxy.password,
},
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu', '--window-size=1920,1080'],
});
return { browser, shouldClose: true };
}
const browser = await BrowserManager.getBrowser({ headless });
return { browser, shouldClose: false };
}
function parseChinaDateFromDateTimeString(dateTime: unknown): Date | null {
if (!dateTime) return null;
const s = String(dateTime).trim();
if (s.length < 10) return null;
const ymd = s.slice(0, 10); // YYYY-MM-DD
const m = ymd.match(/^(\d{4})-(\d{2})-(\d{2})$/);
if (!m) return null;
const yyyy = Number(m[1]);
const mm = Number(m[2]);
const dd = Number(m[3]);
if (!yyyy || !mm || !dd) return null;
const d = new Date(yyyy, mm - 1, dd, 0, 0, 0, 0);
return d;
}
function toNumber(val: unknown, defaultValue = 0): number {
if (typeof val === 'number') return Number.isFinite(val) ? val : defaultValue;
if (typeof val === 'string') {
const n = Number(val);
return Number.isFinite(n) ? n : defaultValue;
}
return defaultValue;
}
function toInt(val: unknown, defaultValue = 0): number {
const n = toNumber(val, NaN);
if (!Number.isFinite(n)) return defaultValue;
return Math.round(n);
}
function normalizePercentString(val: unknown): string | undefined {
const n = toNumber(val, NaN);
if (!Number.isFinite(n)) return undefined;
if (n === 0) return '0';
// 去掉多余的 0:48.730000 -> 48.73
const s = n.toString();
return `${s}%`;
}
function isDouyinLoginExpiredByApi(body: any): boolean {
const code = Number(body?.status_code);
const msg = String(body?.status_msg || '');
if (code === 20001) return true; // user not match
if (msg.includes('user not match')) return true;
if (msg.includes('登录') && msg.includes('失效')) return true;
return false;
}
class DouyinMetricsTrendClient {
private capturedHeaders: Record | null = null;
private buildTrendUrl(itemId: string, metric: string): string {
const base = 'https://creator.douyin.com/janus/douyin/creator/data/item_analysis/metrics_trend';
const params = new URLSearchParams({
aid: '2906',
app_name: 'aweme_creator_platform',
device_platform: 'web',
// referer/user_agent 等埋点参数保留,尽量贴近浏览器请求
referer: 'https://creator.douyin.com/creator-micro/data-center/content',
user_agent:
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36',
cookie_enabled: 'true',
screen_width: '1920',
screen_height: '1080',
browser_language: 'zh-CN',
browser_platform: 'Win32',
browser_name: 'Mozilla',
browser_version:
'5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36',
browser_online: 'true',
timezone_name: 'Asia/Shanghai',
item_id: itemId,
trend_type: '1',
time_unit: '1',
metrics_group: '0,1,3',
metrics: metric,
});
return `${base}?${params.toString()}`;
}
private filterCapturedHeaders(h: Record): Record {
const out: Record = {};
const allowList = new Set([
'accept',
'accept-language',
'agw-js-conv',
'user-agent',
'x-secsdk-csrf-token',
]);
for (const [k, v] of Object.entries(h || {})) {
const key = k.toLowerCase();
if (key === 'cookie') continue;
if (key === 'host') continue;
if (key === 'authority') continue;
if (key === 'content-length') continue;
if (key === 'referer') continue; // 每次按作品详情页动态设置
if (allowList.has(key)) out[key] = v;
// 保留所有 x- 前缀的 header(有些风控字段会放在 x- 里)
else if (key.startsWith('x-')) out[key] = v;
}
return out;
}
private async captureHeadersFromRealRequest(page: Page, metricLabel: string, itemId: string, metric: string): Promise {
// 通过点击 UI 触发一次真实请求,抓取其 headers(尤其是 x-secsdk-csrf-token)
const apiPattern = /\/janus\/douyin\/creator\/data\/item_analysis\/metrics_trend/i;
const wait = page.waitForResponse((res) => {
if (res.request().method() !== 'GET') return false;
const u = res.url();
return apiPattern.test(u) && u.includes(`item_id=${itemId}`) && u.includes(`metrics=${metric}`);
}, { timeout: 25_000 });
// 指标卡片(如:点赞量/播放量/评论量...)
// 允许用 "|" 提供多个候选文案(适配 UI 文案差异)
const labels = metricLabel
.split('|')
.map((s) => s.trim())
.filter(Boolean);
let clicked = false;
for (const label of labels.length ? labels : [metricLabel]) {
const loc = page.getByText(label, { exact: false }).first();
const cnt = await loc.count().catch(() => 0);
if (!cnt) continue;
await loc.waitFor({ state: 'visible', timeout: 10_000 }).catch(() => undefined);
await loc.click().catch(() => undefined);
clicked = true;
break;
}
if (!clicked) {
// 不强制失败:有些情况下 UI 不可点击,但直连请求仍可能成功
logger.warn(`[DY WorkStats] Could not click metric label on page for header capture. label=${metricLabel}`);
}
const res = await wait;
const headers = res.request().headers();
this.capturedHeaders = this.filterCapturedHeaders(headers);
logger.info(`[DY WorkStats] Captured request headers for metrics_trend. keys=${Object.keys(this.capturedHeaders).join(',')}`);
}
async fetchTrend(
ctx: BrowserContext,
page: Page,
itemId: string,
metric: string,
metricLabelForFallback: string,
refererUrl: string
): Promise {
const url = this.buildTrendUrl(itemId, metric);
const headers: Record = {
accept: 'application/json, text/plain, */*',
'accept-language': 'zh-CN,zh;q=0.9',
referer: refererUrl,
'user-agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36',
...(this.capturedHeaders || {}),
};
// 1) 先尝试直接请求(最快)
try {
const res = await ctx.request.get(url, {
headers,
timeout: 25_000,
});
const json = (await res.json().catch(() => null)) as MetricsTrendResponse | null;
if (json && typeof json === 'object') return json;
} catch {
// fallthrough
}
// 2) 如果直连失败,抓一次真实请求 header 后重试
if (!this.capturedHeaders) {
await this.captureHeadersFromRealRequest(page, metricLabelForFallback, itemId, metric).catch(() => undefined);
}
const headers2: Record = {
accept: 'application/json, text/plain, */*',
'accept-language': 'zh-CN,zh;q=0.9',
referer: refererUrl,
'user-agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36',
...(this.capturedHeaders || {}),
};
const res2 = await ctx.request.get(url, {
headers: headers2,
timeout: 25_000,
});
const json2 = (await res2.json().catch(() => null)) as MetricsTrendResponse | null;
if (!json2) throw new Error('metrics_trend 响应不是 JSON');
return json2;
}
}
export class DouyinWorkStatisticsImportService {
private accountRepository = AppDataSource.getRepository(PlatformAccount);
private workRepository = AppDataSource.getRepository(Work);
private workDayStatisticsService = new WorkDayStatisticsService();
static async runDailyImport(): Promise {
const svc = new DouyinWorkStatisticsImportService();
await svc.runDailyImportForAllDouyinAccounts();
}
async runDailyImportForAllDouyinAccounts(): Promise {
const accounts = await this.accountRepository.find({
where: { platform: 'douyin' as any },
});
logger.info(`[DY WorkStats] Start import for ${accounts.length} accounts`);
for (const account of accounts) {
try {
await this.importAccountWorksStatistics(account);
} catch (e) {
logger.error(
`[DY WorkStats] Account failed. accountId=${account.id} name=${account.accountName || ''}`,
e
);
// 单账号失败仅记录日志,不中断循环,其他账号照常同步
}
}
logger.info('[DY WorkStats] All accounts done');
}
/**
* 按账号同步作品日统计。检测到 cookie 失效时:先尝试同步/刷新账号一次;刷新仍失效则标记账号 expired。
* @param isRetry 是否为「刷新账号后的重试」,避免无限递归
*/
private async importAccountWorksStatistics(account: PlatformAccount, isRetry = false): Promise {
const cookies = parseCookiesFromAccount(account.cookieData);
if (!cookies.length) {
logger.warn(`[DY WorkStats] accountId=${account.id} cookieData 为空或无法解析,跳过`);
return;
}
const works = await this.workRepository.find({
where: {
accountId: account.id,
platform: 'douyin' as any,
},
});
if (!works.length) {
logger.info(`[DY WorkStats] accountId=${account.id} 没有作品,跳过`);
return;
}
const { browser, shouldClose } = await createBrowserForAccount(account.proxyConfig);
let context: BrowserContext | null = null;
let closedDueToLoginExpired = false;
try {
context = await browser.newContext({
viewport: { width: 1920, height: 1080 },
locale: 'zh-CN',
timezoneId: 'Asia/Shanghai',
userAgent:
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36',
});
await context.addCookies(cookies as any);
context.setDefaultTimeout(60_000);
if (!context) {
throw new Error('BrowserContext 初始化失败');
}
const ctx = context;
const page = await context.newPage();
const client = new DouyinMetricsTrendClient();
let totalInserted = 0;
let totalUpdated = 0;
for (const work of works) {
const itemId = (work.platformVideoId || '').trim();
if (!itemId) continue;
const detailUrl = `https://creator.douyin.com/creator-micro/work-management/work-detail/${encodeURIComponent(
itemId
)}?enter_from=item_data`;
try {
await page.goto(detailUrl, { waitUntil: 'domcontentloaded' }).catch(() => undefined);
await page.waitForTimeout(1200);
if (page.url().includes('login') || page.url().includes('passport')) {
throw new DouyinLoginExpiredError('work-detail 页面跳转登录,cookie 可能失效');
}
// metrics -> 入库字段映射
const metricsPlan: Array<{
metric: string;
label: string;
apply: (patch: DailyWorkStatPatch, v: unknown) => void;
}> = [
{ metric: 'view_count', label: '播放量', apply: (p, v) => (p.playCount = toInt(v, 0)) },
{ metric: 'like_count', label: '点赞量', apply: (p, v) => (p.likeCount = toInt(v, 0)) },
{ metric: 'comment_count', label: '评论量', apply: (p, v) => (p.commentCount = toInt(v, 0)) },
{ metric: 'share_count', label: '分享量', apply: (p, v) => (p.shareCount = toInt(v, 0)) },
{ metric: 'favorite_count', label: '收藏量|收藏数', apply: (p, v) => (p.collectCount = toInt(v, 0)) },
{ metric: 'subscribe_count', label: '涨粉量|涨粉数', apply: (p, v) => (p.fansIncrease = toInt(v, 0)) },
{
metric: 'completion_rate',
label: '完播率',
apply: (p, v) => {
const s = normalizePercentString(v);
if (s != null) p.completionRate = s;
},
},
{
metric: 'bounce_rate_2s',
label: '2s退出率|2s跳出率|2s跳出',
apply: (p, v) => {
const s = normalizePercentString(v);
if (s != null) p.twoSecondExitRate = s;
},
},
];
const dayMap = new Map();
for (const m of metricsPlan) {
const body = await client.fetchTrend(ctx, page, itemId, m.metric, m.label, detailUrl);
if (!body || typeof body !== 'object') continue;
if (isDouyinLoginExpiredByApi(body)) {
throw new DouyinLoginExpiredError(body.status_msg || 'metrics_trend: user not match');
}
if (Number(body.status_code) !== 0) {
logger.warn(
`[DY WorkStats] metrics_trend 非成功返回. accountId=${account.id} workId=${work.id} itemId=${itemId} metric=${m.metric} code=${body.status_code} msg=${body.status_msg || ''}`
);
continue;
}
const trendMap = body.trend_map || {};
const metricMap = (trendMap as any)[m.metric] as Record | undefined;
if (!metricMap) continue;
// 优先取 group "0"(一般为“总计/全部”),否则兜底合并全部 group
const points = Array.isArray(metricMap['0'])
? metricMap['0']
: Object.values(metricMap).flatMap((arr) => (Array.isArray(arr) ? arr : []));
for (const pt of points) {
const d = parseChinaDateFromDateTimeString(pt?.date_time);
if (!d) continue;
const key = d.getTime();
let entry = dayMap.get(key);
if (!entry) {
entry = { workId: work.id, recordDate: d };
dayMap.set(key, entry);
}
m.apply(entry, pt?.value);
}
}
const patches = Array.from(dayMap.values()).sort(
(a, b) => a.recordDate.getTime() - b.recordDate.getTime()
);
if (!patches.length) continue;
const result = await this.workDayStatisticsService.saveStatisticsForDateBatch(
patches.map((p) => ({
workId: p.workId,
recordDate: p.recordDate,
playCount: p.playCount,
likeCount: p.likeCount,
commentCount: p.commentCount,
shareCount: p.shareCount,
collectCount: p.collectCount,
fansIncrease: p.fansIncrease,
completionRate: p.completionRate,
twoSecondExitRate: p.twoSecondExitRate,
}))
);
totalInserted += result.inserted;
totalUpdated += result.updated;
} catch (e) {
if (e instanceof DouyinLoginExpiredError) {
closedDueToLoginExpired = true;
if (context) {
await context.close().catch(() => undefined);
context = null;
}
if (shouldClose) {
await browser.close().catch(() => undefined);
}
// cookie 过期处理:先刷新一次账号,再决定是否标记 expired
if (!isRetry) {
logger.info(`[DY WorkStats] accountId=${account.id} 登录失效,尝试同步账号后重试...`);
try {
const accountService = new AccountService();
const refreshResult = await accountService.refreshAccount(account.userId, account.id);
if (refreshResult.needReLogin) {
await this.markAccountExpired(account, 'cookie 过期,需要重新登录');
return;
}
const refreshed = await this.accountRepository.findOne({ where: { id: account.id } });
if (refreshed) {
logger.info(`[DY WorkStats] accountId=${account.id} 同步账号成功,重新拉取作品数据`);
return this.importAccountWorksStatistics(refreshed, true);
}
} catch (refreshErr) {
logger.error(`[DY WorkStats] accountId=${account.id} 同步账号失败`, refreshErr);
await this.markAccountExpired(account, '同步账号失败,已标记过期');
return;
}
} else {
await this.markAccountExpired(account, '同步后仍失效,已标记过期');
return;
}
}
logger.error(
`[DY WorkStats] Failed to import work stats. accountId=${account.id} workId=${work.id} itemId=${itemId}`,
e
);
}
}
logger.info(
`[DY WorkStats] accountId=${account.id} completed. inserted=${totalInserted}, updated=${totalUpdated}`
);
} finally {
if (!closedDueToLoginExpired) {
if (context) {
await context.close().catch(() => undefined);
}
if (shouldClose) {
await browser.close().catch(() => undefined);
}
}
}
}
private async markAccountExpired(account: PlatformAccount, reason: string): Promise {
await this.accountRepository.update(account.id, { status: 'expired' as any });
wsManager.sendToUser(account.userId, WS_EVENTS.ACCOUNT_UPDATED, {
account: { id: account.id, status: 'expired', platform: 'douyin' as PlatformType },
});
wsManager.sendToUser(account.userId, WS_EVENTS.SYSTEM_MESSAGE, {
level: 'warning',
message: `抖音账号「${account.accountName || account.accountId || account.id}」登录已失效:${reason}`,
platform: 'douyin',
accountId: account.id,
});
}
}