DouyinWorkStatisticsImportService.ts 29 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776
  1. import { launchBrowser } from '../automation/browserProvider.js';
  2. /// <reference lib="dom" />
  3. import type { Browser, BrowserContext, Page } from 'playwright';
  4. import { AppDataSource, PlatformAccount, Work } from '../models/index.js';
  5. import { logger } from '../utils/logger.js';
  6. import { WorkDayStatisticsService } from './WorkDayStatisticsService.js';
  7. import { AccountService } from './AccountService.js';
  8. import { BrowserManager } from '../automation/browser.js';
  9. import type { ProxyConfig } from '@media-manager/shared';
  10. import { CookieManager } from '../automation/cookie.js';
  11. import { WS_EVENTS } from '@media-manager/shared';
  12. import { wsManager } from '../websocket/index.js';
  13. import type { PlatformType } from '@media-manager/shared';
  14. /** 抖音 metrics_trend 返回 user not match / 未登录时抛出,用于触发「先刷新账号、再决定是否账号失效」 */
  15. export class DouyinLoginExpiredError extends Error {
  16. constructor(message = 'DOUYIN_LOGIN_EXPIRED') {
  17. super(message);
  18. this.name = 'DouyinLoginExpiredError';
  19. }
  20. }
  21. type PlaywrightCookie = {
  22. name: string;
  23. value: string;
  24. domain?: string;
  25. path?: string;
  26. url?: string;
  27. expires?: number;
  28. httpOnly?: boolean;
  29. secure?: boolean;
  30. sameSite?: 'Lax' | 'None' | 'Strict';
  31. };
  32. type TrendPoint = { date_time?: string; value?: string | number };
  33. type MetricsTrendResponse = {
  34. status_code: number;
  35. status_msg?: string;
  36. trend_map?: Record<
  37. string,
  38. Record<string, TrendPoint[]>
  39. >;
  40. };
  41. interface DailyWorkStatPatch {
  42. workId: number;
  43. recordDate: Date;
  44. playCount?: number;
  45. likeCount?: number;
  46. commentCount?: number;
  47. shareCount?: number;
  48. collectCount?: number;
  49. fansIncrease?: number;
  50. completionRate?: string;
  51. twoSecondExitRate?: string;
  52. }
  53. function tryDecryptCookieData(cookieData: string | null): string | null {
  54. if (!cookieData) return null;
  55. const raw = cookieData.trim();
  56. if (!raw) return null;
  57. try {
  58. return CookieManager.decrypt(raw);
  59. } catch {
  60. return raw;
  61. }
  62. }
  63. function parseCookiesFromAccount(cookieData: string | null): PlaywrightCookie[] {
  64. const rawOrDecrypted = tryDecryptCookieData(cookieData);
  65. if (!rawOrDecrypted) return [];
  66. const raw = rawOrDecrypted.trim();
  67. if (!raw) return [];
  68. // 1) JSON array(最常见)
  69. if (raw.startsWith('[') || raw.startsWith('{')) {
  70. try {
  71. const parsed = JSON.parse(raw);
  72. const arr = Array.isArray(parsed) ? parsed : (parsed?.cookies ? parsed.cookies : []);
  73. if (!Array.isArray(arr)) return [];
  74. return arr
  75. .map((c: any) => {
  76. const name = String(c?.name ?? '').trim();
  77. const value = String(c?.value ?? '').trim();
  78. if (!name) return null;
  79. const domain = c?.domain ? String(c.domain) : undefined;
  80. const pathVal = c?.path ? String(c.path) : '/';
  81. const url = !domain ? 'https://creator.douyin.com' : undefined;
  82. const sameSiteRaw = c?.sameSite;
  83. const sameSite =
  84. sameSiteRaw === 'Lax' || sameSiteRaw === 'None' || sameSiteRaw === 'Strict'
  85. ? sameSiteRaw
  86. : undefined;
  87. return {
  88. name,
  89. value,
  90. domain,
  91. path: pathVal,
  92. url,
  93. expires: typeof c?.expires === 'number' ? c.expires : undefined,
  94. httpOnly: typeof c?.httpOnly === 'boolean' ? c.httpOnly : undefined,
  95. secure: typeof c?.secure === 'boolean' ? c.secure : undefined,
  96. sameSite,
  97. } satisfies PlaywrightCookie;
  98. })
  99. .filter(Boolean) as PlaywrightCookie[];
  100. } catch {
  101. // fallthrough
  102. }
  103. }
  104. // 2) "a=b; c=d" 拼接格式
  105. const pairs = raw.split(';').map((p) => p.trim()).filter(Boolean);
  106. const cookies: PlaywrightCookie[] = [];
  107. for (const p of pairs) {
  108. const idx = p.indexOf('=');
  109. if (idx <= 0) continue;
  110. const name = p.slice(0, idx).trim();
  111. const value = p.slice(idx + 1).trim();
  112. if (!name) continue;
  113. cookies.push({ name, value, url: 'https://creator.douyin.com' });
  114. }
  115. return cookies;
  116. }
  117. async function createBrowserForAccount(proxy: ProxyConfig | null): Promise<{ browser: Browser; shouldClose: boolean }> {
  118. const headless = true;
  119. if (proxy?.enabled) {
  120. const server = `${proxy.type}://${proxy.host}:${proxy.port}`;
  121. const browser = await launchBrowser({
  122. headless,
  123. proxy: {
  124. server,
  125. username: proxy.username,
  126. password: proxy.password,
  127. },
  128. args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu', '--window-size=1920,1080'],
  129. });
  130. return { browser, shouldClose: true };
  131. }
  132. const browser = await BrowserManager.getBrowser({ headless });
  133. return { browser, shouldClose: false };
  134. }
  135. function parseChinaDateFromDateTimeString(dateTime: unknown): Date | null {
  136. if (!dateTime) return null;
  137. const s = String(dateTime).trim();
  138. if (s.length < 10) return null;
  139. const ymd = s.slice(0, 10); // YYYY-MM-DD
  140. const m = ymd.match(/^(\d{4})-(\d{2})-(\d{2})$/);
  141. if (!m) return null;
  142. const yyyy = Number(m[1]);
  143. const mm = Number(m[2]);
  144. const dd = Number(m[3]);
  145. if (!yyyy || !mm || !dd) return null;
  146. const d = new Date(yyyy, mm - 1, dd, 0, 0, 0, 0);
  147. return d;
  148. }
  149. function toNumber(val: unknown, defaultValue = 0): number {
  150. if (typeof val === 'number') return Number.isFinite(val) ? val : defaultValue;
  151. if (typeof val === 'string') {
  152. const n = Number(val);
  153. return Number.isFinite(n) ? n : defaultValue;
  154. }
  155. return defaultValue;
  156. }
  157. function toInt(val: unknown, defaultValue = 0): number {
  158. const n = toNumber(val, NaN);
  159. if (!Number.isFinite(n)) return defaultValue;
  160. return Math.round(n);
  161. }
  162. function normalizePercentString(val: unknown): string | undefined {
  163. const n = toNumber(val, NaN);
  164. if (!Number.isFinite(n)) return undefined;
  165. // 小于等于 0 统一记为 "0"
  166. if (n <= 0) return '0';
  167. // 原始值视为 0-1 之间的小数,这里 *100 后四舍五入保留两位小数并加 "%"
  168. const scaled = n * 100;
  169. const rounded = Math.round(scaled * 100) / 100;
  170. const s = rounded.toFixed(2).replace(/\.00$/, '').replace(/(\.\d)0$/, '$1');
  171. return `${s}%`;
  172. }
  173. /** 平均时长等:保留两位小数,四舍五入 */
  174. function toFixed2String(val: unknown): string | undefined {
  175. const n = toNonNegativeNumber(val);
  176. if (n == null) return undefined;
  177. const rounded = Math.round(n * 100) / 100;
  178. return rounded.toFixed(2);
  179. }
  180. function isDouyinLoginExpiredByApi(body: any): boolean {
  181. const code = Number(body?.status_code);
  182. const msg = String(body?.status_msg || '');
  183. if (code === 20001) return true; // user not match
  184. if (msg.includes('user not match')) return true;
  185. if (msg.includes('登录') && msg.includes('失效')) return true;
  186. return false;
  187. }
  188. class DouyinMetricsTrendClient {
  189. private capturedHeaders: Record<string, string> | null = null;
  190. private buildTrendUrl(itemId: string, metric: string): string {
  191. const base = 'https://creator.douyin.com/janus/douyin/creator/data/item_analysis/metrics_trend';
  192. const params = new URLSearchParams({
  193. aid: '2906',
  194. app_name: 'aweme_creator_platform',
  195. device_platform: 'web',
  196. // referer/user_agent 等埋点参数保留,尽量贴近浏览器请求
  197. referer: 'https://creator.douyin.com/creator-micro/data-center/content',
  198. user_agent:
  199. 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36',
  200. cookie_enabled: 'true',
  201. screen_width: '1920',
  202. screen_height: '1080',
  203. browser_language: 'zh-CN',
  204. browser_platform: 'Win32',
  205. browser_name: 'Mozilla',
  206. browser_version:
  207. '5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36',
  208. browser_online: 'true',
  209. timezone_name: 'Asia/Shanghai',
  210. item_id: itemId,
  211. // 按照浏览器抓包使用 trend_type=1,与抖音后台曲线口径保持一致
  212. trend_type: '1',
  213. time_unit: '1',
  214. metrics_group: '0,1,3',
  215. metrics: metric,
  216. });
  217. return `${base}?${params.toString()}`;
  218. }
  219. private filterCapturedHeaders(h: Record<string, string>): Record<string, string> {
  220. const out: Record<string, string> = {};
  221. const allowList = new Set([
  222. 'accept',
  223. 'accept-language',
  224. 'agw-js-conv',
  225. 'user-agent',
  226. 'x-secsdk-csrf-token',
  227. ]);
  228. for (const [k, v] of Object.entries(h || {})) {
  229. const key = k.toLowerCase();
  230. if (key === 'cookie') continue;
  231. if (key === 'host') continue;
  232. if (key === 'authority') continue;
  233. if (key === 'content-length') continue;
  234. if (key === 'referer') continue; // 每次按作品详情页动态设置
  235. if (allowList.has(key)) out[key] = v;
  236. // 保留所有 x- 前缀的 header(有些风控字段会放在 x- 里)
  237. else if (key.startsWith('x-')) out[key] = v;
  238. }
  239. return out;
  240. }
  241. private async captureHeadersFromRealRequest(page: Page, metricLabel: string, itemId: string, metric: string): Promise<void> {
  242. // 通过点击 UI 触发一次真实请求,抓取其 headers(尤其是 x-secsdk-csrf-token)
  243. const apiPattern = /\/janus\/douyin\/creator\/data\/item_analysis\/metrics_trend/i;
  244. const wait = page.waitForResponse((res) => {
  245. if (res.request().method() !== 'GET') return false;
  246. const u = res.url();
  247. return apiPattern.test(u) && u.includes(`item_id=${itemId}`) && u.includes(`metrics=${metric}`);
  248. }, { timeout: 25_000 });
  249. // 指标卡片(如:点赞量/播放量/评论量...)
  250. // 允许用 "|" 提供多个候选文案(适配 UI 文案差异)
  251. const labels = metricLabel
  252. .split('|')
  253. .map((s) => s.trim())
  254. .filter(Boolean);
  255. let clicked = false;
  256. for (const label of labels.length ? labels : [metricLabel]) {
  257. const loc = page.getByText(label, { exact: false }).first();
  258. const cnt = await loc.count().catch(() => 0);
  259. if (!cnt) continue;
  260. await loc.waitFor({ state: 'visible', timeout: 10_000 }).catch(() => undefined);
  261. await loc.click().catch(() => undefined);
  262. clicked = true;
  263. break;
  264. }
  265. if (!clicked) {
  266. // 不强制失败:有些情况下 UI 不可点击,但直连请求仍可能成功
  267. logger.warn(`[DY WorkStats] Could not click metric label on page for header capture. label=${metricLabel}`);
  268. }
  269. const res = await wait;
  270. const headers = res.request().headers();
  271. this.capturedHeaders = this.filterCapturedHeaders(headers);
  272. logger.info(`[DY WorkStats] Captured request headers for metrics_trend. keys=${Object.keys(this.capturedHeaders).join(',')}`);
  273. }
  274. async fetchTrend(
  275. ctx: BrowserContext,
  276. page: Page,
  277. itemId: string,
  278. metric: string,
  279. metricLabelForFallback: string,
  280. refererUrl: string
  281. ): Promise<MetricsTrendResponse> {
  282. const url = this.buildTrendUrl(itemId, metric);
  283. const headers: Record<string, string> = {
  284. accept: 'application/json, text/plain, */*',
  285. 'accept-language': 'zh-CN,zh;q=0.9',
  286. referer: refererUrl,
  287. 'user-agent':
  288. 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36',
  289. ...(this.capturedHeaders || {}),
  290. };
  291. // 1) 先尝试直接请求(最快)
  292. try {
  293. const res = await ctx.request.get(url, {
  294. headers,
  295. timeout: 25_000,
  296. });
  297. const json = (await res.json().catch(() => null)) as MetricsTrendResponse | null;
  298. if (json && typeof json === 'object') return json;
  299. } catch {
  300. // fallthrough
  301. }
  302. // 2) 如果直连失败,抓一次真实请求 header 后重试
  303. if (!this.capturedHeaders) {
  304. await this.captureHeadersFromRealRequest(page, metricLabelForFallback, itemId, metric).catch(() => undefined);
  305. }
  306. const headers2: Record<string, string> = {
  307. accept: 'application/json, text/plain, */*',
  308. 'accept-language': 'zh-CN,zh;q=0.9',
  309. referer: refererUrl,
  310. 'user-agent':
  311. 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36',
  312. ...(this.capturedHeaders || {}),
  313. };
  314. const res2 = await ctx.request.get(url, {
  315. headers: headers2,
  316. timeout: 25_000,
  317. });
  318. const json2 = (await res2.json().catch(() => null)) as MetricsTrendResponse | null;
  319. if (!json2) throw new Error('metrics_trend 响应不是 JSON');
  320. return json2;
  321. }
  322. }
  323. function toNonNegativeNumber(val: unknown): number | undefined {
  324. const n = toNumber(val, NaN);
  325. if (!Number.isFinite(n)) return undefined;
  326. return n < 0 ? 0 : n;
  327. }
  328. /**
  329. * 比率类:0 不加 "%"(返回 "0"),非 0 时 *100 后四舍五入到两位小数并加 "%"
  330. * 例如: 0.12345 -> "12.35%", 0.0 -> "0"
  331. */
  332. function toRatePercentStringFromValue(val: unknown): string | undefined {
  333. const n = toNumber(val, NaN);
  334. if (!Number.isFinite(n)) return undefined;
  335. if (n === 0) return '0';
  336. const scaled = n * 100;
  337. const rounded = Math.round(scaled * 100) / 100; // 保留两位小数,四舍五入
  338. const s = rounded.toFixed(2).replace(/\.00$/, '').replace(/(\.\d)0$/, '$1');
  339. return `${s}%`;
  340. }
  341. export class DouyinWorkStatisticsImportService {
  342. private accountRepository = AppDataSource.getRepository(PlatformAccount);
  343. private workRepository = AppDataSource.getRepository(Work);
  344. private workDayStatisticsService = new WorkDayStatisticsService();
  345. static async runDailyImport(): Promise<void> {
  346. const svc = new DouyinWorkStatisticsImportService();
  347. await svc.runDailyImportForAllDouyinAccounts();
  348. }
  349. async runDailyImportForAllDouyinAccounts(): Promise<void> {
  350. const accounts = await this.accountRepository.find({
  351. where: { platform: 'douyin' as any },
  352. });
  353. logger.info(`[DY WorkStats] Start import for ${accounts.length} accounts`);
  354. for (const account of accounts) {
  355. try {
  356. await this.importAccountWorksStatistics(account);
  357. } catch (e) {
  358. logger.error(
  359. `[DY WorkStats] Account failed. accountId=${account.id} name=${account.accountName || ''}`,
  360. e
  361. );
  362. // 单账号失败仅记录日志,不中断循环,其他账号照常同步
  363. }
  364. }
  365. logger.info('[DY WorkStats] All accounts done');
  366. }
  367. /**
  368. * 按账号同步作品日统计。检测到 cookie 失效时:先尝试同步/刷新账号一次;刷新仍失效则标记账号 expired。
  369. * @param isRetry 是否为「刷新账号后的重试」,避免无限递归
  370. */
  371. async importAccountWorksStatistics(
  372. account: PlatformAccount,
  373. isRetry = false,
  374. options?: {
  375. workIdFilter?: number[];
  376. onProgress?: (payload: { index: number; total: number; work: Work }) => void;
  377. }
  378. ): Promise<void> {
  379. const cookies = parseCookiesFromAccount(account.cookieData);
  380. if (!cookies.length) {
  381. logger.warn(`[DY WorkStats] accountId=${account.id} cookieData 为空或无法解析,跳过`);
  382. return;
  383. }
  384. let works = await this.workRepository.find({
  385. where: {
  386. accountId: account.id,
  387. platform: 'douyin' as any,
  388. },
  389. });
  390. if (options?.workIdFilter && options.workIdFilter.length > 0) {
  391. const filterSet = new Set(
  392. options.workIdFilter.map((id) => Number(id)).filter((n) => Number.isFinite(n) && n > 0)
  393. );
  394. works = works.filter((w) => filterSet.has(w.id));
  395. }
  396. // 同小红书保持一致:按发布时间从近到远处理;无发布时间的排在最后
  397. if (works.length > 1) {
  398. works.sort((a, b) => {
  399. const ta = a.publishTime ? new Date(a.publishTime as any).getTime() : -Infinity;
  400. const tb = b.publishTime ? new Date(b.publishTime as any).getTime() : -Infinity;
  401. if (tb !== ta) return tb - ta;
  402. return (b.id || 0) - (a.id || 0);
  403. });
  404. }
  405. if (!works.length) {
  406. logger.info(`[DY WorkStats] accountId=${account.id} 没有作品,跳过`);
  407. return;
  408. }
  409. const { browser, shouldClose } = await createBrowserForAccount(account.proxyConfig);
  410. let context: BrowserContext | null = null;
  411. let closedDueToLoginExpired = false;
  412. try {
  413. context = await browser.newContext({
  414. viewport: { width: 1920, height: 1080 },
  415. locale: 'zh-CN',
  416. timezoneId: 'Asia/Shanghai',
  417. userAgent:
  418. 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36',
  419. });
  420. await context.addCookies(cookies as any);
  421. context.setDefaultTimeout(60_000);
  422. if (!context) {
  423. throw new Error('BrowserContext 初始化失败');
  424. }
  425. const ctx = context;
  426. const page = await context.newPage();
  427. const client = new DouyinMetricsTrendClient();
  428. let totalInserted = 0;
  429. let totalUpdated = 0;
  430. const total = works.length;
  431. for (let i = 0; i < works.length; i++) {
  432. const work = works[i];
  433. if (options?.onProgress) {
  434. options.onProgress({ index: i + 1, total, work });
  435. }
  436. const itemId = (work.platformVideoId || '').trim();
  437. if (!itemId) continue;
  438. const detailUrl = `https://creator.douyin.com/creator-micro/work-management/work-detail/${encodeURIComponent(
  439. itemId
  440. )}?enter_from=item_data`;
  441. try {
  442. await page.goto(detailUrl, { waitUntil: 'domcontentloaded' }).catch(() => undefined);
  443. await page.waitForTimeout(1200);
  444. if (page.url().includes('login') || page.url().includes('passport')) {
  445. throw new DouyinLoginExpiredError('work-detail 页面跳转登录,cookie 可能失效');
  446. }
  447. // metrics -> 入库字段映射
  448. const metricsPlan: Array<{
  449. metric: string;
  450. label: string;
  451. apply: (patch: DailyWorkStatPatch, v: unknown) => void;
  452. }> = [
  453. { metric: 'view_count', label: '播放量', apply: (p, v) => (p.playCount = toInt(v, 0)) },
  454. { metric: 'like_count', label: '点赞量', apply: (p, v) => (p.likeCount = toInt(v, 0)) },
  455. { metric: 'comment_count', label: '评论量', apply: (p, v) => (p.commentCount = toInt(v, 0)) },
  456. { metric: 'share_count', label: '分享量', apply: (p, v) => (p.shareCount = toInt(v, 0)) },
  457. { metric: 'favorite_count', label: '收藏量|收藏数', apply: (p, v) => (p.collectCount = toInt(v, 0)) },
  458. { metric: 'subscribe_count', label: '涨粉量|涨粉数', apply: (p, v) => (p.fansIncrease = toInt(v, 0)) },
  459. {
  460. metric: 'completion_rate',
  461. label: '完播率',
  462. apply: (p, v) => {
  463. const s = normalizePercentString(v);
  464. if (s != null) p.completionRate = s;
  465. },
  466. },
  467. {
  468. metric: 'bounce_rate_2s',
  469. label: '2s退出率|2s跳出率|2s跳出',
  470. apply: (p, v) => {
  471. const s = normalizePercentString(v);
  472. if (s != null) p.twoSecondExitRate = s;
  473. },
  474. },
  475. ];
  476. const dayMap = new Map<number, DailyWorkStatPatch>();
  477. for (const m of metricsPlan) {
  478. const body = await client.fetchTrend(ctx, page, itemId, m.metric, m.label, detailUrl);
  479. if (!body || typeof body !== 'object') continue;
  480. // 调试:把 metrics_trend 原始返回打印成 JSON,方便和抖音后台对比
  481. if (work.id === 39 && m.metric === 'completion_rate') {
  482. try {
  483. logger.info(
  484. `[DY WorkStats][debug metrics_trend raw] workId=${work.id} itemId=${itemId} metric=${m.metric} body=${JSON.stringify(
  485. body
  486. )}`
  487. );
  488. } catch {
  489. // ignore JSON stringify error
  490. }
  491. }
  492. if (isDouyinLoginExpiredByApi(body)) {
  493. throw new DouyinLoginExpiredError(body.status_msg || 'metrics_trend: user not match');
  494. }
  495. if (Number(body.status_code) !== 0) {
  496. logger.warn(
  497. `[DY WorkStats] metrics_trend 非成功返回. accountId=${account.id} workId=${work.id} itemId=${itemId} metric=${m.metric} code=${body.status_code} msg=${body.status_msg || ''}`
  498. );
  499. continue;
  500. }
  501. const trendMap = body.trend_map || {};
  502. const metricMap = (trendMap as any)[m.metric] as Record<string, TrendPoint[]> | undefined;
  503. if (!metricMap) continue;
  504. // 优先取 group "0"(一般为“总计/全部”),否则兜底合并全部 group
  505. const points = Array.isArray(metricMap['0'])
  506. ? metricMap['0']
  507. : Object.values(metricMap).flatMap((arr) => (Array.isArray(arr) ? arr : []));
  508. // 调试:打印 workId=39 的 completion_rate 全量 points,确认与抖音后台返回是否一致
  509. if (work.id === 39 && m.metric === 'completion_rate') {
  510. try {
  511. logger.info(
  512. `[DY WorkStats][debug completion_rate points] workId=${work.id} itemId=${itemId} points=${JSON.stringify(
  513. points
  514. )}`
  515. );
  516. } catch {
  517. // ignore JSON stringify error
  518. }
  519. }
  520. for (const pt of points) {
  521. // 调试:打印指定作品的完播率原始值
  522. if (work.id === 39 && m.metric === 'completion_rate') {
  523. logger.info(
  524. `[DY WorkStats][debug completion_rate] workId=${work.id} itemId=${itemId} date=${pt?.date_time} raw_value=${pt?.value}`
  525. );
  526. }
  527. const d = parseChinaDateFromDateTimeString(pt?.date_time);
  528. if (!d) continue;
  529. const key = d.getTime();
  530. let entry = dayMap.get(key);
  531. if (!entry) {
  532. entry = { workId: work.id, recordDate: d };
  533. dayMap.set(key, entry);
  534. }
  535. m.apply(entry, pt?.value);
  536. }
  537. }
  538. const patches = Array.from(dayMap.values()).sort(
  539. (a, b) => a.recordDate.getTime() - b.recordDate.getTime()
  540. );
  541. if (!patches.length) continue;
  542. // 同时补充作品级昨日快照(works.yesterday_*),使用 item/mget metrics
  543. try {
  544. await this.applyWorkSnapshotFromItemMget(ctx, itemId, detailUrl, work.id);
  545. } catch (e) {
  546. logger.warn(
  547. `[DY WorkStats] Failed to update works snapshot from item/mget. accountId=${account.id} workId=${work.id} itemId=${itemId}`,
  548. e
  549. );
  550. }
  551. const result = await this.workDayStatisticsService.saveStatisticsForDateBatch(
  552. patches.map((p) => ({
  553. workId: p.workId,
  554. recordDate: p.recordDate,
  555. playCount: p.playCount,
  556. likeCount: p.likeCount,
  557. commentCount: p.commentCount,
  558. shareCount: p.shareCount,
  559. collectCount: p.collectCount,
  560. fansIncrease: p.fansIncrease,
  561. completionRate: p.completionRate,
  562. twoSecondExitRate: p.twoSecondExitRate,
  563. }))
  564. );
  565. totalInserted += result.inserted;
  566. totalUpdated += result.updated;
  567. } catch (e) {
  568. if (e instanceof DouyinLoginExpiredError) {
  569. closedDueToLoginExpired = true;
  570. if (context) {
  571. await context.close().catch(() => undefined);
  572. context = null;
  573. }
  574. if (shouldClose) {
  575. await browser.close().catch(() => undefined);
  576. }
  577. // cookie 过期处理:先刷新一次账号,再决定是否标记 expired
  578. if (!isRetry) {
  579. logger.info(`[DY WorkStats] accountId=${account.id} 登录失效,尝试同步账号后重试...`);
  580. try {
  581. const accountService = new AccountService();
  582. const refreshResult = await accountService.refreshAccount(account.userId, account.id);
  583. if (refreshResult.needReLogin) {
  584. await this.markAccountExpired(account, 'cookie 过期,需要重新登录');
  585. return;
  586. }
  587. const refreshed = await this.accountRepository.findOne({ where: { id: account.id } });
  588. if (refreshed) {
  589. logger.info(`[DY WorkStats] accountId=${account.id} 同步账号成功,重新拉取作品数据`);
  590. return this.importAccountWorksStatistics(refreshed, true);
  591. }
  592. } catch (refreshErr) {
  593. logger.error(`[DY WorkStats] accountId=${account.id} 同步账号失败`, refreshErr);
  594. await this.markAccountExpired(account, '同步账号失败,已标记过期');
  595. return;
  596. }
  597. } else {
  598. await this.markAccountExpired(account, '同步后仍失效,已标记过期');
  599. return;
  600. }
  601. }
  602. logger.error(
  603. `[DY WorkStats] Failed to import work stats. accountId=${account.id} workId=${work.id} itemId=${itemId}`,
  604. e
  605. );
  606. }
  607. }
  608. logger.info(
  609. `[DY WorkStats] accountId=${account.id} completed. inserted=${totalInserted}, updated=${totalUpdated}`
  610. );
  611. } finally {
  612. if (!closedDueToLoginExpired) {
  613. if (context) {
  614. await context.close().catch(() => undefined);
  615. }
  616. if (shouldClose) {
  617. await browser.close().catch(() => undefined);
  618. }
  619. }
  620. }
  621. }
  622. /**
  623. * 使用 item/mget 接口为 works 表补充昨日快照(yesterday_* 字段)
  624. */
  625. private async applyWorkSnapshotFromItemMget(
  626. ctx: BrowserContext,
  627. itemId: string,
  628. refererUrl: string,
  629. workId: number
  630. ): Promise<void> {
  631. const url = `https://creator.douyin.com/web/api/creator/item/mget?ids=${encodeURIComponent(
  632. itemId
  633. )}&fields=metrics%2Creview%2Cplay_info`;
  634. const headers: Record<string, string> = {
  635. accept: '*/*',
  636. 'accept-language': 'zh-CN,zh;q=0.9',
  637. referer: refererUrl,
  638. 'user-agent':
  639. 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/144.0.0.0 Safari/537.36',
  640. };
  641. const res = await ctx.request.get(url, { headers, timeout: 25_000 });
  642. const body = (await res.json().catch(() => null)) as any;
  643. if (!body || typeof body !== 'object' || Number(body.status_code) !== 0) {
  644. return;
  645. }
  646. const items = Array.isArray(body.items) ? body.items : [];
  647. const first = items[0];
  648. if (!first || typeof first !== 'object') return;
  649. const metrics = first.metrics || {};
  650. const patch: Partial<Work> = {};
  651. const viewCount = toNonNegativeNumber(metrics.view_count);
  652. if (viewCount != null) (patch as any).yesterdayPlayCount = Math.trunc(viewCount);
  653. const likeCount = toNonNegativeNumber(metrics.like_count);
  654. if (likeCount != null) (patch as any).yesterdayLikeCount = Math.trunc(likeCount);
  655. const commentCount = toNonNegativeNumber(metrics.comment_count);
  656. if (commentCount != null) (patch as any).yesterdayCommentCount = Math.trunc(commentCount);
  657. const shareCount = toNonNegativeNumber(metrics.share_count);
  658. if (shareCount != null) (patch as any).yesterdayShareCount = Math.trunc(shareCount);
  659. const collectCount = toNonNegativeNumber(metrics.favorite_count);
  660. if (collectCount != null) (patch as any).yesterdayCollectCount = Math.trunc(collectCount);
  661. const fansIncrease = toNonNegativeNumber(metrics.subscribe_count);
  662. if (fansIncrease != null) (patch as any).yesterdayFansIncrease = Math.trunc(fansIncrease);
  663. // 平均观看时长(秒):保留两位小数
  664. const avgWatchStr = toFixed2String(metrics.avg_view_second);
  665. if (avgWatchStr != null) (patch as any).yesterdayAvgWatchDuration = avgWatchStr;
  666. const completionRateStr = toRatePercentStringFromValue(metrics.completion_rate);
  667. if (completionRateStr != null) (patch as any).yesterdayCompletionRate = completionRateStr;
  668. const twoSecondExitRateStr = toRatePercentStringFromValue(metrics.bounce_rate_2s);
  669. if (twoSecondExitRateStr != null) (patch as any).yesterdayTwoSecondExitRate = twoSecondExitRateStr;
  670. const completion5sStr = toRatePercentStringFromValue(metrics.completion_rate_5s);
  671. if (completion5sStr != null) (patch as any).yesterdayCompletionRate5s = completion5sStr;
  672. if (Object.keys(patch).length === 0) return;
  673. await this.workRepository.update(workId, patch as any);
  674. }
  675. private async markAccountExpired(account: PlatformAccount, reason: string): Promise<void> {
  676. await this.accountRepository.update(account.id, { status: 'expired' as any });
  677. wsManager.sendToUser(account.userId, WS_EVENTS.ACCOUNT_UPDATED, {
  678. account: { id: account.id, status: 'expired', platform: 'douyin' as PlatformType },
  679. });
  680. wsManager.sendToUser(account.userId, WS_EVENTS.SYSTEM_MESSAGE, {
  681. level: 'warning',
  682. message: `抖音账号「${account.accountName || account.accountId || account.id}」登录已失效:${reason}`,
  683. platform: 'douyin',
  684. accountId: account.id,
  685. });
  686. }
  687. }