import type {PlatformFieldConfig} from '@/types'; /** * 等待重试机制 */ async function waitForElement(rootDom: ParentNode, selector: string) { let retryCount = 5; for (let i = 0; i < retryCount; i++) { const element = rootDom.querySelector(selector); if (element) { return element; } if (i < retryCount) { await sleep(500); } } return null; } // 睡眠工具,给点击、翻页、异步渲染留出等待时间。 const sleep = (ms?: number) => new Promise((resolve) => window.setTimeout(resolve, ms ?? 1500)); /** * 从元素中提取实际值,默认取文本,也支持 attr、图片 src、链接 href。 */ function extractValue(el: Element | null, config: PlatformFieldConfig): string | null { if (el == null) { return "未找到" } if (config.attr) { return (el.getAttribute(config.attr) || "").trim(); } const tagName = el.tagName.toUpperCase(); if (tagName === 'IMG') { return el.getAttribute('src'); } if (tagName === 'A') { const href = el.getAttribute('href'); return href && !href.startsWith('http') ? window.location.origin + href : href; } return (el.textContent || '').replace(/\n/g, '').trim(); } /** * 自动点击 * 根据字段 condition 配置在指定 DOM 范围内自动点击目标元素。 */ async function autoClick(config: PlatformFieldConfig, rootDom: Element): Promise { if (!config.condition) { return; } for (const condition of config.condition.list) { let targets: HTMLElement[] = Array.from(rootDom.querySelectorAll(condition)) for (const target of targets) { target.click(); await sleep(config?.condition.time); } } } /** * 递归处理字段配置,支持普通字段、嵌套 row、列表和表格。 */ export async function processFields(columns: PlatformFieldConfig[], rootDom: Element) { const result = {} as any; for (const item of columns) { await autoClick(item, rootDom); const element = await waitForElement(rootDom, item.className) if (!element) { result[item.label] = '没找到该元素'; continue; } if (!item.type) { if (item.keys && item.keys.length > 0) { await autoClick(item, element); result[item.label] = await processFields(item.keys, element); } else { await autoClick(item, element); result[item.label] = extractValue(element, item); } continue; } if (item.type === 1) { result[item.label] = await processList(item, rootDom); continue; } if (item.type === 2) { result[item.label] = await processTable(item, element); } } return result; } /** * 提取列表的数据 * @param config 配置 * @param rootDom 父节点 */ async function processList(config: PlatformFieldConfig, rootDom: ParentNode) { const allList = []; let pageCount = 0; while (true) { pageCount += 1; const elements = Array.from(rootDom.querySelectorAll(config.className)); for (const element of elements) { const itemData = await processFields(config.keys ?? [], element); allList.push(itemData); } if (!config.pagination) { console.log('未配置分页信息,抓取单页后结束。'); break; } if (config.pagination.maxPage && pageCount >= config.pagination.maxPage) { console.log('已达到配置的最大页数,停止。'); break; } const nextBtn = document.querySelector(config.pagination.nextBtn); if (!nextBtn) { console.log('未找到下一页按钮,抓取结束。'); break; } const isDisabled = config.pagination.disabledClass ? nextBtn.classList.contains(config.pagination.disabledClass) : nextBtn.hasAttribute('disabled'); if (isDisabled) { console.log('下一页按钮已禁用,抓取结束。'); break; } nextBtn.click(); await sleep(config.pagination.delay); } return allList; } /** * 按表格配置抓取表格行数据,并按分页配置继续翻页。 */ async function processTable(config: PlatformFieldConfig, rootDom: ParentNode) { const allTableData: any[] = []; let pageCount = 0; while (true) { pageCount += 1; const partsNodes: any = {}; config.tableParts!.forEach(part => { partsNodes[part.name as any] = rootDom.querySelectorAll(`${part.select} tr`); }); // //以第一个part的行数为准,进行横向扫描 const rowCount = partsNodes[config.tableParts![0].name!]?.length || 0 for (let i = 0; i < rowCount; i++) { const rowData: any = {}; //遍历keys,根据part映射,取对应的里面找 for (const keyItem of config.keys!) { const targetRowNode = partsNodes[keyItem.part!][i]; if (targetRowNode) { //提取值 if (keyItem.keys) { rowData[keyItem.label] = await processFields(keyItem.keys, targetRowNode) } else { rowData[keyItem.label] = extractValue(targetRowNode.querySelector(keyItem.className), keyItem); } } } allTableData.push(rowData); } if (!config.pagination) { console.log("未配置分页信息,抓取单页后结束。"); break; } if (config.pagination.maxPage && pageCount >= config.pagination.maxPage) { console.log("已达到配置的最大页数,停止。"); break; } const nextBtn = document.querySelector(config.pagination.nextBtn); if (!nextBtn) { console.log('未找到下一页按钮,抓取结束。'); break; } const isDisabled = config.pagination.disabledClass ? nextBtn.classList.contains(config.pagination.disabledClass) : nextBtn.hasAttribute('disabled'); if (isDisabled) { console.log('下一页按钮已禁用,抓取结束。'); break; } nextBtn.click(); await sleep(config.pagination.delay); } return allTableData; }