1
This commit is contained in:
@@ -1,5 +1,7 @@
|
|||||||
import type {PlatformFieldConfig} from '@/types';
|
import type {PlatformFieldConfig} from '@/types';
|
||||||
|
|
||||||
|
export type DomScrapeResult = Record<string, unknown>;
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 等待重试机制
|
* 等待重试机制
|
||||||
*/
|
*/
|
||||||
@@ -225,4 +227,4 @@ async function processTable(config: PlatformFieldConfig, rootDom: ParentNode) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return allTableData;
|
return allTableData;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,13 @@
|
|||||||
import {handleBackgroundCommand, handleWindowRemoved} from './service';
|
import { handleBackgroundCommand, handleInstalled, handleStartup, handleWindowRemoved } from './service';
|
||||||
import type {BackgroundCommand} from './types';
|
import type { BackgroundCommand } from './types';
|
||||||
|
|
||||||
|
chrome.runtime.onInstalled.addListener(() => {
|
||||||
|
void handleInstalled();
|
||||||
|
});
|
||||||
|
|
||||||
|
chrome.runtime.onStartup.addListener(() => {
|
||||||
|
void handleStartup();
|
||||||
|
});
|
||||||
|
|
||||||
chrome.runtime.onMessage.addListener((message: BackgroundCommand, _sender, sendResponse) => {
|
chrome.runtime.onMessage.addListener((message: BackgroundCommand, _sender, sendResponse) => {
|
||||||
void handleBackgroundMessage(message, sendResponse);
|
void handleBackgroundMessage(message, sendResponse);
|
||||||
@@ -12,20 +18,19 @@ chrome.windows.onRemoved.addListener((windowId) => {
|
|||||||
void handleWindowRemoved(windowId);
|
void handleWindowRemoved(windowId);
|
||||||
});
|
});
|
||||||
|
|
||||||
chrome.runtime.onMessageExternal.addListener((message, sender, sendResponse) => {
|
chrome.runtime.onMessageExternal.addListener((message, _sender, sendResponse) => {
|
||||||
if (message.type === "STORE_AI_PING") {
|
if (message.type === 'STORE_AI_PING') {
|
||||||
// 返回版本号等信息
|
|
||||||
sendResponse({
|
sendResponse({
|
||||||
success: true,
|
success: true,
|
||||||
version: chrome.runtime.getManifest().version
|
version: chrome.runtime.getManifest().version,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
// 注意:外部消息处理必须返回 true 才能支持异步 sendResponse
|
|
||||||
return true;
|
return true;
|
||||||
});
|
});
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 统一包装后台消息处理,确保异步错误能回给调用方。
|
* Wrap background command handling so async errors can still be returned to callers.
|
||||||
*/
|
*/
|
||||||
async function handleBackgroundMessage(
|
async function handleBackgroundMessage(
|
||||||
message: BackgroundCommand,
|
message: BackgroundCommand,
|
||||||
@@ -36,6 +41,6 @@ async function handleBackgroundMessage(
|
|||||||
sendResponse(result);
|
sendResponse(result);
|
||||||
} catch (error: unknown) {
|
} catch (error: unknown) {
|
||||||
const messageText = error instanceof Error ? error.message : 'Unknown error';
|
const messageText = error instanceof Error ? error.message : 'Unknown error';
|
||||||
sendResponse({ok: false, error: messageText});
|
sendResponse({ ok: false, data: null, error: messageText });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -2,7 +2,7 @@ import { getPlatformById } from '@/config/platforms';
|
|||||||
import type { CrawlPauseInfo, CrawlProgressStep, CrawlTaskState, PlatformConfig, PlatformStepConfig } from '@/types';
|
import type { CrawlPauseInfo, CrawlProgressStep, CrawlTaskState, PlatformConfig, PlatformStepConfig } from '@/types';
|
||||||
import type { DomScrapeResult } from '../domScraper';
|
import type { DomScrapeResult } from '../domScraper';
|
||||||
import type { CrawlStateResponse } from '../types';
|
import type { CrawlStateResponse } from '../types';
|
||||||
import { getCrawlTaskState, setCrawlTaskState, updateCrawlTaskState } from './taskState';
|
import { clearCrawlTaskState, getCrawlTaskState, setCrawlTaskState, updateCrawlTaskState } from './taskState';
|
||||||
|
|
||||||
interface PageRunnerResponse {
|
interface PageRunnerResponse {
|
||||||
ok: boolean;
|
ok: boolean;
|
||||||
@@ -11,6 +11,8 @@ interface PageRunnerResponse {
|
|||||||
error?: string;
|
error?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const activeCrawlControllers = new Map<string, AbortController>();
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 创建新的爬取任务,打开目标平台窗口,并把初始时间轴状态写入 storage。
|
* 创建新的爬取任务,打开目标平台窗口,并把初始时间轴状态写入 storage。
|
||||||
*/
|
*/
|
||||||
@@ -47,9 +49,13 @@ export async function startCrawl(platformId: string): Promise<CrawlStateResponse
|
|||||||
try {
|
try {
|
||||||
const windowInfo = await createCrawlWindow(firstStep.url);
|
const windowInfo = await createCrawlWindow(firstStep.url);
|
||||||
const stateWithWindow = { ...nextState, windowId: windowInfo.id };
|
const stateWithWindow = { ...nextState, windowId: windowInfo.id };
|
||||||
|
const controller = new AbortController();
|
||||||
|
|
||||||
await setCrawlTaskState(stateWithWindow);
|
await setCrawlTaskState(stateWithWindow);
|
||||||
void runCrawlSteps(platform, stateWithWindow);
|
activeCrawlControllers.set(stateWithWindow.id, controller);
|
||||||
|
void runCrawlSteps(platform, stateWithWindow, controller.signal).finally(() => {
|
||||||
|
activeCrawlControllers.delete(stateWithWindow.id);
|
||||||
|
});
|
||||||
|
|
||||||
return { ok: true, data: stateWithWindow };
|
return { ok: true, data: stateWithWindow };
|
||||||
} catch (error: unknown) {
|
} catch (error: unknown) {
|
||||||
@@ -76,21 +82,15 @@ export async function cancelCrawl(): Promise<CrawlStateResponse> {
|
|||||||
return { ok: true, data: null };
|
return { ok: true, data: null };
|
||||||
}
|
}
|
||||||
|
|
||||||
const canceledState: CrawlTaskState = {
|
abortActiveCrawl(state.id);
|
||||||
...state,
|
|
||||||
status: 'canceled',
|
|
||||||
steps: state.steps.map((step, index) =>
|
|
||||||
index === state.currentStepIndex ? { ...step, status: 'failed', message: '用户已取消' } : step,
|
|
||||||
),
|
|
||||||
};
|
|
||||||
|
|
||||||
await setCrawlTaskState(canceledState);
|
await clearCrawlTaskState();
|
||||||
|
|
||||||
if (state.windowId) {
|
if (state.windowId) {
|
||||||
await chrome.windows.remove(state.windowId).catch(() => undefined);
|
await chrome.windows.remove(state.windowId).catch(() => undefined);
|
||||||
}
|
}
|
||||||
|
|
||||||
return { ok: true, data: canceledState };
|
return { ok: true, data: null };
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -122,10 +122,12 @@ export async function resumeCrawl(): Promise<CrawlStateResponse> {
|
|||||||
export async function cancelCrawlWhenWindowRemoved(windowId: number): Promise<void> {
|
export async function cancelCrawlWhenWindowRemoved(windowId: number): Promise<void> {
|
||||||
const state = await getCrawlTaskState();
|
const state = await getCrawlTaskState();
|
||||||
|
|
||||||
if (state?.windowId !== windowId || state.status !== 'running') {
|
if (state?.windowId !== windowId || !['running', 'paused'].includes(state.status)) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
abortActiveCrawl(state.id);
|
||||||
|
|
||||||
await setCrawlTaskState({
|
await setCrawlTaskState({
|
||||||
...state,
|
...state,
|
||||||
status: 'canceled',
|
status: 'canceled',
|
||||||
@@ -135,10 +137,38 @@ export async function cancelCrawlWhenWindowRemoved(windowId: number): Promise<vo
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export async function cancelStaleCrawlWhenWindowMissing(): Promise<void> {
|
||||||
|
const state = await getCrawlTaskState();
|
||||||
|
|
||||||
|
if (!state || !['running', 'paused'].includes(state.status)) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const isWindowAlive = state.windowId ? await hasWindow(state.windowId) : false;
|
||||||
|
|
||||||
|
if (isWindowAlive) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
abortActiveCrawl(state.id);
|
||||||
|
|
||||||
|
await setCrawlTaskState({
|
||||||
|
...state,
|
||||||
|
status: 'canceled',
|
||||||
|
steps: state.steps.map((step, index) =>
|
||||||
|
index === state.currentStepIndex ? { ...step, status: 'failed', message: '爬取窗口已关闭,任务已取消' } : step,
|
||||||
|
),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function abortActiveCrawl(taskId: string): void {
|
||||||
|
activeCrawlControllers.get(taskId)?.abort();
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 按平台 steps 顺序执行页面跳转、DOM 等待、字段抓取和进度更新。
|
* 按平台 steps 顺序执行页面跳转、DOM 等待、字段抓取和进度更新。
|
||||||
*/
|
*/
|
||||||
async function runCrawlSteps(platform: PlatformConfig, initialState: CrawlTaskState): Promise<void> {
|
async function runCrawlSteps(platform: PlatformConfig, initialState: CrawlTaskState, signal: AbortSignal): Promise<void> {
|
||||||
if (!initialState.windowId) {
|
if (!initialState.windowId) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
@@ -151,12 +181,12 @@ async function runCrawlSteps(platform: PlatformConfig, initialState: CrawlTaskSt
|
|||||||
while (shouldRetryStep) {
|
while (shouldRetryStep) {
|
||||||
const currentState = await getCrawlTaskState();
|
const currentState = await getCrawlTaskState();
|
||||||
|
|
||||||
if (currentState?.id !== initialState.id || currentState.status === 'canceled') {
|
if (signal.aborted || currentState?.id !== initialState.id || currentState.status === 'canceled') {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (currentState.status === 'paused') {
|
if (currentState.status === 'paused') {
|
||||||
const resumed = await waitUntilResumed(initialState.id);
|
const resumed = await waitUntilResumed(initialState.id, signal);
|
||||||
|
|
||||||
if (!resumed) {
|
if (!resumed) {
|
||||||
return;
|
return;
|
||||||
@@ -177,13 +207,21 @@ async function runCrawlSteps(platform: PlatformConfig, initialState: CrawlTaskSt
|
|||||||
|
|
||||||
const tabId = await getWindowActiveTabId(initialState.windowId);
|
const tabId = await getWindowActiveTabId(initialState.windowId);
|
||||||
await chrome.tabs.update(tabId, { url: step.url, active: true });
|
await chrome.tabs.update(tabId, { url: step.url, active: true });
|
||||||
await waitForTabLoaded(tabId);
|
const tabLoaded = await waitForTabLoaded(tabId, signal);
|
||||||
|
|
||||||
const response = await scrapeStepInContent(tabId, step);
|
if (!tabLoaded || signal.aborted) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const response = await scrapeStepInContent(tabId, step, signal);
|
||||||
|
|
||||||
|
if (signal.aborted) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
if (response.interrupt) {
|
if (response.interrupt) {
|
||||||
await pauseForInterrupt(initialState.id, stepIndex, response.interrupt);
|
await pauseForInterrupt(initialState.id, stepIndex, response.interrupt);
|
||||||
const resumed = await waitUntilResumed(initialState.id);
|
const resumed = await waitUntilResumed(initialState.id, signal);
|
||||||
|
|
||||||
if (!resumed) {
|
if (!resumed) {
|
||||||
return;
|
return;
|
||||||
@@ -258,23 +296,33 @@ async function getWindowActiveTabId(windowId: number): Promise<number> {
|
|||||||
/**
|
/**
|
||||||
* 让 content script 直接在目标页面执行检查和抓取。
|
* 让 content script 直接在目标页面执行检查和抓取。
|
||||||
*/
|
*/
|
||||||
async function scrapeStepInContent(tabId: number, step: PlatformStepConfig): Promise<PageRunnerResponse> {
|
async function scrapeStepInContent(
|
||||||
|
tabId: number,
|
||||||
|
step: PlatformStepConfig,
|
||||||
|
signal: AbortSignal,
|
||||||
|
): Promise<PageRunnerResponse> {
|
||||||
const startedAt = Date.now();
|
const startedAt = Date.now();
|
||||||
|
|
||||||
while (Date.now() - startedAt < 20000) {
|
while (Date.now() - startedAt < 20000) {
|
||||||
|
if (signal.aborted) {
|
||||||
|
return { ok: false, error: 'canceled' };
|
||||||
|
}
|
||||||
|
|
||||||
const response = await sendPageRunnerMessage(tabId, {
|
const response = await sendPageRunnerMessage(tabId, {
|
||||||
action: 'SCRAPE_STEP',
|
action: 'SCRAPE_STEP',
|
||||||
payload: {
|
payload: {
|
||||||
fields: step.fields,
|
fields: step.fields,
|
||||||
checkSelector: step.checkSelector,
|
checkSelector: step.checkSelector,
|
||||||
},
|
},
|
||||||
});
|
}, signal);
|
||||||
|
|
||||||
if (response.ok || response.interrupt || !isPageRunnerNotReadyError(response.error)) {
|
if (response.ok || response.interrupt || !isPageRunnerNotReadyError(response.error)) {
|
||||||
return response;
|
return response;
|
||||||
}
|
}
|
||||||
|
|
||||||
await sleep(500);
|
if (!(await sleep(500, signal))) {
|
||||||
|
return { ok: false, error: 'canceled' };
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return { ok: false, error: '页面脚本未响应,请刷新扩展后重试' };
|
return { ok: false, error: '页面脚本未响应,请刷新扩展后重试' };
|
||||||
@@ -283,7 +331,15 @@ async function scrapeStepInContent(tabId: number, step: PlatformStepConfig): Pro
|
|||||||
/**
|
/**
|
||||||
* 给目标页的 content script 发送页面执行消息。
|
* 给目标页的 content script 发送页面执行消息。
|
||||||
*/
|
*/
|
||||||
async function sendPageRunnerMessage(tabId: number, message: unknown): Promise<PageRunnerResponse> {
|
async function sendPageRunnerMessage(tabId: number, message: unknown, signal: AbortSignal): Promise<PageRunnerResponse> {
|
||||||
|
if (signal.aborted) {
|
||||||
|
return { ok: false, error: 'canceled' };
|
||||||
|
}
|
||||||
|
|
||||||
|
return raceWithAbort(sendPageRunnerMessageOnce(tabId, message), signal);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function sendPageRunnerMessageOnce(tabId: number, message: unknown): Promise<PageRunnerResponse> {
|
||||||
try {
|
try {
|
||||||
const response = await chrome.tabs.sendMessage(tabId, message);
|
const response = await chrome.tabs.sendMessage(tabId, message);
|
||||||
|
|
||||||
@@ -326,8 +382,12 @@ async function pauseForInterrupt(taskId: string, stepIndex: number, interrupt: C
|
|||||||
/**
|
/**
|
||||||
* 暂停后等待用户点继续或取消。
|
* 暂停后等待用户点继续或取消。
|
||||||
*/
|
*/
|
||||||
async function waitUntilResumed(taskId: string): Promise<boolean> {
|
async function waitUntilResumed(taskId: string, signal: AbortSignal): Promise<boolean> {
|
||||||
while (true) {
|
while (true) {
|
||||||
|
if (signal.aborted) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
const state = await getCrawlTaskState();
|
const state = await getCrawlTaskState();
|
||||||
|
|
||||||
if (!state || state.id !== taskId || state.status === 'canceled' || state.status === 'failed') {
|
if (!state || state.id !== taskId || state.status === 'canceled' || state.status === 'failed') {
|
||||||
@@ -338,7 +398,9 @@ async function waitUntilResumed(taskId: string): Promise<boolean> {
|
|||||||
return true;
|
return true;
|
||||||
}
|
}
|
||||||
|
|
||||||
await sleep(1000);
|
if (!(await sleep(1000, signal))) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -377,30 +439,123 @@ function createCrawlWindow(url: string): Promise<chrome.windows.Window> {
|
|||||||
/**
|
/**
|
||||||
* 等待 tab 完成页面加载。
|
* 等待 tab 完成页面加载。
|
||||||
*/
|
*/
|
||||||
function waitForTabLoaded(tabId: number): Promise<void> {
|
function waitForTabLoaded(tabId: number, signal: AbortSignal): Promise<boolean> {
|
||||||
return new Promise((resolve) => {
|
return new Promise((resolve) => {
|
||||||
|
if (signal.aborted) {
|
||||||
|
resolve(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
const timeout = globalThis.setTimeout(() => {
|
const timeout = globalThis.setTimeout(() => {
|
||||||
chrome.tabs.onUpdated.removeListener(handleUpdated);
|
cleanup();
|
||||||
resolve();
|
resolve(true);
|
||||||
}, 15000);
|
}, 15000);
|
||||||
|
|
||||||
|
function cleanup() {
|
||||||
|
globalThis.clearTimeout(timeout);
|
||||||
|
chrome.tabs.onUpdated.removeListener(handleUpdated);
|
||||||
|
signal.removeEventListener('abort', handleAbort);
|
||||||
|
}
|
||||||
|
|
||||||
|
function handleAbort() {
|
||||||
|
cleanup();
|
||||||
|
resolve(false);
|
||||||
|
}
|
||||||
|
|
||||||
function handleUpdated(updatedTabId: number, changeInfo: { status?: string }) {
|
function handleUpdated(updatedTabId: number, changeInfo: { status?: string }) {
|
||||||
if (updatedTabId === tabId && changeInfo.status === 'complete') {
|
if (updatedTabId === tabId && changeInfo.status === 'complete') {
|
||||||
globalThis.clearTimeout(timeout);
|
cleanup();
|
||||||
chrome.tabs.onUpdated.removeListener(handleUpdated);
|
resolve(true);
|
||||||
resolve();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
chrome.tabs.onUpdated.addListener(handleUpdated);
|
chrome.tabs.onUpdated.addListener(handleUpdated);
|
||||||
|
signal.addEventListener('abort', handleAbort, { once: true });
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 简单等待工具。
|
* 简单等待工具。
|
||||||
*/
|
*/
|
||||||
function sleep(ms: number): Promise<void> {
|
async function hasWindow(windowId: number): Promise<boolean> {
|
||||||
return new Promise((resolve) => {
|
try {
|
||||||
globalThis.setTimeout(resolve, ms);
|
await chrome.windows.get(windowId);
|
||||||
|
return true;
|
||||||
|
} catch {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function raceWithAbort<T>(promise: Promise<T>, signal: AbortSignal): Promise<T> {
|
||||||
|
return new Promise((resolve, reject) => {
|
||||||
|
if (signal.aborted) {
|
||||||
|
resolve({ ok: false, error: 'canceled' } as T);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
let isSettled = false;
|
||||||
|
|
||||||
|
function cleanup() {
|
||||||
|
signal.removeEventListener('abort', handleAbort);
|
||||||
|
}
|
||||||
|
|
||||||
|
function handleAbort() {
|
||||||
|
if (isSettled) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
isSettled = true;
|
||||||
|
cleanup();
|
||||||
|
resolve({ ok: false, error: 'canceled' } as T);
|
||||||
|
}
|
||||||
|
|
||||||
|
signal.addEventListener('abort', handleAbort, { once: true });
|
||||||
|
|
||||||
|
promise.then(
|
||||||
|
(value) => {
|
||||||
|
if (isSettled) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
isSettled = true;
|
||||||
|
cleanup();
|
||||||
|
resolve(value);
|
||||||
|
},
|
||||||
|
(error) => {
|
||||||
|
if (isSettled) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
isSettled = true;
|
||||||
|
cleanup();
|
||||||
|
reject(error);
|
||||||
|
},
|
||||||
|
);
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function sleep(ms: number, signal?: AbortSignal): Promise<boolean> {
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
if (signal?.aborted) {
|
||||||
|
resolve(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const timeout = globalThis.setTimeout(() => {
|
||||||
|
cleanup();
|
||||||
|
resolve(true);
|
||||||
|
}, ms);
|
||||||
|
|
||||||
|
function cleanup() {
|
||||||
|
globalThis.clearTimeout(timeout);
|
||||||
|
signal?.removeEventListener('abort', handleAbort);
|
||||||
|
}
|
||||||
|
|
||||||
|
function handleAbort() {
|
||||||
|
cleanup();
|
||||||
|
resolve(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
signal?.addEventListener('abort', handleAbort, { once: true });
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
import type { BackgroundCommand, BackgroundResponse, CrawlStateResponse } from '../types';
|
import type { BackgroundCommand, BackgroundResponse, CrawlStateResponse } from '../types';
|
||||||
import { cancelCrawl, cancelCrawlWhenWindowRemoved, resumeCrawl, startCrawl } from './crawlTask';
|
import { cancelCrawl, cancelCrawlWhenWindowRemoved, cancelStaleCrawlWhenWindowMissing, resumeCrawl, startCrawl } from './crawlTask';
|
||||||
import { getCrawlTaskState } from './taskState';
|
import { getCrawlTaskState } from './taskState';
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -14,6 +14,7 @@ export async function handleInstalled(): Promise<void> {
|
|||||||
*/
|
*/
|
||||||
export async function handleStartup(): Promise<void> {
|
export async function handleStartup(): Promise<void> {
|
||||||
console.log('[background] startup');
|
console.log('[background] startup');
|
||||||
|
await cancelStaleCrawlWhenWindowMissing();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -34,6 +35,7 @@ export async function handleBackgroundCommand(
|
|||||||
case 'START_CRAWL':
|
case 'START_CRAWL':
|
||||||
return startCrawl(message.payload.platformId);
|
return startCrawl(message.payload.platformId);
|
||||||
case 'GET_CRAWL_STATE':
|
case 'GET_CRAWL_STATE':
|
||||||
|
await cancelStaleCrawlWhenWindowMissing();
|
||||||
return { ok: true, data: await getCrawlTaskState() };
|
return { ok: true, data: await getCrawlTaskState() };
|
||||||
case 'CANCEL_CRAWL':
|
case 'CANCEL_CRAWL':
|
||||||
return cancelCrawl();
|
return cancelCrawl();
|
||||||
|
|||||||
@@ -1,27 +1,21 @@
|
|||||||
import type { CrawlTaskState } from '@/types';
|
import type { CrawlTaskState } from '@/types';
|
||||||
|
|
||||||
// chrome.storage.local 中保存当前爬取任务状态的键名。
|
|
||||||
const CRAWL_TASK_STORAGE_KEY = 'crawlTaskState';
|
const CRAWL_TASK_STORAGE_KEY = 'crawlTaskState';
|
||||||
|
|
||||||
/**
|
|
||||||
* 从 chrome.storage.local 读取当前爬取任务状态。
|
|
||||||
*/
|
|
||||||
export async function getCrawlTaskState(): Promise<CrawlTaskState | null> {
|
export async function getCrawlTaskState(): Promise<CrawlTaskState | null> {
|
||||||
const result = await chrome.storage.local.get(CRAWL_TASK_STORAGE_KEY);
|
const result = await chrome.storage.local.get(CRAWL_TASK_STORAGE_KEY);
|
||||||
const state = result[CRAWL_TASK_STORAGE_KEY];
|
const state = result[CRAWL_TASK_STORAGE_KEY];
|
||||||
return isCrawlTaskState(state) ? state : null;
|
return isCrawlTaskState(state) ? state : null;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 将最新爬取任务状态写入 chrome.storage.local,供 popup 和 content script 同步读取。
|
|
||||||
*/
|
|
||||||
export async function setCrawlTaskState(state: CrawlTaskState): Promise<void> {
|
export async function setCrawlTaskState(state: CrawlTaskState): Promise<void> {
|
||||||
await chrome.storage.local.set({ [CRAWL_TASK_STORAGE_KEY]: state });
|
await chrome.storage.local.set({ [CRAWL_TASK_STORAGE_KEY]: state });
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
export async function clearCrawlTaskState(): Promise<void> {
|
||||||
* 读取任务状态后执行不可变更新,避免覆盖已取消或已替换的任务。
|
await chrome.storage.local.remove(CRAWL_TASK_STORAGE_KEY);
|
||||||
*/
|
}
|
||||||
|
|
||||||
export async function updateCrawlTaskState(
|
export async function updateCrawlTaskState(
|
||||||
taskId: string,
|
taskId: string,
|
||||||
updater: (state: CrawlTaskState) => CrawlTaskState,
|
updater: (state: CrawlTaskState) => CrawlTaskState,
|
||||||
@@ -35,9 +29,6 @@ export async function updateCrawlTaskState(
|
|||||||
await setCrawlTaskState(updater(state));
|
await setCrawlTaskState(updater(state));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 粗略判断 storage 中读取到的值是否像一个爬取任务状态对象。
|
|
||||||
*/
|
|
||||||
function isCrawlTaskState(value: unknown): value is CrawlTaskState {
|
function isCrawlTaskState(value: unknown): value is CrawlTaskState {
|
||||||
return typeof value === 'object' && value !== null && 'id' in value && 'steps' in value;
|
return typeof value === 'object' && value !== null && 'id' in value && 'steps' in value;
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -18,6 +18,7 @@ const {
|
|||||||
isScanning,
|
isScanning,
|
||||||
crawlState,
|
crawlState,
|
||||||
handleScan,
|
handleScan,
|
||||||
|
handleCancelCrawl,
|
||||||
elapsedSeconds
|
elapsedSeconds
|
||||||
} = useScan()
|
} = useScan()
|
||||||
|
|
||||||
@@ -29,12 +30,6 @@ const shouldShowCrawlProgress = computed<boolean>(() =>
|
|||||||
crawlState.value != null
|
crawlState.value != null
|
||||||
);
|
);
|
||||||
|
|
||||||
/**
|
|
||||||
* 取消爬取
|
|
||||||
*/
|
|
||||||
const handleCancelCrawl = () => {
|
|
||||||
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 获取进度样式
|
* 获取进度样式
|
||||||
|
|||||||
@@ -1,49 +1,87 @@
|
|||||||
import {onMounted, onUnmounted, ref} from "vue";
|
import { onMounted, onUnmounted, ref } from 'vue';
|
||||||
import {platformConfigs} from "@/config/platforms";
|
import { platformConfigs } from '@/config/platforms';
|
||||||
import {CrawlTaskState} from "@/types";
|
import type { CrawlTaskState } from '@/types';
|
||||||
import {sendBackgroundMessage} from "@/shared/message";
|
import { sendBackgroundMessage } from '@/shared/message';
|
||||||
|
|
||||||
|
const CRAWL_TASK_STORAGE_KEY = 'crawlTaskState';
|
||||||
|
const ACTIVE_STATUSES = new Set(['running', 'paused']);
|
||||||
|
|
||||||
export const useScan = () => {
|
export const useScan = () => {
|
||||||
//选中id
|
|
||||||
const selectedPlatformId = ref(platformConfigs[0]?.id ?? '');
|
const selectedPlatformId = ref(platformConfigs[0]?.id ?? '');
|
||||||
//防抖
|
const isScanning = ref<boolean>(false);
|
||||||
const isScanning = ref<boolean>(false)
|
|
||||||
//步骤数据
|
|
||||||
const crawlState = ref<CrawlTaskState | null>(null);
|
const crawlState = ref<CrawlTaskState | null>(null);
|
||||||
//爬取时间
|
const elapsedSeconds = ref<number>(0);
|
||||||
const elapsedSeconds = ref<number>(0)
|
|
||||||
|
|
||||||
|
|
||||||
let timer: number | undefined;
|
let timer: number | undefined;
|
||||||
/**
|
|
||||||
* 开始爬取
|
|
||||||
*/
|
|
||||||
const handleScan = async () => {
|
const handleScan = async () => {
|
||||||
if (isScanning.value) return
|
if (isScanning.value) {
|
||||||
isScanning.value = true
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
isScanning.value = true;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
updateSeconds()
|
ensureElapsedTimer();
|
||||||
//定时器
|
|
||||||
timer = window.setInterval(() => {
|
|
||||||
updateSeconds();
|
|
||||||
}, 1000);
|
|
||||||
//发送
|
|
||||||
const response = await sendBackgroundMessage<CrawlTaskState>({
|
const response = await sendBackgroundMessage<CrawlTaskState>({
|
||||||
action: 'START_CRAWL',
|
action: 'START_CRAWL',
|
||||||
payload: {platformId: selectedPlatformId.value},
|
payload: { platformId: selectedPlatformId.value },
|
||||||
});
|
});
|
||||||
if (response.data) {
|
|
||||||
crawlState.value = response.data;
|
if (response.ok) {
|
||||||
|
syncCrawlState(response.data ?? null);
|
||||||
|
} else {
|
||||||
|
console.error('[crawl] start failed', response.error);
|
||||||
}
|
}
|
||||||
} finally {
|
} finally {
|
||||||
isScanning.value = false;
|
isScanning.value = false;
|
||||||
}
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleCancelCrawl = async () => {
|
||||||
|
const response = await sendBackgroundMessage<CrawlTaskState | null>({ action: 'CANCEL_CRAWL' });
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
syncCrawlState(response.data ?? null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
console.error('[crawl] cancel failed', response.error);
|
||||||
|
await refreshCrawlState();
|
||||||
|
};
|
||||||
|
|
||||||
|
function syncCrawlState(state: CrawlTaskState | null) {
|
||||||
|
crawlState.value = state;
|
||||||
|
updateSeconds();
|
||||||
|
|
||||||
|
if (state && ACTIVE_STATUSES.has(state.status)) {
|
||||||
|
ensureElapsedTimer();
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
clearElapsedTimer();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function ensureElapsedTimer() {
|
||||||
|
if (timer !== undefined) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
timer = window.setInterval(() => {
|
||||||
|
updateSeconds();
|
||||||
|
}, 1000);
|
||||||
|
}
|
||||||
|
|
||||||
|
function clearElapsedTimer() {
|
||||||
|
if (timer === undefined) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
window.clearInterval(timer);
|
||||||
|
timer = undefined;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 更新时间
|
|
||||||
*/
|
|
||||||
function updateSeconds() {
|
function updateSeconds() {
|
||||||
if (!crawlState.value) {
|
if (!crawlState.value) {
|
||||||
elapsedSeconds.value = 0;
|
elapsedSeconds.value = 0;
|
||||||
@@ -53,33 +91,54 @@ export const useScan = () => {
|
|||||||
elapsedSeconds.value = Math.max(0, Math.floor((Date.now() - crawlState.value.startedAt) / 1000));
|
elapsedSeconds.value = Math.max(0, Math.floor((Date.now() - crawlState.value.startedAt) / 1000));
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
|
||||||
* 同步爬取状态
|
|
||||||
*/
|
|
||||||
async function refreshCrawlState() {
|
async function refreshCrawlState() {
|
||||||
const response = await sendBackgroundMessage<CrawlTaskState | null>({action: 'GET_CRAWL_STATE'});
|
const response = await sendBackgroundMessage<CrawlTaskState | null>({ action: 'GET_CRAWL_STATE' });
|
||||||
|
|
||||||
if (response.data) {
|
if (response.ok) {
|
||||||
crawlState.value = response.data ?? null;
|
syncCrawlState(response.data ?? null);
|
||||||
updateSeconds();
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function handleStorageChanged(changes: Record<string, chrome.storage.StorageChange>, areaName: string) {
|
||||||
|
if (areaName !== 'local') {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
const change = changes[CRAWL_TASK_STORAGE_KEY];
|
||||||
|
|
||||||
|
if (!change) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
syncCrawlState(isCrawlTaskState(change.newValue) ? change.newValue : null);
|
||||||
|
}
|
||||||
|
|
||||||
onMounted(async () => {
|
onMounted(async () => {
|
||||||
await refreshCrawlState()
|
await refreshCrawlState();
|
||||||
})
|
|
||||||
|
if (typeof chrome !== 'undefined' && chrome.storage?.onChanged) {
|
||||||
|
chrome.storage.onChanged.addListener(handleStorageChanged);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
onUnmounted(() => {
|
onUnmounted(() => {
|
||||||
if (timer) {
|
clearElapsedTimer();
|
||||||
window.clearInterval(timer);
|
|
||||||
|
if (typeof chrome !== 'undefined' && chrome.storage?.onChanged) {
|
||||||
|
chrome.storage.onChanged.removeListener(handleStorageChanged);
|
||||||
}
|
}
|
||||||
})
|
});
|
||||||
|
|
||||||
return {
|
return {
|
||||||
selectedPlatformId,
|
selectedPlatformId,
|
||||||
isScanning,
|
isScanning,
|
||||||
crawlState,
|
crawlState,
|
||||||
handleScan,
|
handleScan,
|
||||||
|
handleCancelCrawl,
|
||||||
elapsedSeconds,
|
elapsedSeconds,
|
||||||
}
|
};
|
||||||
}
|
};
|
||||||
|
|
||||||
|
function isCrawlTaskState(value: unknown): value is CrawlTaskState {
|
||||||
|
return typeof value === 'object' && value !== null && 'id' in value && 'steps' in value;
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,24 +1,27 @@
|
|||||||
export type MessageAction =
|
export type MessageAction =
|
||||||
| "GET_CRAWL_STATE" // 获取爬虫的当前状态
|
| 'GET_CRAWL_STATE'
|
||||||
| "START_CRAWL" // 开始爬取
|
| 'START_CRAWL'
|
||||||
|
| 'CANCEL_CRAWL'
|
||||||
|
| 'RESUME_CRAWL';
|
||||||
|
|
||||||
interface BackgroundMessage<T = any> {
|
interface BackgroundMessage<T = unknown> {
|
||||||
action: MessageAction; // 标识要执行的操作
|
action: MessageAction;
|
||||||
payload?: T; // 附带的数据
|
payload?: T;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface BackgroundResponse<T = any> {
|
interface BackgroundResponse<T = unknown> {
|
||||||
data: T | null
|
ok: boolean;
|
||||||
|
data: T | null;
|
||||||
|
error?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* 定义发送给 Background Script 的消息类型
|
* Send a command to the background service worker.
|
||||||
*/
|
*/
|
||||||
export function sendBackgroundMessage<T>(data: BackgroundMessage): Promise<BackgroundResponse<T>> {
|
export function sendBackgroundMessage<T>(data: BackgroundMessage): Promise<BackgroundResponse<T>> {
|
||||||
// 检查是否在 Chrome 扩展环境中运行
|
|
||||||
if (typeof chrome === 'undefined' || !chrome.runtime?.sendMessage) {
|
if (typeof chrome === 'undefined' || !chrome.runtime?.sendMessage) {
|
||||||
return Promise.resolve({data: null});
|
return Promise.resolve({ ok: true, data: null });
|
||||||
}
|
}
|
||||||
|
|
||||||
return chrome.runtime.sendMessage(data);
|
return chrome.runtime.sendMessage(data);
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
{"root":["./manifest.config.ts","./message.js","./vite.config.ts","./src/background/domscraper.ts","./src/background/index.ts","./src/background/service.ts","./src/background/types.ts","./src/background/service/crawltask.ts","./src/background/service/lifecycle.ts","./src/background/service/taskstate.ts","./src/config/platforms.ts","./src/content/app.vue","./src/content/main.ts","./src/content/pagerunner.ts","./src/options/app.vue","./src/options/main.ts","./src/popup/app.vue","./src/popup/main.ts","./src/shared/auth.ts","./src/types/crawl.ts","./src/types/index.ts","./src/types/platform.ts","./storeai-extension-v0.1.0/service-worker-loader.js","./storeai-extension-v0.1.0/assets/config-cf-xklo9.js","./storeai-extension-v0.1.0/assets/fetch-hook.ts-bvrghr__.js","./storeai-extension-v0.1.0/assets/index-dxg1qimp.js","./storeai-extension-v0.1.0/assets/index.ts-dirvxn_b.js","./storeai-extension-v0.1.0/assets/orchestrator.ts-bleul1fk.js","./storeai-extension-v0.1.0/assets/orchestrator.ts-loader-drev6v6h.js","./storeai-extension-v0.1.0/assets/popup-dbgvbs2c.js","./storeai-extension-v0.1.0/assets/selectors-xrdds_u0.js"],"version":"5.9.3"}
|
{"root":["./manifest.config.ts","./message.js","./vite.config.ts","./src/background/domscraper.ts","./src/background/index.ts","./src/background/service.ts","./src/background/types.ts","./src/background/service/crawltask.ts","./src/background/service/lifecycle.ts","./src/background/service/taskstate.ts","./src/config/platforms.ts","./src/content/app.vue","./src/content/main.ts","./src/content/pagerunner.ts","./src/options/app.vue","./src/options/main.ts","./src/popup/app.vue","./src/popup/main.ts","./src/popup/hook/use-login.ts","./src/popup/hook/use-scan.ts","./src/shared/auth.ts","./src/shared/message.ts","./src/shared/time_format.ts","./src/types/crawl.ts","./src/types/index.ts","./src/types/platform.ts"],"version":"5.9.3"}
|
||||||
@@ -3,7 +3,7 @@ import {crx} from '@crxjs/vite-plugin'
|
|||||||
import tailwindcss from '@tailwindcss/vite'
|
import tailwindcss from '@tailwindcss/vite'
|
||||||
import vue from '@vitejs/plugin-vue'
|
import vue from '@vitejs/plugin-vue'
|
||||||
import {defineConfig} from 'vite'
|
import {defineConfig} from 'vite'
|
||||||
import manifest from './manifest.config.ts'
|
import manifest from './manifest.config'
|
||||||
|
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
resolve: {
|
resolve: {
|
||||||
|
|||||||
Reference in New Issue
Block a user