From 696abd2e05aa3e4b9ab8ba9a2092032634558948 Mon Sep 17 00:00:00 2001 From: meishibiezb <750783119@qq.com> Date: Sat, 2 May 2026 04:19:32 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=A2=9E=E5=8A=A0=E4=BA=86=E8=87=AA?= =?UTF-8?q?=E5=8A=A8=E5=8A=A0=E8=BD=BD=E6=89=80=E6=9C=89=E5=9B=BE=E7=89=87?= =?UTF-8?q?=E7=9A=84=E9=80=BB=E8=BE=91?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- components/autoLoader.ts | 299 ++++++++++++++++++++++++++++++++++++++ entrypoints/content.ts | 77 +++++++++- entrypoints/popup/main.ts | 35 ++++- 3 files changed, 407 insertions(+), 4 deletions(-) create mode 100644 components/autoLoader.ts diff --git a/components/autoLoader.ts b/components/autoLoader.ts new file mode 100644 index 0000000..30b73ce --- /dev/null +++ b/components/autoLoader.ts @@ -0,0 +1,299 @@ +// ============================================================ +// AutoLoader v2 — 支持页码分页 + 游标分页 +// ============================================================ + +// ---------- 类型定义 ---------- + +/** 分页模式 */ +type PaginationMode = 'page' | 'cursor'; + +/** 游标配置 */ +interface CursorConfig { + /** 游标参数名，如 "max" */ + paramName: string; + /** 从响应项中提取游标值的路径，如 "pin_id" */ + valuePath: string; + /** 首次请求的游标初始值（可选，不传则请求不带该参数） */ + initialValue?: string; +} + +interface AutoLoadConfig { + endpoint: string; + method?: 'GET' | 'POST'; + + /** 分页模式，默认 "page" */ + paginationMode?: PaginationMode; + + /** 页码模式下的参数（paginationMode='page' 时生效） */ + pageParamName?: string; + limitParamName?: string; + limit?: number; + maxPages?: number; + + /** 游标模式下的配置（paginationMode='cursor' 时生效） */ + cursor?: CursorConfig; + + /** 固定参数（每页都带） */ + baseParams?: Record; + + /** 每次请求间隔（毫秒），默认 1000 */ + interval?: number; + + headers?: Record; + credentials?: RequestCredentials; + itemsPath?: string; + + onProgress?: (progress: LoadProgress) => void; + onError?: (error: Error, pageOrCursor: number | string) => void; + /** 每次请求拿到原始 items 后回调（可在回调中写入 storage） */ + onData?: (items: any[]) => void | Promise; +} + +interface LoadProgress { + requestCount: number; + totalItems: number; + newItems: number; + done: boolean; + /** 仅游标模式：上一页最后一个游标值 */ + lastCursor?: string; +} + +// ---------- 主类 ---------- + +export class AutoLoader { + private config: Required> & { cursor?: CursorConfig }; + private requestCount: number; + private totalItems: number; + private active: boolean; + private abortController: AbortController | null; + private currentCursor: string | undefined; + + constructor(config: AutoLoadConfig) { + this.config = { + endpoint: config.endpoint, + method: config.method ?? 'GET', + paginationMode: config.paginationMode ?? 'page', + pageParamName: config.pageParamName ?? 'page', + limitParamName: config.limitParamName ?? 'limit', + limit: config.limit ?? 20, + maxPages: config.maxPages ?? 100, + cursor: config.cursor, + baseParams: config.baseParams ?? {}, + interval: config.interval ?? 1000, + headers: config.headers ?? {}, + credentials: config.credentials ?? 'same-origin', + itemsPath: config.itemsPath ?? 'pins', + onProgress: config.onProgress ?? (() => {}), + onError: config.onError ?? (() => {}), + onData: config.onData ?? (() => {}), + }; + + this.requestCount = 0; + this.totalItems = 0; + this.active = false; + this.abortController = null; + this.currentCursor = this.config.cursor?.initialValue; + } + + // -------- 公开方法 -------- + + async start(): Promise { + if (this.active) return; + this.active = true; + this.abortController = new AbortController(); + + console.log('[AutoLoader] 开始加载...'); + this.config.onProgress({ requestCount: 0, totalItems: 0, newItems: 0, done: false }); + + const isCursor = this.config.paginationMode === 'cursor'; + const maxIterations = isCursor ? 9999 : this.config.maxPages; + + for (let i = 1; i <= maxIterations; i++) { + if (!this.active) break; + + try { + const url = this.buildUrl(i); + const newItems = await this.fetchOnce(url); + this.requestCount = i; + this.totalItems += newItems; + + const done = newItems < (this.config.limit ?? 20); + + this.config.onProgress({ + requestCount: i, + totalItems: this.totalItems, + newItems, + done, + lastCursor: this.currentCursor, + }); + + if (newItems === 0) { + console.log(`[AutoLoader] 无数据，加载完成。共 ${this.totalItems} 条`); + break; + } + + console.log(`[AutoLoader] 第 ${i} 次: +${newItems}，累计 ${this.totalItems}`); + } catch (err) { + if (!this.active) break; + this.config.onError( + err instanceof Error ? err : new Error(String(err)), + isCursor ? (this.currentCursor ?? 'initial') : i + ); + + this.config.onProgress({ + requestCount: i, + totalItems: this.totalItems, + newItems: 0, + done: false, + lastCursor: this.currentCursor, + }); + } + + if (this.active && i < maxIterations) { + await delay(this.config.interval); + } + } + + this.active = false; + this.abortController = null; + console.log('[AutoLoader] 结束'); + } + + stop(): void { + this.active = false; + this.abortController?.abort(); + this.abortController = null; + console.log('[AutoLoader] 已停止'); + } + + get isRunning(): boolean { + return this.active; + } + + // -------- 内部方法 -------- + + private buildUrl(pageOrIndex: number): string { + const { endpoint, method, baseParams, paginationMode, pageParamName, limitParamName, limit, cursor } = + this.config; + + const params: Record = { + ...baseParams, + }; + + if (paginationMode === 'page') { + params[pageParamName] = String(pageOrIndex); + params[limitParamName] = String(limit); + } else if (paginationMode === 'cursor') { + // 游标模式：limit 放在 params，cursor 值由 buildUrl 时注入 + params[limitParamName] = String(limit); + if (this.currentCursor) { + params[cursor!.paramName] = this.currentCursor; + } + } + + if (method === 'GET') { + const qs = new URLSearchParams(params).toString(); + return `${endpoint}${endpoint.includes('?') ? '&' : '?'}${qs}`; + } + return endpoint; + } + + private async fetchOnce(url: string): Promise { + const { method, headers, credentials, itemsPath, paginationMode, cursor } = this.config; + + const fetchOptions: RequestInit = { + method, + headers: method === 'POST' + ? { 'Content-Type': 'application/json', ...headers } + : headers, + credentials, + signal: this.abortController?.signal, + }; + + if (method === 'POST' && paginationMode === 'page') { + // POST 页码模式才传 body；游标模式参数已在 URL + const qs = new URLSearchParams(url.split('?')[1] ?? '').toString(); + const bodyParams: Record = {}; + new URLSearchParams(qs).forEach((v, k) => { bodyParams[k] = v; }); + fetchOptions.body = JSON.stringify(bodyParams); + } + + const response = await fetch(url, fetchOptions); + + if (!response.ok) throw new Error(`HTTP ${response.status}`); + + const json = await response.json(); + const items = getByPath(json, itemsPath); + + if (!Array.isArray(items)) { + console.warn(`[AutoLoader] 路径 "${itemsPath}" 不是数组，已停止`, json); + return 0; + } + + // ★ 游标模式：更新 cursor 为最后一项的值 + if (paginationMode === 'cursor' && cursor && items.length > 0) { + const lastItem = items[items.length - 1]; + this.currentCursor = String(getByPath(lastItem, cursor.valuePath)); + } + + + // 把原始 items 传出去 + if (items.length > 0) { + await this.config.onData(items); + } + + return items.length; + } +} + +// ---------- 便捷工厂方法 ---------- + +/** + * 花瓣画板游标分页 + * + * @param endpoint API 路径，如 "/v3/boards/75606715/pins" + * @param extra 额外固定参数（字段会自动 URL encode） + * + * 使用示例： + * const loader = AutoLoader.forHuabanBoardCursor('/v3/boards/75606715/pins', { + * sort: 'seq', + * fields: 'pins:PIN|board:BOARD_DETAIL|check', + * }); + * loader.start(); + */ +export function forHuabanBoardCursor( + endpoint: string, + extra?: Record +): AutoLoader { + return new AutoLoader({ + endpoint, + method: 'GET', + paginationMode: 'cursor', + limitParamName: 'limit', + limit: 40, + interval: 1000, + credentials: 'same-origin', + itemsPath: 'pins', + cursor: { + paramName: 'max', + valuePath: 'pin_id', + // 首次不带 max + }, + baseParams: { + sort: 'seq', + fields: 'pins:PIN|board:BOARD_DETAIL|check', + ...extra, + }, + }); +} + +// ---------- 工具函数 ---------- + +const delay = (ms: number): Promise => + new Promise((resolve) => setTimeout(resolve, ms)); + +function getByPath(obj: any, path: string): any { + return path.split('.').reduce((acc, key) => acc?.[key], obj); +} + +export type { AutoLoadConfig, LoadProgress, PaginationMode, CursorConfig }; diff --git a/entrypoints/content.ts b/entrypoints/content.ts index 50e05e7..b40bcd7 100644 --- a/entrypoints/content.ts +++ b/entrypoints/content.ts @@ -1,15 +1,19 @@ import { showPageMarker } from "@/components/page-marker"; import { PinCollector } from "#imports"; +import { AutoLoader } from '@/components/autoLoader'; +import type { PinData } from '#imports'; + +let loader: AutoLoader | null = null; export default defineContentScript({ matches: ['*://*.huaban.com/*'], runAt: 'document_idle', + main() { sayHello(); const collector = new PinCollector(); - // 首次加载自动收集并存储 - + // 允许 popup 主动触发重新收集 browser.runtime.onMessage.addListener((msg, sender, sendResponse) => { if (msg.type === 'COLLECT_PINS') { @@ -18,6 +22,62 @@ export default defineContentScript({ return true; // 异步响应 } }); + + // TODO: 首次加载自动收集并存储 + + browser.runtime.onMessage.addListener((msg, _sender, sendResponse) => { + if (msg.action === 'start') { + const id = location.pathname.match(/\/boards\/(\d+)/)?.[1] ?? ''; + + if (!id) { + console.warn('[content] 当前页面不是画板页，无法启动'); + sendResponse({ ok: false, error: '不在画板页' }); + return true; + } + + loader = new AutoLoader({ + endpoint: `https://huaban.com/v3/boards/${id}/pins`, + paginationMode: 'cursor', + limit: 40, + cursor: { paramName: 'max', valuePath: 'pin_id' }, + interval: 1000, + itemsPath: 'pins', + baseParams: { sort: 'seq', fields: 'pins:PIN|board:BOARD_DETAIL|check' }, + onProgress: (p) => { + browser.runtime.sendMessage({ type: 'progress', ...p }); + }, + onError: (err, pageOrCursor) => { + browser.runtime.sendMessage({ + type: 'progress', + requestCount: -1, + totalItems: -1, + newItems: 0, + done: false, + error: err.message, + }).catch(() => { }); + }, + onData: async (items) => { + const newPins: PinData[] = items.map((pin: any) => extractPinData(pin)); + // 读取已有数据，按 pin_id 去重合并 + const stored = await browser.storage.local.get('collectedPins') as { collectedPins?: PinData[] }; + const existing: PinData[] = stored.collectedPins || []; + const existingIds = new Set(existing.map((p) => p.pinId)); + const merged = [...existing, ...newPins.filter((p) => !existingIds.has(p.pinId))]; + await browser.storage.local.set({ collectedPins: merged }); + console.log(`[content] 已写入 storage，累计 ${merged.length} 条`); + }, + }); + loader.start(); + sendResponse({ ok: true }); + return true; + } + + if (msg.action === 'stop') { + loader?.stop(); + sendResponse({ ok: true }); + return true; + } + }); }, }); @@ -32,3 +92,16 @@ function sayHello() { autoRemoveSeconds: 10, }); } + +function extractPinData(pin: any): PinData { + return { + pinId: pin.pin_id ?? '', + imgSmallSrc: pin.file?.url ?? '', + imgSrc: (pin.file?.url ?? '').replace(/_fw\d+webp|_png/, ''), + alt: pin.raw_text ?? '', + author: pin.user?.username ?? '', + time: pin.created_at ?? '', + tags: pin.tags?.map((t: any) => t.tag ?? t) ?? [], + url: `/pins/${pin.pin_id}/`, + }; +} \ No newline at end of file diff --git a/entrypoints/popup/main.ts b/entrypoints/popup/main.ts index 3821bfd..85443cf 100644 --- a/entrypoints/popup/main.ts +++ b/entrypoints/popup/main.ts @@ -7,6 +7,8 @@ document.querySelector('#app')!.innerHTML = `
+ +
@@ -26,8 +28,9 @@ document.getElementById('collectBtn')!.addEventListener('click', async () => { type: 'COLLECT_PINS', }); if (response?.success) { - await browser.storage.local.set({ collectedPins: response.pins }); - renderPins(response.pins as PinData[]); + // await browser.storage.local.set({ collectedPins: response.pins }); + // renderPins(response.pins as PinData[]); + await loadAndRender(); showStatus(`✅ 已收集 ${response.count} 个 Pin`, 'success'); } } catch (e) { @@ -91,3 +94,31 @@ function escapeHtml(str: string): string { div.textContent = str; return div.innerHTML; } + +let activeTabId: number | null = null; +// 获取当前标签页 ID +browser.tabs.query({ active: true, currentWindow: true }, ([tab]) => { + activeTabId = tab.id!; + document.getElementById('start-btn')?.addEventListener('click', () => { + browser.tabs.sendMessage(activeTabId!, { action: 'start' }); + }); + document.getElementById('stop-btn')?.addEventListener('click', () => { + browser.tabs.sendMessage(activeTabId!, { action: 'stop' }); + }); +}); +// 接收进度回调 +browser.runtime.onMessage.addListener(async (msg) => { + if (msg.type === 'progress') { + if (msg.error) { + document.getElementById('status')!.textContent = `❌ 错误: ${msg.error}`; + } else { + document.getElementById('status')!.textContent = + `第 ${msg.requestCount} 次 · 共 ${msg.totalItems} 条`; + } + if (msg.done) { + await loadAndRender(); + document.getElementById('status')!.textContent = + `✅ 加载完成 · 共 ${msg.totalItems} 条`; + } + } +}); \ No newline at end of file