feat: 增加了自动加载所有图片的逻辑

2026-05-02 04:19:32 +08:00
parent dd97df9ede
commit 696abd2e05
3 changed files with 407 additions and 4 deletions
--- a/components/autoLoader.ts
+++ b/components/autoLoader.ts
@@ -0,0 +1,299 @@
+// ============================================================
+// AutoLoader v2 — 支持页码分页 + 游标分页
+// ============================================================
+
+// ---------- 类型定义 ----------
+
+/** 分页模式 */
+type PaginationMode = 'page' | 'cursor';
+
+/** 游标配置 */
+interface CursorConfig {
+  /** 游标参数名，如 "max" */
+  paramName: string;
+  /** 从响应项中提取游标值的路径，如 "pin_id" */
+  valuePath: string;
+  /** 首次请求的游标初始值（可选，不传则请求不带该参数） */
+  initialValue?: string;
+}
+
+interface AutoLoadConfig {
+  endpoint: string;
+  method?: 'GET' | 'POST';
+
+  /** 分页模式，默认 "page" */
+  paginationMode?: PaginationMode;
+
+  /** 页码模式下的参数（paginationMode='page' 时生效） */
+  pageParamName?: string;
+  limitParamName?: string;
+  limit?: number;
+  maxPages?: number;
+
+  /** 游标模式下的配置（paginationMode='cursor' 时生效） */
+  cursor?: CursorConfig;
+
+  /** 固定参数（每页都带） */
+  baseParams?: Record<string, string>;
+
+  /** 每次请求间隔（毫秒），默认 1000 */
+  interval?: number;
+
+  headers?: Record<string, string>;
+  credentials?: RequestCredentials;
+  itemsPath?: string;
+
+  onProgress?: (progress: LoadProgress) => void;
+  onError?: (error: Error, pageOrCursor: number | string) => void;
+  /** 每次请求拿到原始 items 后回调（可在回调中写入 storage） */
+  onData?: (items: any[]) => void | Promise<void>;
+}
+
+interface LoadProgress {
+  requestCount: number;
+  totalItems: number;
+  newItems: number;
+  done: boolean;
+  /** 仅游标模式：上一页最后一个游标值 */
+  lastCursor?: string;
+}
+
+// ---------- 主类 ----------
+
+export class AutoLoader {
+  private config: Required<Omit<AutoLoadConfig, 'cursor'>> & { cursor?: CursorConfig };
+  private requestCount: number;
+  private totalItems: number;
+  private active: boolean;
+  private abortController: AbortController | null;
+  private currentCursor: string | undefined;
+
+  constructor(config: AutoLoadConfig) {
+    this.config = {
+      endpoint: config.endpoint,
+      method: config.method ?? 'GET',
+      paginationMode: config.paginationMode ?? 'page',
+      pageParamName: config.pageParamName ?? 'page',
+      limitParamName: config.limitParamName ?? 'limit',
+      limit: config.limit ?? 20,
+      maxPages: config.maxPages ?? 100,
+      cursor: config.cursor,
+      baseParams: config.baseParams ?? {},
+      interval: config.interval ?? 1000,
+      headers: config.headers ?? {},
+      credentials: config.credentials ?? 'same-origin',
+      itemsPath: config.itemsPath ?? 'pins',
+      onProgress: config.onProgress ?? (() => {}),
+      onError: config.onError ?? (() => {}),
+      onData: config.onData ?? (() => {}),
+    };
+
+    this.requestCount = 0;
+    this.totalItems = 0;
+    this.active = false;
+    this.abortController = null;
+    this.currentCursor = this.config.cursor?.initialValue;
+  }
+
+  // -------- 公开方法 --------
+
+  async start(): Promise<void> {
+    if (this.active) return;
+    this.active = true;
+    this.abortController = new AbortController();
+
+    console.log('[AutoLoader] 开始加载...');
+    this.config.onProgress({ requestCount: 0, totalItems: 0, newItems: 0, done: false });
+
+    const isCursor = this.config.paginationMode === 'cursor';
+    const maxIterations = isCursor ? 9999 : this.config.maxPages;
+
+    for (let i = 1; i <= maxIterations; i++) {
+      if (!this.active) break;
+
+      try {
+        const url = this.buildUrl(i);
+        const newItems = await this.fetchOnce(url);
+        this.requestCount = i;
+        this.totalItems += newItems;
+
+        const done = newItems < (this.config.limit ?? 20);
+
+        this.config.onProgress({
+          requestCount: i,
+          totalItems: this.totalItems,
+          newItems,
+          done,
+          lastCursor: this.currentCursor,
+        });
+
+        if (newItems === 0) {
+          console.log(`[AutoLoader] 无数据，加载完成。共 ${this.totalItems} 条`);
+          break;
+        }
+
+        console.log(`[AutoLoader] 第 ${i} 次: +${newItems}，累计 ${this.totalItems}`);
+      } catch (err) {
+        if (!this.active) break;
+        this.config.onError(
+          err instanceof Error ? err : new Error(String(err)),
+          isCursor ? (this.currentCursor ?? 'initial') : i
+        );
+
+          this.config.onProgress({
+              requestCount: i,
+              totalItems: this.totalItems,
+              newItems: 0,
+              done: false,
+              lastCursor: this.currentCursor,
+          });
+      }
+
+      if (this.active && i < maxIterations) {
+        await delay(this.config.interval);
+      }
+    }
+
+    this.active = false;
+    this.abortController = null;
+    console.log('[AutoLoader] 结束');
+  }
+
+  stop(): void {
+    this.active = false;
+    this.abortController?.abort();
+    this.abortController = null;
+    console.log('[AutoLoader] 已停止');
+  }
+
+  get isRunning(): boolean {
+    return this.active;
+  }
+
+  // -------- 内部方法 --------
+
+  private buildUrl(pageOrIndex: number): string {
+    const { endpoint, method, baseParams, paginationMode, pageParamName, limitParamName, limit, cursor } =
+      this.config;
+
+    const params: Record<string, string> = {
+      ...baseParams,
+    };
+
+    if (paginationMode === 'page') {
+      params[pageParamName] = String(pageOrIndex);
+      params[limitParamName] = String(limit);
+    } else if (paginationMode === 'cursor') {
+      // 游标模式：limit 放在 params，cursor 值由 buildUrl 时注入
+      params[limitParamName] = String(limit);
+      if (this.currentCursor) {
+        params[cursor!.paramName] = this.currentCursor;
+      }
+    }
+
+    if (method === 'GET') {
+      const qs = new URLSearchParams(params).toString();
+      return `${endpoint}${endpoint.includes('?') ? '&' : '?'}${qs}`;
+    }
+    return endpoint;
+  }
+
+  private async fetchOnce(url: string): Promise<number> {
+    const { method, headers, credentials, itemsPath, paginationMode, cursor } = this.config;
+
+    const fetchOptions: RequestInit = {
+      method,
+      headers: method === 'POST'
+        ? { 'Content-Type': 'application/json', ...headers }
+        : headers,
+      credentials,
+      signal: this.abortController?.signal,
+    };
+
+    if (method === 'POST' && paginationMode === 'page') {
+      // POST 页码模式才传 body；游标模式参数已在 URL
+      const qs = new URLSearchParams(url.split('?')[1] ?? '').toString();
+      const bodyParams: Record<string, string> = {};
+      new URLSearchParams(qs).forEach((v, k) => { bodyParams[k] = v; });
+      fetchOptions.body = JSON.stringify(bodyParams);
+    }
+
+    const response = await fetch(url, fetchOptions);
+
+    if (!response.ok) throw new Error(`HTTP ${response.status}`);
+
+    const json = await response.json();
+    const items = getByPath(json, itemsPath);
+
+    if (!Array.isArray(items)) {
+      console.warn(`[AutoLoader] 路径 "${itemsPath}" 不是数组，已停止`, json);
+      return 0;
+    }
+
+    // ★ 游标模式：更新 cursor 为最后一项的值
+    if (paginationMode === 'cursor' && cursor && items.length > 0) {
+      const lastItem = items[items.length - 1];
+      this.currentCursor = String(getByPath(lastItem, cursor.valuePath));
+    }
+
+    
+      // 把原始 items 传出去
+      if (items.length > 0) {
+          await this.config.onData(items);
+      }
+
+    return items.length;
+  }
+}
+
+// ---------- 便捷工厂方法 ----------
+
+/**
+ * 花瓣画板游标分页
+ *
+ * @param endpoint  API 路径，如 "/v3/boards/75606715/pins"
+ * @param extra     额外固定参数（字段会自动 URL encode）
+ *
+ * 使用示例：
+ *   const loader = AutoLoader.forHuabanBoardCursor('/v3/boards/75606715/pins', {
+ *     sort: 'seq',
+ *     fields: 'pins:PIN|board:BOARD_DETAIL|check',
+ *   });
+ *   loader.start();
+ */
+export function forHuabanBoardCursor(
+  endpoint: string,
+  extra?: Record<string, string>
+): AutoLoader {
+  return new AutoLoader({
+    endpoint,
+    method: 'GET',
+    paginationMode: 'cursor',
+    limitParamName: 'limit',
+    limit: 40,
+    interval: 1000,
+    credentials: 'same-origin',
+    itemsPath: 'pins',
+    cursor: {
+      paramName: 'max',
+      valuePath: 'pin_id',
+      // 首次不带 max
+    },
+    baseParams: {
+      sort: 'seq',
+      fields: 'pins:PIN|board:BOARD_DETAIL|check',
+      ...extra,
+    },
+  });
+}
+
+// ---------- 工具函数 ----------
+
+const delay = (ms: number): Promise<void> =>
+  new Promise((resolve) => setTimeout(resolve, ms));
+
+function getByPath(obj: any, path: string): any {
+  return path.split('.').reduce((acc, key) => acc?.[key], obj);
+}
+
+export type { AutoLoadConfig, LoadProgress, PaginationMode, CursorConfig };
--- a/entrypoints/content.ts
+++ b/entrypoints/content.ts
@@ -1,15 +1,19 @@
 import { showPageMarker } from "@/components/page-marker";
 import { PinCollector } from "#imports";
+import { AutoLoader } from '@/components/autoLoader';
+import type { PinData } from '#imports';
+
+let loader: AutoLoader | null = null;

 export default defineContentScript({
  matches: ['*://*.huaban.com/*'],
  runAt: 'document_idle',
+
  main() {
    sayHello();

    const collector = new PinCollector();
-    // 首次加载自动收集并存储
-
+    
    // 允许 popup 主动触发重新收集
    browser.runtime.onMessage.addListener((msg, sender, sendResponse) => {
      if (msg.type === 'COLLECT_PINS') {
@@ -18,6 +22,62 @@ export default defineContentScript({
        return true; // 异步响应
      }
    });
+
+    // TODO: 首次加载自动收集并存储
+
+    browser.runtime.onMessage.addListener((msg, _sender, sendResponse) => {
+      if (msg.action === 'start') {
+        const id = location.pathname.match(/\/boards\/(\d+)/)?.[1] ?? '';
+
+        if (!id) {
+          console.warn('[content] 当前页面不是画板页，无法启动');
+          sendResponse({ ok: false, error: '不在画板页' });
+          return true;
+        }
+
+        loader = new AutoLoader({
+          endpoint: `https://huaban.com/v3/boards/${id}/pins`,
+          paginationMode: 'cursor',
+          limit: 40,
+          cursor: { paramName: 'max', valuePath: 'pin_id' },
+          interval: 1000,
+          itemsPath: 'pins',
+          baseParams: { sort: 'seq', fields: 'pins:PIN|board:BOARD_DETAIL|check' },
+          onProgress: (p) => {
+            browser.runtime.sendMessage({ type: 'progress', ...p });
+          },
+          onError: (err, pageOrCursor) => {
+            browser.runtime.sendMessage({
+              type: 'progress',
+              requestCount: -1,
+              totalItems: -1,
+              newItems: 0,
+              done: false,
+              error: err.message,
+            }).catch(() => { });
+          },
+          onData: async (items) => {
+            const newPins: PinData[] = items.map((pin: any) => extractPinData(pin));
+            // 读取已有数据，按 pin_id 去重合并
+            const stored = await browser.storage.local.get('collectedPins') as { collectedPins?: PinData[] };
+            const existing: PinData[] = stored.collectedPins || [];
+            const existingIds = new Set(existing.map((p) => p.pinId));
+            const merged = [...existing, ...newPins.filter((p) => !existingIds.has(p.pinId))];
+            await browser.storage.local.set({ collectedPins: merged });
+            console.log(`[content] 已写入 storage，累计 ${merged.length} 条`);
+          },
+        });
+        loader.start();
+        sendResponse({ ok: true });
+        return true;
+      }
+      
+      if (msg.action === 'stop') {
+        loader?.stop();
+        sendResponse({ ok: true });
+        return true;
+      }
+    });
  },
 });

@@ -32,3 +92,16 @@ function sayHello() {
    autoRemoveSeconds: 10,
  });
 }
+
+function extractPinData(pin: any): PinData {
+  return {
+    pinId: pin.pin_id ?? '',
+    imgSmallSrc: pin.file?.url ?? '',
+    imgSrc: (pin.file?.url ?? '').replace(/_fw\d+webp|_png/, ''),
+    alt: pin.raw_text ?? '',
+    author: pin.user?.username ?? '',
+    time: pin.created_at ?? '',
+    tags: pin.tags?.map((t: any) => t.tag ?? t) ?? [],
+    url: `/pins/${pin.pin_id}/`,
+  };
+}
--- a/entrypoints/popup/main.ts
+++ b/entrypoints/popup/main.ts
@@ -7,6 +7,8 @@ document.querySelector<HTMLDivElement>('#app')!.innerHTML = `
    <div class="actions">
      <button id="collectBtn">📥 收集</button>
      <button id="clearBtn">🗑 清空</button>
+      <button id="start-btn">start-btn</button>
+      <button id="stop-btn">stop-btn</button>
    </div>
    <div id="status"></div>
    <div id="pinList"></div>
@@ -26,8 +28,9 @@ document.getElementById('collectBtn')!.addEventListener('click', async () => {
      type: 'COLLECT_PINS',
    });
    if (response?.success) {
-      await browser.storage.local.set({ collectedPins: response.pins });
-      renderPins(response.pins as PinData[]);
+      // await browser.storage.local.set({ collectedPins: response.pins });
+      // renderPins(response.pins as PinData[]);
+      await loadAndRender();
      showStatus(`✅ 已收集 ${response.count} 个 Pin`, 'success');
    }
  } catch (e) {
@@ -91,3 +94,31 @@ function escapeHtml(str: string): string {
  div.textContent = str;
  return div.innerHTML;
 }
+
+let activeTabId: number | null = null;
+// 获取当前标签页 ID
+browser.tabs.query({ active: true, currentWindow: true }, ([tab]) => {
+  activeTabId = tab.id!;
+  document.getElementById('start-btn')?.addEventListener('click', () => {
+    browser.tabs.sendMessage(activeTabId!, { action: 'start' });
+  });
+  document.getElementById('stop-btn')?.addEventListener('click', () => {
+    browser.tabs.sendMessage(activeTabId!, { action: 'stop' });
+  });
+});
+// 接收进度回调
+browser.runtime.onMessage.addListener(async (msg) => {
+  if (msg.type === 'progress') {
+    if (msg.error) {
+      document.getElementById('status')!.textContent = `❌ 错误: ${msg.error}`;
+    } else {
+      document.getElementById('status')!.textContent =
+        `第 ${msg.requestCount} 次 · 共 ${msg.totalItems} 条`;
+    }
+    if (msg.done) {
+      await loadAndRender();
+      document.getElementById('status')!.textContent =
+        `✅ 加载完成 · 共 ${msg.totalItems} 条`;
+    }
+  }
+});