fix: resolve 10 code review issues

High priority:
- Fix concurrent race condition for view_count/like_count (atomic update)
- Add route request ID tracking to prevent race conditions
- Filter get_joke by status=approved (no pending content leak)
- Add error feedback for like button

Performance:
- Optimize random joke query (avoid full table sort)
- Limit page_size max to 100 (DoS prevention)

Medium:
- Add localStorage quota error handling
- Handle empty AI response gracefully
- Fix generate content title extraction

Low:
- Add rejected_jokes to stats API
- Update dashboard to show rejected count

crawler/__init__.py

@@ -0,0 +1 @@
+"""AI 提示词模板"""

crawler/ai_service.py

@@ -0,0 +1,71 @@
+"""LLM 处理：调用 NVIDIA NIM（OpenAI 兼容 API）进行笑话提取、改写和分类。"""
+
+import json
+import os
+from openai import OpenAI
+
+
+class AiService:
+    def __init__(self, api_base: str, api_key: str, model_name: str, temperature: float = 0.7, max_tokens: int = 2048):
+        self.client = OpenAI(base_url=api_base, api_key=api_key)
+        self.model = model_name
+        self.temperature = temperature
+        self.max_tokens = max_tokens
+
+    def extract_jokes(self, page_content: str, known_types: list[str], known_crowds: list[str]) -> list[dict]:
+        """从页面内容中提取笑话，返回结构化数据。"""
+        from crawler.prompts import EXTRACTION_SYSTEM_PROMPT, EXTRACTION_USER_PROMPT
+
+        user_prompt = EXTRACTION_USER_PROMPT.format(
+            page_content=page_content[:8000],
+            known_types=", ".join(known_types),
+            known_crowds=", ".join(known_crowds),
+        )
+
+        response = self.client.chat.completions.create(
+            model=self.model,
+            messages=[
+                {"role": "system", "content": EXTRACTION_SYSTEM_PROMPT},
+                {"role": "user", "content": user_prompt},
+            ],
+            temperature=self.temperature,
+            max_tokens=self.max_tokens,
+        )
+
+        raw = response.choices[0].message.content
+        return self._parse_json(raw)
+
+    def rewrite_joke(self, content: str) -> str:
+        """润色单条笑话内容。"""
+        from crawler.prompts import REWRITE_SYSTEM_PROMPT, REWRITE_USER_PROMPT
+
+        response = self.client.chat.completions.create(
+            model=self.model,
+            messages=[
+                {"role": "system", "content": REWRITE_SYSTEM_PROMPT},
+                {"role": "user", "content": REWRITE_USER_PROMPT.format(content=content)},
+            ],
+            temperature=0.8,
+            max_tokens=500,
+        )
+        return response.choices[0].message.content.strip()
+
+    def _parse_json(self, raw: str) -> list[dict]:
+        """安全解析 LLM 返回的 JSON。"""
+        try:
+            data = json.loads(raw)
+            if isinstance(data, dict):
+                jokes = data.get("jokes") or data.get("items") or [data]
+            else:
+                jokes = data
+            return [j for j in jokes if isinstance(j, dict) and j.get("content")]
+        except json.JSONDecodeError:
+            # 尝试提取 markdown 代码块
+            if "```json" in raw:
+                raw = raw.split("```json")[1].split("```")[0]
+            elif "```" in raw:
+                raw = raw.split("```")[1].split("```")[0]
+            try:
+                return json.loads(raw.strip())
+            except Exception:
+                return []

crawler/crawler_service.py

@@ -0,0 +1,439 @@
+"""统一网页获取：搜索笑话站点 + 深度翻页抓取。"""
+
+import asyncio
+import os
+import re
+import sys
+from urllib.parse import quote, urljoin, urlparse
+
+# Windows 下禁用 rich 控制台输出，避免 GBK 编码错误
+if sys.platform == "win32":
+    os.environ["PYTHONIOENCODING"] = "utf-8"
+    os.environ["TERM"] = "dumb"
+
+try:
+    from crawl4ai import AsyncWebCrawler
+    from crawl4ai.async_configs import CrawlerRunConfig, BrowserConfig
+    HAS_CRAWL4AI = True
+except ImportError:
+    HAS_CRAWL4AI = False
+    print("[!] crawl4ai 未安装，将使用 requests 替代（不支持 JS 渲染）")
+
+import httpx
+
+
+class CrawlerService:
+    def __init__(self, headless: bool = True):
+        self.headless = headless
+        self.session = None
+        self._crawler = None
+        # 站点级别容错跟踪
+        self.site_failures: dict[str, int] = {}
+        # 最近一次抓取的原始 HTML（供翻页链接发现使用）
+        self._last_raw_html: str | None = None
+
+    def _get_session(self) -> httpx.Client:
+        if self.session is None:
+            self.session = httpx.Client(
+                headers={
+                    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 Chrome/125.0.0.0 Safari/537.36",
+                    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+                    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+                    "Referer": "https://www.baidu.com/",
+                },
+                timeout=30,
+                follow_redirects=True,
+            )
+        return self.session
+
+    async def _get_crawler(self):
+        """获取或创建 crawl4ai 实例"""
+        if HAS_CRAWL4AI and self._crawler is None:
+            browser_cfg = BrowserConfig(headless=self.headless, verbose=False)
+            self._crawler = AsyncWebCrawler(config=browser_cfg)
+            await self._crawler.__aenter__()
+            if not self.headless:
+                print("  [*] 浏览器窗口已打开（crawl4ai 控制）")
+        return self._crawler
+
+    async def close(self):
+        if self._crawler:
+            await self._crawler.__aexit__(None, None, None)
+            self._crawler = None
+        if self.session:
+            self.session.close()
+            self.session = None
+
+    # ============================================================
+    # 页面抓取（带重试）
+    # ============================================================
+
+    async def crawl_page_with_retry(self, url: str, max_retries: int = 2, timeout: int = 30000) -> tuple[str, bool]:
+        """抓取页面，返回 (html内容, 是否成功)，失败自动重试"""
+        for attempt in range(max_retries + 1):
+            print(f"  [~] 正在获取页面... (尝试 {attempt+1}/{max_retries+1})")
+            try:
+                html = await self._crawl_one(url, timeout)
+                if html:
+                    print(f"  [~] 页面获取成功，内容长度: {len(html)} 字符")
+                    return html, True
+                print(f"  [!] 页面内容为空")
+            except Exception as e:
+                print(f"  [!] 抓取失败 (尝试 {attempt+1}/{max_retries+1}): {url[:60]} - {e}")
+
+            if attempt < max_retries:
+                wait = 3 * (attempt + 1)
+                print(f"  [~] 等待 {wait} 秒后重试...")
+                await asyncio.sleep(wait)
+        return "", False
+
+    async def _crawl_one(self, url: str, timeout: int) -> str:
+        """单次页面抓取，返回纯文本内容（同时保存原始 HTML 供翻页发现）"""
+        if HAS_CRAWL4AI:
+            # crawl4ai 统一处理 headless / visible 两种模式
+            crawler = await self._get_crawler()
+            result = await crawler.arun(url, config=CrawlerRunConfig(verbose=False))
+            if result and result.success:
+                # fit_html 是 AI 清洗后的正文（翻页链接常被清洗掉）
+                # cleaned_html 保留完整结构（用于翻页发现）
+                self._last_raw_html = result.cleaned_html or result.html or ""
+                html = result.fit_html or result.cleaned_html or ""
+                return self._html_to_text(html)
+            return ""
+        else:
+            # 无 crawl4ai 时 fallback 到 requests
+            resp = self._get_session().get(url, timeout=timeout / 1000)
+            if resp.status_code == 200:
+                self._last_raw_html = resp.text
+                return self._html_to_text(resp.text)
+            return ""
+
+    @staticmethod
+    def _html_to_text(html: str) -> str:
+        """简易 HTML 转纯文本"""
+        # 移除 script/style 标签内容
+        text = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL | re.IGNORECASE)
+        text = re.sub(r'<style[^>]*>.*?</style>', '', text, flags=re.DOTALL | re.IGNORECASE)
+        # 移除 HTML 标签
+        text = re.sub(r'<[^>]+>', ' ', text)
+        # 合并空白
+        text = re.sub(r'\s+', ' ', text).strip()
+        return text
+
+    # ============================================================
+    # 搜索笑话站点
+    # ============================================================
+
+    # 已知的笑话聚合站域名黑/白名单
+    JOKE_SITE_KEYWORDS = [
+        "笑话大全", "冷笑话", "搞笑段子", "笑话集锦",
+        "幽默笑话", "爆笑笑话", "成人笑话", "小笑话",
+    ]
+
+    async def search_joke_sites(self, keywords: list[str], max_results: int = 10) -> list[dict]:
+        """搜索笑话站点，返回 [{url, title, domain}]"""
+        all_results = []
+
+        for keyword in keywords:
+            results = await self._search_and_filter(keyword, max_results=max_results)
+            all_results.extend(results)
+
+        # 去重（按域名）
+        seen_domains = set()
+        unique = []
+        for r in all_results:
+            domain = r["domain"]
+            if domain not in seen_domains:
+                seen_domains.add(domain)
+                unique.append(r)
+
+        print(f"  [*] 搜索到 {len(unique)} 个唯一站点")
+        for s in unique:
+            print(f"      - {s['domain']}: {s['title'][:40]}")
+        return unique[:max_results]
+
+    async def _search_and_filter(self, keyword: str, max_results: int = 10) -> list[dict]:
+        """搜索并过滤出疑似笑话聚合站的结果"""
+        # 用多个搜索词提高覆盖率
+        search_queries = [
+            f"{keyword} 网站",
+            f"{keyword} 大全",
+            f"{keyword} 列表",
+        ]
+
+        seen = set()
+        sites = []
+
+        for q in search_queries:
+            if len(sites) >= max_results:
+                break
+
+            search_results = await self._search_bing(q, max_pages=2)
+
+            for r in search_results:
+                if len(sites) >= max_results:
+                    break
+
+                url = r["url"]
+                domain = urlparse(url).netloc.lower()
+
+                if domain in seen:
+                    continue
+                seen.add(domain)
+
+                # 过滤：排除已知的单篇文章站点和搜索引擎
+                if self._is_joke_collection_site(url, r.get("title", "")):
+                    r["domain"] = domain
+                    sites.append(r)
+
+        return sites
+
+    def _is_joke_collection_site(self, url: str, title: str) -> bool:
+        """判断URL是否疑似笑话聚合站（不是单篇文章）"""
+        domain = urlparse(url).netloc.lower()
+        path = urlparse(url).path.lower()
+
+        # 排除项
+        exclude_domains = [
+            "bing.com", "microsoft.com", "baidu.com", "google.com",
+            "sohu.com", "sina.com", "163.com", "qq.com", "toutiao.com",
+            "weibo.com", "zhihu.com", "bilibili.com", "douban.com",
+        ]
+        if any(d in domain for d in exclude_domains):
+            return False
+
+        # 排除明显的单篇文章模式
+        single_article_patterns = [
+            r'/p/\d+', r'/article/\d+', r'/post/\d+', r'/archives/\d+',
+            r'/a/\d+', r'/\d{5,}', r'/detail/\d+', r'/read/\d+',
+            r'\.html$',  # 静态 html 文章页
+        ]
+        # 但如果域名本身含 joke 特征，不排除
+        is_joke_domain = any(kw in domain or kw in title for kw in
+                             ["joke", "xiaohua", "笑话", "段子", "幽默", "搞笑"])
+
+        for p in single_article_patterns:
+            if re.search(p, path) and not is_joke_domain:
+                return False
+
+        # 聚合站特征：域名或标题含特定词，或URL有分类/列表模式
+        collection_patterns = [
+            "joke", "xiaohua", "笑话", "段子", "幽默", "搞笑",
+            "/page/", "/list/", "/category/", "/tag/", "joke",
+        ]
+        for p in collection_patterns:
+            if p in domain or p in path or p.lower() in title:
+                return True
+
+        # 有列表/目录模式的也认为是聚合站
+        if re.search(r'(page|list|category|tag|index)', path):
+            return True
+
+        return False
+
+    # ============================================================
+    # 翻页链接发现
+    # ============================================================
+
+    def discover_page_links(self, html: str, base_url: str) -> list[str]:
+        """从页面 HTML 中发现翻页链接，返回去重排序后的 URL 列表"""
+        # 优先使用 _last_raw_html（完整 HTML 而非纯文本），兜底用传入的 html
+        raw = self._last_raw_html or html
+        base_parsed = urlparse(base_url)
+        base_domain = f"{base_parsed.scheme}://{base_parsed.netloc}"
+
+        links = set()
+
+        # 1. <link rel="next">
+        for m in re.finditer(r'<link[^>]*rel="next"[^>]*href="([^"]+)"', raw, re.IGNORECASE):
+            links.add(urljoin(base_domain, m.group(1)))
+
+        # 2. 翻页文字链接（下一页、下页、>、» 等）
+        page_text_patterns = [
+            r'<a[^>]*href="([^"]*page[^"]*)"[^>]*>\s*(?:下一页|下页|下一页»|»|›|>|Next|last)\s*</a>',
+            r'<a[^>]*>\s*(?:下一页|下页|»|›|>)\s*</a>\s*<a[^>]*href="([^"]*)"',
+        ]
+        for pattern in page_text_patterns:
+            for m in re.finditer(pattern, raw, re.IGNORECASE):
+                href = m.group(1).strip()
+                if href and href not in ("#", "javascript:void(0)"):
+                    links.add(urljoin(base_url, href))
+
+        # 3. 提取所有带数字的翻页 link（?page=N, /page/N/, index_N.html, page_N.html, &page=N）
+        page_link_patterns = [
+            r'href="([^"]*[?&]page=(\d+)[^"]*)"',      # ?page=2  &page=2
+            r'href="([^"]*/page/(\d+)[^"]*)"',           # /page/2/
+            r'href="([^"]*[?&]p=(\d+)[^"]*)"',           # ?p=2
+            r'href="([^"]*[?&]pn=(\d+)[^"]*)"',          # ?pn=2
+            r'href="([^"]*[?&]offset=(\d+)[^"]*)"',      # ?offset=10
+            r'href="([^"]*[?&]start=(\d+)[^"]*)"',       # ?start=10
+            r'href="([^"]*[?&]page_index=(\d+)[^"]*)"',  # ?page_index=2
+            # 匹配 xxx_N.html / page_N.html / list_2.html
+            r'href="([^"]*(?:page|list|index)[-_]?(\d+)\.html?)"',
+            # 匹配 /page_N/ 格式
+            r'href="([^"]*/page[-_]?(\d+)/?)"',
+        ]
+        for pattern in page_link_patterns:
+            for m in re.finditer(pattern, raw, re.IGNORECASE):
+                full_url = urljoin(base_url, m.group(1))
+                links.add(full_url)
+
+        # 4. 翻页数字链接
+        for m in re.finditer(r'<a[^>]*href="([^"]*page=(\d+)[^"]*)"[^>]*>\s*\d+\s*</a>', raw, re.IGNORECASE):
+            links.add(urljoin(base_url, m.group(1)))
+
+        # 过滤：只保留同一域名下的链接
+        result = []
+        for link in links:
+            parsed = urlparse(link)
+            if parsed.netloc and parsed.netloc != base_parsed.netloc:
+                continue  # 跨域排除
+            if parsed.path == base_parsed.path and parsed.query == base_parsed.query:
+                continue  # 排除自身
+            result.append(link)
+
+        # 去重排序
+        return sorted(set(result))
+
+    def _extract_page_number(self, url: str) -> int:
+        """从 URL 中提取页码，用于排序"""
+        nums = re.findall(r'page[=/](\d+)|[?&]p=(\d+)|index[-_]?(\d+)|/page[-_]?(\d+)', url, re.IGNORECASE)
+        for n in nums:
+            for g in n:
+                if g:
+                    return int(g)
+        return 99  # 没识别到页码的排最后
+
+    # ============================================================
+    # 分类/标签链接发现
+    # ============================================================
+
+    def discover_category_links(self, html: str, base_url: str) -> list[str]:
+        """从页面 HTML 中发现分类/标签链接，返回去重排序后的 URL 列表"""
+        raw = self._last_raw_html or html
+        base_parsed = urlparse(base_url)
+        base_domain = f"{base_parsed.scheme}://{base_parsed.netloc}"
+
+        links = set()
+
+        # 1. 匹配分类链接（category-N.html, category-N_M.html, tag-N.html 等）
+        cat_patterns = [
+            r'href="([^"]*/(?:category|cat|sort|type)[-_]?\d+(?:[-_]\d+)?\.html?)"',
+            r'href="([^"]*/(?:tag|tags)/?[-_]?\d*)["\s>]',
+            r'href="([^"]*/tags?[-_]?\d+\.html?)"',
+        ]
+        for pattern in cat_patterns:
+            for m in re.finditer(pattern, raw, re.IGNORECASE):
+                full_url = urljoin(base_domain, m.group(1))
+                parsed = urlparse(full_url)
+                # 只保留同域链接
+                if parsed.netloc and parsed.netloc != base_parsed.netloc:
+                    continue
+                links.add(full_url)
+
+        # 2. 过滤：排除单篇文章、首页、搜索页
+        single_article = re.compile(
+            r'/(?:p|post|article|archives|detail|read|xiaohua)/\d+',
+            re.IGNORECASE,
+        )
+        result = []
+        for link in links:
+            parsed = urlparse(link)
+            path = parsed.path.rstrip("/")
+            # 排除自身
+            if path == base_parsed.path.rstrip("/") and parsed.query == base_parsed.query:
+                continue
+            # 排除单篇文章
+            if single_article.search(path):
+                continue
+            # 排除明显的非分类路径（首页翻页）
+            if re.search(r'/page[-_]?\d+\.html?$', path) and 'category' not in path and 'tag' not in path:
+                continue
+            result.append(link)
+
+        return sorted(set(result))
+
+    # ============================================================
+    # Bing 搜索（复用旧逻辑）
+    # ============================================================
+
+    async def _search_bing(self, keyword: str, max_pages: int = 3) -> list[dict]:
+        """搜索 Bing，返回结果 URL 列表"""
+        results = []
+        session = self._get_session()
+
+        for page in range(max_pages):
+            first = page * 10
+            url = f"https://www.bing.com/search?q={quote(keyword)}&first={first}"
+            print(f"  [*] Bing 搜索: {keyword[:20]}")
+
+            html = None
+            try:
+                if HAS_CRAWL4AI:
+                    crawler = await self._get_crawler()
+                    result = await crawler.arun(url, config=CrawlerRunConfig(verbose=False))
+                    if result.success:
+                        html = result.html if result.html else result.cleaned_html
+                else:
+                    resp = session.get(url)
+                    html = resp.text if resp.status_code == 200 else None
+
+                if html:
+                    urls = self._extract_bing_urls(html)
+                    for item in urls:
+                        item["keyword"] = keyword
+                        results.append(item)
+                else:
+                    print(f"  [!] 获取搜索页面失败")
+
+                await asyncio.sleep(2)
+
+            except Exception as e:
+                print(f"  [!] 搜索异常: {e}")
+                continue
+
+        return results
+
+    def _extract_bing_urls(self, html: str) -> list[dict]:
+        """从 Bing 搜索结果 HTML 中提取链接和标题"""
+        results = []
+        pattern = re.compile(
+            r'<h2[^>]*>\s*<a[^>]*href="(https?[^"]+)"[^>]*>(.*?)</a>',
+            re.DOTALL,
+        )
+        for match in pattern.finditer(html):
+            href = match.group(1).strip()
+            title = re.sub(r'<[^>]+>', '', match.group(2)).strip()
+            if href and title and len(title) > 5 and "bing.com" not in href and "microsoft.com" not in href:
+                results.append({"url": href, "title": title})
+        return results
+
+    # 别名兼容
+    async def search_bing(self, keyword: str, max_pages: int = 3) -> list[dict]:
+        return await self._search_bing(keyword, max_pages)
+
+    async def search(self, keyword: str, max_pages: int = 3) -> list[dict]:
+        return await self._search_bing(keyword, max_pages)
+
+    async def search_baidu(self, keyword: str, max_pages: int = 3) -> list[dict]:
+        return await self._search_bing(keyword, max_pages)
+
+    # ============================================================
+    # 旧接口兼容（单页抓取）
+    # ============================================================
+
+    async def crawl_page(self, url: str) -> str:
+        """抓取单个页面（兼容旧接口）"""
+        content, ok = await self.crawl_page_with_retry(url)
+        if ok:
+            return content[:8000]
+        return ""
+
+    async def crawl_batch(self, urls: list[str]) -> list[tuple[str, str]]:
+        """批量抓取（兼容旧接口）"""
+        contents = []
+        for url in urls:
+            content, ok = await self.crawl_page_with_retry(url)
+            contents.append((url, content[:8000] if ok else ""))
+            await asyncio.sleep(1.5)
+        return contents

crawler/main.py

@@ -0,0 +1,85 @@
+"""
+爬虫持续采集入口（深度采集模式）。
+流程：搜索笑话站点 → 翻页深度采集 → AI 提取 → 入库。
+每批采集 20 条后休息 2-10 分钟，出错休息 3-11 分钟后继续。
+按 Ctrl+C 中断。
+
+用法:
+  python crawler/main.py                     # 默认，无头浏览器
+  python crawler/main.py --no-headless       # 显示浏览器窗口（方便测试）
+"""
+import argparse
+import asyncio
+import os
+import sys
+
+if sys.platform == "win32":
+    import shutil
+    os.environ["PYTHONIOENCODING"] = "utf-8"
+    os.environ["TERM"] = "dumb"
+    try:
+        shutil.get_terminal_size()
+    except Exception:
+        pass
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from crawler.processor import Processor
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="笑话爬虫 - 深度采集模式")
+    parser.add_argument(
+        "--no-headless",
+        action="store_true",
+        help="显示浏览器窗口（默认无头模式）",
+    )
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+    headless = not args.no_headless
+
+    api_base = os.getenv("API_BASE", "http://localhost:8001")
+    username = os.getenv("CRAWL_USERNAME", "admin")
+    password = os.getenv("CRAWL_PASSWORD", "admin123")
+    keyword_str = os.getenv("CRAWL_KEYWORD", "笑话大全,搞笑段子,冷笑话,幽默笑话,爆笑笑话,笑话集锦")
+    keywords = [k.strip() for k in keyword_str.split(",") if k.strip()]
+
+    if not keywords:
+        print("错误: 未设置关键词")
+        return
+
+    print(f"=" * 50)
+    print(f"笑话爬虫 - 深度采集模式")
+    print(f"搜索词: {keywords}")
+    print(f"API 地址: {api_base}")
+    print(f"浏览器: {'显示窗口' if not headless else '无头模式'}")
+    print(f"每批 20 条后休息 2-10 分钟")
+    print(f"出错后休息 3-11 分钟后重试")
+    print(f"按 Ctrl+C 终止")
+    print(f"=" * 50)
+
+    try:
+        import crawl4ai
+        print(f"crawl4ai 版本: {crawl4ai.__version__}")
+    except ImportError:
+        print("错误: crawl4ai 未安装，请先运行: pip install crawl4ai")
+        return
+
+    processor = Processor(
+        api_base=api_base,
+        username=username,
+        password=password,
+        headless=headless,
+    )
+
+    try:
+        asyncio.run(processor.run_continuous(keywords=keywords, max_pages=3, batch_size=20))
+    except KeyboardInterrupt:
+        print("\n用户中断，退出")
+
+
+if __name__ == "__main__":
+    main()

crawler/processor.py

@@ -0,0 +1,470 @@
+"""爬虫流程编排：搜索笑话站点 → 深度翻页采集 → AI 提取 → 入库"""
+
+import asyncio
+import hashlib
+import random
+import re
+import httpx
+from urllib.parse import urljoin, urlparse
+
+from crawler.ai_service import AiService
+from crawler.crawler_service import CrawlerService
+
+
+class Processor:
+    def __init__(self, api_base: str, username: str, password: str, headless: bool = True):
+        self.api_base = api_base.rstrip("/")
+        self.username = username
+        self.password = password
+        self.token = None
+        self.ai = None
+        self.crawler = CrawlerService(headless=headless)
+        self.types = []
+        self.crowds = []
+
+    # === API 认证 ===
+    def _login(self) -> str:
+        resp = httpx.post(
+            f"{self.api_base}/api/auth/login",
+            json={"username": self.username, "password": self.password},
+            timeout=30,
+        )
+        resp.raise_for_status()
+        return resp.json()["access_token"]
+
+    def _get(self, path: str) -> dict | list:
+        resp = httpx.get(
+            f"{self.api_base}{path}",
+            headers={"Authorization": f"Bearer {self.token}"},
+            timeout=30,
+        )
+        resp.raise_for_status()
+        return resp.json()
+
+    def _post(self, path: str, data: dict) -> dict:
+        resp = httpx.post(
+            f"{self.api_base}{path}",
+            json=data,
+            headers={"Authorization": f"Bearer {self.token}"},
+            timeout=30,
+        )
+        resp.raise_for_status()
+        return resp.json()
+
+    # === 初始化 ===
+    def setup(self):
+        print("[*] 正在登录...")
+        self.token = self._login()
+        print("[*] 登录成功")
+
+        ai_config = self._get("/api/admin/settings/active")
+        self.ai = AiService(
+            api_base=ai_config["api_base"],
+            api_key=ai_config["api_key"],
+            model_name=ai_config["model_name"],
+            temperature=ai_config.get("temperature", 0.7),
+            max_tokens=ai_config.get("max_tokens", 2048),
+        )
+        print(f"[*] AI 配置: {ai_config['model_name']}")
+
+        self.types = self._get("/api/categories/types")
+        self.crowds = self._get("/api/categories/crowds")
+        print(f"[*] 分类: {len(self.types)} 种类型, {len(self.crowds)} 种人群")
+
+    def _get_existing_hashes(self) -> set[str]:
+        """获取库里已有笑话的 content hash，用于去重"""
+        try:
+            data = self._get("/api/admin/jokes?page=1&page_size=1000")
+            hashes = set()
+            for j in data.get("items", []):
+                content = j.get("content", "")
+                if content:
+                    hashes.add(hashlib.md5(content.encode()).hexdigest())
+            return hashes
+        except Exception:
+            return set()
+
+    def _submit_joke(self, joke: dict) -> bool:
+        """提交单条笑话至 API（status=pending 待审核）"""
+        try:
+            # Support both old format (type/crowd) and new format (types/crowds arrays)
+            type_names = joke.get("types", [joke.get("type", "")])
+            crowd_names = joke.get("crowds", [joke.get("crowd", "")])
+            if isinstance(type_names, str):
+                type_names = [type_names] if type_names else []
+            if isinstance(crowd_names, str):
+                crowd_names = [crowd_names] if crowd_names else []
+
+            type_ids = []
+            crowd_ids = []
+            for n in type_names:
+                for t in self.types:
+                    if t.get("name") == n:
+                        type_ids.append(t.get("id"))
+                        break
+            for n in crowd_names:
+                for c in self.crowds:
+                    if c.get("name") == n:
+                        crowd_ids.append(c.get("id"))
+                        break
+
+            payload = {
+                "title": joke.get("title", "无标题"),
+                "content": joke.get("content", ""),
+                "type_ids": type_ids if type_ids else None,
+                "crowd_ids": crowd_ids if crowd_ids else None,
+                "status": "pending",
+            }
+            self._post("/api/admin/jokes", payload)
+            return True
+        except Exception as e:
+            print(f"  [!] 提交失败: {e}")
+            return False
+
+    # === 主循环 ===
+
+    async def run_continuous(self, keywords: list[str], max_pages: int = 3, batch_size: int = 20):
+        """持续深度采集循环"""
+        print(f"\n>> 深度采集模式启动")
+        print(f"    搜索词: {keywords}")
+        print(f"    每批目标: {batch_size} 条")
+        print(f"    API: {self.api_base}")
+        print(f"    按 Ctrl+C 中断\n")
+
+        self.setup()
+
+        existing_hashes = self._get_existing_hashes()
+        print(f"[*] 当前库中已有 {len(existing_hashes)} 条笑话（用于去重）")
+
+        type_names = [t.get("name", "") for t in self.types]
+        crowd_names = [c.get("name", "") for c in self.crowds]
+        total_saved = 0
+        visited_urls: set[str] = set()
+
+        try:
+            while True:
+                batch_saved = 0
+                print(f"\n{'='*50}")
+                print(f"  开始新一批深度采集 (已累计 {total_saved} 条)")
+                print(f"{'='*50}")
+
+                try:
+                    # Step 1: 搜索笑话站点
+                    print(f"\n[*] 搜索笑话站点...")
+                    sites = await self.crawler.search_joke_sites(keywords, max_results=8)
+
+                    if not sites:
+                        print(f"  [!] 未找到笑话站点，休息后重试")
+                        rest = random.randint(120, 600)
+                        print(f"  休息 {rest//60} 分 {rest%60} 秒...")
+                        await asyncio.sleep(rest)
+                        continue
+
+                    # Step 2: 逐个站点深度采集
+                    for site in sites:
+                        if batch_saved >= batch_size:
+                            break
+
+                        domain = site["domain"]
+                        site_url = site["url"]
+
+                        print(f"\n{'─'*40}")
+                        print(f"  开始采集站点: {domain}")
+                        print(f"{'─'*40}")
+
+                        # 检查该站点失败次数
+                        fail_count = self.crawler.site_failures.get(domain, 0)
+                        if fail_count >= 3:
+                            print(f"  [!] 站点 {domain} 已连续失败 {fail_count} 次，跳过")
+                            continue
+
+                        saved_from_site = await self._crawl_site_deep(
+                            site_url=site_url,
+                            domain=domain,
+                            type_names=type_names,
+                            crowd_names=crowd_names,
+                            existing_hashes=existing_hashes,
+                            visited_urls=visited_urls,
+                            target=batch_size - batch_saved,
+                        )
+                        batch_saved += saved_from_site
+                        total_saved += saved_from_site
+
+                    # Step 3: 休息
+                    rest = random.randint(120, 600)
+                    print(f"\n[OK] 本批入库 {batch_saved} 条，休息 {rest//60} 分 {rest%60} 秒...")
+                    print(f"     按 Ctrl+C 中断\n")
+
+                except Exception as e:
+                    rest = random.randint(180, 660)
+                    print(f"\n[!] 出错: {e}")
+                    print(f"    休息 {rest//60} 分 {rest%60} 秒后重试...")
+
+                await asyncio.sleep(rest)
+
+        except asyncio.CancelledError:
+            print("\n用户中断，退出")
+        finally:
+            await self.crawler.close()
+
+    async def _crawl_site_deep(
+        self,
+        site_url: str,
+        domain: str,
+        type_names: list[str],
+        crowd_names: list[str],
+        existing_hashes: set[str],
+        visited_urls: set[str],
+        target: int,
+    ) -> int:
+        """深度采集一个站点：抓取首页 → 发现翻页链接 → 逐页抓取提取笑话"""
+        saved = 0
+        pages_to_crawl = []
+
+        # 1. 抓取首页
+        print(f"  [*] 抓取首页: {site_url[:60]}")
+        html, ok = await self.crawler.crawl_page_with_retry(site_url)
+        if not ok:
+            self.crawler.site_failures[domain] = self.crawler.site_failures.get(domain, 0) + 1
+            print(f"  [!] 首页抓取失败 ({self.crawler.site_failures[domain]}/3)")
+            return 0
+
+        visited_urls.add(site_url)
+
+        # 2. 从首页提取笑话
+        try:
+            jokes = self.ai.extract_jokes(html[:8000], type_names, crowd_names)
+            saved += self._save_jokes(jokes, existing_hashes, target - saved)
+            print(f"  [+] 首页提取 {len(jokes)} 条，入库 {saved} 条")
+        except Exception as e:
+            print(f"  [!] 首页 AI 提取失败: {e}")
+
+        if saved >= target:
+            self.crawler.site_failures[domain] = 0
+            return saved
+
+        # 3. 发现翻页链接
+        page_links = self.crawler.discover_page_links(html, site_url)
+        # 过滤已访问的链接
+        page_links = [l for l in page_links if l not in visited_urls]
+        # 按页码排序
+        page_links.sort(key=lambda l: self.crawler._extract_page_number(l))
+
+        # 限制翻页深度，避免无限抓取
+        max_pages_per_site = 30
+        page_links = page_links[:max_pages_per_site]
+        print(f"  [*] 发现 {len(page_links)} 个翻页链接，开始逐页采集...")
+
+        # 4. 逐页翻页采集
+        for idx, page_url in enumerate(page_links):
+            if saved >= target:
+                break
+
+            # 检查站点是否已失效
+            if self.crawler.site_failures.get(domain, 0) >= 3:
+                print(f"  [!] 站点 {domain} 失败过多，跳过")
+                break
+
+            print(f"  [*] 翻页 {idx+1}/{len(page_links)}: {page_url[:60]}")
+
+            html, ok = await self.crawler.crawl_page_with_retry(page_url)
+            visited_urls.add(page_url)
+
+            if not ok:
+                self.crawler.site_failures[domain] = self.crawler.site_failures.get(domain, 0) + 1
+                print(f"  [!] 抓取失败 ({self.crawler.site_failures[domain]}/3)")
+                continue
+
+            # 重置失败计数
+            self.crawler.site_failures[domain] = 0
+
+            try:
+                jokes = self.ai.extract_jokes(html[:8000], type_names, crowd_names)
+                new_saved = self._save_jokes(jokes, existing_hashes, target - saved)
+                if new_saved > 0:
+                    saved += new_saved
+                    print(f"  [+] 提取 {len(jokes)} 条，入库 {new_saved} 条 (累计 {saved}/{target})")
+                else:
+                    print(f"  [*] 提取 {len(jokes)} 条（均为重复）")
+            except Exception as e:
+                print(f"  [!] AI 提取失败: {e}")
+
+            await asyncio.sleep(random.uniform(1, 3))
+
+        self.crawler.site_failures[domain] = 0
+
+        # 5. 发现分类链接并逐个深度采集
+        cat_links = self.crawler.discover_category_links(html, site_url)
+        cat_links = [l for l in cat_links if l not in visited_urls]
+        # 限制分类数量
+        max_categories = 20
+        cat_links = cat_links[:max_categories]
+        if cat_links:
+            print(f"  [*] 发现 {len(cat_links)} 个分类链接，开始逐类采集...")
+            for cat_url in cat_links:
+                if saved >= target:
+                    break
+                if self.crawler.site_failures.get(domain, 0) >= 3:
+                    print(f"  [!] 站点 {domain} 失败过多，跳过分类")
+                    break
+
+                saved += await self._crawl_category(
+                    cat_url=cat_url,
+                    domain=domain,
+                    type_names=type_names,
+                    crowd_names=crowd_names,
+                    existing_hashes=existing_hashes,
+                    visited_urls=visited_urls,
+                    target=target - saved,
+                )
+
+        print(f"  [*] 站点 {domain} 采集完成，共入库 {saved} 条")
+        return saved
+
+    async def _crawl_category(
+        self,
+        cat_url: str,
+        domain: str,
+        type_names: list[str],
+        crowd_names: list[str],
+        existing_hashes: set[str],
+        visited_urls: set[str],
+        target: int,
+    ) -> int:
+        """深度采集一个分类页及其翻页"""
+        saved = 0
+        cat_name = cat_url.split("/")[-1].split(".")[0]
+        print(f"\n  {'─'*36}")
+        print(f"  分类采集 [{cat_name}]: {cat_url[:60]}")
+        print(f"  {'─'*36}")
+
+        # 1. 抓取分类首页
+        html, ok = await self.crawler.crawl_page_with_retry(cat_url)
+        visited_urls.add(cat_url)
+        if not ok:
+            self.crawler.site_failures[domain] = self.crawler.site_failures.get(domain, 0) + 1
+            print(f"  [!] 分类首页抓取失败 ({self.crawler.site_failures[domain]}/3)")
+            return 0
+
+        # 2. AI 提取笑话
+        try:
+            jokes = self.ai.extract_jokes(html[:8000], type_names, crowd_names)
+            saved += self._save_jokes(jokes, existing_hashes, target - saved)
+            print(f"  [+] 分类首页提取 {len(jokes)} 条，入库 {saved} 条")
+        except Exception as e:
+            print(f"  [!] 分类首页 AI 提取失败: {e}")
+
+        if saved >= target:
+            return saved
+
+        # 3. 发现该分类的翻页链接
+        # 先尝试通用翻页模式，再尝试分类特定翻页（category-5_2.html）
+        page_links = self.crawler.discover_page_links(html, cat_url)
+        # 从当前分类 URL 派生出分类翻页模式（e.g. category-5 → category-5_2.html）
+        cat_base = cat_url.split("/")[-1].replace(".html", "")
+        cat_page_pattern = re.compile(
+            rf'href="([^"]*{re.escape(cat_base)}[-_]?(\d+)\.html?)"',
+            re.IGNORECASE,
+        )
+        raw = self.crawler._last_raw_html or html
+        for m in cat_page_pattern.finditer(raw):
+            full_url = urljoin(cat_url, m.group(1))
+            page_links.append(full_url)
+
+        page_links = [l for l in page_links if l not in visited_urls]
+        page_links = list(set(page_links))  # 去重
+        page_links.sort(key=lambda l: self.crawler._extract_page_number(l))
+
+        # 限制翻页深度
+        max_pages_per_cat = 20
+        page_links = page_links[:max_pages_per_cat]
+        if page_links:
+            print(f"  [*] 发现 {len(page_links)} 个翻页链接，开始逐页采集...")
+
+        # 4. 逐页抓取
+        for idx, page_url in enumerate(page_links):
+            if saved >= target:
+                break
+            if self.crawler.site_failures.get(domain, 0) >= 3:
+                print(f"  [!] 站点 {domain} 失败过多，跳过本分类")
+                break
+
+            print(f"  [*] 翻页 {idx+1}/{len(page_links)}: {page_url[:60]}")
+
+            html, ok = await self.crawler.crawl_page_with_retry(page_url)
+            visited_urls.add(page_url)
+
+            if not ok:
+                self.crawler.site_failures[domain] = self.crawler.site_failures.get(domain, 0) + 1
+                print(f"  [!] 分类翻页抓取失败 ({self.crawler.site_failures[domain]}/3)")
+                continue
+
+            self.crawler.site_failures[domain] = 0
+
+            try:
+                jokes = self.ai.extract_jokes(html[:8000], type_names, crowd_names)
+                new_saved = self._save_jokes(jokes, existing_hashes, target - saved)
+                if new_saved > 0:
+                    saved += new_saved
+                    print(f"  [+] 提取 {len(jokes)} 条，入库 {new_saved} 条 (累计 {saved}/{target})")
+                else:
+                    print(f"  [*] 提取 {len(jokes)} 条（均为重复）")
+            except Exception as e:
+                print(f"  [!] AI 提取失败: {e}")
+
+            await asyncio.sleep(random.uniform(1, 3))
+
+        print(f"  [*] 分类 [{cat_name}] 采集完成，入库 {saved} 条")
+        return saved
+
+    def _save_jokes(self, jokes: list[dict], existing_hashes: set[str], limit: int) -> int:
+        """去重并入库笑话，返回成功入库数"""
+        saved = 0
+        for joke in jokes:
+            if saved >= limit:
+                break
+            content_text = joke.get("content", "")
+            if not content_text:
+                continue
+            h = hashlib.md5(content_text.encode()).hexdigest()
+            if h in existing_hashes:
+                continue
+            existing_hashes.add(h)
+            if self._submit_joke(joke):
+                saved += 1
+                print(f"  [+] 入库: {joke.get('title', '')[:30]}")
+        return saved
+
+    # === 单站点采集（供 site_crawler.py 调用） ===
+
+    async def crawl_site(self, site_url: str, batch_size: int = 9999):
+        """初始化后深度采集单个站点"""
+        print(f"\n>> 单站点采集: {site_url}\n")
+        self.setup()
+
+        existing_hashes = self._get_existing_hashes()
+        print(f"[*] 当前库中已有 {len(existing_hashes)} 条笑话（用于去重）")
+
+        type_names = [t.get("name", "") for t in self.types]
+        crowd_names = [c.get("name", "") for c in self.crowds]
+        domain = urlparse(site_url).netloc.lower()
+        visited_urls: set[str] = set()
+
+        try:
+            saved = await self._crawl_site_deep(
+                site_url=site_url,
+                domain=domain,
+                type_names=type_names,
+                crowd_names=crowd_names,
+                existing_hashes=existing_hashes,
+                visited_urls=visited_urls,
+                target=batch_size,
+            )
+            print(f"\n[OK] 站点采集完成，共入库 {saved} 条笑话")
+        finally:
+            await self.crawler.close()
+
+    # === 旧接口兼容 ===
+    async def run(self, keywords: list[str], max_pages: int = 3):
+        """单轮爬取（旧接口，内部调用 run_continuous）"""
+        await self.run_continuous(keywords, max_pages, batch_size=9999)

crawler/prompts.py

@@ -0,0 +1,41 @@
+"""AI 提示词模板。"""
+
+# ===== 笑话提取 =====
+EXTRACTION_SYSTEM_PROMPT = """你是一个笑话提取专家。从给定的网页文本中识别并提取所有笑话、幽默段子或有趣内容。
+要求：
+1. 只返回真正的笑话内容，不要提取普通文章或新闻
+2. 每条笑话需要包含：title（简短标题）、content（完整笑话内容）、type（类型）、crowd（人群）
+3. 如果网页中没有笑话，返回空数组 []
+4. 永远返回合法的 JSON 格式，根节点为数组或包含 jokes 键的对象"""
+
+EXTRACTION_USER_PROMPT = """网页内容：
+---
+{page_content}
+---
+
+已知笑话类型：{known_types}
+已知人群分类：{known_crowds}
+
+请提取所有笑话，以 JSON 格式返回，示例：
+[
+  {{"title": "程序员的幽默", "content": "程序员去相亲...", "types": ["谐音梗", "段子"], "crowds": ["职场", "大学生"]}},
+  {{"title": "...", "content": "...", "types": ["..."], "crowds": ["..."]}}
+]
+
+注意：types 和 crowds 是数组，可以填多个。
+只返回 JSON，不要其他文字。"""
+
+
+# ===== 笑话改写 =====
+REWRITE_SYSTEM_PROMPT = """你是一个幽默作家，负责润色和改写笑话。
+要求：
+1. 保持笑话的核心笑点不变
+2. 语言更通顺、更幽默
+3. 字数控制在原内容的 80%-120% 之间
+4. 不要添加任何解释说明"""
+
+REWRITE_USER_PROMPT = """请润色以下笑话：
+
+{content}
+
+只返回润色后的笑话文字，不要其他内容。"""

crawler/site_crawler.py

@@ -0,0 +1,165 @@
+"""
+单站点深度采集工具。
+从 JSON 文件按编号加载站点，或直接指定 URL，深度采集该站点所有笑话。
+
+用法:
+  python crawler/site_crawler.py --id 1                    # 采集 site_finder 发现的 #1 站点
+  python crawler/site_crawler.py --id 1,2,3               # 批量采集多个站点
+  python crawler/site_crawler.py --url https://...         # 直接采集指定 URL
+  python crawler/site_crawler.py --id 1 --no-headless      # 显示浏览器窗口
+"""
+import argparse
+import asyncio
+import json
+import os
+import sys
+
+if sys.platform == "win32":
+    import shutil
+    os.environ["PYTHONIOENCODING"] = "utf-8"
+    os.environ["TERM"] = "dumb"
+    try:
+        shutil.get_terminal_size()
+    except Exception:
+        pass
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from crawler.processor import Processor
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="单站点深度采集工具")
+    parser.add_argument(
+        "--id",
+        type=str,
+        default=None,
+        help="站点编号（从 site_finder 生成的 JSON 读取），多个用逗号分隔",
+    )
+    parser.add_argument(
+        "--url",
+        type=str,
+        default=None,
+        help="直接指定站点 URL（与 --id 二选一）",
+    )
+    parser.add_argument(
+        "--sites-file",
+        type=str,
+        default="joke_sites.json",
+        help="站点列表 JSON 文件路径（默认: joke_sites.json）",
+    )
+    parser.add_argument(
+        "--api-base",
+        type=str,
+        default=os.getenv("API_BASE", "http://localhost:8001"),
+        help="API 服务地址",
+    )
+    parser.add_argument(
+        "--username",
+        type=str,
+        default=os.getenv("CRAWL_USERNAME", "admin"),
+        help="管理员用户名",
+    )
+    parser.add_argument(
+        "--password",
+        type=str,
+        default=os.getenv("CRAWL_PASSWORD", "admin123"),
+        help="管理员密码",
+    )
+    parser.add_argument(
+        "--no-headless",
+        action="store_true",
+        help="显示浏览器窗口",
+    )
+    return parser.parse_args()
+
+
+def load_sites(path: str) -> list[dict]:
+    """从 JSON 文件加载站点列表"""
+    if not os.path.exists(path):
+        print(f"错误: 站点文件 {path} 不存在，请先运行 site_finder.py")
+        sys.exit(1)
+    try:
+        with open(path, "r", encoding="utf-8") as f:
+            data = json.load(f)
+        if not data:
+            print(f"错误: 站点文件 {path} 为空")
+            sys.exit(1)
+        return data
+    except Exception as e:
+        print(f"错误: 读取站点文件失败: {e}")
+        sys.exit(1)
+
+
+def resolve_sites(args) -> list[str]:
+    """解析 --id 或 --url 参数，返回待采集的 URL 列表"""
+    if args.url:
+        return [args.url]
+
+    if not args.id:
+        print("错误: 请指定 --id 或 --url")
+        print("  例如: python crawler/site_crawler.py --id 1")
+        print("  例如: python crawler/site_crawler.py --url https://xiaohua.com")
+        sys.exit(1)
+
+    # 解析编号列表 "1,2,3" → [1, 2, 3]
+    try:
+        ids = [int(x.strip()) for x in args.id.split(",") if x.strip()]
+    except ValueError:
+        print("错误: --id 参数必须是数字，多个用逗号分隔")
+        sys.exit(1)
+
+    sites = load_sites(args.sites_file)
+    found = []
+    for sid in ids:
+        match = [s for s in sites if s["id"] == sid]
+        if match:
+            found.append(match[0])
+            print(f"  [*] 站点 #{sid}: {match[0]['domain']} — {match[0]['title'][:40]}")
+        else:
+            print(f"  [!] 站点 #{sid} 未找到（可用编号: {[s['id'] for s in sites[:10]]}...)")
+
+    if not found:
+        print("错误: 没有找到有效的站点编号")
+        sys.exit(1)
+
+    return [s["url"] for s in found]
+
+
+def main():
+    args = parse_args()
+    headless = not args.no_headless
+    urls = resolve_sites(args)
+
+    print(f"=" * 50)
+    print(f"单站点深度采集工具")
+    print(f"目标站点: {len(urls)} 个")
+    for u in urls:
+        print(f"  - {u}")
+    print(f"API 地址: {args.api_base}")
+    print(f"浏览器: {'显示窗口' if not headless else '无头模式'}")
+    print(f"=" * 50)
+
+    try:
+        import crawl4ai
+        print(f"crawl4ai 版本: {crawl4ai.__version__}")
+    except ImportError:
+        print("错误: crawl4ai 未安装，请先运行: pip install crawl4ai")
+        return
+
+    processor = Processor(
+        api_base=args.api_base,
+        username=args.username,
+        password=args.password,
+        headless=headless,
+    )
+
+    try:
+        for url in urls:
+            asyncio.run(processor.crawl_site(url))
+    except KeyboardInterrupt:
+        print("\n用户中断，退出")
+
+
+if __name__ == "__main__":
+    main()

crawler/site_finder.py

@@ -0,0 +1,128 @@
+"""
+笑话站点发现工具。
+搜索 Bing 获取笑话聚合网站列表，保存到 JSON 文件。
+
+用法:
+  python crawler/site_finder.py
+  python crawler/site_finder.py --keywords "笑话大全,冷笑话" --output my_sites.json
+  python crawler/site_finder.py --no-headless
+"""
+import argparse
+import asyncio
+import json
+import os
+import sys
+from datetime import datetime
+
+if sys.platform == "win32":
+    import shutil
+    os.environ["PYTHONIOENCODING"] = "utf-8"
+    os.environ["TERM"] = "dumb"
+    try:
+        shutil.get_terminal_size()
+    except Exception:
+        pass
+
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from crawler.crawler_service import CrawlerService
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="笑话站点发现工具")
+    parser.add_argument(
+        "--keywords",
+        type=str,
+        default=os.getenv("CRAWL_KEYWORD", "笑话大全,搞笑段子,冷笑话,幽默笑话,爆笑笑话,笑话集锦"),
+        help="搜索关键词，逗号分隔",
+    )
+    parser.add_argument(
+        "--output",
+        type=str,
+        default="joke_sites.json",
+        help="输出 JSON 文件路径（默认: joke_sites.json）",
+    )
+    parser.add_argument(
+        "--max-sites",
+        type=int,
+        default=15,
+        help="最多保留几个站点（默认: 15）",
+    )
+    parser.add_argument(
+        "--no-headless",
+        action="store_true",
+        help="显示浏览器窗口",
+    )
+    return parser.parse_args()
+
+
+def load_existing_sites(path: str) -> list[dict]:
+    """加载已有站点列表"""
+    if os.path.exists(path):
+        try:
+            with open(path, "r", encoding="utf-8") as f:
+                return json.load(f)
+        except Exception:
+            pass
+    return []
+
+
+def save_sites(path: str, sites: list[dict]):
+    """保存站点列表到 JSON 文件"""
+    with open(path, "w", encoding="utf-8") as f:
+        json.dump(sites, f, ensure_ascii=False, indent=2)
+    print(f"\n[OK] 已保存 {len(sites)} 个站点到 {path}")
+
+
+def main():
+    args = parse_args()
+    headless = not args.no_headless
+    keywords = [k.strip() for k in args.keywords.split(",") if k.strip()]
+
+    print(f"=" * 50)
+    print(f"笑话站点发现工具")
+    print(f"搜索词: {keywords}")
+    print(f"最大站点数: {args.max_sites}")
+    print(f"输出文件: {args.output}")
+    print(f"浏览器: {'显示窗口' if not headless else '无头模式'}")
+    print(f"=" * 50)
+
+    # 加载已存在的站点（保留已有编号）
+    existing = load_existing_sites(args.output)
+    existing_domains = {s["domain"] for s in existing}
+    next_id = max([s["id"] for s in existing], default=0) + 1
+    print(f"[*] 已有 {len(existing)} 个站点记录，新编号从 {next_id} 开始")
+
+    async def run():
+        nonlocal next_id
+        crawler = CrawlerService(headless=headless)
+        try:
+            new_sites = await crawler.search_joke_sites(keywords, max_results=args.max_sites)
+        finally:
+            await crawler.close()
+
+        # 合并新旧站点（去重）
+        added = 0
+        for site in new_sites:
+            domain = site["domain"]
+            if domain not in existing_domains:
+                site["id"] = next_id
+                site["found_at"] = datetime.now().isoformat(timespec="seconds")
+                existing.append(site)
+                existing_domains.add(domain)
+                next_id += 1
+                added += 1
+                print(f"  [+] 新增 #{site['id']}: {site['domain']} — {site['title'][:40]}")
+
+        save_sites(args.output, existing)
+
+        if existing:
+            print(f"\n站点列表：")
+            for s in existing:
+                print(f"  #{s['id']:2d}  {s['domain']:30s} {s['title'][:35]}")
+
+    asyncio.run(run())
+
+
+if __name__ == "__main__":
+    main()