bwstudio
ceed63fcb0
High priority: - Fix concurrent race condition for view_count/like_count (atomic update) - Add route request ID tracking to prevent race conditions - Filter get_joke by status=approved (no pending content leak) - Add error feedback for like button Performance: - Optimize random joke query (avoid full table sort) - Limit page_size max to 100 (DoS prevention) Medium: - Add localStorage quota error handling - Handle empty AI response gracefully - Fix generate content title extraction Low: - Add rejected_jokes to stats API - Update dashboard to show rejected count
128 lines
3.8 KiB
Python
128 lines
3.8 KiB
Python
"""
|
||
笑话站点发现工具。
|
||
搜索 Bing 获取笑话聚合网站列表，保存到 JSON 文件。
|
||
|
||
用法:
|
||
python crawler/site_finder.py
|
||
python crawler/site_finder.py --keywords "笑话大全,冷笑话" --output my_sites.json
|
||
python crawler/site_finder.py --no-headless
|
||
"""
|
||
import argparse
|
||
import asyncio
|
||
import json
|
||
import os
|
||
import sys
|
||
from datetime import datetime
|
||
|
||
if sys.platform == "win32":
|
||
import shutil
|
||
os.environ["PYTHONIOENCODING"] = "utf-8"
|
||
os.environ["TERM"] = "dumb"
|
||
try:
|
||
shutil.get_terminal_size()
|
||
except Exception:
|
||
pass
|
||
|
||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||
|
||
from crawler.crawler_service import CrawlerService
|
||
|
||
|
||
def parse_args():
|
||
parser = argparse.ArgumentParser(description="笑话站点发现工具")
|
||
parser.add_argument(
|
||
"--keywords",
|
||
type=str,
|
||
default=os.getenv("CRAWL_KEYWORD", "笑话大全,搞笑段子,冷笑话,幽默笑话,爆笑笑话,笑话集锦"),
|
||
help="搜索关键词，逗号分隔",
|
||
)
|
||
parser.add_argument(
|
||
"--output",
|
||
type=str,
|
||
default="joke_sites.json",
|
||
help="输出 JSON 文件路径（默认: joke_sites.json）",
|
||
)
|
||
parser.add_argument(
|
||
"--max-sites",
|
||
type=int,
|
||
default=15,
|
||
help="最多保留几个站点（默认: 15）",
|
||
)
|
||
parser.add_argument(
|
||
"--no-headless",
|
||
action="store_true",
|
||
help="显示浏览器窗口",
|
||
)
|
||
return parser.parse_args()
|
||
|
||
|
||
def load_existing_sites(path: str) -> list[dict]:
|
||
"""加载已有站点列表"""
|
||
if os.path.exists(path):
|
||
try:
|
||
with open(path, "r", encoding="utf-8") as f:
|
||
return json.load(f)
|
||
except Exception:
|
||
pass
|
||
return []
|
||
|
||
|
||
def save_sites(path: str, sites: list[dict]):
|
||
"""保存站点列表到 JSON 文件"""
|
||
with open(path, "w", encoding="utf-8") as f:
|
||
json.dump(sites, f, ensure_ascii=False, indent=2)
|
||
print(f"\n[OK] 已保存 {len(sites)} 个站点到 {path}")
|
||
|
||
|
||
def main():
|
||
args = parse_args()
|
||
headless = not args.no_headless
|
||
keywords = [k.strip() for k in args.keywords.split(",") if k.strip()]
|
||
|
||
print(f"=" * 50)
|
||
print(f"笑话站点发现工具")
|
||
print(f"搜索词: {keywords}")
|
||
print(f"最大站点数: {args.max_sites}")
|
||
print(f"输出文件: {args.output}")
|
||
print(f"浏览器: {'显示窗口' if not headless else '无头模式'}")
|
||
print(f"=" * 50)
|
||
|
||
# 加载已存在的站点（保留已有编号）
|
||
existing = load_existing_sites(args.output)
|
||
existing_domains = {s["domain"] for s in existing}
|
||
next_id = max([s["id"] for s in existing], default=0) + 1
|
||
print(f"[*] 已有 {len(existing)} 个站点记录，新编号从 {next_id} 开始")
|
||
|
||
async def run():
|
||
nonlocal next_id
|
||
crawler = CrawlerService(headless=headless)
|
||
try:
|
||
new_sites = await crawler.search_joke_sites(keywords, max_results=args.max_sites)
|
||
finally:
|
||
await crawler.close()
|
||
|
||
# 合并新旧站点（去重）
|
||
added = 0
|
||
for site in new_sites:
|
||
domain = site["domain"]
|
||
if domain not in existing_domains:
|
||
site["id"] = next_id
|
||
site["found_at"] = datetime.now().isoformat(timespec="seconds")
|
||
existing.append(site)
|
||
existing_domains.add(domain)
|
||
next_id += 1
|
||
added += 1
|
||
print(f" [+] 新增 #{site['id']}: {site['domain']} — {site['title'][:40]}")
|
||
|
||
save_sites(args.output, existing)
|
||
|
||
if existing:
|
||
print(f"\n站点列表：")
|
||
for s in existing:
|
||
print(f" #{s['id']:2d} {s['domain']:30s} {s['title'][:35]}")
|
||
|
||
asyncio.run(run())
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main() |