fix: resolve 10 code review issues
High priority: - Fix concurrent race condition for view_count/like_count (atomic update) - Add route request ID tracking to prevent race conditions - Filter get_joke by status=approved (no pending content leak) - Add error feedback for like button Performance: - Optimize random joke query (avoid full table sort) - Limit page_size max to 100 (DoS prevention) Medium: - Add localStorage quota error handling - Handle empty AI response gracefully - Fix generate content title extraction Low: - Add rejected_jokes to stats API - Update dashboard to show rejected count
This commit is contained in:
@@ -0,0 +1,128 @@
|
||||
"""
|
||||
笑话站点发现工具。
|
||||
搜索 Bing 获取笑话聚合网站列表,保存到 JSON 文件。
|
||||
|
||||
用法:
|
||||
python crawler/site_finder.py
|
||||
python crawler/site_finder.py --keywords "笑话大全,冷笑话" --output my_sites.json
|
||||
python crawler/site_finder.py --no-headless
|
||||
"""
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
from datetime import datetime
|
||||
|
||||
if sys.platform == "win32":
|
||||
import shutil
|
||||
os.environ["PYTHONIOENCODING"] = "utf-8"
|
||||
os.environ["TERM"] = "dumb"
|
||||
try:
|
||||
shutil.get_terminal_size()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from crawler.crawler_service import CrawlerService
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description="笑话站点发现工具")
|
||||
parser.add_argument(
|
||||
"--keywords",
|
||||
type=str,
|
||||
default=os.getenv("CRAWL_KEYWORD", "笑话大全,搞笑段子,冷笑话,幽默笑话,爆笑笑话,笑话集锦"),
|
||||
help="搜索关键词,逗号分隔",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=str,
|
||||
default="joke_sites.json",
|
||||
help="输出 JSON 文件路径(默认: joke_sites.json)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-sites",
|
||||
type=int,
|
||||
default=15,
|
||||
help="最多保留几个站点(默认: 15)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-headless",
|
||||
action="store_true",
|
||||
help="显示浏览器窗口",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def load_existing_sites(path: str) -> list[dict]:
|
||||
"""加载已有站点列表"""
|
||||
if os.path.exists(path):
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return json.load(f)
|
||||
except Exception:
|
||||
pass
|
||||
return []
|
||||
|
||||
|
||||
def save_sites(path: str, sites: list[dict]):
|
||||
"""保存站点列表到 JSON 文件"""
|
||||
with open(path, "w", encoding="utf-8") as f:
|
||||
json.dump(sites, f, ensure_ascii=False, indent=2)
|
||||
print(f"\n[OK] 已保存 {len(sites)} 个站点到 {path}")
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
headless = not args.no_headless
|
||||
keywords = [k.strip() for k in args.keywords.split(",") if k.strip()]
|
||||
|
||||
print(f"=" * 50)
|
||||
print(f"笑话站点发现工具")
|
||||
print(f"搜索词: {keywords}")
|
||||
print(f"最大站点数: {args.max_sites}")
|
||||
print(f"输出文件: {args.output}")
|
||||
print(f"浏览器: {'显示窗口' if not headless else '无头模式'}")
|
||||
print(f"=" * 50)
|
||||
|
||||
# 加载已存在的站点(保留已有编号)
|
||||
existing = load_existing_sites(args.output)
|
||||
existing_domains = {s["domain"] for s in existing}
|
||||
next_id = max([s["id"] for s in existing], default=0) + 1
|
||||
print(f"[*] 已有 {len(existing)} 个站点记录,新编号从 {next_id} 开始")
|
||||
|
||||
async def run():
|
||||
nonlocal next_id
|
||||
crawler = CrawlerService(headless=headless)
|
||||
try:
|
||||
new_sites = await crawler.search_joke_sites(keywords, max_results=args.max_sites)
|
||||
finally:
|
||||
await crawler.close()
|
||||
|
||||
# 合并新旧站点(去重)
|
||||
added = 0
|
||||
for site in new_sites:
|
||||
domain = site["domain"]
|
||||
if domain not in existing_domains:
|
||||
site["id"] = next_id
|
||||
site["found_at"] = datetime.now().isoformat(timespec="seconds")
|
||||
existing.append(site)
|
||||
existing_domains.add(domain)
|
||||
next_id += 1
|
||||
added += 1
|
||||
print(f" [+] 新增 #{site['id']}: {site['domain']} — {site['title'][:40]}")
|
||||
|
||||
save_sites(args.output, existing)
|
||||
|
||||
if existing:
|
||||
print(f"\n站点列表:")
|
||||
for s in existing:
|
||||
print(f" #{s['id']:2d} {s['domain']:30s} {s['title'][:35]}")
|
||||
|
||||
asyncio.run(run())
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user