Ubuntu
667 lines
24 KiB
Python
667 lines
24 KiB
Python
"""
|
||
通用采集路由 - 基于 Scrapling 引擎
|
||
结合上层业务逻辑：编码修复、数学公式处理、微信防盗链、图片下载等
|
||
"""
|
||
|
||
import os
|
||
import re
|
||
import sys
|
||
import uuid
|
||
import asyncio
|
||
import tempfile
|
||
import base64
|
||
from datetime import datetime
|
||
from typing import Optional, List
|
||
from urllib.parse import urljoin, urlparse
|
||
from pathlib import Path
|
||
|
||
from fastapi import APIRouter, HTTPException, Depends, UploadFile, File
|
||
from pydantic import BaseModel, HttpUrl
|
||
from sqlalchemy.orm import Session
|
||
from bs4 import BeautifulSoup
|
||
|
||
# Scrapling 引擎
|
||
from scrapling import Fetcher, StealthyFetcher
|
||
|
||
# 策略化重构（v2）：站点提取策略注册表 + 双轨调度
|
||
from .crawl_strategies import (
|
||
get_extractor_for_url,
|
||
is_strategy_first,
|
||
run_extractor,
|
||
is_strategy_success,
|
||
)
|
||
|
||
# 复用 crawl.py 的通用工具函数
|
||
from .crawl import (
|
||
fix_encoding,
|
||
fix_math_formulas,
|
||
html_to_markdown,
|
||
extract_title_from_html,
|
||
extract_main_content,
|
||
download_image,
|
||
extract_images,
|
||
extract_images_from_markdown,
|
||
is_spa_site,
|
||
convert_spa_html_to_markdown,
|
||
CrawlRequest,
|
||
CrawlResponse,
|
||
CrawlImage,
|
||
)
|
||
|
||
|
||
class CrawlSaveRequest(BaseModel):
|
||
"""通用采集保存请求"""
|
||
title: str
|
||
content: str
|
||
content_html: Optional[str] = ""
|
||
images: List[CrawlImage] = []
|
||
summary: Optional[str] = ""
|
||
category_id: Optional[int] = None
|
||
is_draft: bool = True
|
||
is_published: bool = False
|
||
tags: Optional[List[str]] = []
|
||
source_url: Optional[str] = None
|
||
|
||
# 复用图片处理
|
||
from .images import generate_thumbnail, get_image_dimensions
|
||
|
||
# 数据库
|
||
from database import get_db
|
||
from models import Post, Category, Tag, PostTag, Image
|
||
|
||
router = APIRouter(prefix="/api/universal-crawl", tags=["通用采集"])
|
||
|
||
# 确保上传目录存在
|
||
UPLOAD_DIR = Path("uploads")
|
||
UPLOAD_DIR.mkdir(exist_ok=True)
|
||
TEMP_DIR = Path("uploads/temp_crawl")
|
||
TEMP_DIR.mkdir(exist_ok=True)
|
||
|
||
|
||
class UniversalCrawlRequest(BaseModel):
|
||
url: str
|
||
|
||
|
||
def _needs_browser(url: str) -> bool:
|
||
"""判断是否需要使用浏览器渲染"""
|
||
domain = urlparse(url).netloc.lower()
|
||
browser_domains = [
|
||
'mp.weixin.qq.com',
|
||
'juejin.cn',
|
||
'zhihu.com',
|
||
'segmentfault.com',
|
||
'infoq.cn',
|
||
'infoq.com',
|
||
'51cto.com',
|
||
'eeworld.com.cn', # JS WAF 防护，必须用浏览器渲染
|
||
]
|
||
return any(d in domain for d in browser_domains)
|
||
|
||
|
||
def _fetch_with_scrapling(url: str) -> str:
|
||
"""使用 Scrapling Fetcher (curl_cffi) 获取页面"""
|
||
fetcher = Fetcher()
|
||
response = fetcher.get(url)
|
||
if response.status != 200:
|
||
raise Exception(f"HTTP {response.status}: {response.reason}")
|
||
# Scrapling Response 的 text 属性返回解析后的文本，html_content 返回原始 HTML
|
||
return response.html_content if hasattr(response, 'html_content') else response.text
|
||
|
||
|
||
async def _fetch_with_stealth(url: str) -> str:
|
||
"""使用 Scrapling StealthyFetcher 获取页面，失败时回退到原生 Playwright"""
|
||
try:
|
||
from scrapling import StealthyFetcher
|
||
# Scrapling v0.3+ 推荐使用类方法直接调用，无需实例化
|
||
response = await StealthyFetcher.async_fetch(url, wait=3000)
|
||
if not response or response.status != 200:
|
||
raise Exception(f"HTTP {response.status}")
|
||
html = response.html_content if hasattr(response, 'html_content') else response.text
|
||
if not html or len(html) < 100:
|
||
raise Exception("Empty page")
|
||
print(f"[Universal Crawl] StealthyFetcher success: {len(html)} bytes")
|
||
return html
|
||
except Exception as e:
|
||
print(f"[Universal Crawl] StealthyFetcher failed: {e}")
|
||
print("[Universal Crawl] Falling back to native Playwright")
|
||
return await _fetch_with_playwright(url)
|
||
|
||
|
||
async def _fetch_with_playwright(url: str) -> str:
|
||
"""使用原生 Playwright 异步获取页面，带完善反爬配置"""
|
||
from playwright.async_api import async_playwright
|
||
|
||
async with async_playwright() as p:
|
||
browser = await p.chromium.launch(
|
||
headless=True,
|
||
args=[
|
||
"--disable-blink-features=AutomationControlled",
|
||
"--disable-web-security",
|
||
"--disable-features=IsolateOrigins,site-per-process",
|
||
"--disable-dev-shm-usage",
|
||
"--no-sandbox",
|
||
]
|
||
)
|
||
context = await browser.new_context(
|
||
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0",
|
||
viewport={"width": 1920, "height": 1080},
|
||
locale="zh-CN",
|
||
timezone_id="Asia/Shanghai",
|
||
extra_http_headers={
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||
"Accept-Encoding": "gzip, deflate, br",
|
||
"Cache-Control": "max-age=0",
|
||
"Sec-Ch-Ua": '"Not_A Brand";v="8", "Chromium";v="120", "Microsoft Edge";v="120"',
|
||
"Sec-Ch-Ua-Mobile": "?0",
|
||
"Sec-Ch-Ua-Platform": '"Windows"',
|
||
"Sec-Fetch-Dest": "document",
|
||
"Sec-Fetch-Mode": "navigate",
|
||
"Sec-Fetch-Site": "none",
|
||
"Sec-Fetch-User": "?1",
|
||
"Upgrade-Insecure-Requests": "1",
|
||
},
|
||
)
|
||
page = await context.new_page()
|
||
try:
|
||
# SegmentFault 需要先访问首页建立信任，再跳转目标页面
|
||
if 'segmentfault.com' in url:
|
||
await page.goto("https://segmentfault.com", wait_until="domcontentloaded", timeout=15000)
|
||
await asyncio.sleep(2)
|
||
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
||
await asyncio.sleep(5)
|
||
# 知乎需要先访问首页建立 cookie，再跳转目标页面
|
||
elif 'zhihu.com' in url:
|
||
await page.goto("https://www.zhihu.com", wait_until="domcontentloaded", timeout=15000)
|
||
await asyncio.sleep(2)
|
||
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
||
await asyncio.sleep(3)
|
||
# 等待正文加载（知乎专栏文章正文在 .Post-RichTextContainer 或 .RichContent-inner 里）
|
||
try:
|
||
await page.wait_for_selector(".Post-RichTextContainer, .RichContent-inner", timeout=10000)
|
||
await asyncio.sleep(1)
|
||
except Exception:
|
||
pass
|
||
else:
|
||
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
||
await asyncio.sleep(2)
|
||
html = await page.content()
|
||
finally:
|
||
await context.close()
|
||
await browser.close()
|
||
return html
|
||
|
||
|
||
def _bypass_wechat_images(html: str, url: str) -> str:
|
||
"""
|
||
微信图片防盗链绕过：在页面上下文中执行 fetch 获取图片 base64
|
||
这里我们在后端模拟：如果检测到 mmbiz.qpic.cn 图片，
|
||
将其 src 替换为 data-src（微信文章中 data-src 通常包含真实 URL）
|
||
"""
|
||
if 'mp.weixin.qq.com' not in url:
|
||
return html
|
||
|
||
soup = BeautifulSoup(html, 'lxml')
|
||
for img in soup.find_all('img'):
|
||
data_src = img.get('data-src')
|
||
if data_src and 'mmbiz.qpic.cn' in data_src:
|
||
img['src'] = data_src
|
||
return str(soup)
|
||
|
||
|
||
def _extract_infoq_content(html: str) -> str:
|
||
"""提取 InfoQ 文章正文（.ProseMirror 或 article 标签）"""
|
||
soup = BeautifulSoup(html, 'lxml')
|
||
# 优先使用 .ProseMirror（InfoQ 正文编辑器容器）
|
||
container = soup.select_one('.ProseMirror')
|
||
if not container:
|
||
# fallback 到 article 标签
|
||
container = soup.find('article')
|
||
if container:
|
||
return str(container)
|
||
return html
|
||
|
||
|
||
def _extract_infoq_title(html: str) -> Optional[str]:
|
||
"""提取 InfoQ 文章标题，从 <title> 中去掉 ' - InfoQ' 后缀"""
|
||
match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
|
||
if match:
|
||
title = match.group(1).strip()
|
||
# 去掉 " - InfoQ" 后缀
|
||
title = re.sub(r'\s*[-–—]\s*InfoQ\s*$', '', title, flags=re.IGNORECASE).strip()
|
||
if len(title) > 5:
|
||
return title
|
||
return None
|
||
|
||
|
||
def _extract_juejin_content(html: str) -> str:
|
||
"""提取掘金文章正文（article.article 或 .article-viewer）"""
|
||
soup = BeautifulSoup(html, 'lxml')
|
||
# 优先使用 article.article
|
||
container = soup.select_one('article.article')
|
||
if not container:
|
||
# fallback 到 .article-viewer
|
||
container = soup.select_one('.article-viewer')
|
||
if not container:
|
||
# 再 fallback 到 #article-root
|
||
container = soup.select_one('#article-root')
|
||
if container:
|
||
return str(container)
|
||
return html
|
||
|
||
|
||
def _extract_juejin_title(html: str) -> Optional[str]:
|
||
"""提取掘金文章标题，从 <title> 中去掉 ' - 掘金' 后缀"""
|
||
match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
|
||
if match:
|
||
title = match.group(1).strip()
|
||
# 去掉 " - 掘金" 后缀
|
||
title = re.sub(r'\s*[-–—]\s*掘金\s*$', '', title, flags=re.IGNORECASE).strip()
|
||
if len(title) > 5:
|
||
return title
|
||
return None
|
||
|
||
|
||
def _extract_csdn_content(html: str) -> str:
|
||
"""提取 CSDN 文章正文（#content_views 或 .article_content）"""
|
||
soup = BeautifulSoup(html, 'lxml')
|
||
# 优先使用 #content_views（CSDN 正文主容器）
|
||
container = soup.select_one('#content_views')
|
||
if not container:
|
||
# fallback 到 .article_content
|
||
container = soup.select_one('.article_content')
|
||
if container:
|
||
return str(container)
|
||
return html
|
||
|
||
|
||
def _extract_csdn_title(html: str) -> Optional[str]:
|
||
"""提取 CSDN 文章标题，从 <title> 中去掉 '-CSDN博客' 后缀"""
|
||
match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
|
||
if match:
|
||
title = match.group(1).strip()
|
||
# 去掉 " - CSDN博客" / "-CSDN博客" 后缀
|
||
title = re.sub(r'\s*[-–—]?\s*CSDN博客\s*$', '', title, flags=re.IGNORECASE).strip()
|
||
if len(title) > 5:
|
||
return title
|
||
return None
|
||
|
||
|
||
def _filter_csdn_images(images: list) -> list:
|
||
"""过滤掉 CSDN 的 UI 图标图片（csdnimg.cn 的收藏/点赞等图标）"""
|
||
filtered = []
|
||
skip_patterns = [
|
||
'newHeart2023',
|
||
'tobarCollect',
|
||
'tobarCollection',
|
||
'copyright',
|
||
'avatar',
|
||
'blogv2/dist/pc/img',
|
||
]
|
||
for img in images:
|
||
url = img.original_url.lower()
|
||
if any(p in url for p in skip_patterns):
|
||
continue
|
||
filtered.append(img)
|
||
return filtered
|
||
|
||
|
||
def _extract_cnblogs_content(html: str) -> str:
|
||
"""提取博客园文章正文（#cnblogs_post_body 或 .postBody）"""
|
||
soup = BeautifulSoup(html, 'lxml')
|
||
container = soup.select_one('#cnblogs_post_body')
|
||
if not container:
|
||
container = soup.select_one('.postBody')
|
||
if container:
|
||
return str(container)
|
||
return html
|
||
|
||
|
||
def _extract_cnblogs_title(html: str) -> Optional[str]:
|
||
"""提取博客园文章标题，从 <title> 中去掉 ' - 博客园' 后缀"""
|
||
match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
|
||
if match:
|
||
title = match.group(1).strip()
|
||
title = re.sub(r'\s*[-–—]\s*博客园\s*$', '', title, flags=re.IGNORECASE).strip()
|
||
if len(title) > 5:
|
||
return title
|
||
return None
|
||
|
||
|
||
def _filter_cnblogs_images(images: list) -> list:
|
||
"""过滤掉博客园的 UI 图标图片"""
|
||
filtered = []
|
||
skip_patterns = [
|
||
'assets.cnblogs.com/logo',
|
||
'assets.cnblogs.com/icons',
|
||
'skins/custom/images/logo',
|
||
]
|
||
for img in images:
|
||
url = img.original_url.lower()
|
||
if any(p in url for p in skip_patterns):
|
||
continue
|
||
filtered.append(img)
|
||
return filtered
|
||
|
||
|
||
def _extract_51cto_content(html: str) -> str:
|
||
"""提取 51CTO 文章正文（.article-content 或 article）"""
|
||
soup = BeautifulSoup(html, 'lxml')
|
||
container = soup.select_one('.article-content')
|
||
if not container:
|
||
container = soup.find('article')
|
||
if container:
|
||
return str(container)
|
||
return html
|
||
|
||
|
||
def _extract_51cto_title(html: str) -> Optional[str]:
|
||
"""提取 51CTO 文章标题，从 <title> 中去掉 '-51CTO.COM' 后缀"""
|
||
match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
|
||
if match:
|
||
title = match.group(1).strip()
|
||
title = re.sub(r'\s*[-–—]?\s*51CTO\.COM\s*$', '', title, flags=re.IGNORECASE).strip()
|
||
if len(title) > 5:
|
||
return title
|
||
return None
|
||
|
||
|
||
def _filter_51cto_images(images: list) -> list:
|
||
"""过滤掉 51CTO 的 UI 图标和二维码图片"""
|
||
filtered = []
|
||
skip_patterns = [
|
||
'oss/202302', # 公众号矩阵二维码
|
||
'oss/202408', # 软考二维码
|
||
'oss/202506/06', # 顶部横幅
|
||
'oss/202302/07', # APP下载图
|
||
]
|
||
for img in images:
|
||
url = img.original_url.lower()
|
||
if any(p in url for p in skip_patterns):
|
||
continue
|
||
filtered.append(img)
|
||
return filtered
|
||
|
||
|
||
def _legacy_extract(url: str, html: str, images: Optional[list] = None):
|
||
"""旧逻辑（轨A）：现有站点特判提取，逐行保留原实现，不删除。
|
||
|
||
返回 (title, markdown_content, images)，与策略链 ExtractorResult 同构。
|
||
images 为 None 时内部自行调用 extract_images；否则复用调用方已提取的结果。
|
||
原第 6 步的 is_wechat 变量从未被使用，搬迁时省略。
|
||
"""
|
||
# 5. 提取标题（InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取）
|
||
is_infoq = 'infoq.cn' in url or 'infoq.com' in url
|
||
is_juejin = 'juejin.cn' in url
|
||
is_csdn = 'csdn.net' in url or 'csdn.com' in url
|
||
is_cnblogs = 'cnblogs.com' in url
|
||
is_51cto = '51cto.com' in url
|
||
if is_infoq:
|
||
title = _extract_infoq_title(html)
|
||
elif is_juejin:
|
||
title = _extract_juejin_title(html)
|
||
elif is_csdn:
|
||
title = _extract_csdn_title(html)
|
||
elif is_cnblogs:
|
||
title = _extract_cnblogs_title(html)
|
||
elif is_51cto:
|
||
title = _extract_51cto_title(html)
|
||
else:
|
||
title = None
|
||
if not title:
|
||
title = extract_title_from_html(html)
|
||
if not title:
|
||
title = "未获取到标题"
|
||
|
||
# 6. 提取正文和图片
|
||
if images is None:
|
||
images = extract_images(html, url)
|
||
if is_csdn:
|
||
images = _filter_csdn_images(images)
|
||
elif is_cnblogs:
|
||
images = _filter_cnblogs_images(images)
|
||
elif is_51cto:
|
||
images = _filter_51cto_images(images)
|
||
|
||
# 7. 转换为 Markdown（InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取）
|
||
if is_infoq:
|
||
infoq_html = _extract_infoq_content(html)
|
||
markdown_content = html_to_markdown(infoq_html, url)
|
||
elif is_juejin:
|
||
juejin_html = _extract_juejin_content(html)
|
||
markdown_content = html_to_markdown(juejin_html, url)
|
||
elif is_csdn:
|
||
csdn_html = _extract_csdn_content(html)
|
||
markdown_content = html_to_markdown(csdn_html, url)
|
||
elif is_cnblogs:
|
||
cnblogs_html = _extract_cnblogs_content(html)
|
||
markdown_content = html_to_markdown(cnblogs_html, url)
|
||
elif is_51cto:
|
||
cto_html = _extract_51cto_content(html)
|
||
markdown_content = html_to_markdown(cto_html, url)
|
||
elif is_spa_site(html):
|
||
# 正文区 vs 噪音过滤（Pruning 思路）统一在 crawl.py 的
|
||
# convert_spa_html_to_markdown / html_to_markdown 入口处理，此处不重复剪枝
|
||
markdown_content = convert_spa_html_to_markdown(html, images, url)
|
||
else:
|
||
markdown_content = html_to_markdown(html, url)
|
||
|
||
return title, markdown_content, images
|
||
|
||
|
||
@router.post("/fetch", response_model=CrawlResponse)
|
||
async def universal_fetch(request: UniversalCrawlRequest):
|
||
"""通用采集：使用 Scrapling 引擎获取文章"""
|
||
try:
|
||
url = str(request.url)
|
||
print(f"[Universal Crawl] Fetching: {url}")
|
||
|
||
# 1. 判断采集策略
|
||
use_browser = _needs_browser(url)
|
||
|
||
# 2. 使用 Scrapling 获取页面
|
||
if use_browser:
|
||
print(f"[Universal Crawl] Using StealthyFetcher (browser) for {url}")
|
||
html = await _fetch_with_stealth(url)
|
||
else:
|
||
print(f"[Universal Crawl] Using Fetcher (curl_cffi) for {url}")
|
||
html = _fetch_with_scrapling(url)
|
||
|
||
if not html or len(html) < 100:
|
||
raise HTTPException(status_code=400, detail="无法获取页面内容，页面可能为空或需要登录")
|
||
|
||
# 3. 编码修复
|
||
html = fix_encoding(html)
|
||
|
||
# 4. 微信图片防盗链处理
|
||
html = _bypass_wechat_images(html, url)
|
||
|
||
# 5. 提取标题和正文（双轨调度 v2：默认旧逻辑优先；白名单站点新策略优先，质量不达标回退旧逻辑）
|
||
extractor = get_extractor_for_url(url)
|
||
if extractor and is_strategy_first(url):
|
||
images = extract_images(html, url)
|
||
result = run_extractor(extractor, url, html, images)
|
||
if is_strategy_success(result):
|
||
title, markdown_content, images = result.title, result.markdown, result.images
|
||
print(f"[Universal Crawl] 策略链成功: {extractor.name}")
|
||
else:
|
||
print(f"[Universal Crawl] 策略链质量不达标({extractor.name})，回退旧逻辑")
|
||
title, markdown_content, images = _legacy_extract(url, html, images)
|
||
else:
|
||
title, markdown_content, images = _legacy_extract(url, html)
|
||
|
||
# 5.1 只保留正文中实际出现的图片（全页提取会带入头像/侧栏/推荐位等噪音图）
|
||
# 正文无图（如该掘金文章）时 images 会被清空，不再下载噪音图
|
||
def _norm_img_url(u: str) -> str:
|
||
return u.replace('&', '&')
|
||
markdown_images = {_norm_img_url(img.original_url) for img in extract_images_from_markdown(markdown_content, url)}
|
||
images = [img for img in images if _norm_img_url(img.original_url) in markdown_images]
|
||
|
||
# 6. 数学公式修复
|
||
markdown_content = fix_math_formulas(markdown_content)
|
||
|
||
# 7. 清理 Markdown 中的 HTML 标签
|
||
markdown_content = re.sub(r'<[^>]+>', '', markdown_content)
|
||
markdown_content = re.sub(r'\n{3,}', '\n\n', markdown_content)
|
||
markdown_content = markdown_content.strip()
|
||
|
||
return CrawlResponse(
|
||
success=True,
|
||
title=title,
|
||
content=markdown_content,
|
||
images=images,
|
||
)
|
||
|
||
except HTTPException:
|
||
raise
|
||
except Exception as e:
|
||
print(f"[Universal Crawl] Error: {e}")
|
||
import traceback
|
||
traceback.print_exc()
|
||
raise HTTPException(status_code=500, detail=f"采集失败: {str(e)}")
|
||
|
||
|
||
@router.post("/save")
|
||
async def universal_save(request: CrawlSaveRequest, db: Session = Depends(get_db)):
|
||
"""保存通用采集的文章，下载图片到本地并替换内容中的远程 URL"""
|
||
try:
|
||
title = request.title
|
||
content = request.content
|
||
excerpt = request.summary or ""
|
||
category_id = request.category_id
|
||
source_url = request.source_url
|
||
image_list = request.images
|
||
is_draft = request.is_draft
|
||
is_published = request.is_published
|
||
|
||
print(f"[Universal Crawl] Saving article: {title}, images: {len(image_list)}")
|
||
|
||
# 创建临时目录
|
||
temp_dir = tempfile.mkdtemp(dir=TEMP_DIR)
|
||
final_dir = UPLOAD_DIR / "images"
|
||
final_dir.mkdir(exist_ok=True)
|
||
thumb_dir = UPLOAD_DIR / "thumbnails"
|
||
thumb_dir.mkdir(exist_ok=True)
|
||
|
||
# 下载图片
|
||
downloaded_images = []
|
||
images_saved = 0
|
||
images_failed = 0
|
||
for img in image_list:
|
||
try:
|
||
img_url = str(img.original_url)
|
||
if not img_url:
|
||
continue
|
||
filename, filepath, success = download_image(img_url, temp_dir)
|
||
if success and filepath:
|
||
# 移动最终文件
|
||
final_filename = f"crawl_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}{os.path.splitext(filename or 'img.jpg')[1] or '.jpg'}"
|
||
final_path = final_dir / final_filename
|
||
os.rename(filepath, str(final_path))
|
||
|
||
# 生成缩略图
|
||
thumb_filename = f"thumb_{final_filename}"
|
||
thumb_path = thumb_dir / thumb_filename
|
||
thumb_path_db = None
|
||
if generate_thumbnail(str(final_path), str(thumb_path)):
|
||
thumb_path_db = f"/uploads/thumbnails/{thumb_filename}"
|
||
|
||
# 获取尺寸
|
||
width, height = get_image_dimensions(str(final_path))
|
||
|
||
# 保存到数据库
|
||
db_image = Image(
|
||
filename=final_filename,
|
||
original_name=final_filename,
|
||
file_path=f"/uploads/images/{final_filename}",
|
||
file_size=os.path.getsize(str(final_path)),
|
||
width=width,
|
||
height=height,
|
||
thumb_path=thumb_path_db,
|
||
)
|
||
db.add(db_image)
|
||
db.flush()
|
||
|
||
# 替换内容中的图片 URL
|
||
new_url = f"/uploads/images/{final_filename}"
|
||
content = content.replace(img_url, new_url)
|
||
images_saved += 1
|
||
|
||
downloaded_images.append({
|
||
"id": db_image.id,
|
||
"url": new_url,
|
||
"thumb": thumb_path_db,
|
||
})
|
||
except Exception as e:
|
||
images_failed += 1
|
||
print(f"[Universal Crawl] Download image error: {e}")
|
||
continue
|
||
|
||
# 生成摘要
|
||
summary = excerpt if excerpt else content[:300] if len(content) > 300 else content
|
||
summary = re.sub(r'[#*`\[\]\(\)!]', '', summary).strip()[:300]
|
||
|
||
# 创建文章
|
||
now = datetime.now()
|
||
slug = re.sub(r'[^\w\u4e00-\u9fff-]', '-', title)[:50].strip('-')
|
||
slug = f"{slug}-{now.strftime('%Y%m%d%H%M%S')}-{uuid.uuid4().hex[:4]}"
|
||
|
||
# 查找或创建分类
|
||
final_category_id = category_id
|
||
if not final_category_id:
|
||
default_cat = db.query(Category).filter(Category.name == "未分类").first()
|
||
if not default_cat:
|
||
default_cat = Category(name="未分类", slug="uncategorized")
|
||
db.add(default_cat)
|
||
db.flush()
|
||
final_category_id = default_cat.id
|
||
|
||
post = Post(
|
||
title=title,
|
||
slug=slug,
|
||
content=content,
|
||
content_type="markdown",
|
||
summary=summary,
|
||
category_id=final_category_id,
|
||
is_published=is_published,
|
||
is_draft=is_draft,
|
||
view_count=0,
|
||
created_at=now,
|
||
updated_at=now,
|
||
)
|
||
db.add(post)
|
||
db.flush()
|
||
|
||
# 处理标签
|
||
if request.tags:
|
||
for tag_name in request.tags:
|
||
tag_name = tag_name.strip()
|
||
if not tag_name:
|
||
continue
|
||
tag = db.query(Tag).filter(Tag.name == tag_name).first()
|
||
if not tag:
|
||
tag_slug = re.sub(r'[^\w\u4e00-\u9fff-]', '-', tag_name).strip('-')
|
||
tag = Tag(name=tag_name, slug=tag_slug)
|
||
db.add(tag)
|
||
db.flush()
|
||
post_tag = PostTag(post_id=post.id, tag_id=tag.id)
|
||
db.add(post_tag)
|
||
|
||
db.commit()
|
||
|
||
return {
|
||
"success": True,
|
||
"post_id": post.id,
|
||
"slug": slug,
|
||
"message": "文章保存成功",
|
||
"images_saved": images_saved,
|
||
"images_failed": images_failed,
|
||
}
|
||
|
||
except Exception as e:
|
||
db.rollback()
|
||
print(f"[Universal Crawl] Save error: {e}")
|
||
import traceback
|
||
traceback.print_exc()
|
||
raise HTTPException(status_code=500, detail=f"保存失败: {str(e)}")
|