admin/nanocode
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Auto backup \2026-\04-\20

Browse Source
This commit is contained in:
Ubuntu
2026-07-08 16:00:02 +08:00
parent d285680356
commit 065fde366a
73 changed files with 174 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files
personal-blog/backend/routers/universal_crawl.py
+63 -45
View File
@@ -40,6 +40,20 @@ from .crawl import (
CrawlImage,
)
class CrawlSaveRequest(BaseModel):
"""通用采集保存请求"""
title: str
content: str
content_html: Optional[str] = ""
images: List[CrawlImage] = []
summary: Optional[str] = ""
category_id: Optional[int] = None
is_draft: bool = True
is_published: bool = False
tags: Optional[List[str]] = []
source_url: Optional[str] = None
# 复用图片处理
from .images import generate_thumbnail, get_image_dimensions
@@ -71,6 +85,7 @@ def _needs_browser(url: str) -> bool:
'infoq.cn',
'infoq.com',
'51cto.com',
'eeworld.com.cn', # JS WAF 防护，必须用浏览器渲染
]
return any(d in domain for d in browser_domains)
@@ -466,29 +481,19 @@ async def universal_fetch(request: UniversalCrawlRequest):
@router.post("/save")
async def universal_save(
title: str,
content: str,
excerpt: Optional[str] = None,
category_id: Optional[int] = None,
source_url: Optional[str] = None,
images: Optional[str] = None,
db: Session = Depends(get_db)
):
"""保存通用采集的文章（复用 crawl.py 的保存逻辑）"""
async def universal_save(request: CrawlSaveRequest, db: Session = Depends(get_db)):
"""保存通用采集的文章，下载图片到本地并替换内容中的远程 URL"""
try:
print(f"[Universal Crawl] Saving article: {title}")
# 解析图片列表
image_list = []
if images:
try:
import json
image_data = json.loads(images)
for img in image_data:
image_list.append(CrawlImage(**img))
except Exception as e:
print(f"[Universal Crawl] Parse images error: {e}")
title = request.title
content = request.content
excerpt = request.summary or ""
category_id = request.category_id
source_url = request.source_url
image_list = request.images
is_draft = request.is_draft
is_published = request.is_published
print(f"[Universal Crawl] Saving article: {title}, images: {len(image_list)}")
# 创建临时目录
temp_dir = tempfile.mkdtemp(dir=TEMP_DIR)
@@ -499,12 +504,17 @@ async def universal_save(
# 下载图片
downloaded_images = []
images_saved = 0
images_failed = 0
for img in image_list:
try:
filename, filepath, success = download_image(str(img.url), temp_dir)
img_url = str(img.original_url)
if not img_url:
continue
filename, filepath, success = download_image(img_url, temp_dir)
if success and filepath:
# 移动最终文件
final_filename = f"crawl_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}{os.path.splitext(filename)[1]}"
final_filename = f"crawl_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}{os.path.splitext(filename or 'img.jpg')[1] or '.jpg'}"
final_path = final_dir / final_filename
os.rename(filepath, str(final_path))
@@ -521,7 +531,7 @@ async def universal_save(
# 保存到数据库
db_image = Image(
filename=final_filename,
original_filename=img.alt or final_filename,
original_filename=final_filename,
file_path=f"/uploads/images/{final_filename}",
file_size=os.path.getsize(str(final_path)),
mime_type=f"image/{os.path.splitext(final_filename)[1].lstrip('.') or 'jpeg'}",
@@ -533,9 +543,9 @@ async def universal_save(
db.flush()
# 替换内容中的图片 URL
original_url = str(img.url)
new_url = f"/uploads/images/{final_filename}"
content = content.replace(original_url, new_url)
content = content.replace(img_url, new_url)
images_saved += 1
downloaded_images.append({
"id": db_image.id,
@@ -543,25 +553,19 @@ async def universal_save(
"thumb": thumb_path_db,
})
except Exception as e:
images_failed += 1
print(f"[Universal Crawl] Download image error: {e}")
continue
# 清理 excerpt
if excerpt:
excerpt = re.sub(r'<[^>]+>', '', excerpt)
excerpt = excerpt.replace('\n', ' ').strip()[:200]
else:
excerpt = content.replace('\n', ' ').strip()[:200]
# 生成摘要
summary = excerpt if excerpt else content[:300] if len(content) > 300 else content
summary = re.sub(r'[#*`\[\]\(\)!]', '', summary).strip()[:300]
# 创建文章
now = datetime.now()
slug = re.sub(r'[^\w\u4e00-\u9fff-]', '-', title)[:50].strip('-')
slug = f"{slug}-{now.strftime('%Y%m%d%H%M%S')}-{uuid.uuid4().hex[:4]}"
# 生成摘要
summary = content[:300] if len(content) > 300 else content
summary = re.sub(r'[#*`\[\]\(\)!]', '', summary)
# 查找或创建分类
final_category_id = category_id
if not final_category_id:
@@ -576,28 +580,42 @@ async def universal_save(
title=title,
slug=slug,
content=content,
excerpt=excerpt,
content_type="markdown",
summary=summary,
category_id=final_category_id,
author_id=1,
status="draft",
is_published=is_published,
is_draft=is_draft,
view_count=0,
is_original=False,
source_url=source_url,
created_at=now,
updated_at=now,
published_at=None,
)
db.add(post)
db.flush()
# 处理标签
if request.tags:
for tag_name in request.tags:
tag_name = tag_name.strip()
if not tag_name:
continue
tag = db.query(Tag).filter(Tag.name == tag_name).first()
if not tag:
tag_slug = re.sub(r'[^\w\u4e00-\u9fff-]', '-', tag_name).strip('-')
tag = Tag(name=tag_name, slug=tag_slug)
db.add(tag)
db.flush()
post_tag = PostTag(post_id=post.id, tag_id=tag.id)
db.add(post_tag)
db.commit()
return {
"status": "success",
"success": True,
"post_id": post.id,
"slug": slug,
"message": "文章保存成功",
"images": downloaded_images,
"images_saved": images_saved,
"images_failed": images_failed,
}
except Exception as e: