This commit is contained in:
Ubuntu
2026-07-03 16:58:28 +08:00
commit 32b43f85bd
6240 changed files with 1987678 additions and 0 deletions
+78
View File
@@ -0,0 +1,78 @@
"""
个人博客后端 - FastAPI
"""
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
import os
from database import engine, Base
from routers import auth, posts, categories, tags, images, settings, crawl, wechat_crawl, universal_crawl
# 创建数据库表
Base.metadata.create_all(bind=engine)
# 初始化默认管理员
from database import SessionLocal
from routers.auth import init_admin
db = SessionLocal()
init_admin(db)
db.close()
app = FastAPI(
title="Personal Blog API",
description="个人博客后端API",
version="1.0.0",
redirect_slashes=False
)
# CORS配置 - 允许远程访问
app.add_middleware(
CORSMiddleware,
allow_origins=[
"http://localhost:5173",
"http://localhost:3000",
"http://127.0.0.1:5173",
"http://0.0.0.0:5173",
"http://129.211.173.87:8888", # blog 前端
"http://129.211.173.87:9000", # SSO 中心
],
allow_credentials=True,
allow_methods=["*"],
allow_headers=["*"],
)
# 注册路由
app.include_router(auth.router)
app.include_router(posts.router)
app.include_router(categories.router)
app.include_router(tags.router)
app.include_router(images.router)
app.include_router(settings.router)
app.include_router(crawl.router)
app.include_router(wechat_crawl.router)
app.include_router(universal_crawl.router)
# 静态文件服务(图片访问)
uploads_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "uploads")
os.makedirs(uploads_dir, exist_ok=True)
app.mount("/uploads", StaticFiles(directory=uploads_dir), name="uploads")
@app.get("/")
def root():
return {
"message": "Personal Blog API",
"version": "1.0.0",
"docs": "/docs"
}
@app.get("/api/health")
def health_check():
return {"status": "ok"}
if __name__ == "__main__":
import uvicorn
uvicorn.run(app, host="0.0.0.0", port=8888)
Binary file not shown.
Binary file not shown.
+27
View File
@@ -0,0 +1,27 @@
"""
数据库配置 - SQLite
"""
from sqlalchemy import create_engine
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker
import os
# 数据库路径
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
DATABASE_URL = f"sqlite:///{os.path.join(BASE_DIR, 'blog.db')}"
engine = create_engine(
DATABASE_URL,
connect_args={"check_same_thread": False}
)
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
Base = declarative_base()
# 依赖注入
def get_db():
db = SessionLocal()
try:
yield db
finally:
db.close()
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env python3
"""
数据迁移脚本 - 从备份数据库恢复文章数据
"""
import sqlite3
import os
def migrate_posts():
"""迁移文章数据"""
db_path = 'blog.db'
backup_path = 'blog.db.backup'
if not os.path.exists(backup_path):
print("备份文件不存在")
return
# 连接两个数据库
conn_new = sqlite3.connect(db_path)
conn_old = sqlite3.connect(backup_path)
cursor_new = conn_new.cursor()
cursor_old = conn_old.cursor()
# 检查旧表结构
cursor_old.execute("PRAGMA table_info(posts)")
old_columns = [col[1] for col in cursor_old.fetchall()]
print(f"旧表字段: {old_columns}")
# 获取所有文章
cursor_old.execute("SELECT * FROM posts")
posts = cursor_old.fetchall()
print(f"找到 {len(posts)} 篇文章")
# 清空新表并重新插入
cursor_new.execute("DELETE FROM posts")
for post in posts:
# post: (id, title, content, summary, cover_image, is_published, is_draft, view_count, created_at, updated_at, category_id)
cursor_new.execute("""
INSERT INTO posts (id, title, content, summary, cover_image, is_published, is_draft, view_count, created_at, updated_at, category_id)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", post)
print(f" 已恢复: {post[1][:30]}...")
conn_new.commit()
conn_new.close()
conn_old.close()
print(f"\n✅ 成功恢复 {len(posts)} 篇文章")
if __name__ == '__main__':
migrate_posts()
+111
View File
@@ -0,0 +1,111 @@
"""
数据库模型
"""
from sqlalchemy import Column, Integer, String, Text, DateTime, Boolean, ForeignKey
from sqlalchemy.orm import relationship
from datetime import datetime
from database import Base
class Admin(Base):
"""管理员账号"""
__tablename__ = "admins"
id = Column(Integer, primary_key=True, index=True)
username = Column(String, unique=True, index=True, default="admin")
password_hash = Column(String)
created_at = Column(DateTime, default=datetime.utcnow)
class Post(Base):
"""文章"""
__tablename__ = "posts"
id = Column(Integer, primary_key=True, index=True)
title = Column(String(200), nullable=False)
slug = Column(String(200), unique=True, index=True) # URL友好的标识
content = Column(Text, nullable=False)
content_type = Column(String(20), default="markdown") # markdown 或 html
summary = Column(String(500))
cover_image = Column(String(500)) # 封面图路径
is_published = Column(Boolean, default=True)
is_draft = Column(Boolean, default=False)
view_count = Column(Integer, default=0)
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
# 关联
category_id = Column(Integer, ForeignKey("categories.id"))
category = relationship("Category", back_populates="posts")
tags = relationship("Tag", secondary="post_tags", back_populates="posts")
class Category(Base):
"""栏目/分类"""
__tablename__ = "categories"
id = Column(Integer, primary_key=True, index=True)
name = Column(String(50), unique=True, nullable=False)
description = Column(String(200))
created_at = Column(DateTime, default=datetime.utcnow)
posts = relationship("Post", back_populates="category")
class Tag(Base):
"""标签"""
__tablename__ = "tags"
id = Column(Integer, primary_key=True, index=True)
name = Column(String(30), unique=True, nullable=False)
created_at = Column(DateTime, default=datetime.utcnow)
posts = relationship("Post", secondary="post_tags", back_populates="tags")
# 文章-标签关联表
class PostTag(Base):
__tablename__ = "post_tags"
post_id = Column(Integer, ForeignKey("posts.id"), primary_key=True)
tag_id = Column(Integer, ForeignKey("tags.id"), primary_key=True)
class Image(Base):
"""上传的图片"""
__tablename__ = "images"
id = Column(Integer, primary_key=True, index=True)
filename = Column(String(255), nullable=False)
original_name = Column(String(255))
file_path = Column(String(500), nullable=False) # 原图路径 /uploads/xxx.jpg
thumb_path = Column(String(500)) # 缩略图路径 /uploads/thumbnails/xxx.jpg
description = Column(String(500))
file_size = Column(Integer) # 字节
width = Column(Integer) # 图片宽度
height = Column(Integer) # 图片高度
created_at = Column(DateTime, default=datetime.utcnow)
class Setting(Base):
"""博客设置"""
__tablename__ = "settings"
id = Column(Integer, primary_key=True, index=True)
blog_name = Column(String(20), default="我的博客")
blog_description = Column(String(200), default="")
welcome_message = Column(String(300), default="欢迎来到我的博客 👋 分享技术、生活与思考")
blog_logo = Column(String(500))
banner_image = Column(String(500))
banner_type = Column(String(20), default="text") # text或carousel
banner_images = Column(String(1000), default="[]") # JSON数组存储轮播图
theme = Column(String(20), default="neobrutalism") # 默认新粗野主义
layout = Column(String(20), default="card") # card或list
theme_colors = Column(Text, default='{"primary":"#7c3aed","secondary":"#06b6d4","bg":"#ffffff","text":"#1f2937","sidebar":"#1e293b"}')
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
class Page(Base):
"""自定义页面"""
__tablename__ = "pages"
id = Column(Integer, primary_key=True, index=True)
title = Column(String(50), nullable=False)
slug = Column(String(50), unique=True, nullable=False) # URL标识
content = Column(Text, nullable=False)
is_nav = Column(Boolean, default=True) # 是否在导航显示
nav_order = Column(Integer, default=0) # 导航排序
created_at = Column(DateTime, default=datetime.utcnow)
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
+8
View File
@@ -0,0 +1,8 @@
fastapi==0.115.0
uvicorn[standard]==0.32.0
sqlalchemy==2.0.36
pydantic==2.9.2
python-jose[cryptography]==3.3.0
passlib[bcrypt]==1.7.4
python-multipart==0.0.17
pillow==11.0.0
+155
View File
@@ -0,0 +1,155 @@
"""
认证路由
"""
from fastapi import APIRouter, Depends, HTTPException, status, Request, Response
from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
from sqlalchemy.orm import Session
from datetime import datetime, timedelta
from jose import JWTError, jwt
from passlib.context import CryptContext
import os
import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from database import get_db
from models import Admin
from schemas import LoginRequest, LoginResponse, PasswordChange
from sso_client import get_current_user_from_cookie
router = APIRouter(prefix="/api/auth", tags=["认证"])
# 安全配置
SECRET_KEY = "your-secret-key-change-in-production" # 生产环境修改
ALGORITHM = "HS256"
ACCESS_TOKEN_EXPIRE_DAYS = 7
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/auth/login", auto_error=False)
# 与 SSO 中心共享的 Cookie 属性(删除 Cookie 时需一致)
SSO_COOKIE_DOMAIN = "129.211.173.87"
def verify_password(plain_password, hashed_password):
"""验证密码"""
return pwd_context.verify(plain_password, hashed_password)
def get_password_hash(password):
"""生成密码哈希"""
return pwd_context.hash(password)
def create_access_token(data: dict, expires_delta: timedelta = None):
"""创建JWT令牌"""
to_encode = data.copy()
if expires_delta:
expire = datetime.utcnow() + expires_delta
else:
expire = datetime.utcnow() + timedelta(days=ACCESS_TOKEN_EXPIRE_DAYS)
to_encode.update({"exp": expire})
encoded_jwt = jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
return encoded_jwt
async def get_current_admin(
request: Request,
token: str = Depends(oauth2_scheme),
db: Session = Depends(get_db)
):
"""获取当前登录管理员 —— 双模式认证
1. 优先读取 Authorization Header(原有本地登录)
2. 无 Header 则读取 Cookie 中的 SSO token,本地验签后查本地 Admin 表
"""
credentials_exception = HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="无法验证凭据",
headers={"WWW-Authenticate": "Bearer"},
)
username = None
# 模式一:原有 Bearer Token
if token:
try:
payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
username = payload.get("sub")
except JWTError:
pass
# 模式二:SSO Cookie
if not username:
sso_user = get_current_user_from_cookie(request)
if sso_user:
username = sso_user.get("username")
if not username:
raise credentials_exception
admin = db.query(Admin).filter(Admin.username == username).first()
if admin is None:
raise credentials_exception
return admin
@router.post("/login", response_model=LoginResponse)
def login(login_data: LoginRequest, db: Session = Depends(get_db)):
"""管理员登录"""
admin = db.query(Admin).filter(Admin.username == login_data.username).first()
if not admin or not verify_password(login_data.password, admin.password_hash):
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="用户名或密码错误"
)
access_token = create_access_token(data={"sub": admin.username})
return {"access_token": access_token, "token_type": "bearer"}
@router.post("/change-password")
def change_password(
password_data: PasswordChange,
current_admin: Admin = Depends(get_current_admin),
db: Session = Depends(get_db)
):
"""修改密码"""
# 验证原密码
if not verify_password(password_data.old_password, current_admin.password_hash):
raise HTTPException(
status_code=status.HTTP_400_BAD_REQUEST,
detail="原密码错误"
)
# 更新密码
current_admin.password_hash = get_password_hash(password_data.new_password)
db.commit()
return {"message": "密码修改成功"}
@router.get("/me")
def get_me(current_admin: Admin = Depends(get_current_admin)):
"""获取当前管理员信息"""
return {"username": current_admin.username}
@router.post("/logout")
def logout(response: Response):
"""退出登录:清除本地 token 状态并删除 SSO Cookie"""
response.delete_cookie("access_token", domain=SSO_COOKIE_DOMAIN, path="/")
return {"message": "退出成功"}
def init_admin(db: Session):
"""初始化默认管理员账号"""
admin = db.query(Admin).filter(Admin.username == "admin").first()
if not admin:
admin = Admin(
username="admin",
password_hash=get_password_hash("abcd1234")
)
db.add(admin)
db.commit()
print("✅ 默认管理员账号已创建: admin / abcd1234")
+113
View File
@@ -0,0 +1,113 @@
"""
分类管理路由
"""
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy.orm import Session
from typing import List
import sys
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from database import get_db
from models import Category, Post
from schemas import CategoryCreate, CategoryResponse
from routers.auth import get_current_admin
router = APIRouter(prefix="/api/categories", tags=["分类"])
@router.get("", response_model=List[CategoryResponse])
def get_categories(db: Session = Depends(get_db)):
"""获取所有分类"""
categories = db.query(Category).all()
# 添加文章数量
result = []
for cat in categories:
post_count = db.query(Post).filter(Post.category_id == cat.id).count()
cat_dict = {
"id": cat.id,
"name": cat.name,
"description": cat.description,
"created_at": cat.created_at,
"post_count": post_count
}
result.append(cat_dict)
return result
@router.post("", response_model=CategoryResponse)
def create_category(
category_data: CategoryCreate,
current_admin=Depends(get_current_admin),
db: Session = Depends(get_db)
):
"""创建分类"""
# 检查数量限制
count = db.query(Category).count()
if count >= 25:
raise HTTPException(status_code=400, detail="分类数量已达上限(25个)")
# 检查名称重复
existing = db.query(Category).filter(Category.name == category_data.name).first()
if existing:
raise HTTPException(status_code=400, detail="分类名称已存在")
category = Category(**category_data.dict())
db.add(category)
db.commit()
db.refresh(category)
return {
"id": category.id,
"name": category.name,
"description": category.description,
"created_at": category.created_at,
"post_count": 0
}
@router.put("/{category_id}", response_model=CategoryResponse)
def update_category(
category_id: int,
category_data: CategoryCreate,
current_admin=Depends(get_current_admin),
db: Session = Depends(get_db)
):
"""更新分类"""
category = db.query(Category).filter(Category.id == category_id).first()
if not category:
raise HTTPException(status_code=404, detail="分类不存在")
category.name = category_data.name
category.description = category_data.description
db.commit()
db.refresh(category)
post_count = db.query(Post).filter(Post.category_id == category.id).count()
return {
"id": category.id,
"name": category.name,
"description": category.description,
"created_at": category.created_at,
"post_count": post_count
}
@router.delete("/{category_id}")
def delete_category(
category_id: int,
current_admin=Depends(get_current_admin),
db: Session = Depends(get_db)
):
"""删除分类"""
category = db.query(Category).filter(Category.id == category_id).first()
if not category:
raise HTTPException(status_code=404, detail="分类不存在")
# 将该分类下的文章设为未分类
db.query(Post).filter(Post.category_id == category_id).update({"category_id": None})
db.delete(category)
db.commit()
return {"message": "分类已删除"}
File diff suppressed because it is too large Load Diff
+541
View File
@@ -0,0 +1,541 @@
"""
图片上传管理路由
"""
from fastapi import APIRouter, UploadFile, File, Depends, HTTPException, Query
from fastapi.responses import FileResponse
from sqlalchemy.orm import Session
from typing import List, Optional
import os
import uuid
import subprocess
from datetime import datetime
from PIL import Image as PILImage
import sys
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from database import get_db
from models import Image, Post, Setting, Page
from routers.auth import get_current_admin
router = APIRouter(prefix="/api/images", tags=["图片"])
# 上传配置
UPLOAD_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "uploads")
THUMB_DIR = os.path.join(UPLOAD_DIR, "thumbnails")
ALLOWED_EXTENSIONS = {'.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg'}
MAX_FILE_SIZE = 10 * 1024 * 1024 # 10MB
THUMB_WIDTH = 320 # 缩略图宽度
THUMB_QUALITY = 85
# 确保上传目录存在
os.makedirs(UPLOAD_DIR, exist_ok=True)
os.makedirs(THUMB_DIR, exist_ok=True)
def generate_thumbnail(img_path: str, thumb_path: str) -> bool:
"""生成缩略图"""
try:
# 检查是否是 SVG 文件
if img_path.lower().endswith('.svg'):
# SVG 直接复制作为"缩略图"(矢量图不需要缩放)
import shutil
shutil.copy2(img_path, thumb_path)
return True
# 使用 Pillow 生成缩略图(不需要 ImageMagick,更轻量)
with PILImage.open(img_path) as img:
# 转换为 RGB(处理 PNG 透明背景等)
if img.mode in ('RGBA', 'LA', 'P'):
img = img.convert('RGB')
# 计算等比例缩放后的高度
ratio = THUMB_WIDTH / img.width
thumb_height = int(img.height * ratio)
# 生成缩略图
thumb = img.resize((THUMB_WIDTH, thumb_height), PILImage.Resampling.LANCZOS)
# 保存
ext = os.path.splitext(thumb_path)[1].lower()
if ext == '.png':
thumb.save(thumb_path, 'PNG')
else:
thumb.save(thumb_path, 'JPEG', quality=THUMB_QUALITY)
return True
except Exception as e:
print(f"生成缩略图失败: {e}")
return False
def get_image_dimensions(img_path: str) -> tuple:
"""获取图片尺寸"""
try:
# SVG 文件特殊处理
if img_path.lower().endswith('.svg'):
return get_svg_dimensions(img_path)
with PILImage.open(img_path) as img:
return img.width, img.height
except:
return None, None
def get_svg_dimensions(svg_path: str) -> tuple:
"""从 SVG 文件中提取宽度和高度"""
try:
import xml.etree.ElementTree as ET
# 解析 SVG 文件
tree = ET.parse(svg_path)
root = tree.getroot()
# 获取 width 和 height 属性
width = root.get('width', '')
height = root.get('height', '')
# 解析 viewBox 作为备选
viewbox = root.get('viewBox', '')
def parse_value(val):
"""解析数值,移除单位(如 px)"""
if not val:
return None
# 移除常见单位
val = val.strip().replace('px', '').replace('pt', '').replace('em', '').replace('rem', '')
try:
return int(float(val))
except:
return None
w = parse_value(width)
h = parse_value(height)
# 如果没有 width/height,尝试从 viewBox 解析
if (w is None or h is None) and viewbox:
parts = viewbox.replace(',', ' ').split()
if len(parts) >= 4:
# viewBox="minX minY width height"
w = parse_value(parts[2]) if w is None else w
h = parse_value(parts[3]) if h is None else h
# 返回默认值如果无法解析
return w or 100, h or 100
except Exception as e:
print(f"解析 SVG 尺寸失败: {e}")
return 100, 100
def allowed_file(filename):
"""检查文件扩展名"""
return os.path.splitext(filename.lower())[1] in ALLOWED_EXTENSIONS
@router.get("")
def get_images(
skip: int = 0,
limit: int = 50,
db: Session = Depends(get_db),
current_admin=Depends(get_current_admin)
):
"""获取图片列表(按时间倒序)"""
total = db.query(Image).count()
images = db.query(Image).order_by(Image.created_at.desc()).offset(skip).limit(limit).all()
return {
"items": [
{
"id": img.id,
"filename": img.filename,
"original_name": img.original_name,
"file_path": img.file_path,
"thumb_path": img.thumb_path,
"description": img.description,
"file_size": img.file_size,
"width": img.width,
"height": img.height,
"created_at": img.created_at.isoformat() if img.created_at else None
}
for img in images
],
"total": total,
"skip": skip,
"limit": limit
}
@router.post("/upload")
async def upload_image(
file: UploadFile = File(...),
description: str = "",
current_admin=Depends(get_current_admin),
db: Session = Depends(get_db)
):
"""上传图片(自动生成缩略图)"""
# 检查文件类型
if not allowed_file(file.filename):
raise HTTPException(
status_code=400,
detail="不支持的文件格式,仅支持 JPG、PNG、GIF、WebP"
)
# 读取文件内容检查大小
content = await file.read()
if len(content) > MAX_FILE_SIZE:
raise HTTPException(
status_code=400,
detail=f"文件大小超过10MB限制"
)
# 生成唯一文件名(统一转 jpg 以便处理)
ext = os.path.splitext(file.filename.lower())[1]
if ext == '.webp':
ext = '.jpg' # webp 转 jpg
unique_filename = f"{datetime.now().strftime('%Y%m%d')}_{uuid.uuid4().hex[:8]}{ext}"
file_path = os.path.join(UPLOAD_DIR, unique_filename)
thumb_filename = f"thumb_{unique_filename}"
thumb_path_full = os.path.join(THUMB_DIR, thumb_filename)
# 保存原文件
with open(file_path, "wb") as f:
f.write(content)
# 获取图片尺寸
width, height = get_image_dimensions(file_path)
# 生成缩略图
thumb_path_db = None
if generate_thumbnail(file_path, thumb_path_full):
thumb_path_db = f"/uploads/thumbnails/{thumb_filename}"
# 保存到数据库
image = Image(
filename=unique_filename,
original_name=file.filename,
file_path=f"/uploads/{unique_filename}",
thumb_path=thumb_path_db,
description=description,
file_size=len(content),
width=width,
height=height
)
db.add(image)
db.commit()
db.refresh(image)
return {
"id": image.id,
"filename": image.filename,
"original_name": image.original_name,
"description": image.description,
"file_size": image.file_size,
"file_path": image.file_path,
"thumb_path": image.thumb_path,
"width": image.width,
"height": image.height,
"created_at": image.created_at
}
async def process_single_image(file: UploadFile, db: Session):
"""处理单张图片(内部函数,用于批量上传)"""
# 检查文件类型
if not allowed_file(file.filename):
raise ValueError(f"不支持的文件格式")
# 读取内容
content = await file.read()
if len(content) > MAX_FILE_SIZE:
raise ValueError(f"文件大小超过10MB限制")
# 生成唯一文件名
ext = os.path.splitext(file.filename.lower())[1]
if ext == '.webp':
ext = '.jpg'
unique_filename = f"{datetime.now().strftime('%Y%m%d')}_{uuid.uuid4().hex[:8]}{ext}"
file_path = os.path.join(UPLOAD_DIR, unique_filename)
thumb_filename = f"thumb_{unique_filename}"
thumb_path_full = os.path.join(THUMB_DIR, thumb_filename)
# 保存原文件
with open(file_path, "wb") as f:
f.write(content)
# 获取图片尺寸
width, height = get_image_dimensions(file_path)
# 生成缩略图
thumb_path_db = None
if generate_thumbnail(file_path, thumb_path_full):
thumb_path_db = f"/uploads/thumbnails/{thumb_filename}"
# 保存到数据库
image = Image(
filename=unique_filename,
original_name=file.filename,
file_path=f"/uploads/{unique_filename}",
thumb_path=thumb_path_db,
file_size=len(content),
width=width,
height=height
)
db.add(image)
db.commit()
db.refresh(image)
return {
"id": image.id,
"filename": image.filename,
"original_name": image.original_name,
"file_path": image.file_path,
"thumb_path": image.thumb_path,
"width": image.width,
"height": image.height,
"file_size": image.file_size,
"created_at": image.created_at
}
@router.post("/upload-multiple")
async def upload_multiple_images(
files: List[UploadFile] = File(...),
current_admin=Depends(get_current_admin),
db: Session = Depends(get_db)
):
"""批量上传图片(最多10张,自动生成缩略图)"""
if len(files) > 10:
raise HTTPException(status_code=400, detail="一次最多上传10张图片")
uploaded = []
errors = []
for file in files:
try:
result = await process_single_image(file, db)
uploaded.append(result)
except Exception as e:
errors.append(f"{file.filename}: {str(e)}")
return {
"uploaded": uploaded,
"errors": errors,
"total": len(files),
"success": len(uploaded)
}
@router.get("/serve/{filename}")
def serve_image(filename: str):
"""提供原图访问"""
file_path = os.path.join(UPLOAD_DIR, filename)
if not os.path.exists(file_path):
raise HTTPException(status_code=404, detail="图片不存在")
return FileResponse(
file_path,
headers={
"Cache-Control": "no-cache, no-store, must-revalidate",
"Pragma": "no-cache",
"Expires": "0"
}
)
@router.get("/thumb/{filename}")
def serve_thumbnail(filename: str):
"""提供缩略图访问"""
file_path = os.path.join(THUMB_DIR, filename)
if not os.path.exists(file_path):
raise HTTPException(status_code=404, detail="缩略图不存在")
return FileResponse(
file_path,
headers={
"Cache-Control": "no-cache, no-store, must-revalidate",
"Pragma": "no-cache",
"Expires": "0"
}
)
@router.delete("/{image_id}")
def delete_image(
image_id: int,
current_admin=Depends(get_current_admin),
db: Session = Depends(get_db)
):
"""删除图片(同时删除缩略图)"""
image = db.query(Image).filter(Image.id == image_id).first()
if not image:
raise HTTPException(status_code=404, detail="图片不存在")
# 删除原图
file_path = os.path.join(UPLOAD_DIR, image.filename)
if os.path.exists(file_path):
os.remove(file_path)
# 删除缩略图
if image.thumb_path:
thumb_filename = os.path.basename(image.thumb_path)
thumb_path = os.path.join(THUMB_DIR, thumb_filename)
if os.path.exists(thumb_path):
os.remove(thumb_path)
# 删除数据库记录
db.delete(image)
db.commit()
return {"message": "图片已删除"}
@router.post("/cleanup-unused")
def cleanup_unused_images(
current_admin=Depends(get_current_admin),
db: Session = Depends(get_db)
):
"""清理未使用的图片(未被任何文章、页面、设置引用)"""
import json
# 获取所有图片
all_images = db.query(Image).all()
# 收集所有被使用的图片路径
used_paths = set()
# 1. 检查文章content和cover_image
posts = db.query(Post).all()
for post in posts:
if post.cover_image:
used_paths.add(post.cover_image)
if post.content:
# 提取content中的所有图片路径 /uploads/xxx
import re
paths = re.findall(r'/uploads/[^\s"\')<>]+', post.content)
used_paths.update(paths)
# 2. 检查页面content
pages = db.query(Page).all()
for page in pages:
if page.content:
import re
paths = re.findall(r'/uploads/[^\s"\')<>]+', page.content)
used_paths.update(paths)
# 3. 检查设置
settings = db.query(Setting).first()
if settings:
if settings.blog_logo:
used_paths.add(settings.blog_logo)
if settings.banner_image:
used_paths.add(settings.banner_image)
# 轮播图
if settings.banner_images:
try:
banner_list = json.loads(settings.banner_images)
if isinstance(banner_list, list):
used_paths.update(banner_list)
except:
pass
# 找出未使用的图片
unused_images = []
for img in all_images:
if img.file_path not in used_paths:
unused_images.append(img)
# 删除未使用的图片
deleted_count = 0
freed_space = 0
for img in unused_images:
# 删除原图文件
file_path = os.path.join(UPLOAD_DIR, img.filename)
if os.path.exists(file_path):
freed_space += os.path.getsize(file_path)
os.remove(file_path)
# 删除缩略图文件
if img.thumb_path:
thumb_filename = os.path.basename(img.thumb_path)
thumb_path = os.path.join(THUMB_DIR, thumb_filename)
if os.path.exists(thumb_path):
freed_space += os.path.getsize(thumb_path)
os.remove(thumb_path)
# 删除数据库记录
db.delete(img)
deleted_count += 1
db.commit()
return {
"message": f"清理完成,删除 {deleted_count} 张未使用的图片",
"deleted_count": deleted_count,
"freed_space_bytes": freed_space,
"freed_space_mb": round(freed_space / (1024 * 1024), 2)
}
@router.get("/usage-stats")
def get_image_usage_stats(
current_admin=Depends(get_current_admin),
db: Session = Depends(get_db)
):
"""获取图片使用统计"""
import json
total_images = db.query(Image).count()
# 收集所有被使用的图片路径
used_paths = set()
# 检查文章
posts = db.query(Post).all()
for post in posts:
if post.cover_image:
used_paths.add(post.cover_image)
if post.content:
import re
paths = re.findall(r'/uploads/[^\s"\')<>]+', post.content)
used_paths.update(paths)
# 检查页面
pages = db.query(Page).all()
for page in pages:
if page.content:
import re
paths = re.findall(r'/uploads/[^\s"\')<>]+', page.content)
used_paths.update(paths)
# 检查设置
settings = db.query(Setting).first()
if settings:
if settings.blog_logo:
used_paths.add(settings.blog_logo)
if settings.banner_image:
used_paths.add(settings.banner_image)
if settings.banner_images:
try:
banner_list = json.loads(settings.banner_images)
if isinstance(banner_list, list):
used_paths.update(banner_list)
except:
pass
# 计算未使用图片
all_images = db.query(Image).all()
unused_count = 0
unused_space = 0
for img in all_images:
if img.file_path not in used_paths:
unused_count += 1
unused_space += (img.file_size or 0)
return {
"total_images": total_images,
"used_images": len(used_paths),
"unused_images": unused_count,
"unused_space_bytes": unused_space,
"unused_space_mb": round(unused_space / (1024 * 1024), 2)
}
+222
View File
@@ -0,0 +1,222 @@
"""
文章管理路由
"""
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy.orm import Session
from typing import List, Optional
import sys
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from database import get_db
from models import Post, Category, Tag, PostTag, Image
from schemas import PostCreate, PostUpdate, PostResponse, PostListItem
from routers.auth import get_current_admin
import re
router = APIRouter(prefix="/api/posts", tags=["文章"])
@router.get("", response_model=List[PostListItem])
def get_posts(
skip: int = Query(0, ge=0),
limit: int = Query(10, ge=1, le=100),
category_id: Optional[int] = None,
tag_id: Optional[int] = None,
is_draft: Optional[bool] = None,
search: Optional[str] = None,
db: Session = Depends(get_db)
):
"""获取文章列表(支持筛选)"""
query = db.query(Post)
# 筛选条件
if category_id:
query = query.filter(Post.category_id == category_id)
if tag_id:
query = query.join(PostTag).filter(PostTag.tag_id == tag_id)
if is_draft is not None:
query = query.filter(Post.is_draft == is_draft)
if search:
query = query.filter(Post.title.contains(search) | Post.content.contains(search))
# 按时间倒序
query = query.order_by(Post.created_at.desc())
posts = query.offset(skip).limit(limit).all()
return posts
@router.get("/{post_id}", response_model=PostResponse)
def get_post(post_id: str, db: Session = Depends(get_db)):
"""获取单篇文章(支持 id 或 slug)"""
# 尝试作为 id 查询
try:
id_int = int(post_id)
post = db.query(Post).filter(Post.id == id_int).first()
except ValueError:
post = None
# 如果不是 id,尝试作为 slug 查询
if not post:
post = db.query(Post).filter(Post.slug == post_id).first()
if not post:
raise HTTPException(status_code=404, detail="文章不存在")
# 增加浏览量
post.view_count += 1
db.commit()
return post
@router.post("", response_model=PostResponse)
def create_post(
post_data: PostCreate,
current_admin=Depends(get_current_admin),
db: Session = Depends(get_db)
):
"""创建文章"""
# 处理标签
tags = []
if post_data.tag_ids:
tags = db.query(Tag).filter(Tag.id.in_(post_data.tag_ids)).all()
# 创建文章
post = Post(
title=post_data.title,
content=post_data.content,
content_type=post_data.content_type or "markdown",
summary=post_data.summary or post_data.content[:200] + "...",
cover_image=post_data.cover_image,
is_published=post_data.is_published,
is_draft=post_data.is_draft,
category_id=post_data.category_id,
tags=tags
)
db.add(post)
db.commit()
db.refresh(post)
return post
@router.put("/{post_id}", response_model=PostResponse)
def update_post(
post_id: int,
post_data: PostUpdate,
current_admin=Depends(get_current_admin),
db: Session = Depends(get_db)
):
"""更新文章"""
post = db.query(Post).filter(Post.id == post_id).first()
if not post:
raise HTTPException(status_code=404, detail="文章不存在")
# 更新字段
update_data = post_data.dict(exclude_unset=True)
# 处理标签
if "tag_ids" in update_data:
tag_ids = update_data.pop("tag_ids")
if tag_ids is not None:
post.tags = db.query(Tag).filter(Tag.id.in_(tag_ids)).all()
# 更新其他字段
for field, value in update_data.items():
setattr(post, field, value)
# 自动更新摘要
if post_data.content and not post_data.summary:
post.summary = post_data.content[:200] + "..."
db.commit()
db.refresh(post)
return post
def delete_post_images(post: Post, db: Session):
"""删除文章关联的所有图片(物理删除文件+数据库记录)"""
UPLOAD_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "uploads")
THUMB_DIR = os.path.join(UPLOAD_DIR, "thumbnails")
# 收集所有要删除的图片路径
image_paths = set()
# 1. 封面图
if post.cover_image:
image_paths.add(post.cover_image)
# 2. 内容中的图片
if post.content:
paths = re.findall(r'/uploads/[^\s"\')<>]+', post.content)
image_paths.update(paths)
# 删除每张图片
for path in image_paths:
filename = os.path.basename(path)
# 查找数据库记录
image = db.query(Image).filter(Image.file_path == path).first()
if image:
# 删除原图文件
file_path = os.path.join(UPLOAD_DIR, filename)
if os.path.exists(file_path):
os.remove(file_path)
# 删除缩略图文件
if image.thumb_path:
thumb_filename = os.path.basename(image.thumb_path)
thumb_path = os.path.join(THUMB_DIR, thumb_filename)
if os.path.exists(thumb_path):
os.remove(thumb_path)
# 删除数据库记录
db.delete(image)
@router.delete("/{post_id}")
def delete_post(
post_id: int,
current_admin=Depends(get_current_admin),
db: Session = Depends(get_db)
):
"""删除文章(同时删除关联的图片)"""
post = db.query(Post).filter(Post.id == post_id).first()
if not post:
raise HTTPException(status_code=404, detail="文章不存在")
# 先删除关联图片
delete_post_images(post, db)
# 再删除文章
db.delete(post)
db.commit()
return {"message": "文章已删除"}
@router.get("/stats/overview")
def get_stats(db: Session = Depends(get_db)):
"""获取统计数据"""
from sqlalchemy import func
total_posts = db.query(Post).filter(Post.is_draft == False).count()
total_views = db.query(func.sum(Post.view_count)).scalar() or 0
# 按分类统计
category_counts = {}
categories = db.query(Category).all()
for cat in categories:
count = db.query(Post).filter(Post.category_id == cat.id).count()
category_counts[cat.name] = count
# 最近文章
recent_posts = db.query(Post).order_by(Post.created_at.desc()).limit(5).all()
return {
"total_posts": total_posts,
"total_views": total_views,
"category_counts": category_counts,
"recent_posts": recent_posts
}
+155
View File
@@ -0,0 +1,155 @@
"""
博客设置路由
"""
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy.orm import Session
import sys
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from database import get_db
from models import Setting, Page
from schemas import SettingUpdate, SettingResponse, PageCreate, PageUpdate, PageResponse
from routers.auth import get_current_admin
from typing import List
router = APIRouter(prefix="/api", tags=["设置"])
def get_or_create_settings(db: Session):
"""获取或创建设置"""
setting = db.query(Setting).first()
if not setting:
setting = Setting()
db.add(setting)
db.commit()
db.refresh(setting)
return setting
@router.get("/settings", response_model=SettingResponse)
def get_settings(db: Session = Depends(get_db)):
"""获取博客设置"""
setting = get_or_create_settings(db)
return setting
@router.get("/settings/public")
def get_settings_public(db: Session = Depends(get_db)):
"""公开获取博客设置(仅返回前端展示所需字段)"""
setting = get_or_create_settings(db)
import json
try:
colors = json.loads(setting.theme_colors) if setting.theme_colors else {}
except Exception:
colors = {}
return {
"blog_name": setting.blog_name,
"blog_description": setting.blog_description,
"welcome_message": setting.welcome_message,
"blog_logo": setting.blog_logo,
"banner_image": setting.banner_image,
"banner_type": setting.banner_type,
"banner_images": setting.banner_images,
"theme": setting.theme,
"layout": setting.layout,
"theme_colors": colors,
}
@router.put("/settings", response_model=SettingResponse)
def update_settings(
setting_data: SettingUpdate,
current_admin=Depends(get_current_admin),
db: Session = Depends(get_db)
):
"""更新博客设置"""
setting = get_or_create_settings(db)
update_data = setting_data.dict(exclude_unset=True)
for field, value in update_data.items():
setattr(setting, field, value)
db.commit()
db.refresh(setting)
return setting
# ==================== 自定义页面管理 ====================
@router.get("/pages", response_model=List[PageResponse])
def get_pages(db: Session = Depends(get_db)):
"""获取所有自定义页面"""
pages = db.query(Page).order_by(Page.nav_order.asc(), Page.created_at.desc()).all()
return pages
@router.get("/pages/{slug}", response_model=PageResponse)
def get_page_by_slug(slug: str, db: Session = Depends(get_db)):
"""通过slug获取页面"""
page = db.query(Page).filter(Page.slug == slug).first()
if not page:
raise HTTPException(status_code=404, detail="页面不存在")
return page
@router.post("/pages", response_model=PageResponse)
def create_page(
page_data: PageCreate,
current_admin=Depends(get_current_admin),
db: Session = Depends(get_db)
):
"""创建自定义页面"""
# 检查数量限制
count = db.query(Page).count()
if count >= 5:
raise HTTPException(status_code=400, detail="自定义页面数量已达上限(5个)")
# 检查slug重复
existing = db.query(Page).filter(Page.slug == page_data.slug).first()
if existing:
raise HTTPException(status_code=400, detail="页面标识已存在")
page = Page(**page_data.dict())
db.add(page)
db.commit()
db.refresh(page)
return page
@router.put("/pages/{page_id}", response_model=PageResponse)
def update_page(
page_id: int,
page_data: PageUpdate,
current_admin=Depends(get_current_admin),
db: Session = Depends(get_db)
):
"""更新自定义页面"""
page = db.query(Page).filter(Page.id == page_id).first()
if not page:
raise HTTPException(status_code=404, detail="页面不存在")
update_data = page_data.dict(exclude_unset=True)
for field, value in update_data.items():
setattr(page, field, value)
db.commit()
db.refresh(page)
return page
@router.delete("/pages/{page_id}")
def delete_page(
page_id: int,
current_admin=Depends(get_current_admin),
db: Session = Depends(get_db)
):
"""删除自定义页面"""
page = db.query(Page).filter(Page.id == page_id).first()
if not page:
raise HTTPException(status_code=404, detail="页面不存在")
db.delete(page)
db.commit()
return {"message": "页面已删除"}
+64
View File
@@ -0,0 +1,64 @@
"""
标签管理路由
"""
from fastapi import APIRouter, Depends, HTTPException
from sqlalchemy.orm import Session
from typing import List
import sys
import os
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from database import get_db
from models import Tag
from schemas import TagCreate, TagResponse
from routers.auth import get_current_admin
router = APIRouter(prefix="/api/tags", tags=["标签"])
@router.get("/", response_model=List[TagResponse])
def get_tags(db: Session = Depends(get_db)):
"""获取所有标签"""
tags = db.query(Tag).all()
return tags
@router.post("/", response_model=TagResponse)
def create_tag(
tag_data: TagCreate,
current_admin=Depends(get_current_admin),
db: Session = Depends(get_db)
):
"""创建标签"""
# 检查数量限制
count = db.query(Tag).count()
if count >= 20:
raise HTTPException(status_code=400, detail="标签数量已达上限(20个)")
# 检查名称重复
existing = db.query(Tag).filter(Tag.name == tag_data.name).first()
if existing:
raise HTTPException(status_code=400, detail="标签名称已存在")
tag = Tag(**tag_data.dict())
db.add(tag)
db.commit()
db.refresh(tag)
return tag
@router.delete("/{tag_id}")
def delete_tag(
tag_id: int,
current_admin=Depends(get_current_admin),
db: Session = Depends(get_db)
):
"""删除标签"""
tag = db.query(Tag).filter(Tag.id == tag_id).first()
if not tag:
raise HTTPException(status_code=404, detail="标签不存在")
db.delete(tag)
db.commit()
return {"message": "标签已删除"}
@@ -0,0 +1,608 @@
"""
通用采集路由 - 基于 Scrapling 引擎
结合上层业务逻辑:编码修复、数学公式处理、微信防盗链、图片下载等
"""
import os
import re
import sys
import uuid
import asyncio
import tempfile
import base64
from datetime import datetime
from typing import Optional, List
from urllib.parse import urljoin, urlparse
from pathlib import Path
from fastapi import APIRouter, HTTPException, Depends, UploadFile, File
from pydantic import BaseModel, HttpUrl
from sqlalchemy.orm import Session
from bs4 import BeautifulSoup
# Scrapling 引擎
from scrapling import Fetcher, StealthyFetcher
# 复用 crawl.py 的通用工具函数
from .crawl import (
fix_encoding,
fix_math_formulas,
html_to_markdown,
extract_title_from_html,
extract_main_content,
download_image,
extract_images,
extract_images_from_markdown,
is_spa_site,
convert_spa_html_to_markdown,
CrawlRequest,
CrawlResponse,
CrawlImage,
)
# 复用图片处理
from .images import generate_thumbnail, get_image_dimensions
# 数据库
from database import get_db
from models import Post, Category, Tag, PostTag, Image
router = APIRouter(prefix="/api/universal-crawl", tags=["通用采集"])
# 确保上传目录存在
UPLOAD_DIR = Path("uploads")
UPLOAD_DIR.mkdir(exist_ok=True)
TEMP_DIR = Path("uploads/temp_crawl")
TEMP_DIR.mkdir(exist_ok=True)
class UniversalCrawlRequest(BaseModel):
url: str
def _needs_browser(url: str) -> bool:
"""判断是否需要使用浏览器渲染"""
domain = urlparse(url).netloc.lower()
browser_domains = [
'mp.weixin.qq.com',
'juejin.cn',
'zhihu.com',
'segmentfault.com',
'infoq.cn',
'infoq.com',
'51cto.com',
]
return any(d in domain for d in browser_domains)
def _fetch_with_scrapling(url: str) -> str:
"""使用 Scrapling Fetcher (curl_cffi) 获取页面"""
fetcher = Fetcher()
response = fetcher.get(url)
if response.status != 200:
raise Exception(f"HTTP {response.status}: {response.reason}")
# Scrapling Response 的 text 属性返回解析后的文本,html_content 返回原始 HTML
return response.html_content if hasattr(response, 'html_content') else response.text
async def _fetch_with_stealth(url: str) -> str:
"""使用 Scrapling StealthyFetcher 获取页面,失败时回退到原生 Playwright"""
try:
from scrapling import StealthyFetcher
# Scrapling v0.3+ 推荐使用类方法直接调用,无需实例化
response = await StealthyFetcher.async_fetch(url, wait=3000)
if not response or response.status != 200:
raise Exception(f"HTTP {response.status}")
html = response.html_content if hasattr(response, 'html_content') else response.text
if not html or len(html) < 100:
raise Exception("Empty page")
print(f"[Universal Crawl] StealthyFetcher success: {len(html)} bytes")
return html
except Exception as e:
print(f"[Universal Crawl] StealthyFetcher failed: {e}")
print("[Universal Crawl] Falling back to native Playwright")
return await _fetch_with_playwright(url)
async def _fetch_with_playwright(url: str) -> str:
"""使用原生 Playwright 异步获取页面,带完善反爬配置"""
from playwright.async_api import async_playwright
async with async_playwright() as p:
browser = await p.chromium.launch(
headless=True,
args=[
"--disable-blink-features=AutomationControlled",
"--disable-web-security",
"--disable-features=IsolateOrigins,site-per-process",
"--disable-dev-shm-usage",
"--no-sandbox",
]
)
context = await browser.new_context(
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0",
viewport={"width": 1920, "height": 1080},
locale="zh-CN",
timezone_id="Asia/Shanghai",
extra_http_headers={
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
"Accept-Encoding": "gzip, deflate, br",
"Cache-Control": "max-age=0",
"Sec-Ch-Ua": '"Not_A Brand";v="8", "Chromium";v="120", "Microsoft Edge";v="120"',
"Sec-Ch-Ua-Mobile": "?0",
"Sec-Ch-Ua-Platform": '"Windows"',
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Upgrade-Insecure-Requests": "1",
},
)
page = await context.new_page()
try:
# SegmentFault 需要先访问首页建立信任,再跳转目标页面
if 'segmentfault.com' in url:
await page.goto("https://segmentfault.com", wait_until="domcontentloaded", timeout=15000)
await asyncio.sleep(2)
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
await asyncio.sleep(5)
# 知乎需要先访问首页建立 cookie,再跳转目标页面
elif 'zhihu.com' in url:
await page.goto("https://www.zhihu.com", wait_until="domcontentloaded", timeout=15000)
await asyncio.sleep(2)
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
await asyncio.sleep(3)
# 等待正文加载(知乎专栏文章正文在 .Post-RichTextContainer 或 .RichContent-inner 里)
try:
await page.wait_for_selector(".Post-RichTextContainer, .RichContent-inner", timeout=10000)
await asyncio.sleep(1)
except Exception:
pass
else:
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
await asyncio.sleep(2)
html = await page.content()
finally:
await context.close()
await browser.close()
return html
def _bypass_wechat_images(html: str, url: str) -> str:
"""
微信图片防盗链绕过:在页面上下文中执行 fetch 获取图片 base64
这里我们在后端模拟:如果检测到 mmbiz.qpic.cn 图片,
将其 src 替换为 data-src(微信文章中 data-src 通常包含真实 URL)
"""
if 'mp.weixin.qq.com' not in url:
return html
soup = BeautifulSoup(html, 'lxml')
for img in soup.find_all('img'):
data_src = img.get('data-src')
if data_src and 'mmbiz.qpic.cn' in data_src:
img['src'] = data_src
return str(soup)
def _extract_infoq_content(html: str) -> str:
"""提取 InfoQ 文章正文(.ProseMirror 或 article 标签)"""
soup = BeautifulSoup(html, 'lxml')
# 优先使用 .ProseMirror(InfoQ 正文编辑器容器)
container = soup.select_one('.ProseMirror')
if not container:
# fallback 到 article 标签
container = soup.find('article')
if container:
return str(container)
return html
def _extract_infoq_title(html: str) -> Optional[str]:
"""提取 InfoQ 文章标题,从 <title> 中去掉 ' - InfoQ' 后缀"""
match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
if match:
title = match.group(1).strip()
# 去掉 " - InfoQ" 后缀
title = re.sub(r'\s*[-–—]\s*InfoQ\s*$', '', title, flags=re.IGNORECASE).strip()
if len(title) > 5:
return title
return None
def _extract_juejin_content(html: str) -> str:
"""提取掘金文章正文(article.article 或 .article-viewer)"""
soup = BeautifulSoup(html, 'lxml')
# 优先使用 article.article
container = soup.select_one('article.article')
if not container:
# fallback 到 .article-viewer
container = soup.select_one('.article-viewer')
if not container:
# 再 fallback 到 #article-root
container = soup.select_one('#article-root')
if container:
return str(container)
return html
def _extract_juejin_title(html: str) -> Optional[str]:
"""提取掘金文章标题,从 <title> 中去掉 ' - 掘金' 后缀"""
match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
if match:
title = match.group(1).strip()
# 去掉 " - 掘金" 后缀
title = re.sub(r'\s*[-–—]\s*掘金\s*$', '', title, flags=re.IGNORECASE).strip()
if len(title) > 5:
return title
return None
def _extract_csdn_content(html: str) -> str:
"""提取 CSDN 文章正文(#content_views 或 .article_content)"""
soup = BeautifulSoup(html, 'lxml')
# 优先使用 #content_views(CSDN 正文主容器)
container = soup.select_one('#content_views')
if not container:
# fallback 到 .article_content
container = soup.select_one('.article_content')
if container:
return str(container)
return html
def _extract_csdn_title(html: str) -> Optional[str]:
"""提取 CSDN 文章标题,从 <title> 中去掉 '-CSDN博客' 后缀"""
match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
if match:
title = match.group(1).strip()
# 去掉 " - CSDN博客" / "-CSDN博客" 后缀
title = re.sub(r'\s*[-–—]?\s*CSDN博客\s*$', '', title, flags=re.IGNORECASE).strip()
if len(title) > 5:
return title
return None
def _filter_csdn_images(images: list) -> list:
"""过滤掉 CSDN 的 UI 图标图片(csdnimg.cn 的收藏/点赞等图标)"""
filtered = []
skip_patterns = [
'newHeart2023',
'tobarCollect',
'tobarCollection',
'copyright',
'avatar',
'blogv2/dist/pc/img',
]
for img in images:
url = img.original_url.lower()
if any(p in url for p in skip_patterns):
continue
filtered.append(img)
return filtered
def _extract_cnblogs_content(html: str) -> str:
"""提取博客园文章正文(#cnblogs_post_body 或 .postBody)"""
soup = BeautifulSoup(html, 'lxml')
container = soup.select_one('#cnblogs_post_body')
if not container:
container = soup.select_one('.postBody')
if container:
return str(container)
return html
def _extract_cnblogs_title(html: str) -> Optional[str]:
"""提取博客园文章标题,从 <title> 中去掉 ' - 博客园' 后缀"""
match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
if match:
title = match.group(1).strip()
title = re.sub(r'\s*[-–—]\s*博客园\s*$', '', title, flags=re.IGNORECASE).strip()
if len(title) > 5:
return title
return None
def _filter_cnblogs_images(images: list) -> list:
"""过滤掉博客园的 UI 图标图片"""
filtered = []
skip_patterns = [
'assets.cnblogs.com/logo',
'assets.cnblogs.com/icons',
'skins/custom/images/logo',
]
for img in images:
url = img.original_url.lower()
if any(p in url for p in skip_patterns):
continue
filtered.append(img)
return filtered
def _extract_51cto_content(html: str) -> str:
"""提取 51CTO 文章正文(.article-content 或 article)"""
soup = BeautifulSoup(html, 'lxml')
container = soup.select_one('.article-content')
if not container:
container = soup.find('article')
if container:
return str(container)
return html
def _extract_51cto_title(html: str) -> Optional[str]:
"""提取 51CTO 文章标题,从 <title> 中去掉 '-51CTO.COM' 后缀"""
match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
if match:
title = match.group(1).strip()
title = re.sub(r'\s*[-–—]?\s*51CTO\.COM\s*$', '', title, flags=re.IGNORECASE).strip()
if len(title) > 5:
return title
return None
def _filter_51cto_images(images: list) -> list:
"""过滤掉 51CTO 的 UI 图标和二维码图片"""
filtered = []
skip_patterns = [
'oss/202302', # 公众号矩阵二维码
'oss/202408', # 软考二维码
'oss/202506/06', # 顶部横幅
'oss/202302/07', # APP下载图
]
for img in images:
url = img.original_url.lower()
if any(p in url for p in skip_patterns):
continue
filtered.append(img)
return filtered
@router.post("/fetch", response_model=CrawlResponse)
async def universal_fetch(request: UniversalCrawlRequest):
"""通用采集:使用 Scrapling 引擎获取文章"""
try:
url = str(request.url)
print(f"[Universal Crawl] Fetching: {url}")
# 1. 判断采集策略
use_browser = _needs_browser(url)
# 2. 使用 Scrapling 获取页面
if use_browser:
print(f"[Universal Crawl] Using StealthyFetcher (browser) for {url}")
html = await _fetch_with_stealth(url)
else:
print(f"[Universal Crawl] Using Fetcher (curl_cffi) for {url}")
html = _fetch_with_scrapling(url)
if not html or len(html) < 100:
raise HTTPException(status_code=400, detail="无法获取页面内容,页面可能为空或需要登录")
# 3. 编码修复
html = fix_encoding(html)
# 4. 微信图片防盗链处理
html = _bypass_wechat_images(html, url)
# 5. 提取标题(InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取)
is_infoq = 'infoq.cn' in url or 'infoq.com' in url
is_juejin = 'juejin.cn' in url
is_csdn = 'csdn.net' in url or 'csdn.com' in url
is_cnblogs = 'cnblogs.com' in url
is_51cto = '51cto.com' in url
if is_infoq:
title = _extract_infoq_title(html)
elif is_juejin:
title = _extract_juejin_title(html)
elif is_csdn:
title = _extract_csdn_title(html)
elif is_cnblogs:
title = _extract_cnblogs_title(html)
elif is_51cto:
title = _extract_51cto_title(html)
else:
title = None
if not title:
title = extract_title_from_html(html)
if not title:
title = "未获取到标题"
# 6. 提取正文和图片
is_wechat = 'mp.weixin.qq.com' in url
images = extract_images(html, url)
if is_csdn:
images = _filter_csdn_images(images)
elif is_cnblogs:
images = _filter_cnblogs_images(images)
elif is_51cto:
images = _filter_51cto_images(images)
# 7. 转换为 Markdown(InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取)
if is_infoq:
infoq_html = _extract_infoq_content(html)
markdown_content = html_to_markdown(infoq_html, url)
elif is_juejin:
juejin_html = _extract_juejin_content(html)
markdown_content = html_to_markdown(juejin_html, url)
elif is_csdn:
csdn_html = _extract_csdn_content(html)
markdown_content = html_to_markdown(csdn_html, url)
elif is_cnblogs:
cnblogs_html = _extract_cnblogs_content(html)
markdown_content = html_to_markdown(cnblogs_html, url)
elif is_51cto:
cto_html = _extract_51cto_content(html)
markdown_content = html_to_markdown(cto_html, url)
elif is_spa_site(html):
markdown_content = convert_spa_html_to_markdown(html, images, url)
else:
markdown_content = html_to_markdown(html, url)
# 8. 数学公式修复
markdown_content = fix_math_formulas(markdown_content)
# 9. 清理 Markdown 中的 HTML 标签
markdown_content = re.sub(r'<[^>]+>', '', markdown_content)
markdown_content = re.sub(r'\n{3,}', '\n\n', markdown_content)
markdown_content = markdown_content.strip()
return CrawlResponse(
success=True,
title=title,
content=markdown_content,
images=images,
)
except HTTPException:
raise
except Exception as e:
print(f"[Universal Crawl] Error: {e}")
import traceback
traceback.print_exc()
raise HTTPException(status_code=500, detail=f"采集失败: {str(e)}")
@router.post("/save")
async def universal_save(
title: str,
content: str,
excerpt: Optional[str] = None,
category_id: Optional[int] = None,
source_url: Optional[str] = None,
images: Optional[str] = None,
db: Session = Depends(get_db)
):
"""保存通用采集的文章(复用 crawl.py 的保存逻辑)"""
try:
print(f"[Universal Crawl] Saving article: {title}")
# 解析图片列表
image_list = []
if images:
try:
import json
image_data = json.loads(images)
for img in image_data:
image_list.append(CrawlImage(**img))
except Exception as e:
print(f"[Universal Crawl] Parse images error: {e}")
# 创建临时目录
temp_dir = tempfile.mkdtemp(dir=TEMP_DIR)
final_dir = UPLOAD_DIR / "images"
final_dir.mkdir(exist_ok=True)
thumb_dir = UPLOAD_DIR / "thumbnails"
thumb_dir.mkdir(exist_ok=True)
# 下载图片
downloaded_images = []
for img in image_list:
try:
filename, filepath, success = download_image(str(img.url), temp_dir)
if success and filepath:
# 移动最终文件
final_filename = f"crawl_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}{os.path.splitext(filename)[1]}"
final_path = final_dir / final_filename
os.rename(filepath, str(final_path))
# 生成缩略图
thumb_filename = f"thumb_{final_filename}"
thumb_path = thumb_dir / thumb_filename
thumb_path_db = None
if generate_thumbnail(str(final_path), str(thumb_path)):
thumb_path_db = f"/uploads/thumbnails/{thumb_filename}"
# 获取尺寸
width, height = get_image_dimensions(str(final_path))
# 保存到数据库
db_image = Image(
filename=final_filename,
original_filename=img.alt or final_filename,
file_path=f"/uploads/images/{final_filename}",
file_size=os.path.getsize(str(final_path)),
mime_type=f"image/{os.path.splitext(final_filename)[1].lstrip('.') or 'jpeg'}",
width=width,
height=height,
thumbnail_path=thumb_path_db,
)
db.add(db_image)
db.flush()
# 替换内容中的图片 URL
original_url = str(img.url)
new_url = f"/uploads/images/{final_filename}"
content = content.replace(original_url, new_url)
downloaded_images.append({
"id": db_image.id,
"url": new_url,
"thumb": thumb_path_db,
})
except Exception as e:
print(f"[Universal Crawl] Download image error: {e}")
continue
# 清理 excerpt
if excerpt:
excerpt = re.sub(r'<[^>]+>', '', excerpt)
excerpt = excerpt.replace('\n', ' ').strip()[:200]
else:
excerpt = content.replace('\n', ' ').strip()[:200]
# 创建文章
now = datetime.now()
slug = re.sub(r'[^\w\u4e00-\u9fff-]', '-', title)[:50].strip('-')
slug = f"{slug}-{now.strftime('%Y%m%d%H%M%S')}-{uuid.uuid4().hex[:4]}"
# 生成摘要
summary = content[:300] if len(content) > 300 else content
summary = re.sub(r'[#*`\[\]\(\)!]', '', summary)
# 查找或创建分类
final_category_id = category_id
if not final_category_id:
default_cat = db.query(Category).filter(Category.name == "未分类").first()
if not default_cat:
default_cat = Category(name="未分类", slug="uncategorized")
db.add(default_cat)
db.flush()
final_category_id = default_cat.id
post = Post(
title=title,
slug=slug,
content=content,
excerpt=excerpt,
summary=summary,
category_id=final_category_id,
author_id=1,
status="draft",
view_count=0,
is_original=False,
source_url=source_url,
created_at=now,
updated_at=now,
published_at=None,
)
db.add(post)
db.flush()
db.commit()
return {
"status": "success",
"post_id": post.id,
"message": "文章保存成功",
"images": downloaded_images,
}
except Exception as e:
db.rollback()
print(f"[Universal Crawl] Save error: {e}")
import traceback
traceback.print_exc()
raise HTTPException(status_code=500, detail=f"保存失败: {str(e)}")
@@ -0,0 +1,474 @@
"""
微信公众号文章采集路由
使用 Playwright + Stealth 绕过反爬虫
支持 Markdown 转换、图片下载、预览编辑、确认入库
"""
from fastapi import APIRouter, Depends, HTTPException
from pydantic import BaseModel
from sqlalchemy.orm import Session
from typing import Optional, List
import httpx
import re
import os
import uuid
import asyncio
from urllib.parse import urlparse
from datetime import datetime
from bs4 import BeautifulSoup, NavigableString
from database import get_db
from models import Post, Tag, Category
from schemas import PostResponse
from routers.auth import get_current_admin
try:
from playwright.async_api import async_playwright
PLAYWRIGHT_AVAILABLE = True
except ImportError:
PLAYWRIGHT_AVAILABLE = False
print("[WeChat] Playwright not available")
try:
from playwright_stealth import stealth_async
STEALTH_AVAILABLE = True
except ImportError:
STEALTH_AVAILABLE = False
print("[WeChat] Stealth not available")
router = APIRouter(prefix="/api/wechat-crawl", tags=["wechat-crawl"])
UPLOADS_DIR = os.path.join(os.path.dirname(__file__), "..", "uploads")
os.makedirs(UPLOADS_DIR, exist_ok=True)
os.makedirs(os.path.join(UPLOADS_DIR, "thumbnails"), exist_ok=True)
class WechatCrawlRequest(BaseModel):
url: str
class WechatSaveRequest(BaseModel):
title: str
content: str
summary: Optional[str] = ""
cover_image: Optional[str] = ""
category_id: Optional[int] = None
tag_names: Optional[List[str]] = []
source_url: Optional[str] = ""
author: Optional[str] = ""
is_draft: Optional[bool] = False
# ============================================================
# 辅助函数
# ============================================================
def is_wechat_url(url: str) -> bool:
return "mp.weixin.qq.com" in url.lower()
def generate_slug(title: str, suffix: str = "") -> str:
slug = re.sub(r'[^\w\u4e00-\u9fa5]+', '-', title.strip())
slug = slug.strip('-').lower()
if len(slug) > 80:
slug = slug[:80]
if not slug:
slug = "wechat-article"
if suffix:
slug = f"{slug}-{suffix}"
return slug
def remove_empty_paragraphs(soup):
"""移除空的 p 标签"""
for p in soup.find_all('p'):
txt = p.get_text(strip=True)
if not txt:
imgs = p.find_all('img')
if not imgs:
p.decompose()
return soup
def fix_wechat_code_blocks(soup):
"""
修复微信代码块:
<pre class="code-snippet__js"> 里的 <code> 下直接是文本
或多个 <span class="code-snippet__line">...
按行拆成多个 code block 以便 markdownify 正确处理
"""
for pre in soup.find_all('pre', class_=re.compile('code-snippet')):
code_tag = pre.find('code')
if not code_tag:
continue
# 收集纯文本
lines = []
for span in code_tag.find_all('span', class_='code-snippet__line'):
line_text = span.get_text(separator='', strip=False)
if line_text:
lines.append(line_text)
if lines:
code_tag.clear()
for line in lines:
code_tag.append(NavigableString(line + '\n'))
return soup
def fix_wechat_images(soup):
"""
修复微信图片:
微信文章使用 data-src 存放真实图片URL,src 往往是 svg 占位图。
把 data-src 复制到 src,让 markdownify 正确识别图片。
"""
for img in soup.find_all('img'):
data_src = img.get('data-src', '')
src = img.get('src', '')
# data-src 是真实微信图片URL,且 src 不是有效的 http URL
if data_src and data_src.startswith('http') and (
not src or src.startswith('data:image') or not src.startswith('http')
):
img['src'] = data_src
# 清理无意义的占位属性
if 'data-src' in img.attrs:
del img['data-src']
return soup
def convert_html_to_markdown(html: str) -> str:
"""使用 markdownify 将 HTML 转为 Markdown"""
try:
import markdownify
except ImportError:
# 降级:用 BeautifulSoup 提取纯文本
soup = BeautifulSoup(html, 'html.parser')
return soup.get_text(separator='\n\n', strip=True)
soup = BeautifulSoup(html, 'html.parser')
soup = fix_wechat_images(soup)
soup = fix_wechat_code_blocks(soup)
soup = remove_empty_paragraphs(soup)
# markdownify 配置
md = markdownify.markdownify(
str(soup),
heading_style="ATX",
bullets="-",
strip=['script', 'style', 'iframe', 'noscript']
)
# 清理多余空行
md = re.sub(r'\n{3,}', '\n\n', md)
return md.strip()
def generate_filename(original_url: str, article_id: str = "") -> str:
"""生成唯一文件名"""
ext = os.path.splitext(urlparse(original_url).path)[1].lower()
if ext not in ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp']:
ext = '.jpg'
uid = str(uuid.uuid4())[:8]
return f"wechat_{article_id}_{uid}{ext}"
async def download_image(url: str, filepath: str, headers: dict) -> bool:
"""下载单张图片"""
try:
async with httpx.AsyncClient(timeout=30) as client:
r = await client.get(url, headers=headers, follow_redirects=True)
if r.status_code == 200:
with open(filepath, 'wb') as f:
f.write(r.content)
return True
except Exception as e:
print(f"[WeChat] 下载图片失败 {url}: {e}")
return False
async def download_all_images(image_tasks: list, headers: dict):
"""并发下载所有图片"""
async def task_wrapper(task):
url, filepath = task
success = await download_image(url, filepath, headers)
return url, filepath, success
return await asyncio.gather(*(task_wrapper(t) for t in image_tasks))
# ============================================================
# 核心采集函数
# ============================================================
async def crawl_wechat_article(url: str) -> dict:
"""采集微信公众号文章"""
if not PLAYWRIGHT_AVAILABLE:
raise Exception("Playwright 未安装,无法采集微信文章")
async with async_playwright() as p:
browser = await p.chromium.launch(
headless=True,
args=[
'--disable-blink-features=AutomationControlled',
'--disable-web-security',
'--disable-features=IsolateOrigins,site-per-process',
'--disable-dev-shm-usage',
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-accelerated-2d-canvas',
'--disable-gpu',
'--window-size=1920,1080',
]
)
context = await browser.new_context(
viewport={'width': 1920, 'height': 1080},
user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
locale='zh-CN',
timezone_id='Asia/Shanghai',
java_script_enabled=True,
)
await context.add_init_script("""
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5] });
window.chrome = { runtime: {} };
""")
page = await context.new_page()
if STEALTH_AVAILABLE:
await stealth_async(page)
try:
print(f"[WeChat] 访问: {url}")
await page.goto(url, wait_until="networkidle", timeout=60000)
await page.wait_for_timeout(3000)
# 获取标题
title = ""
try:
title = await page.title()
if title in ["微信公众平台", ""]:
h2 = await page.query_selector("#activity_name")
if h2:
title = await h2.inner_text()
except:
pass
title = title.strip() or "未命名文章"
# 获取公众号名称
author = ""
try:
name_el = await page.query_selector("#js_name")
if name_el:
author = await name_el.inner_text()
author = author.strip()
except:
pass
# 获取正文 HTML
content_html = ""
try:
content_el = await page.query_selector("#js_content")
if content_el:
content_html = await content_el.inner_html()
except:
pass
# 获取封面图
cover_image = ""
try:
meta = await page.query_selector('meta[property="og:image"]')
if meta:
cover_image = await meta.get_attribute("content")
except:
pass
# 获取正文中的图片
images = []
if content_html:
soup = BeautifulSoup(content_html, 'html.parser')
for img in soup.find_all('img'):
src = img.get('data-src') or img.get('src', '')
if src and ('mmbiz.qpic.cn' in src or 'mmbiz.qlogo.cn' in src):
images.append({
'original_url': src,
'description': img.get('alt', '')
})
print(f"[WeChat] 标题: {title[:50]}")
print(f"[WeChat] 作者: {author}")
print(f"[WeChat] 内容长度: {len(content_html)}")
print(f"[WeChat] 图片数: {len(images)}")
return {
"title": title,
"content_html": content_html,
"author": author,
"cover_image": cover_image,
"images": images,
}
finally:
await browser.close()
# ============================================================
# 图片下载处理
# ============================================================
async def process_images(images: list, article_id: str) -> dict:
"""下载图片并返回 URL 映射"""
if not images:
return {}
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
"Referer": "https://mp.weixin.qq.com/",
}
tasks = []
url_map = {}
for img in images:
original_url = img['original_url']
filename = generate_filename(original_url, article_id)
filepath = os.path.join(UPLOADS_DIR, filename)
tasks.append((original_url, filepath))
url_map[original_url] = f"/uploads/{filename}"
results = await download_all_images(tasks, headers)
# 过滤失败的
for original_url, filepath, success in results:
if not success and os.path.exists(filepath):
os.remove(filepath)
if not success:
# 保留原 URL
url_map[original_url] = original_url
return url_map
# ============================================================
# API 端点
# ============================================================
@router.post("/preview")
async def preview_wechat(request: WechatCrawlRequest):
"""第1步:采集并返回预览数据"""
url = request.url.strip()
if not url:
raise HTTPException(status_code=400, detail="URL 不能为空")
if not is_wechat_url(url):
raise HTTPException(status_code=400, detail="不是微信公众号文章链接")
try:
result = await crawl_wechat_article(url)
except Exception as e:
raise HTTPException(status_code=500, detail=f"采集失败: {str(e)}")
if not result['content_html']:
raise HTTPException(status_code=500, detail="未能获取到文章内容,可能需要等待页面加载或文章已被删除")
# 转换为 Markdown(预览阶段不下载图片,保留原始 URL)
md_content = convert_html_to_markdown(result['content_html'])
# 提取摘要(前200字符)
plain = BeautifulSoup(result['content_html'], 'html.parser').get_text(strip=True)
summary = plain[:200] if len(plain) > 200 else plain
return {
"title": result['title'],
"content_md": md_content,
"summary": summary,
"author": result['author'],
"cover_image": result['cover_image'],
"images": result['images'],
"source_url": url,
}
@router.post("/save")
async def save_wechat(request: WechatSaveRequest, db: Session = Depends(get_db), current_admin = Depends(get_current_admin)):
"""第2步:确认入库(下载图片、替换URL、保存文章)"""
# 生成唯一 article_id 用于图片命名
article_id = str(uuid.uuid4())[:8]
# 从 content 中提取所有微信图片 URL
image_urls = re.findall(
r'https?://mmbiz\.(?:qpic|qlogo)\.cn/[^\s\)"\'>\]]+',
request.content
)
# 去重
unique_urls = list(set(image_urls))
images_info = [{'original_url': u, 'description': ''} for u in unique_urls]
# 下载图片
url_map = await process_images(images_info, article_id)
# 替换 content 中的图片 URL
final_content = request.content
for original_url, local_url in url_map.items():
final_content = final_content.replace(original_url, local_url)
# 替换封面图
final_cover = request.cover_image
if final_cover and 'mmbiz' in final_cover:
# 封面图也下载
cover_filename = generate_filename(final_cover, article_id)
cover_filepath = os.path.join(UPLOADS_DIR, cover_filename)
headers = {
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
"Referer": "https://mp.weixin.qq.com/",
}
success = await download_image(final_cover, cover_filepath, headers)
if success:
final_cover = f"/uploads/{cover_filename}"
# 生成 slug
base_slug = generate_slug(request.title, article_id)
slug = base_slug
counter = 1
while db.query(Post).filter(Post.slug == slug).first():
slug = f"{base_slug}-{counter}"
counter += 1
# 创建文章
is_draft = request.is_draft if request.is_draft is not None else False
post = Post(
title=request.title,
slug=slug,
content=final_content,
summary=request.summary or "",
cover_image=final_cover or "",
is_published=not is_draft,
is_draft=is_draft,
)
# 分类
if request.category_id:
category = db.query(Category).filter(Category.id == request.category_id).first()
if category:
post.category = category
# 标签
if request.tag_names:
for tag_name in request.tag_names:
tag_name = tag_name.strip()
if not tag_name:
continue
tag = db.query(Tag).filter(Tag.name == tag_name).first()
if not tag:
tag = Tag(name=tag_name)
db.add(tag)
db.flush()
post.tags.append(tag)
db.add(post)
db.commit()
db.refresh(post)
return {
"success": True,
"message": "文章已保存",
"post": {
"id": post.id,
"title": post.title,
"slug": post.slug,
}
}
+204
View File
@@ -0,0 +1,204 @@
"""
Pydantic 数据验证模型
"""
from pydantic import BaseModel, Field
from typing import Optional, List
from datetime import datetime
# ==================== 认证相关 ====================
class LoginRequest(BaseModel):
username: str
password: str
class LoginResponse(BaseModel):
access_token: str
token_type: str = "bearer"
class PasswordChange(BaseModel):
old_password: str
new_password: str = Field(..., min_length=8, max_length=20)
# ==================== 文章相关 ====================
class TagBase(BaseModel):
name: str = Field(..., max_length=30)
class TagCreate(TagBase):
pass
class TagResponse(TagBase):
id: int
created_at: datetime
class Config:
from_attributes = True
class CategoryBase(BaseModel):
name: str = Field(..., max_length=50)
description: Optional[str] = Field(None, max_length=200)
class CategoryCreate(CategoryBase):
pass
class CategoryResponse(CategoryBase):
id: int
created_at: datetime
post_count: int = 0
class Config:
from_attributes = True
class PostBase(BaseModel):
title: str = Field(..., max_length=200)
slug: Optional[str] = Field(None, max_length=200)
content: str
content_type: Optional[str] = Field(default="markdown", max_length=20) # markdown 或 html
summary: Optional[str] = Field(None, max_length=500)
cover_image: Optional[str] = None
is_published: bool = True
is_draft: bool = False
category_id: Optional[int] = None
tag_ids: List[int] = []
class PostCreate(PostBase):
pass
class PostUpdate(BaseModel):
title: Optional[str] = Field(None, max_length=200)
slug: Optional[str] = Field(None, max_length=200)
content: Optional[str] = None
content_type: Optional[str] = Field(None, max_length=20)
summary: Optional[str] = Field(None, max_length=500)
cover_image: Optional[str] = None
is_published: Optional[bool] = None
is_draft: Optional[bool] = None
category_id: Optional[int] = None
tag_ids: Optional[List[int]] = None
class PostResponse(BaseModel):
id: int
title: str
slug: Optional[str] = None
content: str
content_type: Optional[str] = None
summary: Optional[str]
cover_image: Optional[str]
is_published: bool
is_draft: bool
view_count: int
created_at: datetime
updated_at: datetime
category: Optional[CategoryResponse]
tags: List[TagResponse]
class Config:
from_attributes = True
class PostListItem(BaseModel):
id: int
title: str
slug: Optional[str] = None
summary: Optional[str]
cover_image: Optional[str]
content_type: Optional[str] = None
is_published: bool
view_count: int
created_at: datetime
category: Optional[CategoryResponse]
tags: List[TagResponse]
class Config:
from_attributes = True
# ==================== 图片相关 ====================
class ImageResponse(BaseModel):
id: int
filename: str
original_name: str
description: Optional[str]
file_size: int
file_path: str
thumb_path: Optional[str] = None
width: Optional[int] = None
height: Optional[int] = None
created_at: datetime
class Config:
from_attributes = True
class ImageListResponse(BaseModel):
items: List[ImageResponse]
total: int
skip: int
limit: int
# ==================== 设置相关 ====================
class SettingBase(BaseModel):
blog_name: str = Field(..., max_length=20)
blog_description: Optional[str] = Field(None, max_length=200)
welcome_message: Optional[str] = Field(default="欢迎来到我的博客 👋 分享技术、生活与思考", max_length=300)
blog_logo: Optional[str] = None
banner_image: Optional[str] = None
banner_type: str = Field(default="text") # text或carousel
banner_images: Optional[str] = Field(default="[]") # JSON数组
theme: str = Field(default="neobrutalism")
layout: str = Field(default="card")
theme_colors: Optional[str] = Field(default='{"primary":"#7c3aed","secondary":"#06b6d4","bg":"#ffffff","text":"#1f2937","sidebar":"#1e293b"}')
class SettingUpdate(BaseModel):
blog_name: Optional[str] = Field(None, max_length=20)
blog_description: Optional[str] = Field(None, max_length=200)
welcome_message: Optional[str] = Field(None, max_length=300)
blog_logo: Optional[str] = None
banner_image: Optional[str] = None
banner_type: Optional[str] = None
banner_images: Optional[str] = None
theme: Optional[str] = None
layout: Optional[str] = None
theme_colors: Optional[str] = None
class SettingResponse(SettingBase):
id: int
updated_at: datetime
class Config:
from_attributes = True
# ==================== 页面相关 ====================
class PageBase(BaseModel):
title: str = Field(..., max_length=50)
slug: str = Field(..., max_length=50)
content: str
is_nav: bool = True
nav_order: int = 0
class PageCreate(PageBase):
pass
class PageUpdate(BaseModel):
title: Optional[str] = Field(None, max_length=50)
slug: Optional[str] = Field(None, max_length=50)
content: Optional[str] = None
is_nav: Optional[bool] = None
nav_order: Optional[int] = None
class PageResponse(PageBase):
id: int
created_at: datetime
updated_at: datetime
class Config:
from_attributes = True
# ==================== 统计相关 ====================
class StatsResponse(BaseModel):
total_posts: int
total_views: int
today_views: int
category_counts: dict
recent_posts: List[PostListItem]
+22
View File
@@ -0,0 +1,22 @@
"""SSO Cookie 本地验签 —— 与 SSO 中心共享 SECRET_KEY"""
from fastapi import Request, HTTPException
from jose import jwt, JWTError
# 必须与 SSO 中心 sso_config.py 完全一致
SECRET_KEY = "your-secret-key-change-in-production"
ALGORITHM = "HS256"
def get_current_user_from_cookie(request: Request) -> dict:
"""从请求 Cookie 中读取 access_token,本地验签后返回用户信息"""
token = request.cookies.get("access_token")
if not token:
return None
try:
payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
username: str = payload.get("sub")
if not username:
return None
return {"username": username}
except JWTError:
return None
+17
View File
@@ -0,0 +1,17 @@
#!/usr/bin/env python3
"""测试 wechat-crawl API 端点"""
import sys
sys.path.insert(0, '/home/ubuntu/nanocode/personal-blog/backend')
from fastapi.testclient import TestClient
from app import app
client = TestClient(app)
url = "https://mp.weixin.qq.com/s/_YqGx7J9XnyPe0GFhv4LYA"
print(f"测试 API: /api/wechat-crawl/preview")
print(f"URL: {url}\n")
res = client.post("/api/wechat-crawl/preview", json={"url": url})
print(f"状态码: {res.status_code}")
print(f"响应: {res.text[:2000]}")
+41
View File
@@ -0,0 +1,41 @@
#!/usr/bin/env python3
"""本地测试采集功能,绕过 HTTP API 和数据库"""
import asyncio
import sys
sys.path.insert(0, '/home/ubuntu/nanocode/personal-blog/backend')
from routers.crawl import crawl_with_playwright, convert_spa_html_to_markdown
from routers.crawl import CrawlImage
async def main():
url = "https://mp.weixin.qq.com/s/_YqGx7J9XnyPe0GFhv4LYA"
print(f"开始测试采集: {url}\n")
result = await crawl_with_playwright(url)
print(f"标题: {result['title']}")
print(f"图片数: {len(result['images'])}")
print(f"内容HTML长度: {len(result['content_html'])}")
markdown = convert_spa_html_to_markdown(
result['content_html'],
result['images'],
url
)
print(f"\nMarkdown长度: {len(markdown)}")
print("\n=== Markdown 前 3000 字符 ===")
print(markdown[:3000])
print("\n=== Markdown 末尾 1000 字符 ===")
print(markdown[-1000:])
# 检查是否有代码块
if '```' in markdown:
print("\n✅ 检测到代码块!")
code_blocks = markdown.split('```')
print(f"代码块数量: {len(code_blocks)//2}")
else:
print("\n⚠️ 未检测到代码块")
if __name__ == "__main__":
asyncio.run(main())
Binary file not shown.

After

Width:  |  Height:  |  Size: 494 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.4 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 966 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 966 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.3 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 755 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 828 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 104 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.1 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.1 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 889 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 868 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1001 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 988 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 925 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 925 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 447 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 970 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 970 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 860 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 860 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.1 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.1 MiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 63 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 404 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 5.0 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 10 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 672 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 72 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 23 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 32 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 24 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 48 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 64 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 83 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 68 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 56 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 84 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 21 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 45 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 36 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 14 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 2.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 153 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 9.3 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 34 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 112 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 98 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 146 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 7.5 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 28 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 141 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 225 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 106 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 217 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 43 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 528 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 58 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.7 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 24 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 8.4 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 98 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 19 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 25 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 1.2 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 33 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 65 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 16 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 126 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 45 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 41 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 56 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 270 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 71 KiB

Some files were not shown because too many files have changed in this diff Show More