a
@@ -0,0 +1,78 @@
|
||||
"""
|
||||
个人博客后端 - FastAPI
|
||||
"""
|
||||
from fastapi import FastAPI
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
import os
|
||||
|
||||
from database import engine, Base
|
||||
from routers import auth, posts, categories, tags, images, settings, crawl, wechat_crawl, universal_crawl
|
||||
|
||||
# 创建数据库表
|
||||
Base.metadata.create_all(bind=engine)
|
||||
|
||||
# 初始化默认管理员
|
||||
from database import SessionLocal
|
||||
from routers.auth import init_admin
|
||||
db = SessionLocal()
|
||||
init_admin(db)
|
||||
db.close()
|
||||
|
||||
app = FastAPI(
|
||||
title="Personal Blog API",
|
||||
description="个人博客后端API",
|
||||
version="1.0.0",
|
||||
redirect_slashes=False
|
||||
)
|
||||
|
||||
# CORS配置 - 允许远程访问
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=[
|
||||
"http://localhost:5173",
|
||||
"http://localhost:3000",
|
||||
"http://127.0.0.1:5173",
|
||||
"http://0.0.0.0:5173",
|
||||
"http://129.211.173.87:8888", # blog 前端
|
||||
"http://129.211.173.87:9000", # SSO 中心
|
||||
],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
# 注册路由
|
||||
app.include_router(auth.router)
|
||||
app.include_router(posts.router)
|
||||
app.include_router(categories.router)
|
||||
app.include_router(tags.router)
|
||||
app.include_router(images.router)
|
||||
app.include_router(settings.router)
|
||||
app.include_router(crawl.router)
|
||||
app.include_router(wechat_crawl.router)
|
||||
app.include_router(universal_crawl.router)
|
||||
|
||||
# 静态文件服务(图片访问)
|
||||
uploads_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)), "uploads")
|
||||
os.makedirs(uploads_dir, exist_ok=True)
|
||||
app.mount("/uploads", StaticFiles(directory=uploads_dir), name="uploads")
|
||||
|
||||
|
||||
@app.get("/")
|
||||
def root():
|
||||
return {
|
||||
"message": "Personal Blog API",
|
||||
"version": "1.0.0",
|
||||
"docs": "/docs"
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/health")
|
||||
def health_check():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run(app, host="0.0.0.0", port=8888)
|
||||
@@ -0,0 +1,27 @@
|
||||
"""
|
||||
数据库配置 - SQLite
|
||||
"""
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.ext.declarative import declarative_base
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
import os
|
||||
|
||||
# 数据库路径
|
||||
BASE_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
DATABASE_URL = f"sqlite:///{os.path.join(BASE_DIR, 'blog.db')}"
|
||||
|
||||
engine = create_engine(
|
||||
DATABASE_URL,
|
||||
connect_args={"check_same_thread": False}
|
||||
)
|
||||
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
||||
|
||||
Base = declarative_base()
|
||||
|
||||
# 依赖注入
|
||||
def get_db():
|
||||
db = SessionLocal()
|
||||
try:
|
||||
yield db
|
||||
finally:
|
||||
db.close()
|
||||
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
数据迁移脚本 - 从备份数据库恢复文章数据
|
||||
"""
|
||||
import sqlite3
|
||||
import os
|
||||
|
||||
def migrate_posts():
|
||||
"""迁移文章数据"""
|
||||
db_path = 'blog.db'
|
||||
backup_path = 'blog.db.backup'
|
||||
|
||||
if not os.path.exists(backup_path):
|
||||
print("备份文件不存在")
|
||||
return
|
||||
|
||||
# 连接两个数据库
|
||||
conn_new = sqlite3.connect(db_path)
|
||||
conn_old = sqlite3.connect(backup_path)
|
||||
|
||||
cursor_new = conn_new.cursor()
|
||||
cursor_old = conn_old.cursor()
|
||||
|
||||
# 检查旧表结构
|
||||
cursor_old.execute("PRAGMA table_info(posts)")
|
||||
old_columns = [col[1] for col in cursor_old.fetchall()]
|
||||
print(f"旧表字段: {old_columns}")
|
||||
|
||||
# 获取所有文章
|
||||
cursor_old.execute("SELECT * FROM posts")
|
||||
posts = cursor_old.fetchall()
|
||||
print(f"找到 {len(posts)} 篇文章")
|
||||
|
||||
# 清空新表并重新插入
|
||||
cursor_new.execute("DELETE FROM posts")
|
||||
|
||||
for post in posts:
|
||||
# post: (id, title, content, summary, cover_image, is_published, is_draft, view_count, created_at, updated_at, category_id)
|
||||
cursor_new.execute("""
|
||||
INSERT INTO posts (id, title, content, summary, cover_image, is_published, is_draft, view_count, created_at, updated_at, category_id)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""", post)
|
||||
print(f" 已恢复: {post[1][:30]}...")
|
||||
|
||||
conn_new.commit()
|
||||
conn_new.close()
|
||||
conn_old.close()
|
||||
|
||||
print(f"\n✅ 成功恢复 {len(posts)} 篇文章")
|
||||
|
||||
if __name__ == '__main__':
|
||||
migrate_posts()
|
||||
@@ -0,0 +1,111 @@
|
||||
"""
|
||||
数据库模型
|
||||
"""
|
||||
from sqlalchemy import Column, Integer, String, Text, DateTime, Boolean, ForeignKey
|
||||
from sqlalchemy.orm import relationship
|
||||
from datetime import datetime
|
||||
from database import Base
|
||||
|
||||
class Admin(Base):
|
||||
"""管理员账号"""
|
||||
__tablename__ = "admins"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
username = Column(String, unique=True, index=True, default="admin")
|
||||
password_hash = Column(String)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
class Post(Base):
|
||||
"""文章"""
|
||||
__tablename__ = "posts"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
title = Column(String(200), nullable=False)
|
||||
slug = Column(String(200), unique=True, index=True) # URL友好的标识
|
||||
content = Column(Text, nullable=False)
|
||||
content_type = Column(String(20), default="markdown") # markdown 或 html
|
||||
summary = Column(String(500))
|
||||
cover_image = Column(String(500)) # 封面图路径
|
||||
is_published = Column(Boolean, default=True)
|
||||
is_draft = Column(Boolean, default=False)
|
||||
view_count = Column(Integer, default=0)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
# 关联
|
||||
category_id = Column(Integer, ForeignKey("categories.id"))
|
||||
category = relationship("Category", back_populates="posts")
|
||||
tags = relationship("Tag", secondary="post_tags", back_populates="posts")
|
||||
|
||||
class Category(Base):
|
||||
"""栏目/分类"""
|
||||
__tablename__ = "categories"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
name = Column(String(50), unique=True, nullable=False)
|
||||
description = Column(String(200))
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
posts = relationship("Post", back_populates="category")
|
||||
|
||||
class Tag(Base):
|
||||
"""标签"""
|
||||
__tablename__ = "tags"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
name = Column(String(30), unique=True, nullable=False)
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
posts = relationship("Post", secondary="post_tags", back_populates="tags")
|
||||
|
||||
# 文章-标签关联表
|
||||
class PostTag(Base):
|
||||
__tablename__ = "post_tags"
|
||||
|
||||
post_id = Column(Integer, ForeignKey("posts.id"), primary_key=True)
|
||||
tag_id = Column(Integer, ForeignKey("tags.id"), primary_key=True)
|
||||
|
||||
class Image(Base):
|
||||
"""上传的图片"""
|
||||
__tablename__ = "images"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
filename = Column(String(255), nullable=False)
|
||||
original_name = Column(String(255))
|
||||
file_path = Column(String(500), nullable=False) # 原图路径 /uploads/xxx.jpg
|
||||
thumb_path = Column(String(500)) # 缩略图路径 /uploads/thumbnails/xxx.jpg
|
||||
description = Column(String(500))
|
||||
file_size = Column(Integer) # 字节
|
||||
width = Column(Integer) # 图片宽度
|
||||
height = Column(Integer) # 图片高度
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
|
||||
class Setting(Base):
|
||||
"""博客设置"""
|
||||
__tablename__ = "settings"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
blog_name = Column(String(20), default="我的博客")
|
||||
blog_description = Column(String(200), default="")
|
||||
welcome_message = Column(String(300), default="欢迎来到我的博客 👋 分享技术、生活与思考")
|
||||
blog_logo = Column(String(500))
|
||||
banner_image = Column(String(500))
|
||||
banner_type = Column(String(20), default="text") # text或carousel
|
||||
banner_images = Column(String(1000), default="[]") # JSON数组存储轮播图
|
||||
theme = Column(String(20), default="neobrutalism") # 默认新粗野主义
|
||||
layout = Column(String(20), default="card") # card或list
|
||||
theme_colors = Column(Text, default='{"primary":"#7c3aed","secondary":"#06b6d4","bg":"#ffffff","text":"#1f2937","sidebar":"#1e293b"}')
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
|
||||
class Page(Base):
|
||||
"""自定义页面"""
|
||||
__tablename__ = "pages"
|
||||
|
||||
id = Column(Integer, primary_key=True, index=True)
|
||||
title = Column(String(50), nullable=False)
|
||||
slug = Column(String(50), unique=True, nullable=False) # URL标识
|
||||
content = Column(Text, nullable=False)
|
||||
is_nav = Column(Boolean, default=True) # 是否在导航显示
|
||||
nav_order = Column(Integer, default=0) # 导航排序
|
||||
created_at = Column(DateTime, default=datetime.utcnow)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow)
|
||||
@@ -0,0 +1,8 @@
|
||||
fastapi==0.115.0
|
||||
uvicorn[standard]==0.32.0
|
||||
sqlalchemy==2.0.36
|
||||
pydantic==2.9.2
|
||||
python-jose[cryptography]==3.3.0
|
||||
passlib[bcrypt]==1.7.4
|
||||
python-multipart==0.0.17
|
||||
pillow==11.0.0
|
||||
@@ -0,0 +1,155 @@
|
||||
"""
|
||||
认证路由
|
||||
"""
|
||||
from fastapi import APIRouter, Depends, HTTPException, status, Request, Response
|
||||
from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
|
||||
from sqlalchemy.orm import Session
|
||||
from datetime import datetime, timedelta
|
||||
from jose import JWTError, jwt
|
||||
from passlib.context import CryptContext
|
||||
import os
|
||||
|
||||
import sys
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from database import get_db
|
||||
from models import Admin
|
||||
from schemas import LoginRequest, LoginResponse, PasswordChange
|
||||
from sso_client import get_current_user_from_cookie
|
||||
|
||||
router = APIRouter(prefix="/api/auth", tags=["认证"])
|
||||
|
||||
# 安全配置
|
||||
SECRET_KEY = "your-secret-key-change-in-production" # 生产环境修改
|
||||
ALGORITHM = "HS256"
|
||||
ACCESS_TOKEN_EXPIRE_DAYS = 7
|
||||
|
||||
pwd_context = CryptContext(schemes=["bcrypt"], deprecated="auto")
|
||||
oauth2_scheme = OAuth2PasswordBearer(tokenUrl="/api/auth/login", auto_error=False)
|
||||
|
||||
# 与 SSO 中心共享的 Cookie 属性(删除 Cookie 时需一致)
|
||||
SSO_COOKIE_DOMAIN = "129.211.173.87"
|
||||
|
||||
|
||||
def verify_password(plain_password, hashed_password):
|
||||
"""验证密码"""
|
||||
return pwd_context.verify(plain_password, hashed_password)
|
||||
|
||||
|
||||
def get_password_hash(password):
|
||||
"""生成密码哈希"""
|
||||
return pwd_context.hash(password)
|
||||
|
||||
|
||||
def create_access_token(data: dict, expires_delta: timedelta = None):
|
||||
"""创建JWT令牌"""
|
||||
to_encode = data.copy()
|
||||
if expires_delta:
|
||||
expire = datetime.utcnow() + expires_delta
|
||||
else:
|
||||
expire = datetime.utcnow() + timedelta(days=ACCESS_TOKEN_EXPIRE_DAYS)
|
||||
to_encode.update({"exp": expire})
|
||||
encoded_jwt = jwt.encode(to_encode, SECRET_KEY, algorithm=ALGORITHM)
|
||||
return encoded_jwt
|
||||
|
||||
|
||||
async def get_current_admin(
|
||||
request: Request,
|
||||
token: str = Depends(oauth2_scheme),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""获取当前登录管理员 —— 双模式认证
|
||||
1. 优先读取 Authorization Header(原有本地登录)
|
||||
2. 无 Header 则读取 Cookie 中的 SSO token,本地验签后查本地 Admin 表
|
||||
"""
|
||||
credentials_exception = HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="无法验证凭据",
|
||||
headers={"WWW-Authenticate": "Bearer"},
|
||||
)
|
||||
|
||||
username = None
|
||||
|
||||
# 模式一:原有 Bearer Token
|
||||
if token:
|
||||
try:
|
||||
payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
|
||||
username = payload.get("sub")
|
||||
except JWTError:
|
||||
pass
|
||||
|
||||
# 模式二:SSO Cookie
|
||||
if not username:
|
||||
sso_user = get_current_user_from_cookie(request)
|
||||
if sso_user:
|
||||
username = sso_user.get("username")
|
||||
|
||||
if not username:
|
||||
raise credentials_exception
|
||||
|
||||
admin = db.query(Admin).filter(Admin.username == username).first()
|
||||
if admin is None:
|
||||
raise credentials_exception
|
||||
return admin
|
||||
|
||||
|
||||
@router.post("/login", response_model=LoginResponse)
|
||||
def login(login_data: LoginRequest, db: Session = Depends(get_db)):
|
||||
"""管理员登录"""
|
||||
admin = db.query(Admin).filter(Admin.username == login_data.username).first()
|
||||
|
||||
if not admin or not verify_password(login_data.password, admin.password_hash):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="用户名或密码错误"
|
||||
)
|
||||
|
||||
access_token = create_access_token(data={"sub": admin.username})
|
||||
return {"access_token": access_token, "token_type": "bearer"}
|
||||
|
||||
|
||||
@router.post("/change-password")
|
||||
def change_password(
|
||||
password_data: PasswordChange,
|
||||
current_admin: Admin = Depends(get_current_admin),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""修改密码"""
|
||||
# 验证原密码
|
||||
if not verify_password(password_data.old_password, current_admin.password_hash):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_400_BAD_REQUEST,
|
||||
detail="原密码错误"
|
||||
)
|
||||
|
||||
# 更新密码
|
||||
current_admin.password_hash = get_password_hash(password_data.new_password)
|
||||
db.commit()
|
||||
|
||||
return {"message": "密码修改成功"}
|
||||
|
||||
|
||||
@router.get("/me")
|
||||
def get_me(current_admin: Admin = Depends(get_current_admin)):
|
||||
"""获取当前管理员信息"""
|
||||
return {"username": current_admin.username}
|
||||
|
||||
|
||||
@router.post("/logout")
|
||||
def logout(response: Response):
|
||||
"""退出登录:清除本地 token 状态并删除 SSO Cookie"""
|
||||
response.delete_cookie("access_token", domain=SSO_COOKIE_DOMAIN, path="/")
|
||||
return {"message": "退出成功"}
|
||||
|
||||
|
||||
def init_admin(db: Session):
|
||||
"""初始化默认管理员账号"""
|
||||
admin = db.query(Admin).filter(Admin.username == "admin").first()
|
||||
if not admin:
|
||||
admin = Admin(
|
||||
username="admin",
|
||||
password_hash=get_password_hash("abcd1234")
|
||||
)
|
||||
db.add(admin)
|
||||
db.commit()
|
||||
print("✅ 默认管理员账号已创建: admin / abcd1234")
|
||||
@@ -0,0 +1,113 @@
|
||||
"""
|
||||
分类管理路由
|
||||
"""
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import List
|
||||
import sys
|
||||
import os
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from database import get_db
|
||||
from models import Category, Post
|
||||
from schemas import CategoryCreate, CategoryResponse
|
||||
from routers.auth import get_current_admin
|
||||
|
||||
router = APIRouter(prefix="/api/categories", tags=["分类"])
|
||||
|
||||
|
||||
@router.get("", response_model=List[CategoryResponse])
|
||||
def get_categories(db: Session = Depends(get_db)):
|
||||
"""获取所有分类"""
|
||||
categories = db.query(Category).all()
|
||||
# 添加文章数量
|
||||
result = []
|
||||
for cat in categories:
|
||||
post_count = db.query(Post).filter(Post.category_id == cat.id).count()
|
||||
cat_dict = {
|
||||
"id": cat.id,
|
||||
"name": cat.name,
|
||||
"description": cat.description,
|
||||
"created_at": cat.created_at,
|
||||
"post_count": post_count
|
||||
}
|
||||
result.append(cat_dict)
|
||||
return result
|
||||
|
||||
|
||||
@router.post("", response_model=CategoryResponse)
|
||||
def create_category(
|
||||
category_data: CategoryCreate,
|
||||
current_admin=Depends(get_current_admin),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""创建分类"""
|
||||
# 检查数量限制
|
||||
count = db.query(Category).count()
|
||||
if count >= 25:
|
||||
raise HTTPException(status_code=400, detail="分类数量已达上限(25个)")
|
||||
|
||||
# 检查名称重复
|
||||
existing = db.query(Category).filter(Category.name == category_data.name).first()
|
||||
if existing:
|
||||
raise HTTPException(status_code=400, detail="分类名称已存在")
|
||||
|
||||
category = Category(**category_data.dict())
|
||||
db.add(category)
|
||||
db.commit()
|
||||
db.refresh(category)
|
||||
|
||||
return {
|
||||
"id": category.id,
|
||||
"name": category.name,
|
||||
"description": category.description,
|
||||
"created_at": category.created_at,
|
||||
"post_count": 0
|
||||
}
|
||||
|
||||
|
||||
@router.put("/{category_id}", response_model=CategoryResponse)
|
||||
def update_category(
|
||||
category_id: int,
|
||||
category_data: CategoryCreate,
|
||||
current_admin=Depends(get_current_admin),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""更新分类"""
|
||||
category = db.query(Category).filter(Category.id == category_id).first()
|
||||
if not category:
|
||||
raise HTTPException(status_code=404, detail="分类不存在")
|
||||
|
||||
category.name = category_data.name
|
||||
category.description = category_data.description
|
||||
db.commit()
|
||||
db.refresh(category)
|
||||
|
||||
post_count = db.query(Post).filter(Post.category_id == category.id).count()
|
||||
return {
|
||||
"id": category.id,
|
||||
"name": category.name,
|
||||
"description": category.description,
|
||||
"created_at": category.created_at,
|
||||
"post_count": post_count
|
||||
}
|
||||
|
||||
|
||||
@router.delete("/{category_id}")
|
||||
def delete_category(
|
||||
category_id: int,
|
||||
current_admin=Depends(get_current_admin),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""删除分类"""
|
||||
category = db.query(Category).filter(Category.id == category_id).first()
|
||||
if not category:
|
||||
raise HTTPException(status_code=404, detail="分类不存在")
|
||||
|
||||
# 将该分类下的文章设为未分类
|
||||
db.query(Post).filter(Post.category_id == category_id).update({"category_id": None})
|
||||
|
||||
db.delete(category)
|
||||
db.commit()
|
||||
return {"message": "分类已删除"}
|
||||
@@ -0,0 +1,541 @@
|
||||
"""
|
||||
图片上传管理路由
|
||||
"""
|
||||
from fastapi import APIRouter, UploadFile, File, Depends, HTTPException, Query
|
||||
from fastapi.responses import FileResponse
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import List, Optional
|
||||
import os
|
||||
import uuid
|
||||
import subprocess
|
||||
from datetime import datetime
|
||||
from PIL import Image as PILImage
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from database import get_db
|
||||
from models import Image, Post, Setting, Page
|
||||
from routers.auth import get_current_admin
|
||||
|
||||
router = APIRouter(prefix="/api/images", tags=["图片"])
|
||||
|
||||
# 上传配置
|
||||
UPLOAD_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "uploads")
|
||||
THUMB_DIR = os.path.join(UPLOAD_DIR, "thumbnails")
|
||||
ALLOWED_EXTENSIONS = {'.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg'}
|
||||
MAX_FILE_SIZE = 10 * 1024 * 1024 # 10MB
|
||||
THUMB_WIDTH = 320 # 缩略图宽度
|
||||
THUMB_QUALITY = 85
|
||||
|
||||
# 确保上传目录存在
|
||||
os.makedirs(UPLOAD_DIR, exist_ok=True)
|
||||
os.makedirs(THUMB_DIR, exist_ok=True)
|
||||
|
||||
|
||||
def generate_thumbnail(img_path: str, thumb_path: str) -> bool:
|
||||
"""生成缩略图"""
|
||||
try:
|
||||
# 检查是否是 SVG 文件
|
||||
if img_path.lower().endswith('.svg'):
|
||||
# SVG 直接复制作为"缩略图"(矢量图不需要缩放)
|
||||
import shutil
|
||||
shutil.copy2(img_path, thumb_path)
|
||||
return True
|
||||
|
||||
# 使用 Pillow 生成缩略图(不需要 ImageMagick,更轻量)
|
||||
with PILImage.open(img_path) as img:
|
||||
# 转换为 RGB(处理 PNG 透明背景等)
|
||||
if img.mode in ('RGBA', 'LA', 'P'):
|
||||
img = img.convert('RGB')
|
||||
|
||||
# 计算等比例缩放后的高度
|
||||
ratio = THUMB_WIDTH / img.width
|
||||
thumb_height = int(img.height * ratio)
|
||||
|
||||
# 生成缩略图
|
||||
thumb = img.resize((THUMB_WIDTH, thumb_height), PILImage.Resampling.LANCZOS)
|
||||
|
||||
# 保存
|
||||
ext = os.path.splitext(thumb_path)[1].lower()
|
||||
if ext == '.png':
|
||||
thumb.save(thumb_path, 'PNG')
|
||||
else:
|
||||
thumb.save(thumb_path, 'JPEG', quality=THUMB_QUALITY)
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"生成缩略图失败: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def get_image_dimensions(img_path: str) -> tuple:
|
||||
"""获取图片尺寸"""
|
||||
try:
|
||||
# SVG 文件特殊处理
|
||||
if img_path.lower().endswith('.svg'):
|
||||
return get_svg_dimensions(img_path)
|
||||
|
||||
with PILImage.open(img_path) as img:
|
||||
return img.width, img.height
|
||||
except:
|
||||
return None, None
|
||||
|
||||
|
||||
def get_svg_dimensions(svg_path: str) -> tuple:
|
||||
"""从 SVG 文件中提取宽度和高度"""
|
||||
try:
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
# 解析 SVG 文件
|
||||
tree = ET.parse(svg_path)
|
||||
root = tree.getroot()
|
||||
|
||||
# 获取 width 和 height 属性
|
||||
width = root.get('width', '')
|
||||
height = root.get('height', '')
|
||||
|
||||
# 解析 viewBox 作为备选
|
||||
viewbox = root.get('viewBox', '')
|
||||
|
||||
def parse_value(val):
|
||||
"""解析数值,移除单位(如 px)"""
|
||||
if not val:
|
||||
return None
|
||||
# 移除常见单位
|
||||
val = val.strip().replace('px', '').replace('pt', '').replace('em', '').replace('rem', '')
|
||||
try:
|
||||
return int(float(val))
|
||||
except:
|
||||
return None
|
||||
|
||||
w = parse_value(width)
|
||||
h = parse_value(height)
|
||||
|
||||
# 如果没有 width/height,尝试从 viewBox 解析
|
||||
if (w is None or h is None) and viewbox:
|
||||
parts = viewbox.replace(',', ' ').split()
|
||||
if len(parts) >= 4:
|
||||
# viewBox="minX minY width height"
|
||||
w = parse_value(parts[2]) if w is None else w
|
||||
h = parse_value(parts[3]) if h is None else h
|
||||
|
||||
# 返回默认值如果无法解析
|
||||
return w or 100, h or 100
|
||||
except Exception as e:
|
||||
print(f"解析 SVG 尺寸失败: {e}")
|
||||
return 100, 100
|
||||
|
||||
|
||||
def allowed_file(filename):
|
||||
"""检查文件扩展名"""
|
||||
return os.path.splitext(filename.lower())[1] in ALLOWED_EXTENSIONS
|
||||
|
||||
|
||||
@router.get("")
|
||||
def get_images(
|
||||
skip: int = 0,
|
||||
limit: int = 50,
|
||||
db: Session = Depends(get_db),
|
||||
current_admin=Depends(get_current_admin)
|
||||
):
|
||||
"""获取图片列表(按时间倒序)"""
|
||||
total = db.query(Image).count()
|
||||
images = db.query(Image).order_by(Image.created_at.desc()).offset(skip).limit(limit).all()
|
||||
|
||||
return {
|
||||
"items": [
|
||||
{
|
||||
"id": img.id,
|
||||
"filename": img.filename,
|
||||
"original_name": img.original_name,
|
||||
"file_path": img.file_path,
|
||||
"thumb_path": img.thumb_path,
|
||||
"description": img.description,
|
||||
"file_size": img.file_size,
|
||||
"width": img.width,
|
||||
"height": img.height,
|
||||
"created_at": img.created_at.isoformat() if img.created_at else None
|
||||
}
|
||||
for img in images
|
||||
],
|
||||
"total": total,
|
||||
"skip": skip,
|
||||
"limit": limit
|
||||
}
|
||||
|
||||
|
||||
@router.post("/upload")
|
||||
async def upload_image(
|
||||
file: UploadFile = File(...),
|
||||
description: str = "",
|
||||
current_admin=Depends(get_current_admin),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""上传图片(自动生成缩略图)"""
|
||||
# 检查文件类型
|
||||
if not allowed_file(file.filename):
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="不支持的文件格式,仅支持 JPG、PNG、GIF、WebP"
|
||||
)
|
||||
|
||||
# 读取文件内容检查大小
|
||||
content = await file.read()
|
||||
if len(content) > MAX_FILE_SIZE:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=f"文件大小超过10MB限制"
|
||||
)
|
||||
|
||||
# 生成唯一文件名(统一转 jpg 以便处理)
|
||||
ext = os.path.splitext(file.filename.lower())[1]
|
||||
if ext == '.webp':
|
||||
ext = '.jpg' # webp 转 jpg
|
||||
unique_filename = f"{datetime.now().strftime('%Y%m%d')}_{uuid.uuid4().hex[:8]}{ext}"
|
||||
file_path = os.path.join(UPLOAD_DIR, unique_filename)
|
||||
thumb_filename = f"thumb_{unique_filename}"
|
||||
thumb_path_full = os.path.join(THUMB_DIR, thumb_filename)
|
||||
|
||||
# 保存原文件
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(content)
|
||||
|
||||
# 获取图片尺寸
|
||||
width, height = get_image_dimensions(file_path)
|
||||
|
||||
# 生成缩略图
|
||||
thumb_path_db = None
|
||||
if generate_thumbnail(file_path, thumb_path_full):
|
||||
thumb_path_db = f"/uploads/thumbnails/{thumb_filename}"
|
||||
|
||||
# 保存到数据库
|
||||
image = Image(
|
||||
filename=unique_filename,
|
||||
original_name=file.filename,
|
||||
file_path=f"/uploads/{unique_filename}",
|
||||
thumb_path=thumb_path_db,
|
||||
description=description,
|
||||
file_size=len(content),
|
||||
width=width,
|
||||
height=height
|
||||
)
|
||||
db.add(image)
|
||||
db.commit()
|
||||
db.refresh(image)
|
||||
|
||||
return {
|
||||
"id": image.id,
|
||||
"filename": image.filename,
|
||||
"original_name": image.original_name,
|
||||
"description": image.description,
|
||||
"file_size": image.file_size,
|
||||
"file_path": image.file_path,
|
||||
"thumb_path": image.thumb_path,
|
||||
"width": image.width,
|
||||
"height": image.height,
|
||||
"created_at": image.created_at
|
||||
}
|
||||
|
||||
|
||||
async def process_single_image(file: UploadFile, db: Session):
|
||||
"""处理单张图片(内部函数,用于批量上传)"""
|
||||
# 检查文件类型
|
||||
if not allowed_file(file.filename):
|
||||
raise ValueError(f"不支持的文件格式")
|
||||
|
||||
# 读取内容
|
||||
content = await file.read()
|
||||
if len(content) > MAX_FILE_SIZE:
|
||||
raise ValueError(f"文件大小超过10MB限制")
|
||||
|
||||
# 生成唯一文件名
|
||||
ext = os.path.splitext(file.filename.lower())[1]
|
||||
if ext == '.webp':
|
||||
ext = '.jpg'
|
||||
unique_filename = f"{datetime.now().strftime('%Y%m%d')}_{uuid.uuid4().hex[:8]}{ext}"
|
||||
file_path = os.path.join(UPLOAD_DIR, unique_filename)
|
||||
thumb_filename = f"thumb_{unique_filename}"
|
||||
thumb_path_full = os.path.join(THUMB_DIR, thumb_filename)
|
||||
|
||||
# 保存原文件
|
||||
with open(file_path, "wb") as f:
|
||||
f.write(content)
|
||||
|
||||
# 获取图片尺寸
|
||||
width, height = get_image_dimensions(file_path)
|
||||
|
||||
# 生成缩略图
|
||||
thumb_path_db = None
|
||||
if generate_thumbnail(file_path, thumb_path_full):
|
||||
thumb_path_db = f"/uploads/thumbnails/{thumb_filename}"
|
||||
|
||||
# 保存到数据库
|
||||
image = Image(
|
||||
filename=unique_filename,
|
||||
original_name=file.filename,
|
||||
file_path=f"/uploads/{unique_filename}",
|
||||
thumb_path=thumb_path_db,
|
||||
file_size=len(content),
|
||||
width=width,
|
||||
height=height
|
||||
)
|
||||
db.add(image)
|
||||
db.commit()
|
||||
db.refresh(image)
|
||||
|
||||
return {
|
||||
"id": image.id,
|
||||
"filename": image.filename,
|
||||
"original_name": image.original_name,
|
||||
"file_path": image.file_path,
|
||||
"thumb_path": image.thumb_path,
|
||||
"width": image.width,
|
||||
"height": image.height,
|
||||
"file_size": image.file_size,
|
||||
"created_at": image.created_at
|
||||
}
|
||||
|
||||
|
||||
@router.post("/upload-multiple")
|
||||
async def upload_multiple_images(
|
||||
files: List[UploadFile] = File(...),
|
||||
current_admin=Depends(get_current_admin),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""批量上传图片(最多10张,自动生成缩略图)"""
|
||||
if len(files) > 10:
|
||||
raise HTTPException(status_code=400, detail="一次最多上传10张图片")
|
||||
|
||||
uploaded = []
|
||||
errors = []
|
||||
|
||||
for file in files:
|
||||
try:
|
||||
result = await process_single_image(file, db)
|
||||
uploaded.append(result)
|
||||
except Exception as e:
|
||||
errors.append(f"{file.filename}: {str(e)}")
|
||||
|
||||
return {
|
||||
"uploaded": uploaded,
|
||||
"errors": errors,
|
||||
"total": len(files),
|
||||
"success": len(uploaded)
|
||||
}
|
||||
|
||||
|
||||
@router.get("/serve/{filename}")
|
||||
def serve_image(filename: str):
|
||||
"""提供原图访问"""
|
||||
file_path = os.path.join(UPLOAD_DIR, filename)
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(status_code=404, detail="图片不存在")
|
||||
|
||||
return FileResponse(
|
||||
file_path,
|
||||
headers={
|
||||
"Cache-Control": "no-cache, no-store, must-revalidate",
|
||||
"Pragma": "no-cache",
|
||||
"Expires": "0"
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.get("/thumb/{filename}")
|
||||
def serve_thumbnail(filename: str):
|
||||
"""提供缩略图访问"""
|
||||
file_path = os.path.join(THUMB_DIR, filename)
|
||||
if not os.path.exists(file_path):
|
||||
raise HTTPException(status_code=404, detail="缩略图不存在")
|
||||
|
||||
return FileResponse(
|
||||
file_path,
|
||||
headers={
|
||||
"Cache-Control": "no-cache, no-store, must-revalidate",
|
||||
"Pragma": "no-cache",
|
||||
"Expires": "0"
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
@router.delete("/{image_id}")
|
||||
def delete_image(
|
||||
image_id: int,
|
||||
current_admin=Depends(get_current_admin),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""删除图片(同时删除缩略图)"""
|
||||
image = db.query(Image).filter(Image.id == image_id).first()
|
||||
if not image:
|
||||
raise HTTPException(status_code=404, detail="图片不存在")
|
||||
|
||||
# 删除原图
|
||||
file_path = os.path.join(UPLOAD_DIR, image.filename)
|
||||
if os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
|
||||
# 删除缩略图
|
||||
if image.thumb_path:
|
||||
thumb_filename = os.path.basename(image.thumb_path)
|
||||
thumb_path = os.path.join(THUMB_DIR, thumb_filename)
|
||||
if os.path.exists(thumb_path):
|
||||
os.remove(thumb_path)
|
||||
|
||||
# 删除数据库记录
|
||||
db.delete(image)
|
||||
db.commit()
|
||||
|
||||
return {"message": "图片已删除"}
|
||||
|
||||
|
||||
@router.post("/cleanup-unused")
|
||||
def cleanup_unused_images(
|
||||
current_admin=Depends(get_current_admin),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""清理未使用的图片(未被任何文章、页面、设置引用)"""
|
||||
import json
|
||||
|
||||
# 获取所有图片
|
||||
all_images = db.query(Image).all()
|
||||
|
||||
# 收集所有被使用的图片路径
|
||||
used_paths = set()
|
||||
|
||||
# 1. 检查文章content和cover_image
|
||||
posts = db.query(Post).all()
|
||||
for post in posts:
|
||||
if post.cover_image:
|
||||
used_paths.add(post.cover_image)
|
||||
if post.content:
|
||||
# 提取content中的所有图片路径 /uploads/xxx
|
||||
import re
|
||||
paths = re.findall(r'/uploads/[^\s"\')<>]+', post.content)
|
||||
used_paths.update(paths)
|
||||
|
||||
# 2. 检查页面content
|
||||
pages = db.query(Page).all()
|
||||
for page in pages:
|
||||
if page.content:
|
||||
import re
|
||||
paths = re.findall(r'/uploads/[^\s"\')<>]+', page.content)
|
||||
used_paths.update(paths)
|
||||
|
||||
# 3. 检查设置
|
||||
settings = db.query(Setting).first()
|
||||
if settings:
|
||||
if settings.blog_logo:
|
||||
used_paths.add(settings.blog_logo)
|
||||
if settings.banner_image:
|
||||
used_paths.add(settings.banner_image)
|
||||
# 轮播图
|
||||
if settings.banner_images:
|
||||
try:
|
||||
banner_list = json.loads(settings.banner_images)
|
||||
if isinstance(banner_list, list):
|
||||
used_paths.update(banner_list)
|
||||
except:
|
||||
pass
|
||||
|
||||
# 找出未使用的图片
|
||||
unused_images = []
|
||||
for img in all_images:
|
||||
if img.file_path not in used_paths:
|
||||
unused_images.append(img)
|
||||
|
||||
# 删除未使用的图片
|
||||
deleted_count = 0
|
||||
freed_space = 0
|
||||
|
||||
for img in unused_images:
|
||||
# 删除原图文件
|
||||
file_path = os.path.join(UPLOAD_DIR, img.filename)
|
||||
if os.path.exists(file_path):
|
||||
freed_space += os.path.getsize(file_path)
|
||||
os.remove(file_path)
|
||||
|
||||
# 删除缩略图文件
|
||||
if img.thumb_path:
|
||||
thumb_filename = os.path.basename(img.thumb_path)
|
||||
thumb_path = os.path.join(THUMB_DIR, thumb_filename)
|
||||
if os.path.exists(thumb_path):
|
||||
freed_space += os.path.getsize(thumb_path)
|
||||
os.remove(thumb_path)
|
||||
|
||||
# 删除数据库记录
|
||||
db.delete(img)
|
||||
deleted_count += 1
|
||||
|
||||
db.commit()
|
||||
|
||||
return {
|
||||
"message": f"清理完成,删除 {deleted_count} 张未使用的图片",
|
||||
"deleted_count": deleted_count,
|
||||
"freed_space_bytes": freed_space,
|
||||
"freed_space_mb": round(freed_space / (1024 * 1024), 2)
|
||||
}
|
||||
|
||||
|
||||
@router.get("/usage-stats")
|
||||
def get_image_usage_stats(
|
||||
current_admin=Depends(get_current_admin),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""获取图片使用统计"""
|
||||
import json
|
||||
|
||||
total_images = db.query(Image).count()
|
||||
|
||||
# 收集所有被使用的图片路径
|
||||
used_paths = set()
|
||||
|
||||
# 检查文章
|
||||
posts = db.query(Post).all()
|
||||
for post in posts:
|
||||
if post.cover_image:
|
||||
used_paths.add(post.cover_image)
|
||||
if post.content:
|
||||
import re
|
||||
paths = re.findall(r'/uploads/[^\s"\')<>]+', post.content)
|
||||
used_paths.update(paths)
|
||||
|
||||
# 检查页面
|
||||
pages = db.query(Page).all()
|
||||
for page in pages:
|
||||
if page.content:
|
||||
import re
|
||||
paths = re.findall(r'/uploads/[^\s"\')<>]+', page.content)
|
||||
used_paths.update(paths)
|
||||
|
||||
# 检查设置
|
||||
settings = db.query(Setting).first()
|
||||
if settings:
|
||||
if settings.blog_logo:
|
||||
used_paths.add(settings.blog_logo)
|
||||
if settings.banner_image:
|
||||
used_paths.add(settings.banner_image)
|
||||
if settings.banner_images:
|
||||
try:
|
||||
banner_list = json.loads(settings.banner_images)
|
||||
if isinstance(banner_list, list):
|
||||
used_paths.update(banner_list)
|
||||
except:
|
||||
pass
|
||||
|
||||
# 计算未使用图片
|
||||
all_images = db.query(Image).all()
|
||||
unused_count = 0
|
||||
unused_space = 0
|
||||
|
||||
for img in all_images:
|
||||
if img.file_path not in used_paths:
|
||||
unused_count += 1
|
||||
unused_space += (img.file_size or 0)
|
||||
|
||||
return {
|
||||
"total_images": total_images,
|
||||
"used_images": len(used_paths),
|
||||
"unused_images": unused_count,
|
||||
"unused_space_bytes": unused_space,
|
||||
"unused_space_mb": round(unused_space / (1024 * 1024), 2)
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
"""
|
||||
文章管理路由
|
||||
"""
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import List, Optional
|
||||
import sys
|
||||
import os
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from database import get_db
|
||||
from models import Post, Category, Tag, PostTag, Image
|
||||
from schemas import PostCreate, PostUpdate, PostResponse, PostListItem
|
||||
from routers.auth import get_current_admin
|
||||
import re
|
||||
|
||||
router = APIRouter(prefix="/api/posts", tags=["文章"])
|
||||
|
||||
|
||||
@router.get("", response_model=List[PostListItem])
|
||||
def get_posts(
|
||||
skip: int = Query(0, ge=0),
|
||||
limit: int = Query(10, ge=1, le=100),
|
||||
category_id: Optional[int] = None,
|
||||
tag_id: Optional[int] = None,
|
||||
is_draft: Optional[bool] = None,
|
||||
search: Optional[str] = None,
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""获取文章列表(支持筛选)"""
|
||||
query = db.query(Post)
|
||||
|
||||
# 筛选条件
|
||||
if category_id:
|
||||
query = query.filter(Post.category_id == category_id)
|
||||
if tag_id:
|
||||
query = query.join(PostTag).filter(PostTag.tag_id == tag_id)
|
||||
if is_draft is not None:
|
||||
query = query.filter(Post.is_draft == is_draft)
|
||||
if search:
|
||||
query = query.filter(Post.title.contains(search) | Post.content.contains(search))
|
||||
|
||||
# 按时间倒序
|
||||
query = query.order_by(Post.created_at.desc())
|
||||
|
||||
posts = query.offset(skip).limit(limit).all()
|
||||
return posts
|
||||
|
||||
|
||||
@router.get("/{post_id}", response_model=PostResponse)
|
||||
def get_post(post_id: str, db: Session = Depends(get_db)):
|
||||
"""获取单篇文章(支持 id 或 slug)"""
|
||||
# 尝试作为 id 查询
|
||||
try:
|
||||
id_int = int(post_id)
|
||||
post = db.query(Post).filter(Post.id == id_int).first()
|
||||
except ValueError:
|
||||
post = None
|
||||
|
||||
# 如果不是 id,尝试作为 slug 查询
|
||||
if not post:
|
||||
post = db.query(Post).filter(Post.slug == post_id).first()
|
||||
if not post:
|
||||
raise HTTPException(status_code=404, detail="文章不存在")
|
||||
|
||||
# 增加浏览量
|
||||
post.view_count += 1
|
||||
db.commit()
|
||||
|
||||
return post
|
||||
|
||||
|
||||
@router.post("", response_model=PostResponse)
|
||||
def create_post(
|
||||
post_data: PostCreate,
|
||||
current_admin=Depends(get_current_admin),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""创建文章"""
|
||||
# 处理标签
|
||||
tags = []
|
||||
if post_data.tag_ids:
|
||||
tags = db.query(Tag).filter(Tag.id.in_(post_data.tag_ids)).all()
|
||||
|
||||
# 创建文章
|
||||
post = Post(
|
||||
title=post_data.title,
|
||||
content=post_data.content,
|
||||
content_type=post_data.content_type or "markdown",
|
||||
summary=post_data.summary or post_data.content[:200] + "...",
|
||||
cover_image=post_data.cover_image,
|
||||
is_published=post_data.is_published,
|
||||
is_draft=post_data.is_draft,
|
||||
category_id=post_data.category_id,
|
||||
tags=tags
|
||||
)
|
||||
|
||||
db.add(post)
|
||||
db.commit()
|
||||
db.refresh(post)
|
||||
return post
|
||||
|
||||
|
||||
@router.put("/{post_id}", response_model=PostResponse)
|
||||
def update_post(
|
||||
post_id: int,
|
||||
post_data: PostUpdate,
|
||||
current_admin=Depends(get_current_admin),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""更新文章"""
|
||||
post = db.query(Post).filter(Post.id == post_id).first()
|
||||
if not post:
|
||||
raise HTTPException(status_code=404, detail="文章不存在")
|
||||
|
||||
# 更新字段
|
||||
update_data = post_data.dict(exclude_unset=True)
|
||||
|
||||
# 处理标签
|
||||
if "tag_ids" in update_data:
|
||||
tag_ids = update_data.pop("tag_ids")
|
||||
if tag_ids is not None:
|
||||
post.tags = db.query(Tag).filter(Tag.id.in_(tag_ids)).all()
|
||||
|
||||
# 更新其他字段
|
||||
for field, value in update_data.items():
|
||||
setattr(post, field, value)
|
||||
|
||||
# 自动更新摘要
|
||||
if post_data.content and not post_data.summary:
|
||||
post.summary = post_data.content[:200] + "..."
|
||||
|
||||
db.commit()
|
||||
db.refresh(post)
|
||||
return post
|
||||
|
||||
|
||||
def delete_post_images(post: Post, db: Session):
|
||||
"""删除文章关联的所有图片(物理删除文件+数据库记录)"""
|
||||
UPLOAD_DIR = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "uploads")
|
||||
THUMB_DIR = os.path.join(UPLOAD_DIR, "thumbnails")
|
||||
|
||||
# 收集所有要删除的图片路径
|
||||
image_paths = set()
|
||||
|
||||
# 1. 封面图
|
||||
if post.cover_image:
|
||||
image_paths.add(post.cover_image)
|
||||
|
||||
# 2. 内容中的图片
|
||||
if post.content:
|
||||
paths = re.findall(r'/uploads/[^\s"\')<>]+', post.content)
|
||||
image_paths.update(paths)
|
||||
|
||||
# 删除每张图片
|
||||
for path in image_paths:
|
||||
filename = os.path.basename(path)
|
||||
|
||||
# 查找数据库记录
|
||||
image = db.query(Image).filter(Image.file_path == path).first()
|
||||
if image:
|
||||
# 删除原图文件
|
||||
file_path = os.path.join(UPLOAD_DIR, filename)
|
||||
if os.path.exists(file_path):
|
||||
os.remove(file_path)
|
||||
|
||||
# 删除缩略图文件
|
||||
if image.thumb_path:
|
||||
thumb_filename = os.path.basename(image.thumb_path)
|
||||
thumb_path = os.path.join(THUMB_DIR, thumb_filename)
|
||||
if os.path.exists(thumb_path):
|
||||
os.remove(thumb_path)
|
||||
|
||||
# 删除数据库记录
|
||||
db.delete(image)
|
||||
|
||||
|
||||
@router.delete("/{post_id}")
|
||||
def delete_post(
|
||||
post_id: int,
|
||||
current_admin=Depends(get_current_admin),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""删除文章(同时删除关联的图片)"""
|
||||
post = db.query(Post).filter(Post.id == post_id).first()
|
||||
if not post:
|
||||
raise HTTPException(status_code=404, detail="文章不存在")
|
||||
|
||||
# 先删除关联图片
|
||||
delete_post_images(post, db)
|
||||
|
||||
# 再删除文章
|
||||
db.delete(post)
|
||||
db.commit()
|
||||
return {"message": "文章已删除"}
|
||||
|
||||
|
||||
@router.get("/stats/overview")
|
||||
def get_stats(db: Session = Depends(get_db)):
|
||||
"""获取统计数据"""
|
||||
from sqlalchemy import func
|
||||
|
||||
total_posts = db.query(Post).filter(Post.is_draft == False).count()
|
||||
total_views = db.query(func.sum(Post.view_count)).scalar() or 0
|
||||
|
||||
# 按分类统计
|
||||
category_counts = {}
|
||||
categories = db.query(Category).all()
|
||||
for cat in categories:
|
||||
count = db.query(Post).filter(Post.category_id == cat.id).count()
|
||||
category_counts[cat.name] = count
|
||||
|
||||
# 最近文章
|
||||
recent_posts = db.query(Post).order_by(Post.created_at.desc()).limit(5).all()
|
||||
|
||||
return {
|
||||
"total_posts": total_posts,
|
||||
"total_views": total_views,
|
||||
"category_counts": category_counts,
|
||||
"recent_posts": recent_posts
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
"""
|
||||
博客设置路由
|
||||
"""
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
import sys
|
||||
import os
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from database import get_db
|
||||
from models import Setting, Page
|
||||
from schemas import SettingUpdate, SettingResponse, PageCreate, PageUpdate, PageResponse
|
||||
from routers.auth import get_current_admin
|
||||
from typing import List
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["设置"])
|
||||
|
||||
|
||||
def get_or_create_settings(db: Session):
|
||||
"""获取或创建设置"""
|
||||
setting = db.query(Setting).first()
|
||||
if not setting:
|
||||
setting = Setting()
|
||||
db.add(setting)
|
||||
db.commit()
|
||||
db.refresh(setting)
|
||||
return setting
|
||||
|
||||
|
||||
@router.get("/settings", response_model=SettingResponse)
|
||||
def get_settings(db: Session = Depends(get_db)):
|
||||
"""获取博客设置"""
|
||||
setting = get_or_create_settings(db)
|
||||
return setting
|
||||
|
||||
|
||||
@router.get("/settings/public")
|
||||
def get_settings_public(db: Session = Depends(get_db)):
|
||||
"""公开获取博客设置(仅返回前端展示所需字段)"""
|
||||
setting = get_or_create_settings(db)
|
||||
import json
|
||||
try:
|
||||
colors = json.loads(setting.theme_colors) if setting.theme_colors else {}
|
||||
except Exception:
|
||||
colors = {}
|
||||
return {
|
||||
"blog_name": setting.blog_name,
|
||||
"blog_description": setting.blog_description,
|
||||
"welcome_message": setting.welcome_message,
|
||||
"blog_logo": setting.blog_logo,
|
||||
"banner_image": setting.banner_image,
|
||||
"banner_type": setting.banner_type,
|
||||
"banner_images": setting.banner_images,
|
||||
"theme": setting.theme,
|
||||
"layout": setting.layout,
|
||||
"theme_colors": colors,
|
||||
}
|
||||
|
||||
|
||||
@router.put("/settings", response_model=SettingResponse)
|
||||
def update_settings(
|
||||
setting_data: SettingUpdate,
|
||||
current_admin=Depends(get_current_admin),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""更新博客设置"""
|
||||
setting = get_or_create_settings(db)
|
||||
|
||||
update_data = setting_data.dict(exclude_unset=True)
|
||||
for field, value in update_data.items():
|
||||
setattr(setting, field, value)
|
||||
|
||||
db.commit()
|
||||
db.refresh(setting)
|
||||
return setting
|
||||
|
||||
|
||||
# ==================== 自定义页面管理 ====================
|
||||
|
||||
@router.get("/pages", response_model=List[PageResponse])
|
||||
def get_pages(db: Session = Depends(get_db)):
|
||||
"""获取所有自定义页面"""
|
||||
pages = db.query(Page).order_by(Page.nav_order.asc(), Page.created_at.desc()).all()
|
||||
return pages
|
||||
|
||||
|
||||
@router.get("/pages/{slug}", response_model=PageResponse)
|
||||
def get_page_by_slug(slug: str, db: Session = Depends(get_db)):
|
||||
"""通过slug获取页面"""
|
||||
page = db.query(Page).filter(Page.slug == slug).first()
|
||||
if not page:
|
||||
raise HTTPException(status_code=404, detail="页面不存在")
|
||||
return page
|
||||
|
||||
|
||||
@router.post("/pages", response_model=PageResponse)
|
||||
def create_page(
|
||||
page_data: PageCreate,
|
||||
current_admin=Depends(get_current_admin),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""创建自定义页面"""
|
||||
# 检查数量限制
|
||||
count = db.query(Page).count()
|
||||
if count >= 5:
|
||||
raise HTTPException(status_code=400, detail="自定义页面数量已达上限(5个)")
|
||||
|
||||
# 检查slug重复
|
||||
existing = db.query(Page).filter(Page.slug == page_data.slug).first()
|
||||
if existing:
|
||||
raise HTTPException(status_code=400, detail="页面标识已存在")
|
||||
|
||||
page = Page(**page_data.dict())
|
||||
db.add(page)
|
||||
db.commit()
|
||||
db.refresh(page)
|
||||
return page
|
||||
|
||||
|
||||
@router.put("/pages/{page_id}", response_model=PageResponse)
|
||||
def update_page(
|
||||
page_id: int,
|
||||
page_data: PageUpdate,
|
||||
current_admin=Depends(get_current_admin),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""更新自定义页面"""
|
||||
page = db.query(Page).filter(Page.id == page_id).first()
|
||||
if not page:
|
||||
raise HTTPException(status_code=404, detail="页面不存在")
|
||||
|
||||
update_data = page_data.dict(exclude_unset=True)
|
||||
for field, value in update_data.items():
|
||||
setattr(page, field, value)
|
||||
|
||||
db.commit()
|
||||
db.refresh(page)
|
||||
return page
|
||||
|
||||
|
||||
@router.delete("/pages/{page_id}")
|
||||
def delete_page(
|
||||
page_id: int,
|
||||
current_admin=Depends(get_current_admin),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""删除自定义页面"""
|
||||
page = db.query(Page).filter(Page.id == page_id).first()
|
||||
if not page:
|
||||
raise HTTPException(status_code=404, detail="页面不存在")
|
||||
|
||||
db.delete(page)
|
||||
db.commit()
|
||||
return {"message": "页面已删除"}
|
||||
@@ -0,0 +1,64 @@
|
||||
"""
|
||||
标签管理路由
|
||||
"""
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import List
|
||||
import sys
|
||||
import os
|
||||
|
||||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from database import get_db
|
||||
from models import Tag
|
||||
from schemas import TagCreate, TagResponse
|
||||
from routers.auth import get_current_admin
|
||||
|
||||
router = APIRouter(prefix="/api/tags", tags=["标签"])
|
||||
|
||||
|
||||
@router.get("/", response_model=List[TagResponse])
|
||||
def get_tags(db: Session = Depends(get_db)):
|
||||
"""获取所有标签"""
|
||||
tags = db.query(Tag).all()
|
||||
return tags
|
||||
|
||||
|
||||
@router.post("/", response_model=TagResponse)
|
||||
def create_tag(
|
||||
tag_data: TagCreate,
|
||||
current_admin=Depends(get_current_admin),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""创建标签"""
|
||||
# 检查数量限制
|
||||
count = db.query(Tag).count()
|
||||
if count >= 20:
|
||||
raise HTTPException(status_code=400, detail="标签数量已达上限(20个)")
|
||||
|
||||
# 检查名称重复
|
||||
existing = db.query(Tag).filter(Tag.name == tag_data.name).first()
|
||||
if existing:
|
||||
raise HTTPException(status_code=400, detail="标签名称已存在")
|
||||
|
||||
tag = Tag(**tag_data.dict())
|
||||
db.add(tag)
|
||||
db.commit()
|
||||
db.refresh(tag)
|
||||
return tag
|
||||
|
||||
|
||||
@router.delete("/{tag_id}")
|
||||
def delete_tag(
|
||||
tag_id: int,
|
||||
current_admin=Depends(get_current_admin),
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""删除标签"""
|
||||
tag = db.query(Tag).filter(Tag.id == tag_id).first()
|
||||
if not tag:
|
||||
raise HTTPException(status_code=404, detail="标签不存在")
|
||||
|
||||
db.delete(tag)
|
||||
db.commit()
|
||||
return {"message": "标签已删除"}
|
||||
@@ -0,0 +1,608 @@
|
||||
"""
|
||||
通用采集路由 - 基于 Scrapling 引擎
|
||||
结合上层业务逻辑:编码修复、数学公式处理、微信防盗链、图片下载等
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import uuid
|
||||
import asyncio
|
||||
import tempfile
|
||||
import base64
|
||||
from datetime import datetime
|
||||
from typing import Optional, List
|
||||
from urllib.parse import urljoin, urlparse
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import APIRouter, HTTPException, Depends, UploadFile, File
|
||||
from pydantic import BaseModel, HttpUrl
|
||||
from sqlalchemy.orm import Session
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
# Scrapling 引擎
|
||||
from scrapling import Fetcher, StealthyFetcher
|
||||
|
||||
# 复用 crawl.py 的通用工具函数
|
||||
from .crawl import (
|
||||
fix_encoding,
|
||||
fix_math_formulas,
|
||||
html_to_markdown,
|
||||
extract_title_from_html,
|
||||
extract_main_content,
|
||||
download_image,
|
||||
extract_images,
|
||||
extract_images_from_markdown,
|
||||
is_spa_site,
|
||||
convert_spa_html_to_markdown,
|
||||
CrawlRequest,
|
||||
CrawlResponse,
|
||||
CrawlImage,
|
||||
)
|
||||
|
||||
# 复用图片处理
|
||||
from .images import generate_thumbnail, get_image_dimensions
|
||||
|
||||
# 数据库
|
||||
from database import get_db
|
||||
from models import Post, Category, Tag, PostTag, Image
|
||||
|
||||
router = APIRouter(prefix="/api/universal-crawl", tags=["通用采集"])
|
||||
|
||||
# 确保上传目录存在
|
||||
UPLOAD_DIR = Path("uploads")
|
||||
UPLOAD_DIR.mkdir(exist_ok=True)
|
||||
TEMP_DIR = Path("uploads/temp_crawl")
|
||||
TEMP_DIR.mkdir(exist_ok=True)
|
||||
|
||||
|
||||
class UniversalCrawlRequest(BaseModel):
|
||||
url: str
|
||||
|
||||
|
||||
def _needs_browser(url: str) -> bool:
|
||||
"""判断是否需要使用浏览器渲染"""
|
||||
domain = urlparse(url).netloc.lower()
|
||||
browser_domains = [
|
||||
'mp.weixin.qq.com',
|
||||
'juejin.cn',
|
||||
'zhihu.com',
|
||||
'segmentfault.com',
|
||||
'infoq.cn',
|
||||
'infoq.com',
|
||||
'51cto.com',
|
||||
]
|
||||
return any(d in domain for d in browser_domains)
|
||||
|
||||
|
||||
def _fetch_with_scrapling(url: str) -> str:
|
||||
"""使用 Scrapling Fetcher (curl_cffi) 获取页面"""
|
||||
fetcher = Fetcher()
|
||||
response = fetcher.get(url)
|
||||
if response.status != 200:
|
||||
raise Exception(f"HTTP {response.status}: {response.reason}")
|
||||
# Scrapling Response 的 text 属性返回解析后的文本,html_content 返回原始 HTML
|
||||
return response.html_content if hasattr(response, 'html_content') else response.text
|
||||
|
||||
|
||||
async def _fetch_with_stealth(url: str) -> str:
|
||||
"""使用 Scrapling StealthyFetcher 获取页面,失败时回退到原生 Playwright"""
|
||||
try:
|
||||
from scrapling import StealthyFetcher
|
||||
# Scrapling v0.3+ 推荐使用类方法直接调用,无需实例化
|
||||
response = await StealthyFetcher.async_fetch(url, wait=3000)
|
||||
if not response or response.status != 200:
|
||||
raise Exception(f"HTTP {response.status}")
|
||||
html = response.html_content if hasattr(response, 'html_content') else response.text
|
||||
if not html or len(html) < 100:
|
||||
raise Exception("Empty page")
|
||||
print(f"[Universal Crawl] StealthyFetcher success: {len(html)} bytes")
|
||||
return html
|
||||
except Exception as e:
|
||||
print(f"[Universal Crawl] StealthyFetcher failed: {e}")
|
||||
print("[Universal Crawl] Falling back to native Playwright")
|
||||
return await _fetch_with_playwright(url)
|
||||
|
||||
|
||||
async def _fetch_with_playwright(url: str) -> str:
|
||||
"""使用原生 Playwright 异步获取页面,带完善反爬配置"""
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(
|
||||
headless=True,
|
||||
args=[
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--disable-web-security",
|
||||
"--disable-features=IsolateOrigins,site-per-process",
|
||||
"--disable-dev-shm-usage",
|
||||
"--no-sandbox",
|
||||
]
|
||||
)
|
||||
context = await browser.new_context(
|
||||
user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36 Edg/120.0.0.0",
|
||||
viewport={"width": 1920, "height": 1080},
|
||||
locale="zh-CN",
|
||||
timezone_id="Asia/Shanghai",
|
||||
extra_http_headers={
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
|
||||
"Accept-Encoding": "gzip, deflate, br",
|
||||
"Cache-Control": "max-age=0",
|
||||
"Sec-Ch-Ua": '"Not_A Brand";v="8", "Chromium";v="120", "Microsoft Edge";v="120"',
|
||||
"Sec-Ch-Ua-Mobile": "?0",
|
||||
"Sec-Ch-Ua-Platform": '"Windows"',
|
||||
"Sec-Fetch-Dest": "document",
|
||||
"Sec-Fetch-Mode": "navigate",
|
||||
"Sec-Fetch-Site": "none",
|
||||
"Sec-Fetch-User": "?1",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
},
|
||||
)
|
||||
page = await context.new_page()
|
||||
try:
|
||||
# SegmentFault 需要先访问首页建立信任,再跳转目标页面
|
||||
if 'segmentfault.com' in url:
|
||||
await page.goto("https://segmentfault.com", wait_until="domcontentloaded", timeout=15000)
|
||||
await asyncio.sleep(2)
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
||||
await asyncio.sleep(5)
|
||||
# 知乎需要先访问首页建立 cookie,再跳转目标页面
|
||||
elif 'zhihu.com' in url:
|
||||
await page.goto("https://www.zhihu.com", wait_until="domcontentloaded", timeout=15000)
|
||||
await asyncio.sleep(2)
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
||||
await asyncio.sleep(3)
|
||||
# 等待正文加载(知乎专栏文章正文在 .Post-RichTextContainer 或 .RichContent-inner 里)
|
||||
try:
|
||||
await page.wait_for_selector(".Post-RichTextContainer, .RichContent-inner", timeout=10000)
|
||||
await asyncio.sleep(1)
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
|
||||
await asyncio.sleep(2)
|
||||
html = await page.content()
|
||||
finally:
|
||||
await context.close()
|
||||
await browser.close()
|
||||
return html
|
||||
|
||||
|
||||
def _bypass_wechat_images(html: str, url: str) -> str:
|
||||
"""
|
||||
微信图片防盗链绕过:在页面上下文中执行 fetch 获取图片 base64
|
||||
这里我们在后端模拟:如果检测到 mmbiz.qpic.cn 图片,
|
||||
将其 src 替换为 data-src(微信文章中 data-src 通常包含真实 URL)
|
||||
"""
|
||||
if 'mp.weixin.qq.com' not in url:
|
||||
return html
|
||||
|
||||
soup = BeautifulSoup(html, 'lxml')
|
||||
for img in soup.find_all('img'):
|
||||
data_src = img.get('data-src')
|
||||
if data_src and 'mmbiz.qpic.cn' in data_src:
|
||||
img['src'] = data_src
|
||||
return str(soup)
|
||||
|
||||
|
||||
def _extract_infoq_content(html: str) -> str:
|
||||
"""提取 InfoQ 文章正文(.ProseMirror 或 article 标签)"""
|
||||
soup = BeautifulSoup(html, 'lxml')
|
||||
# 优先使用 .ProseMirror(InfoQ 正文编辑器容器)
|
||||
container = soup.select_one('.ProseMirror')
|
||||
if not container:
|
||||
# fallback 到 article 标签
|
||||
container = soup.find('article')
|
||||
if container:
|
||||
return str(container)
|
||||
return html
|
||||
|
||||
|
||||
def _extract_infoq_title(html: str) -> Optional[str]:
|
||||
"""提取 InfoQ 文章标题,从 <title> 中去掉 ' - InfoQ' 后缀"""
|
||||
match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
|
||||
if match:
|
||||
title = match.group(1).strip()
|
||||
# 去掉 " - InfoQ" 后缀
|
||||
title = re.sub(r'\s*[-–—]\s*InfoQ\s*$', '', title, flags=re.IGNORECASE).strip()
|
||||
if len(title) > 5:
|
||||
return title
|
||||
return None
|
||||
|
||||
|
||||
def _extract_juejin_content(html: str) -> str:
|
||||
"""提取掘金文章正文(article.article 或 .article-viewer)"""
|
||||
soup = BeautifulSoup(html, 'lxml')
|
||||
# 优先使用 article.article
|
||||
container = soup.select_one('article.article')
|
||||
if not container:
|
||||
# fallback 到 .article-viewer
|
||||
container = soup.select_one('.article-viewer')
|
||||
if not container:
|
||||
# 再 fallback 到 #article-root
|
||||
container = soup.select_one('#article-root')
|
||||
if container:
|
||||
return str(container)
|
||||
return html
|
||||
|
||||
|
||||
def _extract_juejin_title(html: str) -> Optional[str]:
|
||||
"""提取掘金文章标题,从 <title> 中去掉 ' - 掘金' 后缀"""
|
||||
match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
|
||||
if match:
|
||||
title = match.group(1).strip()
|
||||
# 去掉 " - 掘金" 后缀
|
||||
title = re.sub(r'\s*[-–—]\s*掘金\s*$', '', title, flags=re.IGNORECASE).strip()
|
||||
if len(title) > 5:
|
||||
return title
|
||||
return None
|
||||
|
||||
|
||||
def _extract_csdn_content(html: str) -> str:
|
||||
"""提取 CSDN 文章正文(#content_views 或 .article_content)"""
|
||||
soup = BeautifulSoup(html, 'lxml')
|
||||
# 优先使用 #content_views(CSDN 正文主容器)
|
||||
container = soup.select_one('#content_views')
|
||||
if not container:
|
||||
# fallback 到 .article_content
|
||||
container = soup.select_one('.article_content')
|
||||
if container:
|
||||
return str(container)
|
||||
return html
|
||||
|
||||
|
||||
def _extract_csdn_title(html: str) -> Optional[str]:
|
||||
"""提取 CSDN 文章标题,从 <title> 中去掉 '-CSDN博客' 后缀"""
|
||||
match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
|
||||
if match:
|
||||
title = match.group(1).strip()
|
||||
# 去掉 " - CSDN博客" / "-CSDN博客" 后缀
|
||||
title = re.sub(r'\s*[-–—]?\s*CSDN博客\s*$', '', title, flags=re.IGNORECASE).strip()
|
||||
if len(title) > 5:
|
||||
return title
|
||||
return None
|
||||
|
||||
|
||||
def _filter_csdn_images(images: list) -> list:
|
||||
"""过滤掉 CSDN 的 UI 图标图片(csdnimg.cn 的收藏/点赞等图标)"""
|
||||
filtered = []
|
||||
skip_patterns = [
|
||||
'newHeart2023',
|
||||
'tobarCollect',
|
||||
'tobarCollection',
|
||||
'copyright',
|
||||
'avatar',
|
||||
'blogv2/dist/pc/img',
|
||||
]
|
||||
for img in images:
|
||||
url = img.original_url.lower()
|
||||
if any(p in url for p in skip_patterns):
|
||||
continue
|
||||
filtered.append(img)
|
||||
return filtered
|
||||
|
||||
|
||||
def _extract_cnblogs_content(html: str) -> str:
|
||||
"""提取博客园文章正文(#cnblogs_post_body 或 .postBody)"""
|
||||
soup = BeautifulSoup(html, 'lxml')
|
||||
container = soup.select_one('#cnblogs_post_body')
|
||||
if not container:
|
||||
container = soup.select_one('.postBody')
|
||||
if container:
|
||||
return str(container)
|
||||
return html
|
||||
|
||||
|
||||
def _extract_cnblogs_title(html: str) -> Optional[str]:
|
||||
"""提取博客园文章标题,从 <title> 中去掉 ' - 博客园' 后缀"""
|
||||
match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
|
||||
if match:
|
||||
title = match.group(1).strip()
|
||||
title = re.sub(r'\s*[-–—]\s*博客园\s*$', '', title, flags=re.IGNORECASE).strip()
|
||||
if len(title) > 5:
|
||||
return title
|
||||
return None
|
||||
|
||||
|
||||
def _filter_cnblogs_images(images: list) -> list:
|
||||
"""过滤掉博客园的 UI 图标图片"""
|
||||
filtered = []
|
||||
skip_patterns = [
|
||||
'assets.cnblogs.com/logo',
|
||||
'assets.cnblogs.com/icons',
|
||||
'skins/custom/images/logo',
|
||||
]
|
||||
for img in images:
|
||||
url = img.original_url.lower()
|
||||
if any(p in url for p in skip_patterns):
|
||||
continue
|
||||
filtered.append(img)
|
||||
return filtered
|
||||
|
||||
|
||||
def _extract_51cto_content(html: str) -> str:
|
||||
"""提取 51CTO 文章正文(.article-content 或 article)"""
|
||||
soup = BeautifulSoup(html, 'lxml')
|
||||
container = soup.select_one('.article-content')
|
||||
if not container:
|
||||
container = soup.find('article')
|
||||
if container:
|
||||
return str(container)
|
||||
return html
|
||||
|
||||
|
||||
def _extract_51cto_title(html: str) -> Optional[str]:
|
||||
"""提取 51CTO 文章标题,从 <title> 中去掉 '-51CTO.COM' 后缀"""
|
||||
match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
|
||||
if match:
|
||||
title = match.group(1).strip()
|
||||
title = re.sub(r'\s*[-–—]?\s*51CTO\.COM\s*$', '', title, flags=re.IGNORECASE).strip()
|
||||
if len(title) > 5:
|
||||
return title
|
||||
return None
|
||||
|
||||
|
||||
def _filter_51cto_images(images: list) -> list:
|
||||
"""过滤掉 51CTO 的 UI 图标和二维码图片"""
|
||||
filtered = []
|
||||
skip_patterns = [
|
||||
'oss/202302', # 公众号矩阵二维码
|
||||
'oss/202408', # 软考二维码
|
||||
'oss/202506/06', # 顶部横幅
|
||||
'oss/202302/07', # APP下载图
|
||||
]
|
||||
for img in images:
|
||||
url = img.original_url.lower()
|
||||
if any(p in url for p in skip_patterns):
|
||||
continue
|
||||
filtered.append(img)
|
||||
return filtered
|
||||
|
||||
|
||||
@router.post("/fetch", response_model=CrawlResponse)
|
||||
async def universal_fetch(request: UniversalCrawlRequest):
|
||||
"""通用采集:使用 Scrapling 引擎获取文章"""
|
||||
try:
|
||||
url = str(request.url)
|
||||
print(f"[Universal Crawl] Fetching: {url}")
|
||||
|
||||
# 1. 判断采集策略
|
||||
use_browser = _needs_browser(url)
|
||||
|
||||
# 2. 使用 Scrapling 获取页面
|
||||
if use_browser:
|
||||
print(f"[Universal Crawl] Using StealthyFetcher (browser) for {url}")
|
||||
html = await _fetch_with_stealth(url)
|
||||
else:
|
||||
print(f"[Universal Crawl] Using Fetcher (curl_cffi) for {url}")
|
||||
html = _fetch_with_scrapling(url)
|
||||
|
||||
if not html or len(html) < 100:
|
||||
raise HTTPException(status_code=400, detail="无法获取页面内容,页面可能为空或需要登录")
|
||||
|
||||
# 3. 编码修复
|
||||
html = fix_encoding(html)
|
||||
|
||||
# 4. 微信图片防盗链处理
|
||||
html = _bypass_wechat_images(html, url)
|
||||
|
||||
# 5. 提取标题(InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取)
|
||||
is_infoq = 'infoq.cn' in url or 'infoq.com' in url
|
||||
is_juejin = 'juejin.cn' in url
|
||||
is_csdn = 'csdn.net' in url or 'csdn.com' in url
|
||||
is_cnblogs = 'cnblogs.com' in url
|
||||
is_51cto = '51cto.com' in url
|
||||
if is_infoq:
|
||||
title = _extract_infoq_title(html)
|
||||
elif is_juejin:
|
||||
title = _extract_juejin_title(html)
|
||||
elif is_csdn:
|
||||
title = _extract_csdn_title(html)
|
||||
elif is_cnblogs:
|
||||
title = _extract_cnblogs_title(html)
|
||||
elif is_51cto:
|
||||
title = _extract_51cto_title(html)
|
||||
else:
|
||||
title = None
|
||||
if not title:
|
||||
title = extract_title_from_html(html)
|
||||
if not title:
|
||||
title = "未获取到标题"
|
||||
|
||||
# 6. 提取正文和图片
|
||||
is_wechat = 'mp.weixin.qq.com' in url
|
||||
images = extract_images(html, url)
|
||||
if is_csdn:
|
||||
images = _filter_csdn_images(images)
|
||||
elif is_cnblogs:
|
||||
images = _filter_cnblogs_images(images)
|
||||
elif is_51cto:
|
||||
images = _filter_51cto_images(images)
|
||||
|
||||
# 7. 转换为 Markdown(InfoQ / 掘金 / CSDN / 博客园 / 51CTO 使用专用提取)
|
||||
if is_infoq:
|
||||
infoq_html = _extract_infoq_content(html)
|
||||
markdown_content = html_to_markdown(infoq_html, url)
|
||||
elif is_juejin:
|
||||
juejin_html = _extract_juejin_content(html)
|
||||
markdown_content = html_to_markdown(juejin_html, url)
|
||||
elif is_csdn:
|
||||
csdn_html = _extract_csdn_content(html)
|
||||
markdown_content = html_to_markdown(csdn_html, url)
|
||||
elif is_cnblogs:
|
||||
cnblogs_html = _extract_cnblogs_content(html)
|
||||
markdown_content = html_to_markdown(cnblogs_html, url)
|
||||
elif is_51cto:
|
||||
cto_html = _extract_51cto_content(html)
|
||||
markdown_content = html_to_markdown(cto_html, url)
|
||||
elif is_spa_site(html):
|
||||
markdown_content = convert_spa_html_to_markdown(html, images, url)
|
||||
else:
|
||||
markdown_content = html_to_markdown(html, url)
|
||||
|
||||
# 8. 数学公式修复
|
||||
markdown_content = fix_math_formulas(markdown_content)
|
||||
|
||||
# 9. 清理 Markdown 中的 HTML 标签
|
||||
markdown_content = re.sub(r'<[^>]+>', '', markdown_content)
|
||||
markdown_content = re.sub(r'\n{3,}', '\n\n', markdown_content)
|
||||
markdown_content = markdown_content.strip()
|
||||
|
||||
return CrawlResponse(
|
||||
success=True,
|
||||
title=title,
|
||||
content=markdown_content,
|
||||
images=images,
|
||||
)
|
||||
|
||||
except HTTPException:
|
||||
raise
|
||||
except Exception as e:
|
||||
print(f"[Universal Crawl] Error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
raise HTTPException(status_code=500, detail=f"采集失败: {str(e)}")
|
||||
|
||||
|
||||
@router.post("/save")
|
||||
async def universal_save(
|
||||
title: str,
|
||||
content: str,
|
||||
excerpt: Optional[str] = None,
|
||||
category_id: Optional[int] = None,
|
||||
source_url: Optional[str] = None,
|
||||
images: Optional[str] = None,
|
||||
db: Session = Depends(get_db)
|
||||
):
|
||||
"""保存通用采集的文章(复用 crawl.py 的保存逻辑)"""
|
||||
try:
|
||||
print(f"[Universal Crawl] Saving article: {title}")
|
||||
|
||||
# 解析图片列表
|
||||
image_list = []
|
||||
if images:
|
||||
try:
|
||||
import json
|
||||
image_data = json.loads(images)
|
||||
for img in image_data:
|
||||
image_list.append(CrawlImage(**img))
|
||||
except Exception as e:
|
||||
print(f"[Universal Crawl] Parse images error: {e}")
|
||||
|
||||
# 创建临时目录
|
||||
temp_dir = tempfile.mkdtemp(dir=TEMP_DIR)
|
||||
final_dir = UPLOAD_DIR / "images"
|
||||
final_dir.mkdir(exist_ok=True)
|
||||
thumb_dir = UPLOAD_DIR / "thumbnails"
|
||||
thumb_dir.mkdir(exist_ok=True)
|
||||
|
||||
# 下载图片
|
||||
downloaded_images = []
|
||||
for img in image_list:
|
||||
try:
|
||||
filename, filepath, success = download_image(str(img.url), temp_dir)
|
||||
if success and filepath:
|
||||
# 移动最终文件
|
||||
final_filename = f"crawl_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}{os.path.splitext(filename)[1]}"
|
||||
final_path = final_dir / final_filename
|
||||
os.rename(filepath, str(final_path))
|
||||
|
||||
# 生成缩略图
|
||||
thumb_filename = f"thumb_{final_filename}"
|
||||
thumb_path = thumb_dir / thumb_filename
|
||||
thumb_path_db = None
|
||||
if generate_thumbnail(str(final_path), str(thumb_path)):
|
||||
thumb_path_db = f"/uploads/thumbnails/{thumb_filename}"
|
||||
|
||||
# 获取尺寸
|
||||
width, height = get_image_dimensions(str(final_path))
|
||||
|
||||
# 保存到数据库
|
||||
db_image = Image(
|
||||
filename=final_filename,
|
||||
original_filename=img.alt or final_filename,
|
||||
file_path=f"/uploads/images/{final_filename}",
|
||||
file_size=os.path.getsize(str(final_path)),
|
||||
mime_type=f"image/{os.path.splitext(final_filename)[1].lstrip('.') or 'jpeg'}",
|
||||
width=width,
|
||||
height=height,
|
||||
thumbnail_path=thumb_path_db,
|
||||
)
|
||||
db.add(db_image)
|
||||
db.flush()
|
||||
|
||||
# 替换内容中的图片 URL
|
||||
original_url = str(img.url)
|
||||
new_url = f"/uploads/images/{final_filename}"
|
||||
content = content.replace(original_url, new_url)
|
||||
|
||||
downloaded_images.append({
|
||||
"id": db_image.id,
|
||||
"url": new_url,
|
||||
"thumb": thumb_path_db,
|
||||
})
|
||||
except Exception as e:
|
||||
print(f"[Universal Crawl] Download image error: {e}")
|
||||
continue
|
||||
|
||||
# 清理 excerpt
|
||||
if excerpt:
|
||||
excerpt = re.sub(r'<[^>]+>', '', excerpt)
|
||||
excerpt = excerpt.replace('\n', ' ').strip()[:200]
|
||||
else:
|
||||
excerpt = content.replace('\n', ' ').strip()[:200]
|
||||
|
||||
# 创建文章
|
||||
now = datetime.now()
|
||||
slug = re.sub(r'[^\w\u4e00-\u9fff-]', '-', title)[:50].strip('-')
|
||||
slug = f"{slug}-{now.strftime('%Y%m%d%H%M%S')}-{uuid.uuid4().hex[:4]}"
|
||||
|
||||
# 生成摘要
|
||||
summary = content[:300] if len(content) > 300 else content
|
||||
summary = re.sub(r'[#*`\[\]\(\)!]', '', summary)
|
||||
|
||||
# 查找或创建分类
|
||||
final_category_id = category_id
|
||||
if not final_category_id:
|
||||
default_cat = db.query(Category).filter(Category.name == "未分类").first()
|
||||
if not default_cat:
|
||||
default_cat = Category(name="未分类", slug="uncategorized")
|
||||
db.add(default_cat)
|
||||
db.flush()
|
||||
final_category_id = default_cat.id
|
||||
|
||||
post = Post(
|
||||
title=title,
|
||||
slug=slug,
|
||||
content=content,
|
||||
excerpt=excerpt,
|
||||
summary=summary,
|
||||
category_id=final_category_id,
|
||||
author_id=1,
|
||||
status="draft",
|
||||
view_count=0,
|
||||
is_original=False,
|
||||
source_url=source_url,
|
||||
created_at=now,
|
||||
updated_at=now,
|
||||
published_at=None,
|
||||
)
|
||||
db.add(post)
|
||||
db.flush()
|
||||
|
||||
db.commit()
|
||||
|
||||
return {
|
||||
"status": "success",
|
||||
"post_id": post.id,
|
||||
"message": "文章保存成功",
|
||||
"images": downloaded_images,
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
db.rollback()
|
||||
print(f"[Universal Crawl] Save error: {e}")
|
||||
import traceback
|
||||
traceback.print_exc()
|
||||
raise HTTPException(status_code=500, detail=f"保存失败: {str(e)}")
|
||||
@@ -0,0 +1,474 @@
|
||||
"""
|
||||
微信公众号文章采集路由
|
||||
使用 Playwright + Stealth 绕过反爬虫
|
||||
支持 Markdown 转换、图片下载、预览编辑、确认入库
|
||||
"""
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy.orm import Session
|
||||
from typing import Optional, List
|
||||
import httpx
|
||||
import re
|
||||
import os
|
||||
import uuid
|
||||
import asyncio
|
||||
from urllib.parse import urlparse
|
||||
from datetime import datetime
|
||||
from bs4 import BeautifulSoup, NavigableString
|
||||
|
||||
from database import get_db
|
||||
from models import Post, Tag, Category
|
||||
from schemas import PostResponse
|
||||
from routers.auth import get_current_admin
|
||||
|
||||
try:
|
||||
from playwright.async_api import async_playwright
|
||||
PLAYWRIGHT_AVAILABLE = True
|
||||
except ImportError:
|
||||
PLAYWRIGHT_AVAILABLE = False
|
||||
print("[WeChat] Playwright not available")
|
||||
|
||||
try:
|
||||
from playwright_stealth import stealth_async
|
||||
STEALTH_AVAILABLE = True
|
||||
except ImportError:
|
||||
STEALTH_AVAILABLE = False
|
||||
print("[WeChat] Stealth not available")
|
||||
|
||||
router = APIRouter(prefix="/api/wechat-crawl", tags=["wechat-crawl"])
|
||||
|
||||
UPLOADS_DIR = os.path.join(os.path.dirname(__file__), "..", "uploads")
|
||||
os.makedirs(UPLOADS_DIR, exist_ok=True)
|
||||
os.makedirs(os.path.join(UPLOADS_DIR, "thumbnails"), exist_ok=True)
|
||||
|
||||
|
||||
class WechatCrawlRequest(BaseModel):
|
||||
url: str
|
||||
|
||||
|
||||
class WechatSaveRequest(BaseModel):
|
||||
title: str
|
||||
content: str
|
||||
summary: Optional[str] = ""
|
||||
cover_image: Optional[str] = ""
|
||||
category_id: Optional[int] = None
|
||||
tag_names: Optional[List[str]] = []
|
||||
source_url: Optional[str] = ""
|
||||
author: Optional[str] = ""
|
||||
is_draft: Optional[bool] = False
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 辅助函数
|
||||
# ============================================================
|
||||
|
||||
def is_wechat_url(url: str) -> bool:
|
||||
return "mp.weixin.qq.com" in url.lower()
|
||||
|
||||
|
||||
def generate_slug(title: str, suffix: str = "") -> str:
|
||||
slug = re.sub(r'[^\w\u4e00-\u9fa5]+', '-', title.strip())
|
||||
slug = slug.strip('-').lower()
|
||||
if len(slug) > 80:
|
||||
slug = slug[:80]
|
||||
if not slug:
|
||||
slug = "wechat-article"
|
||||
if suffix:
|
||||
slug = f"{slug}-{suffix}"
|
||||
return slug
|
||||
|
||||
|
||||
def remove_empty_paragraphs(soup):
|
||||
"""移除空的 p 标签"""
|
||||
for p in soup.find_all('p'):
|
||||
txt = p.get_text(strip=True)
|
||||
if not txt:
|
||||
imgs = p.find_all('img')
|
||||
if not imgs:
|
||||
p.decompose()
|
||||
return soup
|
||||
|
||||
|
||||
def fix_wechat_code_blocks(soup):
|
||||
"""
|
||||
修复微信代码块:
|
||||
<pre class="code-snippet__js"> 里的 <code> 下直接是文本
|
||||
或多个 <span class="code-snippet__line">...
|
||||
按行拆成多个 code block 以便 markdownify 正确处理
|
||||
"""
|
||||
for pre in soup.find_all('pre', class_=re.compile('code-snippet')):
|
||||
code_tag = pre.find('code')
|
||||
if not code_tag:
|
||||
continue
|
||||
# 收集纯文本
|
||||
lines = []
|
||||
for span in code_tag.find_all('span', class_='code-snippet__line'):
|
||||
line_text = span.get_text(separator='', strip=False)
|
||||
if line_text:
|
||||
lines.append(line_text)
|
||||
if lines:
|
||||
code_tag.clear()
|
||||
for line in lines:
|
||||
code_tag.append(NavigableString(line + '\n'))
|
||||
return soup
|
||||
|
||||
|
||||
def fix_wechat_images(soup):
|
||||
"""
|
||||
修复微信图片:
|
||||
微信文章使用 data-src 存放真实图片URL,src 往往是 svg 占位图。
|
||||
把 data-src 复制到 src,让 markdownify 正确识别图片。
|
||||
"""
|
||||
for img in soup.find_all('img'):
|
||||
data_src = img.get('data-src', '')
|
||||
src = img.get('src', '')
|
||||
# data-src 是真实微信图片URL,且 src 不是有效的 http URL
|
||||
if data_src and data_src.startswith('http') and (
|
||||
not src or src.startswith('data:image') or not src.startswith('http')
|
||||
):
|
||||
img['src'] = data_src
|
||||
# 清理无意义的占位属性
|
||||
if 'data-src' in img.attrs:
|
||||
del img['data-src']
|
||||
return soup
|
||||
|
||||
|
||||
def convert_html_to_markdown(html: str) -> str:
|
||||
"""使用 markdownify 将 HTML 转为 Markdown"""
|
||||
try:
|
||||
import markdownify
|
||||
except ImportError:
|
||||
# 降级:用 BeautifulSoup 提取纯文本
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
return soup.get_text(separator='\n\n', strip=True)
|
||||
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
soup = fix_wechat_images(soup)
|
||||
soup = fix_wechat_code_blocks(soup)
|
||||
soup = remove_empty_paragraphs(soup)
|
||||
|
||||
# markdownify 配置
|
||||
md = markdownify.markdownify(
|
||||
str(soup),
|
||||
heading_style="ATX",
|
||||
bullets="-",
|
||||
strip=['script', 'style', 'iframe', 'noscript']
|
||||
)
|
||||
# 清理多余空行
|
||||
md = re.sub(r'\n{3,}', '\n\n', md)
|
||||
return md.strip()
|
||||
|
||||
|
||||
def generate_filename(original_url: str, article_id: str = "") -> str:
|
||||
"""生成唯一文件名"""
|
||||
ext = os.path.splitext(urlparse(original_url).path)[1].lower()
|
||||
if ext not in ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.bmp']:
|
||||
ext = '.jpg'
|
||||
uid = str(uuid.uuid4())[:8]
|
||||
return f"wechat_{article_id}_{uid}{ext}"
|
||||
|
||||
|
||||
async def download_image(url: str, filepath: str, headers: dict) -> bool:
|
||||
"""下载单张图片"""
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=30) as client:
|
||||
r = await client.get(url, headers=headers, follow_redirects=True)
|
||||
if r.status_code == 200:
|
||||
with open(filepath, 'wb') as f:
|
||||
f.write(r.content)
|
||||
return True
|
||||
except Exception as e:
|
||||
print(f"[WeChat] 下载图片失败 {url}: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def download_all_images(image_tasks: list, headers: dict):
|
||||
"""并发下载所有图片"""
|
||||
async def task_wrapper(task):
|
||||
url, filepath = task
|
||||
success = await download_image(url, filepath, headers)
|
||||
return url, filepath, success
|
||||
|
||||
return await asyncio.gather(*(task_wrapper(t) for t in image_tasks))
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 核心采集函数
|
||||
# ============================================================
|
||||
|
||||
async def crawl_wechat_article(url: str) -> dict:
|
||||
"""采集微信公众号文章"""
|
||||
if not PLAYWRIGHT_AVAILABLE:
|
||||
raise Exception("Playwright 未安装,无法采集微信文章")
|
||||
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(
|
||||
headless=True,
|
||||
args=[
|
||||
'--disable-blink-features=AutomationControlled',
|
||||
'--disable-web-security',
|
||||
'--disable-features=IsolateOrigins,site-per-process',
|
||||
'--disable-dev-shm-usage',
|
||||
'--no-sandbox',
|
||||
'--disable-setuid-sandbox',
|
||||
'--disable-accelerated-2d-canvas',
|
||||
'--disable-gpu',
|
||||
'--window-size=1920,1080',
|
||||
]
|
||||
)
|
||||
context = await browser.new_context(
|
||||
viewport={'width': 1920, 'height': 1080},
|
||||
user_agent='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
locale='zh-CN',
|
||||
timezone_id='Asia/Shanghai',
|
||||
java_script_enabled=True,
|
||||
)
|
||||
await context.add_init_script("""
|
||||
Object.defineProperty(navigator, 'webdriver', { get: () => undefined });
|
||||
Object.defineProperty(navigator, 'plugins', { get: () => [1,2,3,4,5] });
|
||||
window.chrome = { runtime: {} };
|
||||
""")
|
||||
page = await context.new_page()
|
||||
if STEALTH_AVAILABLE:
|
||||
await stealth_async(page)
|
||||
|
||||
try:
|
||||
print(f"[WeChat] 访问: {url}")
|
||||
await page.goto(url, wait_until="networkidle", timeout=60000)
|
||||
await page.wait_for_timeout(3000)
|
||||
|
||||
# 获取标题
|
||||
title = ""
|
||||
try:
|
||||
title = await page.title()
|
||||
if title in ["微信公众平台", ""]:
|
||||
h2 = await page.query_selector("#activity_name")
|
||||
if h2:
|
||||
title = await h2.inner_text()
|
||||
except:
|
||||
pass
|
||||
title = title.strip() or "未命名文章"
|
||||
|
||||
# 获取公众号名称
|
||||
author = ""
|
||||
try:
|
||||
name_el = await page.query_selector("#js_name")
|
||||
if name_el:
|
||||
author = await name_el.inner_text()
|
||||
author = author.strip()
|
||||
except:
|
||||
pass
|
||||
|
||||
# 获取正文 HTML
|
||||
content_html = ""
|
||||
try:
|
||||
content_el = await page.query_selector("#js_content")
|
||||
if content_el:
|
||||
content_html = await content_el.inner_html()
|
||||
except:
|
||||
pass
|
||||
|
||||
# 获取封面图
|
||||
cover_image = ""
|
||||
try:
|
||||
meta = await page.query_selector('meta[property="og:image"]')
|
||||
if meta:
|
||||
cover_image = await meta.get_attribute("content")
|
||||
except:
|
||||
pass
|
||||
|
||||
# 获取正文中的图片
|
||||
images = []
|
||||
if content_html:
|
||||
soup = BeautifulSoup(content_html, 'html.parser')
|
||||
for img in soup.find_all('img'):
|
||||
src = img.get('data-src') or img.get('src', '')
|
||||
if src and ('mmbiz.qpic.cn' in src or 'mmbiz.qlogo.cn' in src):
|
||||
images.append({
|
||||
'original_url': src,
|
||||
'description': img.get('alt', '')
|
||||
})
|
||||
|
||||
print(f"[WeChat] 标题: {title[:50]}")
|
||||
print(f"[WeChat] 作者: {author}")
|
||||
print(f"[WeChat] 内容长度: {len(content_html)}")
|
||||
print(f"[WeChat] 图片数: {len(images)}")
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"content_html": content_html,
|
||||
"author": author,
|
||||
"cover_image": cover_image,
|
||||
"images": images,
|
||||
}
|
||||
|
||||
finally:
|
||||
await browser.close()
|
||||
|
||||
|
||||
# ============================================================
|
||||
# 图片下载处理
|
||||
# ============================================================
|
||||
|
||||
async def process_images(images: list, article_id: str) -> dict:
|
||||
"""下载图片并返回 URL 映射"""
|
||||
if not images:
|
||||
return {}
|
||||
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
|
||||
"Referer": "https://mp.weixin.qq.com/",
|
||||
}
|
||||
|
||||
tasks = []
|
||||
url_map = {}
|
||||
for img in images:
|
||||
original_url = img['original_url']
|
||||
filename = generate_filename(original_url, article_id)
|
||||
filepath = os.path.join(UPLOADS_DIR, filename)
|
||||
tasks.append((original_url, filepath))
|
||||
url_map[original_url] = f"/uploads/{filename}"
|
||||
|
||||
results = await download_all_images(tasks, headers)
|
||||
|
||||
# 过滤失败的
|
||||
for original_url, filepath, success in results:
|
||||
if not success and os.path.exists(filepath):
|
||||
os.remove(filepath)
|
||||
if not success:
|
||||
# 保留原 URL
|
||||
url_map[original_url] = original_url
|
||||
|
||||
return url_map
|
||||
|
||||
|
||||
# ============================================================
|
||||
# API 端点
|
||||
# ============================================================
|
||||
|
||||
@router.post("/preview")
|
||||
async def preview_wechat(request: WechatCrawlRequest):
|
||||
"""第1步:采集并返回预览数据"""
|
||||
url = request.url.strip()
|
||||
if not url:
|
||||
raise HTTPException(status_code=400, detail="URL 不能为空")
|
||||
if not is_wechat_url(url):
|
||||
raise HTTPException(status_code=400, detail="不是微信公众号文章链接")
|
||||
|
||||
try:
|
||||
result = await crawl_wechat_article(url)
|
||||
except Exception as e:
|
||||
raise HTTPException(status_code=500, detail=f"采集失败: {str(e)}")
|
||||
|
||||
if not result['content_html']:
|
||||
raise HTTPException(status_code=500, detail="未能获取到文章内容,可能需要等待页面加载或文章已被删除")
|
||||
|
||||
# 转换为 Markdown(预览阶段不下载图片,保留原始 URL)
|
||||
md_content = convert_html_to_markdown(result['content_html'])
|
||||
|
||||
# 提取摘要(前200字符)
|
||||
plain = BeautifulSoup(result['content_html'], 'html.parser').get_text(strip=True)
|
||||
summary = plain[:200] if len(plain) > 200 else plain
|
||||
|
||||
return {
|
||||
"title": result['title'],
|
||||
"content_md": md_content,
|
||||
"summary": summary,
|
||||
"author": result['author'],
|
||||
"cover_image": result['cover_image'],
|
||||
"images": result['images'],
|
||||
"source_url": url,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/save")
|
||||
async def save_wechat(request: WechatSaveRequest, db: Session = Depends(get_db), current_admin = Depends(get_current_admin)):
|
||||
"""第2步:确认入库(下载图片、替换URL、保存文章)"""
|
||||
|
||||
# 生成唯一 article_id 用于图片命名
|
||||
article_id = str(uuid.uuid4())[:8]
|
||||
|
||||
# 从 content 中提取所有微信图片 URL
|
||||
image_urls = re.findall(
|
||||
r'https?://mmbiz\.(?:qpic|qlogo)\.cn/[^\s\)"\'>\]]+',
|
||||
request.content
|
||||
)
|
||||
|
||||
# 去重
|
||||
unique_urls = list(set(image_urls))
|
||||
images_info = [{'original_url': u, 'description': ''} for u in unique_urls]
|
||||
|
||||
# 下载图片
|
||||
url_map = await process_images(images_info, article_id)
|
||||
|
||||
# 替换 content 中的图片 URL
|
||||
final_content = request.content
|
||||
for original_url, local_url in url_map.items():
|
||||
final_content = final_content.replace(original_url, local_url)
|
||||
|
||||
# 替换封面图
|
||||
final_cover = request.cover_image
|
||||
if final_cover and 'mmbiz' in final_cover:
|
||||
# 封面图也下载
|
||||
cover_filename = generate_filename(final_cover, article_id)
|
||||
cover_filepath = os.path.join(UPLOADS_DIR, cover_filename)
|
||||
headers = {
|
||||
"User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36",
|
||||
"Referer": "https://mp.weixin.qq.com/",
|
||||
}
|
||||
success = await download_image(final_cover, cover_filepath, headers)
|
||||
if success:
|
||||
final_cover = f"/uploads/{cover_filename}"
|
||||
|
||||
# 生成 slug
|
||||
base_slug = generate_slug(request.title, article_id)
|
||||
slug = base_slug
|
||||
counter = 1
|
||||
while db.query(Post).filter(Post.slug == slug).first():
|
||||
slug = f"{base_slug}-{counter}"
|
||||
counter += 1
|
||||
|
||||
# 创建文章
|
||||
is_draft = request.is_draft if request.is_draft is not None else False
|
||||
post = Post(
|
||||
title=request.title,
|
||||
slug=slug,
|
||||
content=final_content,
|
||||
summary=request.summary or "",
|
||||
cover_image=final_cover or "",
|
||||
is_published=not is_draft,
|
||||
is_draft=is_draft,
|
||||
)
|
||||
|
||||
# 分类
|
||||
if request.category_id:
|
||||
category = db.query(Category).filter(Category.id == request.category_id).first()
|
||||
if category:
|
||||
post.category = category
|
||||
|
||||
# 标签
|
||||
if request.tag_names:
|
||||
for tag_name in request.tag_names:
|
||||
tag_name = tag_name.strip()
|
||||
if not tag_name:
|
||||
continue
|
||||
tag = db.query(Tag).filter(Tag.name == tag_name).first()
|
||||
if not tag:
|
||||
tag = Tag(name=tag_name)
|
||||
db.add(tag)
|
||||
db.flush()
|
||||
post.tags.append(tag)
|
||||
|
||||
db.add(post)
|
||||
db.commit()
|
||||
db.refresh(post)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"message": "文章已保存",
|
||||
"post": {
|
||||
"id": post.id,
|
||||
"title": post.title,
|
||||
"slug": post.slug,
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
"""
|
||||
Pydantic 数据验证模型
|
||||
"""
|
||||
from pydantic import BaseModel, Field
|
||||
from typing import Optional, List
|
||||
from datetime import datetime
|
||||
|
||||
# ==================== 认证相关 ====================
|
||||
|
||||
class LoginRequest(BaseModel):
|
||||
username: str
|
||||
password: str
|
||||
|
||||
class LoginResponse(BaseModel):
|
||||
access_token: str
|
||||
token_type: str = "bearer"
|
||||
|
||||
class PasswordChange(BaseModel):
|
||||
old_password: str
|
||||
new_password: str = Field(..., min_length=8, max_length=20)
|
||||
|
||||
# ==================== 文章相关 ====================
|
||||
|
||||
class TagBase(BaseModel):
|
||||
name: str = Field(..., max_length=30)
|
||||
|
||||
class TagCreate(TagBase):
|
||||
pass
|
||||
|
||||
class TagResponse(TagBase):
|
||||
id: int
|
||||
created_at: datetime
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
class CategoryBase(BaseModel):
|
||||
name: str = Field(..., max_length=50)
|
||||
description: Optional[str] = Field(None, max_length=200)
|
||||
|
||||
class CategoryCreate(CategoryBase):
|
||||
pass
|
||||
|
||||
class CategoryResponse(CategoryBase):
|
||||
id: int
|
||||
created_at: datetime
|
||||
post_count: int = 0
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
class PostBase(BaseModel):
|
||||
title: str = Field(..., max_length=200)
|
||||
slug: Optional[str] = Field(None, max_length=200)
|
||||
content: str
|
||||
content_type: Optional[str] = Field(default="markdown", max_length=20) # markdown 或 html
|
||||
summary: Optional[str] = Field(None, max_length=500)
|
||||
cover_image: Optional[str] = None
|
||||
is_published: bool = True
|
||||
is_draft: bool = False
|
||||
category_id: Optional[int] = None
|
||||
tag_ids: List[int] = []
|
||||
|
||||
class PostCreate(PostBase):
|
||||
pass
|
||||
|
||||
class PostUpdate(BaseModel):
|
||||
title: Optional[str] = Field(None, max_length=200)
|
||||
slug: Optional[str] = Field(None, max_length=200)
|
||||
content: Optional[str] = None
|
||||
content_type: Optional[str] = Field(None, max_length=20)
|
||||
summary: Optional[str] = Field(None, max_length=500)
|
||||
cover_image: Optional[str] = None
|
||||
is_published: Optional[bool] = None
|
||||
is_draft: Optional[bool] = None
|
||||
category_id: Optional[int] = None
|
||||
tag_ids: Optional[List[int]] = None
|
||||
|
||||
class PostResponse(BaseModel):
|
||||
id: int
|
||||
title: str
|
||||
slug: Optional[str] = None
|
||||
content: str
|
||||
content_type: Optional[str] = None
|
||||
summary: Optional[str]
|
||||
cover_image: Optional[str]
|
||||
is_published: bool
|
||||
is_draft: bool
|
||||
view_count: int
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
category: Optional[CategoryResponse]
|
||||
tags: List[TagResponse]
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
class PostListItem(BaseModel):
|
||||
id: int
|
||||
title: str
|
||||
slug: Optional[str] = None
|
||||
summary: Optional[str]
|
||||
cover_image: Optional[str]
|
||||
content_type: Optional[str] = None
|
||||
is_published: bool
|
||||
view_count: int
|
||||
created_at: datetime
|
||||
category: Optional[CategoryResponse]
|
||||
tags: List[TagResponse]
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
# ==================== 图片相关 ====================
|
||||
|
||||
class ImageResponse(BaseModel):
|
||||
id: int
|
||||
filename: str
|
||||
original_name: str
|
||||
description: Optional[str]
|
||||
file_size: int
|
||||
file_path: str
|
||||
thumb_path: Optional[str] = None
|
||||
width: Optional[int] = None
|
||||
height: Optional[int] = None
|
||||
created_at: datetime
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
class ImageListResponse(BaseModel):
|
||||
items: List[ImageResponse]
|
||||
total: int
|
||||
skip: int
|
||||
limit: int
|
||||
|
||||
# ==================== 设置相关 ====================
|
||||
|
||||
class SettingBase(BaseModel):
|
||||
blog_name: str = Field(..., max_length=20)
|
||||
blog_description: Optional[str] = Field(None, max_length=200)
|
||||
welcome_message: Optional[str] = Field(default="欢迎来到我的博客 👋 分享技术、生活与思考", max_length=300)
|
||||
blog_logo: Optional[str] = None
|
||||
banner_image: Optional[str] = None
|
||||
banner_type: str = Field(default="text") # text或carousel
|
||||
banner_images: Optional[str] = Field(default="[]") # JSON数组
|
||||
theme: str = Field(default="neobrutalism")
|
||||
layout: str = Field(default="card")
|
||||
theme_colors: Optional[str] = Field(default='{"primary":"#7c3aed","secondary":"#06b6d4","bg":"#ffffff","text":"#1f2937","sidebar":"#1e293b"}')
|
||||
|
||||
class SettingUpdate(BaseModel):
|
||||
blog_name: Optional[str] = Field(None, max_length=20)
|
||||
blog_description: Optional[str] = Field(None, max_length=200)
|
||||
welcome_message: Optional[str] = Field(None, max_length=300)
|
||||
blog_logo: Optional[str] = None
|
||||
banner_image: Optional[str] = None
|
||||
banner_type: Optional[str] = None
|
||||
banner_images: Optional[str] = None
|
||||
theme: Optional[str] = None
|
||||
layout: Optional[str] = None
|
||||
theme_colors: Optional[str] = None
|
||||
|
||||
class SettingResponse(SettingBase):
|
||||
id: int
|
||||
updated_at: datetime
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
# ==================== 页面相关 ====================
|
||||
|
||||
class PageBase(BaseModel):
|
||||
title: str = Field(..., max_length=50)
|
||||
slug: str = Field(..., max_length=50)
|
||||
content: str
|
||||
is_nav: bool = True
|
||||
nav_order: int = 0
|
||||
|
||||
class PageCreate(PageBase):
|
||||
pass
|
||||
|
||||
class PageUpdate(BaseModel):
|
||||
title: Optional[str] = Field(None, max_length=50)
|
||||
slug: Optional[str] = Field(None, max_length=50)
|
||||
content: Optional[str] = None
|
||||
is_nav: Optional[bool] = None
|
||||
nav_order: Optional[int] = None
|
||||
|
||||
class PageResponse(PageBase):
|
||||
id: int
|
||||
created_at: datetime
|
||||
updated_at: datetime
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
|
||||
# ==================== 统计相关 ====================
|
||||
|
||||
class StatsResponse(BaseModel):
|
||||
total_posts: int
|
||||
total_views: int
|
||||
today_views: int
|
||||
category_counts: dict
|
||||
recent_posts: List[PostListItem]
|
||||
@@ -0,0 +1,22 @@
|
||||
"""SSO Cookie 本地验签 —— 与 SSO 中心共享 SECRET_KEY"""
|
||||
from fastapi import Request, HTTPException
|
||||
from jose import jwt, JWTError
|
||||
|
||||
# 必须与 SSO 中心 sso_config.py 完全一致
|
||||
SECRET_KEY = "your-secret-key-change-in-production"
|
||||
ALGORITHM = "HS256"
|
||||
|
||||
|
||||
def get_current_user_from_cookie(request: Request) -> dict:
|
||||
"""从请求 Cookie 中读取 access_token,本地验签后返回用户信息"""
|
||||
token = request.cookies.get("access_token")
|
||||
if not token:
|
||||
return None
|
||||
try:
|
||||
payload = jwt.decode(token, SECRET_KEY, algorithms=[ALGORITHM])
|
||||
username: str = payload.get("sub")
|
||||
if not username:
|
||||
return None
|
||||
return {"username": username}
|
||||
except JWTError:
|
||||
return None
|
||||
@@ -0,0 +1,17 @@
|
||||
#!/usr/bin/env python3
|
||||
"""测试 wechat-crawl API 端点"""
|
||||
import sys
|
||||
sys.path.insert(0, '/home/ubuntu/nanocode/personal-blog/backend')
|
||||
|
||||
from fastapi.testclient import TestClient
|
||||
from app import app
|
||||
|
||||
client = TestClient(app)
|
||||
|
||||
url = "https://mp.weixin.qq.com/s/_YqGx7J9XnyPe0GFhv4LYA"
|
||||
print(f"测试 API: /api/wechat-crawl/preview")
|
||||
print(f"URL: {url}\n")
|
||||
|
||||
res = client.post("/api/wechat-crawl/preview", json={"url": url})
|
||||
print(f"状态码: {res.status_code}")
|
||||
print(f"响应: {res.text[:2000]}")
|
||||
@@ -0,0 +1,41 @@
|
||||
#!/usr/bin/env python3
|
||||
"""本地测试采集功能,绕过 HTTP API 和数据库"""
|
||||
import asyncio
|
||||
import sys
|
||||
sys.path.insert(0, '/home/ubuntu/nanocode/personal-blog/backend')
|
||||
|
||||
from routers.crawl import crawl_with_playwright, convert_spa_html_to_markdown
|
||||
from routers.crawl import CrawlImage
|
||||
|
||||
async def main():
|
||||
url = "https://mp.weixin.qq.com/s/_YqGx7J9XnyPe0GFhv4LYA"
|
||||
print(f"开始测试采集: {url}\n")
|
||||
|
||||
result = await crawl_with_playwright(url)
|
||||
|
||||
print(f"标题: {result['title']}")
|
||||
print(f"图片数: {len(result['images'])}")
|
||||
print(f"内容HTML长度: {len(result['content_html'])}")
|
||||
|
||||
markdown = convert_spa_html_to_markdown(
|
||||
result['content_html'],
|
||||
result['images'],
|
||||
url
|
||||
)
|
||||
|
||||
print(f"\nMarkdown长度: {len(markdown)}")
|
||||
print("\n=== Markdown 前 3000 字符 ===")
|
||||
print(markdown[:3000])
|
||||
print("\n=== Markdown 末尾 1000 字符 ===")
|
||||
print(markdown[-1000:])
|
||||
|
||||
# 检查是否有代码块
|
||||
if '```' in markdown:
|
||||
print("\n✅ 检测到代码块!")
|
||||
code_blocks = markdown.split('```')
|
||||
print(f"代码块数量: {len(code_blocks)//2}")
|
||||
else:
|
||||
print("\n⚠️ 未检测到代码块")
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
|
After Width: | Height: | Size: 494 KiB |
|
After Width: | Height: | Size: 1.4 MiB |
|
After Width: | Height: | Size: 966 KiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 966 KiB |
|
After Width: | Height: | Size: 1.3 MiB |
|
After Width: | Height: | Size: 755 KiB |
|
After Width: | Height: | Size: 828 KiB |
|
After Width: | Height: | Size: 104 KiB |
|
After Width: | Height: | Size: 1.1 MiB |
|
After Width: | Height: | Size: 1.1 MiB |
|
After Width: | Height: | Size: 889 KiB |
|
After Width: | Height: | Size: 868 KiB |
|
After Width: | Height: | Size: 1001 KiB |
|
After Width: | Height: | Size: 988 KiB |
|
After Width: | Height: | Size: 925 KiB |
|
After Width: | Height: | Size: 925 KiB |
|
After Width: | Height: | Size: 447 KiB |
|
After Width: | Height: | Size: 1.2 MiB |
|
After Width: | Height: | Size: 1.2 MiB |
|
After Width: | Height: | Size: 970 KiB |
|
After Width: | Height: | Size: 970 KiB |
|
After Width: | Height: | Size: 860 KiB |
|
After Width: | Height: | Size: 860 KiB |
|
After Width: | Height: | Size: 1.2 MiB |
|
After Width: | Height: | Size: 1.1 MiB |
|
After Width: | Height: | Size: 1.1 MiB |
|
After Width: | Height: | Size: 63 KiB |
|
After Width: | Height: | Size: 404 KiB |
|
After Width: | Height: | Size: 5.0 KiB |
|
After Width: | Height: | Size: 10 KiB |
|
After Width: | Height: | Size: 672 KiB |
|
After Width: | Height: | Size: 9.7 KiB |
|
After Width: | Height: | Size: 72 KiB |
|
After Width: | Height: | Size: 23 KiB |
|
After Width: | Height: | Size: 32 KiB |
|
After Width: | Height: | Size: 24 KiB |
|
After Width: | Height: | Size: 48 KiB |
|
After Width: | Height: | Size: 64 KiB |
|
After Width: | Height: | Size: 83 KiB |
|
After Width: | Height: | Size: 68 KiB |
|
After Width: | Height: | Size: 56 KiB |
|
After Width: | Height: | Size: 84 KiB |
|
After Width: | Height: | Size: 21 KiB |
|
After Width: | Height: | Size: 45 KiB |
|
After Width: | Height: | Size: 36 KiB |
|
After Width: | Height: | Size: 14 KiB |
|
After Width: | Height: | Size: 2.4 KiB |
|
After Width: | Height: | Size: 153 KiB |
|
After Width: | Height: | Size: 9.3 KiB |
|
After Width: | Height: | Size: 34 KiB |
|
After Width: | Height: | Size: 112 KiB |
|
After Width: | Height: | Size: 98 KiB |
|
After Width: | Height: | Size: 146 KiB |
|
After Width: | Height: | Size: 7.5 KiB |
|
After Width: | Height: | Size: 28 KiB |
|
After Width: | Height: | Size: 141 KiB |
|
After Width: | Height: | Size: 225 KiB |
|
After Width: | Height: | Size: 106 KiB |
|
After Width: | Height: | Size: 217 KiB |
|
After Width: | Height: | Size: 43 KiB |
|
After Width: | Height: | Size: 528 KiB |
|
After Width: | Height: | Size: 16 KiB |
|
After Width: | Height: | Size: 58 KiB |
|
After Width: | Height: | Size: 1.7 KiB |
|
After Width: | Height: | Size: 24 KiB |
|
After Width: | Height: | Size: 8.4 KiB |
|
After Width: | Height: | Size: 98 KiB |
|
After Width: | Height: | Size: 19 KiB |
|
After Width: | Height: | Size: 25 KiB |
|
After Width: | Height: | Size: 1.2 KiB |
|
After Width: | Height: | Size: 33 KiB |
|
After Width: | Height: | Size: 65 KiB |
|
After Width: | Height: | Size: 16 KiB |
|
After Width: | Height: | Size: 126 KiB |
|
After Width: | Height: | Size: 45 KiB |
|
After Width: | Height: | Size: 41 KiB |
|
After Width: | Height: | Size: 56 KiB |
|
After Width: | Height: | Size: 270 KiB |
|
After Width: | Height: | Size: 71 KiB |