Ubuntu
1165 lines
46 KiB
Python
1165 lines
46 KiB
Python
"""
|
||
文章采集路由 - 从网页URL采集内容
|
||
支持SPA网站采集
|
||
"""
|
||
import os
|
||
import re
|
||
import uuid
|
||
import requests
|
||
from datetime import datetime
|
||
from typing import Optional, List
|
||
from urllib.parse import urljoin, urlparse
|
||
|
||
try:
|
||
from bs4 import BeautifulSoup
|
||
BS4_AVAILABLE = True
|
||
except ImportError:
|
||
BS4_AVAILABLE = False
|
||
|
||
# Playwright用于SPA采集
|
||
try:
|
||
from playwright.async_api import async_playwright
|
||
PLAYWRIGHT_AVAILABLE = True
|
||
except ImportError:
|
||
PLAYWRIGHT_AVAILABLE = False
|
||
|
||
from fastapi import APIRouter, Depends, HTTPException
|
||
from pydantic import BaseModel
|
||
from sqlalchemy.orm import Session
|
||
|
||
import sys
|
||
sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||
|
||
from database import get_db
|
||
from models import Image
|
||
from routers.auth import get_current_admin
|
||
from routers.images import generate_thumbnail, get_image_dimensions
|
||
|
||
router = APIRouter(prefix="/api/crawl", tags=["文章采集"])
|
||
|
||
|
||
class CrawlRequest(BaseModel):
|
||
url: str
|
||
|
||
|
||
class CrawlImage(BaseModel):
|
||
id: Optional[int] = None # 数据库ID
|
||
original_url: str
|
||
local_path: Optional[str] = None
|
||
filename: Optional[str] = None
|
||
success: bool = False
|
||
|
||
|
||
class CrawlResponse(BaseModel):
|
||
success: bool
|
||
title: Optional[str] = None
|
||
content: Optional[str] = None # Markdown内容
|
||
summary: Optional[str] = None
|
||
images: List[CrawlImage] = []
|
||
error: Optional[str] = None
|
||
|
||
|
||
def clean_html_tags(text: str) -> str:
|
||
"""移除文本中的HTML标签"""
|
||
# 移除所有HTML标签
|
||
text = re.sub(r'<[^>]+>', '', text)
|
||
# 解码HTML实体
|
||
text = text.replace(' ', ' ')
|
||
text = text.replace('<', '<')
|
||
text = text.replace('>', '>')
|
||
text = text.replace('&', '&')
|
||
text = text.replace('"', '"')
|
||
text = text.replace(''', "'")
|
||
return text.strip()
|
||
|
||
|
||
def fix_math_formulas(text: str) -> str:
|
||
"""修复数学公式格式，处理混合格式问题"""
|
||
import re
|
||
|
||
# 1. 清理已渲染的数学符号（保留LaTeX代码）
|
||
# 移除 Unicode 数学运算符，但保留后面的LaTeX
|
||
text = re.sub(r'[⁰¹²³⁴⁵⁶⁷⁸⁹⁺⁻⁼⁽⁾ⁿⁱ]+', '', text)
|
||
|
||
# 2. 规范化反斜杠（双重转义变单重）
|
||
text = text.replace('\\\\', '\\')
|
||
|
||
# 3. 检测并包裹行内公式
|
||
# 匹配常见的行内数学模式：\mathbf{x}, \alpha, x_{i}, x^{2}, \sum 等
|
||
inline_patterns = [
|
||
r'\\[a-zA-Z]+\{[^}]*\}', # \mathbf{x}, \frac{a}{b}
|
||
r'\\[a-zA-Z]+', # \alpha, \beta
|
||
r'[a-zA-Z]_[a-zA-Z0-9]+', # x_i, y_j
|
||
r'[a-zA-Z]\^[a-zA-Z0-9]+', # x^2, y^n
|
||
]
|
||
|
||
# 4. 检测块级公式（多行或包含 array, matrix 等）
|
||
block_keywords = ['\\begin{array}', '\\begin{matrix}', '\\begin{equation}',
|
||
'\\sum_', '\\int_', '\\prod_', '\\lim_']
|
||
|
||
lines = text.split('\n')
|
||
result_lines = []
|
||
in_block = False
|
||
block_content = []
|
||
|
||
for line in lines:
|
||
stripped = line.strip()
|
||
|
||
# 检测块级公式开始
|
||
if any(kw in stripped for kw in block_keywords):
|
||
if not in_block:
|
||
in_block = True
|
||
block_content = []
|
||
block_content.append(line)
|
||
continue
|
||
|
||
# 如果在块内，继续收集
|
||
if in_block:
|
||
block_content.append(line)
|
||
# 检测块结束
|
||
if '\\end{array}' in stripped or '\\end{matrix}' in stripped or '\\end{equation}' in stripped:
|
||
# 包裹整个块
|
||
block_text = '\n'.join(block_content)
|
||
if not block_text.startswith('$$') and not block_text.startswith('\\['):
|
||
result_lines.append('$$' + block_text + '$$')
|
||
else:
|
||
result_lines.append(block_text)
|
||
in_block = False
|
||
block_content = []
|
||
continue
|
||
|
||
# 处理行内公式
|
||
# 如果一行中有明显的LaTeX命令，但还没有被包裹
|
||
if re.search(r'\\[a-zA-Z]+', line) and not re.search(r'\\\(|\\\)|\\\[|\\\]', line):
|
||
# 简单处理：如果包含复杂结构，作为块级；否则作为行内
|
||
if re.search(r'\\(frac|sum|int|prod|lim)\{', line):
|
||
line = '$$' + line + '$$'
|
||
else:
|
||
line = '\\(' + line + '\\)'
|
||
|
||
result_lines.append(line)
|
||
|
||
# 处理未闭合的块
|
||
if in_block and block_content:
|
||
block_text = '\n'.join(block_content)
|
||
result_lines.append('$$' + block_text + '$$')
|
||
|
||
return '\n'.join(result_lines)
|
||
|
||
|
||
def extract_title_from_html(html: str) -> str:
|
||
"""从HTML中提取标题 - 优先提取文章主标题"""
|
||
|
||
# 1. 尝试提取 Open Graph title (og:title) - 通常是文章主标题
|
||
og_title_match = re.search(
|
||
r'<meta[^>]+property=["\']og:title["\'][^>]+content=["\']([^"\']+)["\']',
|
||
html, re.IGNORECASE
|
||
)
|
||
if not og_title_match:
|
||
og_title_match = re.search(
|
||
r'<meta[^>]+content=["\']([^"\']+)["\'][^>]+property=["\']og:title["\']',
|
||
html, re.IGNORECASE
|
||
)
|
||
if og_title_match:
|
||
title = clean_html_tags(og_title_match.group(1))
|
||
if len(title) > 5: # 确保不是太短的无意义标题
|
||
# 清理常见后缀
|
||
title = re.sub(r'\s+[–—-]\s+.*$', '', title).strip()
|
||
return title
|
||
|
||
# 1.5 博客园/CSDN等特殊平台处理（放在h1之前，因为这些平台的h1可能是博客名而非文章标题）
|
||
# 博客园：h1.postTitle 或 a#cb_post_title_url
|
||
cnblog_title = re.search(r'<h1[^>]+class=["\'][^"\']*postTitle[^"\']*["\'][^>]*>(.*?)</h1>', html, re.IGNORECASE | re.DOTALL)
|
||
if cnblog_title:
|
||
title = clean_html_tags(cnblog_title.group(1))
|
||
if len(title) > 5:
|
||
return title
|
||
|
||
cnblog_title2 = re.search(r'<a[^>]+id=["\']cb_post_title_url["\'][^>]*>(.*?)</a>', html, re.IGNORECASE | re.DOTALL)
|
||
if cnblog_title2:
|
||
title = clean_html_tags(cnblog_title2.group(1))
|
||
if len(title) > 5:
|
||
return title
|
||
|
||
# 1.6 人人都是产品经理 (woshipm.com)
|
||
woshipm_title = re.search(r'<[^>]+class=["\'][^"\']*article--title[^"\']*["\'][^>]*>(.*?)</[^>]+>', html, re.IGNORECASE | re.DOTALL)
|
||
if woshipm_title:
|
||
title = clean_html_tags(woshipm_title.group(1))
|
||
if len(title) > 5:
|
||
return title
|
||
|
||
# 1.7 菜鸟教程：从article-intro内的h1提取标题，或使用title标签
|
||
runoob_title = re.search(r'<div[^>]*class=["\']article-intro["\'][^>]*>.*?<h1[^>]*>(.*?)</h1>', html, re.IGNORECASE | re.DOTALL)
|
||
if runoob_title:
|
||
title = clean_html_tags(runoob_title.group(1))
|
||
if len(title) > 5 and "菜鸟教程" not in title: # 排除网站名
|
||
return title
|
||
|
||
# 从title标签提取（菜鸟教程等网站）
|
||
title_match = re.search(r'<title[^>]*>(.*?)</title>', html, re.IGNORECASE | re.DOTALL)
|
||
if title_match:
|
||
title = clean_html_tags(title_match.group(1))
|
||
# 移除常见的分隔符和后缀（但保留技术词汇中的连字符如Objective-C）
|
||
# 使用更严格的匹配，确保是标题分隔符而不是单词中的连字符
|
||
title = re.sub(r'\s+[|│┃]\s+.*$', '', title).strip() # | 分隔符
|
||
title = re.sub(r'\s+[-–—]\s+[^\s]*教程.*$', '', title).strip() # 匹配 " - 菜鸟教程"
|
||
title = re.sub(r'\s+[-–—]\s+[^\s]*博客.*$', '', title).strip() # 匹配 " - XXX博客"
|
||
if len(title) > 5:
|
||
return title
|
||
|
||
# 2. 尝试提取 h1 标题（文章主标题通常是页面第一个h1）
|
||
# 先清理HTML获取纯文本
|
||
h1_matches = re.findall(r'<h1[^>]*>(.*?)</h1>', html, re.IGNORECASE | re.DOTALL)
|
||
for h1 in h1_matches:
|
||
title = clean_html_tags(h1)
|
||
# 过滤掉太短的标题
|
||
if len(title) > 5:
|
||
return title
|
||
|
||
# 3. 尝试从 article/main 内容区域提取 h2 或 h3（作为备选）
|
||
content_patterns = [
|
||
r'<article[^>]*>.*?<h[23][^>]*>(.*?)</h[23]>.*?</article>',
|
||
r'<main[^>]*>.*?<h[23][^>]*>(.*?)</h[23]>.*?</main>',
|
||
r'<div[^>]*class=["\'][^"\']*(?:content|article|post|entry)[^"\']*["\'][^>]*>.*?<h[23][^>]*>(.*?)</h[23]>',
|
||
]
|
||
for pattern in content_patterns:
|
||
match = re.search(pattern, html, re.IGNORECASE | re.DOTALL)
|
||
if match:
|
||
title = clean_html_tags(match.group(1))
|
||
if len(title) > 10: # 内容区标题通常较长
|
||
return title
|
||
|
||
# 4. 尝试提取 Twitter Card title
|
||
twitter_title_match = re.search(
|
||
r'<meta[^>]+name=["\']twitter:title["\'][^>]+content=["\']([^"\']+)["\']',
|
||
html, re.IGNORECASE
|
||
)
|
||
if twitter_title_match:
|
||
title = clean_html_tags(twitter_title_match.group(1))
|
||
if len(title) > 5:
|
||
return title
|
||
|
||
# 5. 最后回退到 <title> 标签
|
||
title_match = re.search(r'<title[^>]*>([^<]+)</title>', html, re.IGNORECASE)
|
||
if title_match:
|
||
title = clean_html_tags(title_match.group(1))
|
||
# 清理常见后缀（网站名、分类等）
|
||
# 处理各种分隔符: | - – — :: · •
|
||
title = re.sub(r'\s*[\|\-–—:·•]\s*.+$', '', title)
|
||
# 清理特定后缀词
|
||
title = re.sub(r'\s*[-–]\s*(Home|首页|主页|官网|官方网站|新闻|News|Press Release).*?$', '', title, flags=re.IGNORECASE)
|
||
if title:
|
||
return title
|
||
|
||
return "未命名文章"
|
||
|
||
|
||
def extract_main_content(html: str) -> str:
|
||
"""从HTML中提取主要内容（简化版）"""
|
||
# 使用 BeautifulSoup 如果可用，更精确可靠
|
||
if BS4_AVAILABLE:
|
||
try:
|
||
soup = BeautifulSoup(html, 'html.parser')
|
||
|
||
# 移除 script 和 style
|
||
for tag in soup(['script', 'style']):
|
||
tag.decompose()
|
||
|
||
# 按优先级尝试不同的内容选择器
|
||
content_selectors = [
|
||
# 人人都是产品经理
|
||
'.article--content.grap',
|
||
# 博客园
|
||
'#cnblogs_post_body',
|
||
# 菜鸟教程
|
||
'.article-intro',
|
||
# 掘金
|
||
'article',
|
||
# 通用
|
||
'article', 'main',
|
||
'.post-content', '.article-content', '.entry-content',
|
||
'#content', '#main-content',
|
||
]
|
||
|
||
for selector in content_selectors:
|
||
elem = soup.select_one(selector)
|
||
if elem:
|
||
# 返回元素的HTML内容
|
||
return str(elem)
|
||
|
||
# 如果都没找到，返回body内容
|
||
body = soup.find('body')
|
||
if body:
|
||
return str(body)
|
||
except Exception as e:
|
||
print(f"BeautifulSoup提取失败，回退到正则: {e}")
|
||
|
||
# 回退到正则表达式方式
|
||
# 移除 script 和 style
|
||
html = re.sub(r'<script[^>]*>.*?</script>', '', html, flags=re.DOTALL | re.IGNORECASE)
|
||
html = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL | re.IGNORECASE)
|
||
|
||
# 尝试找到主要内容区域
|
||
content = ""
|
||
|
||
# 常见的文章容器
|
||
patterns = [
|
||
# 人人都是产品经理 (woshipm.com)
|
||
r'<div[^>]*class=["\']article--content grap["\'][^>]*>(.*?)</div>\s*</div>\s*<div[^>]*class=["\']article--footer',
|
||
r'<div[^>]*class=["\']article--content grap["\'][^>]*>(.*?)</div>\s*</div>\s*<div[^>]*class=["\']article-bottom-meta',
|
||
r'<div[^>]*class=["\']article--content grap["\'][^>]*>(.*?)</div>\s*<div[^>]*class=["\']u-marginTop30',
|
||
# 菜鸟教程
|
||
r'<div[^>]*class=["\']article-intro["\'][^>]*>(.*?)<div[^>]*class=["\'](?:google-auto-placed|pre|next)',
|
||
r'<div[^>]*class=["\']article-intro["\'][^>]*>(.*?)</div>\s*(?:<div|<!--|\Z)',
|
||
r'<div[^>]*class=["\']article-intro["\'][^>]*>(.*?)</div>\s*$',
|
||
r'<div[^>]*class=["\']article-intro["\'][^>]*>(.*)',
|
||
# 博客园
|
||
r'<div[^>]+id=["\']cnblogs_post_body["\'][^>]*>(.*?)</div>\s*<div[^>]+id=["\']blog_post_info_wrap["\']',
|
||
r'<div[^>]+id=["\']cnblogs_post_body["\'][^>]*>(.*?)</div>',
|
||
# 通用
|
||
r'<article[^>]*>(.*?)</article>',
|
||
r'<main[^>]*>(.*?)</main>',
|
||
r'<div[^>]*class=["\'][^"\']*(?:content|article|post|entry)[^"\']*["\'][^>]*>(.*?)</div>',
|
||
r'<div[^>]*id=["\'][^"\']*(?:content|article|post|entry)[^"\']*["\'][^>]*>(.*?)</div>',
|
||
]
|
||
|
||
for pattern in patterns:
|
||
match = re.search(pattern, html, re.DOTALL | re.IGNORECASE)
|
||
if match:
|
||
content = match.group(1)
|
||
break
|
||
|
||
# 如果没找到，尝试提取 body
|
||
if not content:
|
||
body_match = re.search(r'<body[^>]*>(.*?)</body>', html, re.DOTALL | re.IGNORECASE)
|
||
if body_match:
|
||
content = body_match.group(1)
|
||
|
||
return content if content else html
|
||
|
||
|
||
def html_to_markdown(html: str, base_url: str = "") -> str:
|
||
"""HTML转Markdown，保留图片在原始位置"""
|
||
|
||
# 检查base_url是否为HTTPS
|
||
parsed_base = urlparse(base_url) if base_url else None
|
||
base_is_https = parsed_base and parsed_base.scheme == 'https'
|
||
|
||
def make_absolute_url(src: str) -> str:
|
||
"""将相对URL转换为绝对URL"""
|
||
if not src:
|
||
return src
|
||
if src.startswith('//'):
|
||
src = 'https:' + src
|
||
elif src.startswith('/') and base_url:
|
||
src = f"{parsed_base.scheme}://{parsed_base.netloc}{src}"
|
||
elif not src.startswith(('http://', 'https://')) and base_url:
|
||
src = urljoin(base_url, src)
|
||
# 修复混合内容：页面HTTPS时图片也用HTTPS
|
||
if base_is_https and src.startswith('http://'):
|
||
src = 'https://' + src[7:]
|
||
return src
|
||
|
||
if BS4_AVAILABLE:
|
||
# 使用BeautifulSoup更精确地处理
|
||
soup = BeautifulSoup(html, 'html.parser')
|
||
|
||
# 处理所有图片，保留在原位置
|
||
for img in soup.find_all('img'):
|
||
src = img.get('src', '')
|
||
alt = img.get('alt', '')
|
||
if src:
|
||
src = make_absolute_url(src)
|
||
# 创建Markdown图片标记
|
||
md_img = f''
|
||
img.replace_with(md_img)
|
||
|
||
# 处理标题
|
||
for tag in soup.find_all(['h1', 'h2', 'h3', 'h4', 'h5', 'h6']):
|
||
level = int(tag.name[1])
|
||
tag.replace_with(f'\n{"#" * level} {tag.get_text().strip()}\n')
|
||
|
||
# 处理段落
|
||
for p in soup.find_all('p'):
|
||
text = p.get_text()
|
||
# 保留段落中的Markdown图片标记
|
||
for content in p.contents:
|
||
if isinstance(content, str) and content.startswith('!['):
|
||
text = text.replace(content, content)
|
||
p.replace_with(f'\n{text}\n')
|
||
|
||
# 处理换行
|
||
for br in soup.find_all('br'):
|
||
br.replace_with('\n')
|
||
|
||
# 处理代码块 <pre><code>...</code></pre>
|
||
for pre in soup.find_all('pre'):
|
||
code = pre.find('code')
|
||
if code:
|
||
# 获取代码内容，保留原始格式
|
||
code_text = code.get_text()
|
||
# 检测语言（从class属性）
|
||
lang = ''
|
||
code_classes = code.get('class', [])
|
||
for cls in code_classes:
|
||
if 'language-' in cls or 'lang-' in cls:
|
||
lang = cls.replace('language-', '').replace('lang-', '')
|
||
break
|
||
# 创建Markdown代码块
|
||
md_code = f'\n```{lang}\n{code_text}\n```\n'
|
||
pre.replace_with(md_code)
|
||
else:
|
||
# 纯pre标签
|
||
pre_text = pre.get_text()
|
||
md_code = f'\n```\n{pre_text}\n```\n'
|
||
pre.replace_with(md_code)
|
||
|
||
# 处理行内代码 <code>...</code>
|
||
for code in soup.find_all('code'):
|
||
code_text = code.get_text()
|
||
# 检查是否在pre内（已处理过）
|
||
if not code.find_parent('pre'):
|
||
code.replace_with(f'`{code_text}`')
|
||
|
||
# 处理粗体
|
||
for tag in soup.find_all(['strong', 'b']):
|
||
tag.replace_with(f'**{tag.get_text()}**')
|
||
|
||
# 处理斜体
|
||
for tag in soup.find_all(['em', 'i']):
|
||
tag.replace_with(f'*{tag.get_text()}*')
|
||
|
||
# 获取文本
|
||
text = soup.get_text()
|
||
else:
|
||
# 回退到正则方式
|
||
def convert_img_to_markdown(match):
|
||
src = match.group(1)
|
||
alt = match.group(2) if match.lastindex and match.lastindex >= 2 else ""
|
||
src = make_absolute_url(src)
|
||
return f''
|
||
|
||
text = html
|
||
# 先处理图片
|
||
text = re.sub(
|
||
r'<img[^>]+src=["\']([^"\']+)["\'][^>]*alt=["\']([^"\']*)["\'][^>]*>',
|
||
convert_img_to_markdown, text, flags=re.IGNORECASE
|
||
)
|
||
text = re.sub(
|
||
r'<img[^>]+src=["\']([^"\']+)["\'][^>]*>',
|
||
lambda m: f')})',
|
||
text, flags=re.IGNORECASE
|
||
)
|
||
|
||
# 处理其他标签
|
||
text = re.sub(r'<h1[^>]*>(.*?)</h1>', r'\n# \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<h2[^>]*>(.*?)</h2>', r'\n## \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<h3[^>]*>(.*?)</h3>', r'\n### \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<h4[^>]*>(.*?)</h4>', r'\n#### \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<p[^>]*>(.*?)</p>', r'\n\1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<br\s*/?>', '\n', text, flags=re.IGNORECASE)
|
||
# 处理代码块 <pre><code>...</code></pre>
|
||
def convert_code_block(match):
|
||
code_content = match.group(1)
|
||
# 提取语言
|
||
lang_match = re.search(r'class=["\'][^"\']*language-([^"\'\s]+)["\']', match.group(0), re.IGNORECASE)
|
||
lang = lang_match.group(1) if lang_match else ''
|
||
return f'\n```{lang}\n{code_content}\n```\n'
|
||
|
||
text = re.sub(r'<pre[^>]*>\s*<code[^>]*>(.*?)</code>\s*</pre>', convert_code_block,
|
||
text, flags=re.DOTALL | re.IGNORECASE)
|
||
# 处理纯pre标签
|
||
text = re.sub(r'<pre[^>]*>(.*?)</pre>', r'\n```\n\1\n```\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
# 处理行内代码
|
||
text = re.sub(r'<code[^>]*>(.*?)</code>', r'`\1`', text, flags=re.DOTALL | re.IGNORECASE)
|
||
|
||
text = re.sub(r'<strong[^>]*>(.*?)</strong>', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<b[^>]*>(.*?)</b>', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<em[^>]*>(.*?)</em>', r'*\1*', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<i[^>]*>(.*?)</i>', r'*\1*', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<[^>]+>', '', text)
|
||
|
||
# 解码HTML实体
|
||
text = text.replace(' ', ' ')
|
||
text = text.replace('<', '<')
|
||
text = text.replace('>', '>')
|
||
text = text.replace('&', '&')
|
||
text = text.replace('"', '"')
|
||
|
||
# 修复数学公式格式
|
||
text = fix_math_formulas(text)
|
||
|
||
# 清理多余空白行
|
||
lines = text.split('\n')
|
||
cleaned_lines = []
|
||
prev_empty = False
|
||
for line in lines:
|
||
stripped = line.strip()
|
||
if stripped == '':
|
||
if not prev_empty:
|
||
cleaned_lines.append('')
|
||
prev_empty = True
|
||
else:
|
||
cleaned_lines.append(line)
|
||
prev_empty = False
|
||
|
||
return '\n'.join(cleaned_lines).strip()
|
||
|
||
|
||
def extract_images(html: str, base_url: str) -> List[CrawlImage]:
|
||
"""从HTML中提取图片URL"""
|
||
images = []
|
||
parsed_base = urlparse(base_url)
|
||
base_is_https = parsed_base.scheme == 'https'
|
||
|
||
# 匹配 <img src="...">
|
||
img_pattern = r'<img[^>]+src=["\']([^"\']+)["\'][^>]*>'
|
||
for match in re.finditer(img_pattern, html, re.IGNORECASE):
|
||
img_url = match.group(1)
|
||
# 转换为绝对URL
|
||
if img_url.startswith('//'):
|
||
img_url = 'https:' + img_url
|
||
elif img_url.startswith('/'):
|
||
img_url = f"{parsed_base.scheme}://{parsed_base.netloc}{img_url}"
|
||
elif not img_url.startswith(('http://', 'https://')):
|
||
img_url = urljoin(base_url, img_url)
|
||
|
||
# 修复混合内容问题：如果页面是HTTPS，图片也要用HTTPS
|
||
if base_is_https and img_url.startswith('http://'):
|
||
img_url = 'https://' + img_url[7:]
|
||
|
||
images.append(CrawlImage(original_url=img_url))
|
||
|
||
return images
|
||
|
||
|
||
def extract_images_from_markdown(markdown_content: str, base_url: str) -> List[CrawlImage]:
|
||
"""从Markdown内容中提取图片URL（确保只提取实际在文章中的图片）"""
|
||
images = []
|
||
parsed_base = urlparse(base_url)
|
||
base_is_https = parsed_base.scheme == 'https'
|
||
|
||
# 匹配Markdown图片语法: 
|
||
img_pattern = r'!\[([^\]]*)\]\(([^)]+)\)'
|
||
for match in re.finditer(img_pattern, markdown_content):
|
||
img_url = match.group(2)
|
||
# 跳过已经是本地路径的图片
|
||
if img_url.startswith('/uploads/'):
|
||
continue
|
||
# 转换为绝对URL
|
||
if img_url.startswith('//'):
|
||
img_url = 'https:' + img_url
|
||
elif img_url.startswith('/'):
|
||
img_url = f"{parsed_base.scheme}://{parsed_base.netloc}{img_url}"
|
||
elif not img_url.startswith(('http://', 'https://')):
|
||
img_url = urljoin(base_url, img_url)
|
||
# 修复混合内容问题
|
||
if base_is_https and img_url.startswith('http://'):
|
||
img_url = 'https://' + img_url[7:]
|
||
|
||
images.append(CrawlImage(original_url=img_url))
|
||
|
||
return images
|
||
|
||
|
||
def download_image(url: str, temp_dir: str) -> tuple:
|
||
"""下载图片到临时目录"""
|
||
try:
|
||
headers = {
|
||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
|
||
}
|
||
|
||
# 微信图片需要referer
|
||
if 'mmbiz.qpic.cn' in url or 'mmbiz.qlogo.cn' in url:
|
||
headers['Referer'] = 'https://mp.weixin.qq.com/'
|
||
|
||
response = requests.get(url, headers=headers, timeout=30, stream=True)
|
||
response.raise_for_status()
|
||
|
||
# 获取文件扩展名
|
||
content_type = response.headers.get('content-type', '')
|
||
ext = '.jpg'
|
||
if 'png' in content_type:
|
||
ext = '.png'
|
||
elif 'gif' in content_type:
|
||
ext = '.gif'
|
||
elif 'webp' in content_type:
|
||
ext = '.webp'
|
||
else:
|
||
# 从URL获取扩展名
|
||
parsed = urlparse(url)
|
||
path_ext = os.path.splitext(parsed.path)[1]
|
||
if path_ext:
|
||
ext = path_ext
|
||
|
||
# 生成文件名
|
||
filename = f"crawl_{datetime.now().strftime('%Y%m%d_%H%M%S')}_{uuid.uuid4().hex[:8]}{ext}"
|
||
filepath = os.path.join(temp_dir, filename)
|
||
|
||
# 保存文件
|
||
with open(filepath, 'wb') as f:
|
||
for chunk in response.iter_content(chunk_size=8192):
|
||
f.write(chunk)
|
||
|
||
return filename, filepath, True
|
||
except Exception as e:
|
||
print(f"下载图片失败 {url}: {e}")
|
||
return None, None, False
|
||
|
||
|
||
def is_spa_site(html: str) -> bool:
|
||
"""检测是否为SPA网站"""
|
||
# 检测标志
|
||
spa_indicators = [
|
||
# React/Vue/Angular常见标记
|
||
'<div id="root"></div>',
|
||
'<div id="app"></div>',
|
||
'<div id="__next">',
|
||
'<div id="__nuxt">',
|
||
'You need to enable JavaScript',
|
||
# 加载中标记
|
||
'Please wait',
|
||
'Loading...',
|
||
# 常见SPA框架
|
||
'react-root',
|
||
'ng-app',
|
||
'data-reactroot',
|
||
# 动态加载标记
|
||
'window.__INITIAL_STATE__',
|
||
'window.__DATA__',
|
||
]
|
||
html_lower = html.lower()
|
||
for indicator in spa_indicators:
|
||
if indicator.lower() in html_lower:
|
||
return True
|
||
|
||
# 检查内容是否极少（只有JS没有实际内容）
|
||
text_content = re.sub(r'<[^>]+>', '', html)
|
||
text_content = re.sub(r'\s+', '', text_content)
|
||
if len(text_content) < 200:
|
||
return True
|
||
|
||
return False
|
||
|
||
|
||
async def crawl_with_playwright(url: str) -> dict:
|
||
"""使用Playwright异步采集SPA页面"""
|
||
if not PLAYWRIGHT_AVAILABLE:
|
||
raise Exception("Playwright未安装，无法采集SPA网站")
|
||
|
||
# 检测是否为微信文章
|
||
is_wechat = 'mp.weixin.qq.com' in url
|
||
|
||
async with async_playwright() as p:
|
||
browser = await p.chromium.launch(headless=True)
|
||
page = await browser.new_page()
|
||
|
||
# 打开页面
|
||
await page.goto(url, wait_until='domcontentloaded', timeout=30000)
|
||
|
||
# 微信文章特殊处理
|
||
if is_wechat:
|
||
await page.wait_for_timeout(5000) # 微信需要更长等待时间
|
||
|
||
# 提取标题 - 微信文章标题在 #activity_name
|
||
title = ""
|
||
try:
|
||
if await page.locator('#activity_name').count() > 0:
|
||
title = await page.locator('#activity_name').first.inner_text()
|
||
title = title.strip()
|
||
except:
|
||
pass
|
||
if not title:
|
||
title = await page.title()
|
||
|
||
# 提取正文 - 微信文章正文在 #js_content
|
||
content_html = ""
|
||
try:
|
||
if await page.locator('#js_content').count() > 0:
|
||
content_html = await page.locator('#js_content').first.inner_html()
|
||
except:
|
||
pass
|
||
|
||
# 提取图片 - 微信图片用data-src
|
||
images = []
|
||
try:
|
||
img_elements = await page.locator('#js_content img').all()
|
||
for img in img_elements[:10]:
|
||
try:
|
||
src = await img.get_attribute('data-src') or await img.get_attribute('src')
|
||
alt = await img.get_attribute('alt') or ''
|
||
if src and src.startswith('http'):
|
||
images.append(CrawlImage(original_url=src))
|
||
except:
|
||
pass
|
||
except:
|
||
pass
|
||
|
||
await browser.close()
|
||
return {
|
||
'title': title if title else "未命名文章",
|
||
'content_html': content_html,
|
||
'images': images
|
||
}
|
||
|
||
# 普通SPA网站处理
|
||
# 检测平台
|
||
is_juejin = 'juejin.cn' in url
|
||
is_tencent_cloud = 'cloud.tencent.com' in url
|
||
|
||
# 等待内容加载
|
||
if is_juejin:
|
||
# 掘金需要等待article标签
|
||
try:
|
||
await page.wait_for_selector('article', timeout=15000)
|
||
except:
|
||
pass
|
||
await page.wait_for_timeout(3000)
|
||
elif is_tencent_cloud:
|
||
# 腾讯云开发者社区等待markdown内容
|
||
try:
|
||
await page.wait_for_selector('[class*="markdown"]', timeout=15000)
|
||
except:
|
||
pass
|
||
await page.wait_for_timeout(3000)
|
||
else:
|
||
try:
|
||
await page.wait_for_selector('h1, .article-title, .content, article, main', timeout=10000)
|
||
except:
|
||
pass
|
||
await page.wait_for_timeout(5000)
|
||
|
||
# 提取标题
|
||
title = ""
|
||
if is_juejin:
|
||
# 掘金标题选择器
|
||
for selector in ['article h1', 'h1']:
|
||
try:
|
||
if await page.locator(selector).count() > 0:
|
||
title = await page.locator(selector).first.inner_text()
|
||
if title.strip():
|
||
break
|
||
except:
|
||
continue
|
||
elif is_tencent_cloud:
|
||
# 腾讯云开发者社区标题
|
||
for selector in ['h1', '.article-title', '[class*="title"]']:
|
||
try:
|
||
if await page.locator(selector).count() > 0:
|
||
title = await page.locator(selector).first.inner_text()
|
||
if title.strip():
|
||
break
|
||
except:
|
||
continue
|
||
|
||
if not title:
|
||
for selector in ['h1', '.article-title', '[class*="title"]']:
|
||
try:
|
||
if await page.locator(selector).count() > 0:
|
||
title = await page.locator(selector).first.inner_text()
|
||
if title.strip():
|
||
break
|
||
except:
|
||
continue
|
||
if not title:
|
||
title = await page.title()
|
||
|
||
# 提取正文HTML
|
||
content_html = ""
|
||
if is_juejin:
|
||
# 掘金内容在 .markdown-body 中
|
||
try:
|
||
if await page.locator('.markdown-body').count() > 0:
|
||
content_html = await page.locator('.markdown-body').first.inner_html()
|
||
elif await page.locator('article').count() > 0:
|
||
content_html = await page.locator('article').first.inner_html()
|
||
except:
|
||
pass
|
||
|
||
# 清理掘金内容：移除 style 和 script 标签（掘金有大量内嵌CSS）
|
||
if content_html:
|
||
import re
|
||
# 移除 style 标签及其内容
|
||
content_html = re.sub(r'<style[^>]*>.*?</style>', '', content_html, flags=re.DOTALL | re.IGNORECASE)
|
||
# 移除 script 标签及其内容
|
||
content_html = re.sub(r'<script[^>]*>.*?</script>', '', content_html, flags=re.DOTALL | re.IGNORECASE)
|
||
# 移除所有元素的 style 属性
|
||
content_html = re.sub(r'\s+style="[^"]*"', '', content_html, flags=re.IGNORECASE)
|
||
# 移除 class 属性（减少无用属性）
|
||
content_html = re.sub(r'\s+class="[^"]*"', '', content_html, flags=re.IGNORECASE)
|
||
elif is_tencent_cloud:
|
||
# 腾讯云开发者社区内容
|
||
try:
|
||
if await page.locator('.mod-content__markdown').count() > 0:
|
||
content_html = await page.locator('.mod-content__markdown').first.inner_html()
|
||
except:
|
||
pass
|
||
|
||
if not content_html:
|
||
selectors = ['article', '.article-content', '.content', '.post-content',
|
||
'.detail-content', '.news-content', 'main', '[class*="content"]']
|
||
for selector in selectors:
|
||
try:
|
||
if await page.locator(selector).count() > 0:
|
||
content_html = await page.locator(selector).first.inner_html()
|
||
if len(content_html) > 500:
|
||
break
|
||
except:
|
||
continue
|
||
|
||
# 提取图片
|
||
images = []
|
||
parsed_url = urlparse(url)
|
||
url_is_https = parsed_url.scheme == 'https'
|
||
try:
|
||
img_elements = await page.locator('img').all()
|
||
for img in img_elements[:10]:
|
||
try:
|
||
src = await img.get_attribute('src')
|
||
alt = await img.get_attribute('alt') or ''
|
||
if src and not src.startswith('data:') and len(src) > 5:
|
||
# 转为绝对URL
|
||
if src.startswith('//'):
|
||
src = 'https:' + src
|
||
elif src.startswith('/'):
|
||
src = f"{parsed_url.scheme}://{parsed_url.netloc}{src}"
|
||
elif not src.startswith(('http://', 'https://')):
|
||
src = urljoin(url, src)
|
||
# 修复混合内容：页面HTTPS时图片也用HTTPS
|
||
if url_is_https and src.startswith('http://'):
|
||
src = 'https://' + src[7:]
|
||
images.append(CrawlImage(original_url=src))
|
||
except:
|
||
pass
|
||
except:
|
||
pass
|
||
|
||
await browser.close()
|
||
|
||
return {
|
||
'title': title.strip() if title else "未命名文章",
|
||
'content_html': content_html,
|
||
'images': images
|
||
}
|
||
|
||
|
||
def convert_spa_html_to_markdown(html: str, images: List[CrawlImage], base_url: str) -> str:
|
||
"""将SPA采集的HTML转为Markdown"""
|
||
# 先清理 style 和 script 标签
|
||
text = re.sub(r'<style[^>]*>.*?</style>', '', html, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<script[^>]*>.*?</script>', '', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'\s+style="[^"]*"', '', text, flags=re.IGNORECASE)
|
||
text = re.sub(r'\s+class="[^"]*"', '', text, flags=re.IGNORECASE)
|
||
parsed_base = urlparse(base_url)
|
||
base_is_https = parsed_base.scheme == 'https'
|
||
is_wechat = 'mp.weixin.qq.com' in base_url
|
||
|
||
# 首先将img标签转为Markdown
|
||
def img_to_markdown(match):
|
||
src = match.group(1)
|
||
alt = match.group(2) if match.lastindex and match.lastindex >= 2 else ""
|
||
# 处理URL
|
||
if src.startswith('//'):
|
||
src = 'https:' + src
|
||
elif src.startswith('/'):
|
||
src = f"{parsed_base.scheme}://{parsed_base.netloc}{src}"
|
||
# 修复混合内容：页面HTTPS时图片也用HTTPS
|
||
if base_is_https and src.startswith('http://'):
|
||
src = 'https://' + src[7:]
|
||
return f''
|
||
|
||
# 微信文章特殊处理：图片使用data-src
|
||
if is_wechat:
|
||
text = re.sub(
|
||
r'<img[^>]+data-src=["\']([^"\']+)["\'][^>]*>',
|
||
lambda m: f'})',
|
||
text, flags=re.IGNORECASE
|
||
)
|
||
|
||
# 匹配普通img标签
|
||
text = re.sub(
|
||
r'<img[^>]+src=["\']([^"\']+)["\'][^>]*alt=["\']([^"\']*)["\'][^>]*>',
|
||
img_to_markdown, text, flags=re.IGNORECASE
|
||
)
|
||
# 匹配没有alt的img标签 - 需要处理URL转换！
|
||
def img_to_markdown_no_alt(match):
|
||
src = match.group(1)
|
||
# 处理URL（与img_to_markdown保持一致）
|
||
if src.startswith('//'):
|
||
src = 'https:' + src
|
||
elif src.startswith('/'):
|
||
src = f"{parsed_base.scheme}://{parsed_base.netloc}{src}"
|
||
# 修复混合内容
|
||
if base_is_https and src.startswith('http://'):
|
||
src = 'https://' + src[7:]
|
||
return f''
|
||
|
||
text = re.sub(
|
||
r'<img[^>]+src=["\']([^"\']+)["\'][^>]*>',
|
||
img_to_markdown_no_alt,
|
||
text, flags=re.IGNORECASE
|
||
)
|
||
|
||
# 处理代码块 <pre><code>...</code></pre> - 必须在其他标签之前处理
|
||
def convert_code_block(match):
|
||
code_content = match.group(1)
|
||
# 提取语言
|
||
lang_match = re.search(r'class=["\'][^"\']*language-([^"\'\s]+)["\']', match.group(0), re.IGNORECASE)
|
||
lang = lang_match.group(1) if lang_match else ''
|
||
return f'\n```{lang}\n{code_content}\n```\n'
|
||
|
||
text = re.sub(r'<pre[^>]*>\s*<code[^>]*>(.*?)</code>\s*</pre>', convert_code_block,
|
||
text, flags=re.DOTALL | re.IGNORECASE)
|
||
# 处理纯pre标签
|
||
text = re.sub(r'<pre[^>]*>(.*?)</pre>', r'\n```\n\1\n```\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
# 处理行内代码
|
||
text = re.sub(r'<code[^>]*>(.*?)</code>', r'`\1`', text, flags=re.DOTALL | re.IGNORECASE)
|
||
|
||
# 转换其他标签
|
||
text = re.sub(r'<h1[^>]*>(.*?)</h1>', r'\n# \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<h2[^>]*>(.*?)</h2>', r'\n## \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<h3[^>]*>(.*?)</h3>', r'\n### \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<h4[^>]*>(.*?)</h4>', r'\n#### \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<p[^>]*>(.*?)</p>', r'\n\1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<br\s*/?>', '\n', text, flags=re.IGNORECASE)
|
||
text = re.sub(r'<strong[^>]*>(.*?)</strong>', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<b[^>]*>(.*?)</b>', r'**\1**', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<em[^>]*>(.*?)</em>', r'*\1*', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<li[^>]*>(.*?)</li>', r'- \1\n', text, flags=re.DOTALL | re.IGNORECASE)
|
||
text = re.sub(r'<[^>]+>', '', text)
|
||
|
||
# 解码HTML实体
|
||
text = text.replace(' ', ' ').replace('<', '<').replace('>', '>')
|
||
text = text.replace('&', '&').replace('"', '"')
|
||
|
||
# 修复数学公式格式（掘金等平台）
|
||
text = fix_math_formulas(text)
|
||
|
||
# 清理空行
|
||
lines = [line for line in text.split('\n')]
|
||
cleaned = []
|
||
prev_empty = False
|
||
for line in lines:
|
||
stripped = line.strip()
|
||
if stripped == '':
|
||
if not prev_empty:
|
||
cleaned.append('')
|
||
prev_empty = True
|
||
else:
|
||
cleaned.append(line)
|
||
prev_empty = False
|
||
|
||
return '\n'.join(cleaned).strip()
|
||
|
||
|
||
@router.post("")
|
||
async def crawl_article(request: CrawlRequest, db: Session = Depends(get_db), current_admin = Depends(get_current_admin)):
|
||
"""
|
||
采集文章（自动检测SPA并使用Playwright）
|
||
"""
|
||
try:
|
||
# 验证URL
|
||
if not request.url.startswith(('http://', 'https://')):
|
||
raise HTTPException(status_code=400, detail="无效的URL")
|
||
|
||
# 微信文章特殊处理：直接用Playwright（requests会被拦截）
|
||
is_wechat = 'mp.weixin.qq.com' in request.url
|
||
|
||
if is_wechat:
|
||
print(f"检测到微信文章，使用Playwright采集: {request.url}")
|
||
|
||
if not PLAYWRIGHT_AVAILABLE:
|
||
return CrawlResponse(
|
||
success=False,
|
||
error="微信文章需要Playwright才能采集。请联系管理员安装: pip install playwright && playwright install chromium"
|
||
)
|
||
|
||
# 使用Playwright采集微信文章
|
||
spa_result = await crawl_with_playwright(request.url)
|
||
|
||
title = spa_result['title']
|
||
images = spa_result['images']
|
||
content_html = spa_result['content_html']
|
||
|
||
# 微信文章使用专门的转换
|
||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||
|
||
else:
|
||
# 第一步：尝试普通requests采集
|
||
headers = {
|
||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
|
||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||
}
|
||
|
||
response = requests.get(request.url, headers=headers, timeout=30)
|
||
response.raise_for_status()
|
||
|
||
# 智能编码处理：优先使用声明的编码，而不是猜测
|
||
# 1. 先检查HTTP响应头中的编码
|
||
declared_encoding = None
|
||
if response.encoding:
|
||
declared_encoding = response.encoding
|
||
|
||
# 2. 如果响应头没有明确编码，从HTML meta标签中提取
|
||
if not declared_encoding or declared_encoding.lower() in ['iso-8859-1', 'latin-1']:
|
||
# 尝试从HTML中提取charset
|
||
raw_content = response.content
|
||
charset_match = re.search(rb'<meta[^>]+charset=["\']?([^"\'>\s]+)', raw_content, re.IGNORECASE)
|
||
if charset_match:
|
||
declared_encoding = charset_match.group(1).decode('ascii', errors='ignore')
|
||
|
||
# 3. 使用声明的编码，如果没有才用 apparent_encoding 猜测
|
||
if declared_encoding and declared_encoding.lower() not in ['iso-8859-1', 'latin-1']:
|
||
response.encoding = declared_encoding
|
||
else:
|
||
response.encoding = response.apparent_encoding or 'utf-8'
|
||
|
||
html = response.text
|
||
|
||
# 检测是否为SPA
|
||
if is_spa_site(html):
|
||
print(f"检测到SPA网站，使用Playwright采集: {request.url}")
|
||
|
||
if not PLAYWRIGHT_AVAILABLE:
|
||
return CrawlResponse(
|
||
success=False,
|
||
error="该网站为SPA单页应用，需要安装Playwright才能采集。请联系管理员安装: pip install playwright && playwright install chromium"
|
||
)
|
||
|
||
# 使用Playwright采集
|
||
spa_result = await crawl_with_playwright(request.url)
|
||
|
||
title = spa_result['title']
|
||
images = spa_result['images']
|
||
content_html = spa_result['content_html']
|
||
|
||
# 转换HTML为Markdown
|
||
markdown_content = convert_spa_html_to_markdown(content_html, images, request.url)
|
||
|
||
else:
|
||
# 普通网站，使用原有逻辑
|
||
print(f"普通网站，使用requests采集: {request.url}")
|
||
|
||
# 提取标题
|
||
title = extract_title_from_html(html)
|
||
|
||
# 提取主要内容
|
||
main_content = extract_main_content(html)
|
||
|
||
# 转换HTML为Markdown（必须先执行，用于图片提取）
|
||
markdown_content = html_to_markdown(main_content, request.url)
|
||
|
||
# 从Markdown内容中提取图片（确保只提取实际在文章中的图片）
|
||
images = extract_images_from_markdown(markdown_content, request.url)
|
||
|
||
# 创建临时目录
|
||
temp_dir = "/tmp/blog_crawl"
|
||
os.makedirs(temp_dir, exist_ok=True)
|
||
|
||
# uploads 目录配置
|
||
backend_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
upload_dir = os.path.join(backend_dir, "uploads")
|
||
thumb_dir = os.path.join(upload_dir, "thumbnails")
|
||
os.makedirs(upload_dir, exist_ok=True)
|
||
os.makedirs(thumb_dir, exist_ok=True)
|
||
|
||
# 下载图片并保存到数据库（最多5张）
|
||
downloaded_images = []
|
||
for i, img in enumerate(images[:5]):
|
||
filename, filepath, success = download_image(img.original_url, temp_dir)
|
||
if success:
|
||
try:
|
||
# 生成新的文件名（避免冲突）
|
||
ext = os.path.splitext(filename)[1].lower()
|
||
if ext not in ['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg']:
|
||
ext = '.jpg'
|
||
new_filename = f"crawl_{datetime.now().strftime('%Y%m%d')}_{uuid.uuid4().hex[:8]}{ext}"
|
||
|
||
# 移动文件到 uploads 目录
|
||
new_filepath = os.path.join(upload_dir, new_filename)
|
||
with open(filepath, 'rb') as src:
|
||
with open(new_filepath, 'wb') as dst:
|
||
dst.write(src.read())
|
||
|
||
# 删除临时文件
|
||
os.remove(filepath)
|
||
|
||
# 生成缩略图
|
||
thumb_filename = f"thumb_{new_filename}"
|
||
thumb_filepath = os.path.join(thumb_dir, thumb_filename)
|
||
thumb_path_db = None
|
||
if generate_thumbnail(new_filepath, thumb_filepath):
|
||
thumb_path_db = f"/uploads/thumbnails/{thumb_filename}"
|
||
|
||
# 获取图片尺寸
|
||
width, height = get_image_dimensions(new_filepath)
|
||
|
||
# 获取文件大小
|
||
file_size = os.path.getsize(new_filepath)
|
||
|
||
# 保存到数据库 images 表
|
||
image_record = Image(
|
||
filename=new_filename,
|
||
original_name=filename,
|
||
file_path=f"/uploads/{new_filename}",
|
||
thumb_path=thumb_path_db,
|
||
description=f"采集自 {request.url}",
|
||
file_size=file_size,
|
||
width=width,
|
||
height=height
|
||
)
|
||
db.add(image_record)
|
||
db.commit()
|
||
db.refresh(image_record)
|
||
|
||
# 更新返回的图片信息
|
||
img.local_path = f"/uploads/{new_filename}"
|
||
img.filename = new_filename
|
||
img.success = True
|
||
img.id = image_record.id # 保存数据库ID供前端使用
|
||
downloaded_images.append(img)
|
||
|
||
# 将 content 中的原始URL替换为本地路径
|
||
if img.original_url in markdown_content:
|
||
markdown_content = markdown_content.replace(
|
||
img.original_url,
|
||
f"/uploads/{new_filename}"
|
||
)
|
||
print(f"✅ 已将内容中的图片URL替换: {img.original_url[:60]}... -> /uploads/{new_filename}")
|
||
|
||
print(f"图片已保存到数据库: id={image_record.id}, filename={new_filename}")
|
||
|
||
except Exception as e:
|
||
print(f"保存图片到数据库失败: {e}")
|
||
# 如果数据库保存失败，仍然保留临时文件信息
|
||
img.local_path = filepath
|
||
img.filename = filename
|
||
img.success = True
|
||
downloaded_images.append(img)
|
||
|
||
# 生成摘要（前200字符）
|
||
summary = markdown_content[:200].replace('#', '').replace('*', '').strip()
|
||
|
||
return CrawlResponse(
|
||
success=True,
|
||
title=title,
|
||
content=markdown_content,
|
||
summary=summary,
|
||
images=downloaded_images
|
||
)
|
||
|
||
except requests.RequestException as e:
|
||
return CrawlResponse(
|
||
success=False,
|
||
error=f"网络请求失败: {str(e)}"
|
||
)
|
||
except Exception as e:
|
||
import traceback
|
||
print(f"采集失败: {e}")
|
||
print(traceback.format_exc())
|
||
return CrawlResponse(
|
||
success=False,
|
||
error=f"采集失败: {str(e)}"
|
||
)
|