Ubuntu
329 lines
11 KiB
Python
329 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
采集静态页面到本地目录
|
||
用法: python3 crawl_static_page.py <url> <output_dir>
|
||
示例: python3 crawl_static_page.py https://mubaitea.cn/zsjm /var/www/new-media-platform/website-new/static/
|
||
"""
|
||
|
||
import os
|
||
import re
|
||
import sys
|
||
import hashlib
|
||
import requests
|
||
from datetime import datetime
|
||
from urllib.parse import urljoin, urlparse
|
||
from playwright.sync_api import sync_playwright
|
||
from bs4 import BeautifulSoup
|
||
|
||
HEADERS = {
|
||
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36",
|
||
"Accept": "*/*",
|
||
}
|
||
|
||
|
||
def slugify(url):
|
||
"""从 URL 生成文件名"""
|
||
path = urlparse(url).path.strip('/')
|
||
if path:
|
||
name = path.split('/')[-1]
|
||
if '.' in name:
|
||
name = name.rsplit('.', 1)[0]
|
||
return name or 'index'
|
||
return 'index'
|
||
|
||
|
||
def download_file(url, dest_dir, skip_existing=True):
|
||
"""下载文件到指定目录，返回本地相对路径"""
|
||
if not url or url.startswith('data:'):
|
||
return None
|
||
|
||
# 补全协议
|
||
if url.startswith('//'):
|
||
url = 'https:' + url
|
||
|
||
try:
|
||
# 生成文件名
|
||
parsed = urlparse(url)
|
||
path_parts = parsed.path.strip('/').split('/')
|
||
filename = path_parts[-1] if path_parts else 'file'
|
||
|
||
# 如果文件名没有扩展名或太泛，用hash重命名
|
||
ext = os.path.splitext(filename)[1].lower()
|
||
if not ext or len(filename) < 5:
|
||
url_hash = hashlib.md5(url.encode()).hexdigest()[:10]
|
||
filename = f"{url_hash}{ext or '.bin'}"
|
||
|
||
dest_path = os.path.join(dest_dir, filename)
|
||
|
||
if skip_existing and os.path.exists(dest_path):
|
||
return filename
|
||
|
||
resp = requests.get(url, headers=HEADERS, timeout=30)
|
||
if resp.status_code != 200:
|
||
print(f" [下载失败] {url} HTTP {resp.status_code}")
|
||
return None
|
||
|
||
os.makedirs(dest_dir, exist_ok=True)
|
||
with open(dest_path, 'wb') as f:
|
||
f.write(resp.content)
|
||
|
||
print(f" [已下载] {url} -> {filename} ({len(resp.content)} bytes)")
|
||
return filename
|
||
|
||
except Exception as e:
|
||
print(f" [下载失败] {url}: {e}")
|
||
return None
|
||
|
||
|
||
def extract_resource_urls(html_snippet, base_url):
|
||
"""从 HTML 片段中提取所有资源 URL"""
|
||
soup = BeautifulSoup(html_snippet, 'html.parser')
|
||
|
||
images = set()
|
||
scripts = set()
|
||
stylesheets = set()
|
||
|
||
# 图片
|
||
for tag in soup.find_all(['img', 'image']):
|
||
for attr in ['src', '_src', 'data-src', 'data-original']:
|
||
src = tag.get(attr)
|
||
if src and not src.startswith('data:'):
|
||
full = urljoin(base_url, src)
|
||
if any(ext in full.lower() for ext in ['.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg', '.bmp', '.ico']):
|
||
images.add(full)
|
||
|
||
# CSS background-image (inline style)
|
||
for tag in soup.find_all(style=True):
|
||
style = tag['style']
|
||
urls = re.findall(r'url\([\"\']?([^\"\'()]+)[\"\']?\)', style)
|
||
for u in urls:
|
||
if not u.startswith('data:'):
|
||
full = urljoin(base_url, u)
|
||
images.add(full)
|
||
|
||
return {'images': list(images), 'scripts': list(scripts), 'stylesheets': list(stylesheets)}
|
||
|
||
|
||
def crawl_static_page(url, output_dir):
|
||
"""主函数：采集页面到本地目录"""
|
||
js_dir = os.path.join(output_dir, 'js')
|
||
images_dir = os.path.join(output_dir, 'images')
|
||
css_dir = os.path.join(output_dir, 'css')
|
||
|
||
page_name = slugify(url)
|
||
html_path = os.path.join(output_dir, f'{page_name}.html')
|
||
|
||
print(f"目标 URL: {url}")
|
||
print(f"输出目录: {output_dir}")
|
||
print(f"HTML 文件: {html_path}")
|
||
print()
|
||
|
||
with sync_playwright() as p:
|
||
browser = p.chromium.launch(headless=True)
|
||
page = browser.new_page(viewport={'width': 1920, 'height': 1080})
|
||
page.goto(url, wait_until='networkidle', timeout=30000)
|
||
page.wait_for_timeout(5000)
|
||
|
||
html = page.content()
|
||
soup = BeautifulSoup(html, 'html.parser')
|
||
|
||
# === 1. 提取正文内容（ctrl-location="main"） ===
|
||
main_elements = soup.find_all(attrs={'ctrl-location': 'main'})
|
||
print(f"找到 {len(main_elements)} 个正文元素 (ctrl-location=main)")
|
||
|
||
# 取顶层 main 元素（父级不是 main 的）
|
||
top_level_mains = []
|
||
for el in main_elements:
|
||
parent = el.parent
|
||
if parent and parent.get('ctrl-location') != 'main':
|
||
top_level_mains.append(el)
|
||
|
||
print(f"其中顶层元素: {len(top_level_mains)} 个")
|
||
|
||
# 收集 main 内容的 HTML
|
||
main_html_parts = []
|
||
for el in top_level_mains:
|
||
main_html_parts.append(str(el))
|
||
|
||
main_content_html = '\n'.join(main_html_parts)
|
||
|
||
# === 2. 提取页面 CSS 引用 ===
|
||
css_links = []
|
||
for link in soup.find_all('link', rel='stylesheet'):
|
||
href = link.get('href', '')
|
||
if href:
|
||
css_links.append(href)
|
||
|
||
# 页面内联 style
|
||
inline_styles = []
|
||
for style_tag in soup.find_all('style'):
|
||
inline_styles.append(str(style_tag))
|
||
|
||
# === 3. 提取页面 JS 引用 ===
|
||
script_srcs = []
|
||
for script in soup.find_all('script', src=True):
|
||
src = script['src']
|
||
if src:
|
||
script_srcs.append(src)
|
||
|
||
# === 4. 提取 SVG symbol 图标（通常在 body 开头） ===
|
||
svg_sprite = ''
|
||
body = soup.find('body', attrs={'area': 'main'})
|
||
if body:
|
||
svg = body.find('svg', recursive=False)
|
||
if svg:
|
||
svg_sprite = str(svg)
|
||
print("找到 SVG 图标精灵")
|
||
|
||
# === 5. 提取 head 中的 meta/viewport ===
|
||
head_meta = ''
|
||
head = soup.find('head')
|
||
if head:
|
||
meta_tags = head.find_all('meta')
|
||
for meta in meta_tags:
|
||
head_meta += str(meta) + '\n'
|
||
# 也保留 title
|
||
title_tag = head.find('title')
|
||
if title_tag:
|
||
head_meta += str(title_tag) + '\n'
|
||
|
||
browser.close()
|
||
|
||
# === 6. 下载图片 ===
|
||
print("\n=== 下载图片 ===")
|
||
resources = extract_resource_urls(main_content_html, url)
|
||
image_map = {} # original_url -> local_filename
|
||
|
||
for img_url in resources['images']:
|
||
local_name = download_file(img_url, images_dir)
|
||
if local_name:
|
||
image_map[img_url] = f'images/{local_name}'
|
||
|
||
# 也下载 main 内容中 img 标签的图片（更可靠的方式）
|
||
main_soup = BeautifulSoup(main_content_html, 'html.parser')
|
||
for img in main_soup.find_all('img'):
|
||
for attr in ['src', '_src', 'data-src', 'data-original']:
|
||
src = img.get(attr)
|
||
if src and not src.startswith('data:'):
|
||
full = urljoin(url, src)
|
||
if full not in image_map:
|
||
local_name = download_file(full, images_dir)
|
||
if local_name:
|
||
image_map[full] = f'images/{local_name}'
|
||
|
||
# === 7. 下载 JS 文件 ===
|
||
print("\n=== 下载 JS 文件 ===")
|
||
js_map = {}
|
||
for js_url in script_srcs:
|
||
local_name = download_file(js_url, js_dir)
|
||
if local_name:
|
||
js_map[js_url] = f'js/{local_name}'
|
||
|
||
# === 8. 下载 CSS 文件 ===
|
||
print("\n=== 下载 CSS 文件 ===")
|
||
css_map = {}
|
||
css_contents = []
|
||
for css_url in css_links:
|
||
local_name = download_file(css_url, css_dir)
|
||
if local_name:
|
||
css_map[css_url] = f'css/{local_name}'
|
||
# 读取 CSS 内容以便内联 background-image 中的 URL
|
||
css_path = os.path.join(css_dir, local_name)
|
||
try:
|
||
with open(css_path, 'r', encoding='utf-8', errors='replace') as f:
|
||
css_contents.append(f.read())
|
||
except:
|
||
pass
|
||
|
||
# === 9. 替换 HTML 中的资源路径 ===
|
||
print("\n=== 替换资源路径 ===")
|
||
|
||
# 替换图片
|
||
for orig_url, local_path in image_map.items():
|
||
# 处理各种 URL 格式
|
||
main_content_html = main_content_html.replace(orig_url, local_path)
|
||
# 处理 protocol-relative
|
||
if orig_url.startswith('https://'):
|
||
proto_rel = orig_url.replace('https:', '')
|
||
main_content_html = main_content_html.replace(proto_rel, local_path)
|
||
|
||
# 替换 JS
|
||
for orig_url, local_path in js_map.items():
|
||
main_content_html = main_content_html.replace(orig_url, local_path)
|
||
if orig_url.startswith('https://'):
|
||
proto_rel = orig_url.replace('https:', '')
|
||
main_content_html = main_content_html.replace(proto_rel, local_path)
|
||
|
||
# 替换 CSS
|
||
for orig_url, local_path in css_map.items():
|
||
main_content_html = main_content_html.replace(orig_url, local_path)
|
||
if orig_url.startswith('https://'):
|
||
proto_rel = orig_url.replace('https:', '')
|
||
main_content_html = main_content_html.replace(proto_rel, local_path)
|
||
|
||
# 构建 CSS link 标签（使用本地路径）
|
||
local_css_links = ''
|
||
for orig_url, local_path in css_map.items():
|
||
local_css_links += f' <link rel="stylesheet" href="{local_path}">\n'
|
||
|
||
# 构建 JS script 标签（使用本地路径）
|
||
local_js_scripts = ''
|
||
for orig_url, local_path in js_map.items():
|
||
local_js_scripts += f' <script src="{local_path}"></script>\n'
|
||
|
||
# === 10. 构建完整 HTML ===
|
||
full_html = f'''<!DOCTYPE html>
|
||
<html lang="zh-CN">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="viewport" content="width=1200, initial-scale=1.0">
|
||
{head_meta}
|
||
{local_css_links}
|
||
{chr(10).join(f' <style>{s}</style>' for s in inline_styles if s.strip())}
|
||
<style>
|
||
/* 确保主内容区域正确显示 */
|
||
body {{
|
||
margin: 0;
|
||
padding: 0;
|
||
background: #fff;
|
||
font-family: "Microsoft YaHei", "微软雅黑", Arial, sans-serif;
|
||
}}
|
||
.main-content-wrapper {{
|
||
position: relative;
|
||
width: 1200px;
|
||
margin: 0 auto;
|
||
min-height: 100vh;
|
||
}}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
{svg_sprite}
|
||
<div class="main-content-wrapper">
|
||
{main_content_html}
|
||
</div>
|
||
{local_js_scripts}
|
||
</body>
|
||
</html>'''
|
||
|
||
# 写入文件
|
||
with open(html_path, 'w', encoding='utf-8') as f:
|
||
f.write(full_html)
|
||
|
||
print(f"\n=== 完成 ===")
|
||
print(f"HTML: {html_path}")
|
||
print(f"图片: {len(image_map)} 个 -> {images_dir}")
|
||
print(f"JS: {len(js_map)} 个 -> {js_dir}")
|
||
print(f"CSS: {len(css_map)} 个 -> {css_dir}")
|
||
print(f"文件大小: {len(full_html)} 字节")
|
||
|
||
|
||
if __name__ == '__main__':
|
||
if len(sys.argv) < 3:
|
||
print("用法: python3 crawl_static_page.py <url> <output_dir>")
|
||
print("示例: python3 crawl_static_page.py https://mubaitea.cn/zsjm /var/www/new-media-platform/website-new/static/")
|
||
sys.exit(1)
|
||
|
||
target_url = sys.argv[1]
|
||
output_dir = sys.argv[2].rstrip('/')
|
||
crawl_static_page(target_url, output_dir)
|