Ubuntu
550 lines
19 KiB
Python
550 lines
19 KiB
Python
"""Xiaohongshu API client with proper signing for both Main and Creator APIs."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import mimetypes
|
|
import random
|
|
import time
|
|
from typing import Any
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
|
|
from .signing import MainApiSigner, sign_creator, XHSHOW_AVAILABLE, CRYPTO_AVAILABLE
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# API constants
|
|
USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
|
|
CHROME_VERSION = "135"
|
|
HOME_URL = "https://www.xiaohongshu.com"
|
|
EDITH_HOST = "https://edith.xiaohongshu.com"
|
|
CREATOR_HOST = "https://creator.xiaohongshu.com"
|
|
UPLOAD_HOST = "https://ros-upload.xiaohongshu.com"
|
|
|
|
|
|
class XhsCookieClient:
|
|
"""Xiaohongshu API client with proper signing."""
|
|
|
|
def __init__(
|
|
self,
|
|
cookie_file: str | Path = "/home/ubuntu/xiaohongshu_cookies.txt",
|
|
timeout: float = 30.0,
|
|
request_delay: float = 1.0,
|
|
max_retries: int = 3,
|
|
):
|
|
self.cookie_file = Path(cookie_file)
|
|
self.cookies = self._load_cookies()
|
|
self._http = httpx.Client(timeout=timeout, follow_redirects=True)
|
|
self._request_delay = request_delay
|
|
self._base_request_delay = request_delay
|
|
self._max_retries = max_retries
|
|
self._last_request_time = 0.0
|
|
self._request_count = 0
|
|
self._verify_count = 0
|
|
|
|
# Initialize main API signer
|
|
if XHSHOW_AVAILABLE:
|
|
try:
|
|
self._main_signer = MainApiSigner()
|
|
logger.info("Using xhshow library for main API signing")
|
|
except Exception as e:
|
|
logger.warning("Failed to initialize xhshow: %s", e)
|
|
self._main_signer = None
|
|
else:
|
|
logger.warning("xhshow not available, main API will fail")
|
|
self._main_signer = None
|
|
|
|
# Check creator signing
|
|
if not CRYPTO_AVAILABLE:
|
|
logger.warning("pycryptodome not available, creator API will fail")
|
|
|
|
def _load_cookies(self) -> dict[str, str]:
|
|
"""Load cookies from Chrome extension export format."""
|
|
if not self.cookie_file.exists():
|
|
raise FileNotFoundError(f"Cookie file not found: {self.cookie_file}")
|
|
|
|
data = json.loads(self.cookie_file.read_text())
|
|
|
|
# Chrome extension format: list of cookie objects
|
|
if isinstance(data, list):
|
|
cookies = {}
|
|
for item in data:
|
|
if isinstance(item, dict) and "name" in item and "value" in item:
|
|
cookies[item["name"]] = item["value"]
|
|
logger.info("Loaded %d cookies from %s", len(cookies), self.cookie_file)
|
|
return cookies
|
|
|
|
# Simple dict format
|
|
if isinstance(data, dict):
|
|
return data
|
|
|
|
raise ValueError(f"Unknown cookie format in {self.cookie_file}")
|
|
|
|
def _cookies_to_string(self) -> str:
|
|
"""Convert cookies dict to header string."""
|
|
return "; ".join(f"{k}={v}" for k, v in self.cookies.items())
|
|
|
|
def _rate_limit_delay(self) -> None:
|
|
"""Rate limiting with jitter."""
|
|
if self._request_delay <= 0:
|
|
return
|
|
elapsed = time.time() - self._last_request_time
|
|
if elapsed < self._request_delay:
|
|
jitter = max(0, random.gauss(0.3, 0.15))
|
|
if random.random() < 0.05:
|
|
jitter += random.uniform(2.0, 5.0)
|
|
sleep_time = self._request_delay - elapsed + jitter
|
|
logger.debug("Rate-limit delay: %.2fs", sleep_time)
|
|
time.sleep(sleep_time)
|
|
|
|
def _mark_request(self) -> None:
|
|
self._last_request_time = time.time()
|
|
self._request_count += 1
|
|
|
|
def _base_headers(self) -> dict[str, str]:
|
|
"""Build base request headers."""
|
|
return {
|
|
"user-agent": USER_AGENT,
|
|
"content-type": "application/json;charset=UTF-8",
|
|
"cookie": self._cookies_to_string(),
|
|
"origin": HOME_URL,
|
|
"referer": f"{HOME_URL}/",
|
|
"sec-ch-ua": f'"Google Chrome";v="{CHROME_VERSION}", "Chromium";v="{CHROME_VERSION}"',
|
|
"sec-ch-ua-mobile": "?0",
|
|
"sec-ch-ua-platform": '"macOS"',
|
|
"sec-fetch-dest": "empty",
|
|
"sec-fetch-mode": "cors",
|
|
"sec-fetch-site": "same-site",
|
|
"accept": "application/json, text/plain, */*",
|
|
"accept-language": "zh-CN,zh;q=0.9,en;q=0.8",
|
|
"dnt": "1",
|
|
"priority": "u=1, i",
|
|
}
|
|
|
|
def _merge_response_cookies(self, resp: httpx.Response) -> None:
|
|
"""Persist response cookies back into session."""
|
|
for name, value in resp.cookies.items():
|
|
if value:
|
|
self.cookies[name] = value
|
|
|
|
def _request_with_retry(self, method: str, url: str, **kwargs) -> httpx.Response:
|
|
"""Make request with retry logic."""
|
|
self._rate_limit_delay()
|
|
last_exc: Exception | None = None
|
|
|
|
for attempt in range(self._max_retries):
|
|
try:
|
|
resp = self._http.request(method, url, **kwargs)
|
|
self._merge_response_cookies(resp)
|
|
self._mark_request()
|
|
|
|
# Handle rate limiting
|
|
if resp.status_code in (429, 500, 502, 503, 504):
|
|
wait = (2 ** attempt) + random.uniform(0, 1)
|
|
logger.warning(
|
|
"HTTP %d from %s, retrying in %.1fs (attempt %d/%d)",
|
|
resp.status_code, url[:80], wait, attempt + 1, self._max_retries
|
|
)
|
|
time.sleep(wait)
|
|
continue
|
|
|
|
return resp
|
|
|
|
except (httpx.TimeoutException, httpx.NetworkError) as exc:
|
|
last_exc = exc
|
|
wait = (2 ** attempt) + random.uniform(0, 1)
|
|
logger.warning(
|
|
"Network error: %s, retrying in %.1fs (attempt %d/%d)",
|
|
exc, wait, attempt + 1, self._max_retries
|
|
)
|
|
time.sleep(wait)
|
|
|
|
if last_exc:
|
|
raise Exception(f"Request failed after {self._max_retries} retries: {last_exc}")
|
|
raise Exception(f"Request failed after {self._max_retries} retries")
|
|
|
|
def _handle_response(self, resp: httpx.Response) -> Any:
|
|
"""Parse and validate API response."""
|
|
# Handle verification/captcha
|
|
if resp.status_code in (461, 471):
|
|
self._verify_count += 1
|
|
cooldown = min(30, 5 * (2 ** (self._verify_count - 1)))
|
|
logger.warning("Captcha triggered, cooling down %.0fs", cooldown)
|
|
self._request_delay = max(self._request_delay, self._base_request_delay * 2)
|
|
time.sleep(cooldown)
|
|
raise Exception(f"Verification required (status {resp.status_code})")
|
|
|
|
self._verify_count = 0
|
|
text = resp.text
|
|
if not text:
|
|
return None
|
|
|
|
try:
|
|
data = json.loads(text)
|
|
except json.JSONDecodeError:
|
|
raise Exception(f"Non-JSON response: {text[:200]}")
|
|
|
|
if data.get("success"):
|
|
return data.get("data", data)
|
|
|
|
code = data.get("code")
|
|
if code == 300012:
|
|
raise Exception("IP blocked")
|
|
if code == 300015:
|
|
raise Exception(f"Signature error: {data.get('msg', 'unknown')}")
|
|
if code == 300011:
|
|
raise Exception(f"Account error: {data.get('msg', 'account exception')}")
|
|
if code == -100:
|
|
raise Exception("Session expired - need fresh cookies")
|
|
|
|
raise Exception(f"API error: {json.dumps(data)[:300]}")
|
|
|
|
# ============ Main API Methods ============
|
|
|
|
def _main_api_get(self, uri: str, params: dict[str, Any] | None = None) -> Any:
|
|
"""Make signed GET request to Main API."""
|
|
if not self._main_signer:
|
|
raise Exception("Main API signer not available")
|
|
|
|
sign_headers = self._main_signer.sign_get(uri, self.cookies, params=params)
|
|
full_uri = self._main_signer.build_uri(uri, params)
|
|
url = f"{EDITH_HOST}{full_uri}"
|
|
|
|
resp = self._request_with_retry("GET", url, headers={**self._base_headers(), **sign_headers})
|
|
return self._handle_response(resp)
|
|
|
|
def _main_api_post(
|
|
self,
|
|
uri: str,
|
|
data: dict[str, Any],
|
|
header_overrides: dict[str, str] | None = None,
|
|
) -> Any:
|
|
"""Make signed POST request to Main API."""
|
|
if not self._main_signer:
|
|
raise Exception("Main API signer not available")
|
|
|
|
sign_headers = self._main_signer.sign_post(uri, self.cookies, payload=data)
|
|
url = f"{EDITH_HOST}{uri}"
|
|
|
|
headers = {**self._base_headers(), **sign_headers}
|
|
if header_overrides:
|
|
headers.update(header_overrides)
|
|
|
|
body = json.dumps(data, separators=(",", ":"), ensure_ascii=False)
|
|
resp = self._request_with_retry("POST", url, headers=headers, content=body)
|
|
return self._handle_response(resp)
|
|
|
|
# ============ Creator API Methods ============
|
|
|
|
def _creator_host(self, uri: str) -> str:
|
|
"""Determine host for creator API."""
|
|
return CREATOR_HOST if uri.startswith("/api/galaxy/") else EDITH_HOST
|
|
|
|
def _creator_get(self, uri: str, params: dict[str, Any] | None = None) -> Any:
|
|
"""Make signed GET request to Creator API."""
|
|
if not CRYPTO_AVAILABLE:
|
|
raise Exception("pycryptodome not available for creator signing")
|
|
|
|
if "a1" not in self.cookies:
|
|
raise Exception("a1 cookie required for creator API")
|
|
|
|
# Build full URI with params
|
|
if params:
|
|
query = "&".join(f"{k}={v}" for k, v in params.items())
|
|
full_uri = f"{uri}?{query}"
|
|
else:
|
|
full_uri = uri
|
|
|
|
sign = sign_creator(f"url={full_uri}", None, self.cookies["a1"])
|
|
host = self._creator_host(uri)
|
|
url = f"{host}{full_uri}"
|
|
|
|
headers = {
|
|
**self._base_headers(),
|
|
"x-s": sign["x-s"],
|
|
"x-t": sign["x-t"],
|
|
"origin": CREATOR_HOST,
|
|
"referer": f"{CREATOR_HOST}/",
|
|
}
|
|
|
|
resp = self._request_with_retry("GET", url, headers=headers)
|
|
return self._handle_response(resp)
|
|
|
|
def _creator_post(self, uri: str, data: dict[str, Any]) -> Any:
|
|
"""Make signed POST request to Creator API."""
|
|
if not CRYPTO_AVAILABLE:
|
|
raise Exception("pycryptodome not available for creator signing")
|
|
|
|
if "a1" not in self.cookies:
|
|
raise Exception("a1 cookie required for creator API")
|
|
|
|
sign = sign_creator(f"url={uri}", data, self.cookies["a1"])
|
|
host = self._creator_host(uri)
|
|
url = f"{host}{uri}"
|
|
|
|
headers = {
|
|
**self._base_headers(),
|
|
"x-s": sign["x-s"],
|
|
"x-t": sign["x-t"],
|
|
"origin": CREATOR_HOST,
|
|
"referer": f"{CREATOR_HOST}/",
|
|
}
|
|
|
|
body = json.dumps(data, separators=(",", ":"), ensure_ascii=False)
|
|
resp = self._request_with_retry("POST", url, headers=headers, content=body)
|
|
return self._handle_response(resp)
|
|
|
|
def close(self) -> None:
|
|
self._http.close()
|
|
|
|
def __enter__(self):
|
|
return self
|
|
|
|
def __exit__(self, *args):
|
|
self.close()
|
|
|
|
# ============ API Methods ============
|
|
|
|
def search_notes(
|
|
self,
|
|
keyword: str,
|
|
page: int = 1,
|
|
page_size: int = 20,
|
|
sort: str = "general",
|
|
) -> list[dict]:
|
|
"""Search notes by keyword."""
|
|
search_id = self._generate_search_id()
|
|
|
|
result = self._main_api_post("/api/sns/web/v1/search/notes", {
|
|
"keyword": keyword,
|
|
"page": page,
|
|
"page_size": page_size,
|
|
"search_id": search_id,
|
|
"sort": sort,
|
|
"note_type": 0,
|
|
})
|
|
|
|
items = result.get("items", [])
|
|
return [
|
|
{
|
|
"id": item.get("id"),
|
|
"title": item.get("note_card", {}).get("title", ""),
|
|
"desc": item.get("note_card", {}).get("desc", "")[:100],
|
|
"likes": item.get("note_card", {}).get("interact_info", {}).get("liked_count", 0),
|
|
}
|
|
for item in items
|
|
if item.get("model_type") == "note"
|
|
]
|
|
|
|
def get_note_detail(self, note_id: str) -> dict[str, Any]:
|
|
"""Get note detail by ID."""
|
|
return self._main_api_post("/api/sns/web/v1/feed", {
|
|
"source_note_id": note_id,
|
|
"image_formats": ["jpg", "webp", "avif"],
|
|
"extra": {"need_body_topic": 1},
|
|
})
|
|
|
|
def get_feed(self, num: int = 20, category: str = "homefeed.recommend") -> list[dict]:
|
|
"""Get home feed."""
|
|
result = self._main_api_post("/api/sns/web/v1/homefeed", {
|
|
"cursor_score": "",
|
|
"num": num,
|
|
"refresh_type": 1,
|
|
"note_index": 0,
|
|
"unread_begin_note_id": "",
|
|
"unread_end_note_id": "",
|
|
"unread_note_count": 0,
|
|
"category": category,
|
|
})
|
|
|
|
items = result.get("items", [])
|
|
return [
|
|
{
|
|
"id": item.get("id"),
|
|
"title": item.get("note_card", {}).get("title", ""),
|
|
"author": item.get("note_card", {}).get("user", {}).get("nickname", ""),
|
|
"likes": item.get("note_card", {}).get("interact_info", {}).get("liked_count", 0),
|
|
}
|
|
for item in items
|
|
if item.get("model_type") == "note"
|
|
]
|
|
|
|
def like_note(self, note_id: str) -> bool:
|
|
"""Like a note."""
|
|
result = self._main_api_post("/api/sns/web/v1/note/like", {"note_id": note_id})
|
|
return result.get("success", False)
|
|
|
|
def unlike_note(self, note_id: str) -> bool:
|
|
"""Unlike a note."""
|
|
result = self._main_api_post("/api/sns/web/v1/note/unlike", {"note_id": note_id})
|
|
return result.get("success", False)
|
|
|
|
def get_user_info(self, user_id: str) -> dict[str, Any]:
|
|
"""Get user info by ID."""
|
|
return self._main_api_get("/api/sns/web/v1/user/otherinfo", {"target_user_id": user_id})
|
|
|
|
def get_user_notes(self, user_id: str, num: int = 20) -> list[dict]:
|
|
"""Get notes by user ID."""
|
|
result = self._main_api_post("/api/sns/web/v1/user_posted", {
|
|
"user_id": user_id,
|
|
"num": num,
|
|
"cursor": "",
|
|
})
|
|
|
|
notes = result.get("notes", [])
|
|
return [
|
|
{
|
|
"id": note.get("note_id"),
|
|
"title": note.get("title", ""),
|
|
"likes": note.get("interact_info", {}).get("liked_count", 0),
|
|
"time": note.get("time"),
|
|
}
|
|
for note in notes
|
|
]
|
|
|
|
def get_comments(self, note_id: str, num: int = 20) -> list[dict]:
|
|
"""Get comments for a note."""
|
|
result = self._main_api_post("/api/sns/web/v2/comment/page", {
|
|
"note_id": note_id,
|
|
"cursor": "",
|
|
"top_comment_id": "",
|
|
"image_formats": ["jpg", "webp", "avif"],
|
|
"num": num,
|
|
})
|
|
|
|
return result.get("comments", [])
|
|
|
|
# ============ Creator API Methods ============
|
|
|
|
def search_topics(self, keyword: str) -> dict[str, Any]:
|
|
"""Search topics by keyword."""
|
|
return self._creator_post("/web_api/sns/v1/search/topic", {
|
|
"keyword": keyword,
|
|
"suggest_topic_request": {"title": "", "desc": ""},
|
|
"page": {"page_size": 20, "page": 1},
|
|
})
|
|
|
|
def get_upload_permit(self, file_type: str = "image", count: int = 1) -> dict[str, str]:
|
|
"""Get upload permit for images/videos."""
|
|
result = self._creator_get("/api/media/v1/upload/web/permit", {
|
|
"biz_name": "spectrum",
|
|
"scene": file_type,
|
|
"file_count": count,
|
|
"version": 1,
|
|
"source": "web",
|
|
})
|
|
|
|
permit = result.get("uploadTempPermits", [{}])[0]
|
|
return {
|
|
"fileId": permit.get("fileIds", [""])[0],
|
|
"token": permit.get("token", ""),
|
|
"uploadUrl": permit.get("uploadUrl", ""),
|
|
}
|
|
|
|
def upload_file(
|
|
self,
|
|
file_id: str,
|
|
token: str,
|
|
file_path: str,
|
|
content_type: str | None = None,
|
|
) -> None:
|
|
"""Upload a file to cloud storage."""
|
|
url = f"{UPLOAD_HOST}/{file_id}"
|
|
content_type = content_type or mimetypes.guess_type(file_path)[0] or "application/octet-stream"
|
|
|
|
with open(file_path, "rb") as f:
|
|
resp = self._http.put(
|
|
url,
|
|
content=f.read(),
|
|
headers={
|
|
"X-Cos-Security-Token": token,
|
|
"Content-Type": content_type,
|
|
},
|
|
)
|
|
resp.raise_for_status()
|
|
|
|
def create_image_note(
|
|
self,
|
|
title: str,
|
|
desc: str,
|
|
image_file_ids: list[str],
|
|
topics: list[dict[str, str]] | None = None,
|
|
is_private: bool = False,
|
|
) -> dict:
|
|
"""Create and publish an image note."""
|
|
images = [{"file_id": fid, "metadata": {"source": -1}} for fid in image_file_ids]
|
|
business_binds = {
|
|
"version": 1,
|
|
"noteId": 0,
|
|
"noteOrderBind": {},
|
|
"notePostTiming": {"postTime": None},
|
|
"noteCollectionBind": {"id": ""},
|
|
}
|
|
|
|
payload = {
|
|
"common": {
|
|
"type": "normal",
|
|
"title": title,
|
|
"note_id": "",
|
|
"desc": desc,
|
|
"source": '{"type":"web","ids":"","extraInfo":"{\\"subType\\":\\"official\\"}"}',
|
|
"business_binds": json.dumps(business_binds),
|
|
"ats": [],
|
|
"hash_tag": topics or [],
|
|
"post_loc": {},
|
|
"privacy_info": {"op_type": 1, "type": 1 if is_private else 0},
|
|
},
|
|
"image_info": {"images": images},
|
|
"video_info": None,
|
|
}
|
|
|
|
result = self._main_api_post(
|
|
"/web_api/sns/v2/note",
|
|
payload,
|
|
header_overrides={
|
|
"origin": CREATOR_HOST,
|
|
"referer": f"{CREATOR_HOST}/",
|
|
},
|
|
)
|
|
return {"success": True, "data": result}
|
|
|
|
def get_creator_note_list(self, tab: int = 0, page: int = 0) -> dict[str, Any]:
|
|
"""Get my published notes from creator platform."""
|
|
return self._creator_get("/api/galaxy/v2/creator/note/user/posted", {
|
|
"tab": tab,
|
|
"page": page,
|
|
})
|
|
|
|
def get_my_notes(self, num: int = 20) -> list[dict]:
|
|
"""Get my published notes."""
|
|
result = self.get_creator_note_list(tab=0, page=0)
|
|
notes = result.get("notes", [])
|
|
return [
|
|
{
|
|
"id": note.get("note_id"),
|
|
"title": note.get("title", ""),
|
|
"likes": note.get("interact_info", {}).get("liked_count", 0),
|
|
"time": note.get("time"),
|
|
}
|
|
for note in notes[:num]
|
|
]
|
|
|
|
@staticmethod
|
|
def _generate_search_id() -> str:
|
|
"""Generate a unique search ID (base36 of timestamp << 64 + random)."""
|
|
e = int(time.time() * 1000) << 64
|
|
t = random.randint(0, 2147483646)
|
|
num = e + t
|
|
|
|
alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
|
if num == 0:
|
|
return "0"
|
|
result = ""
|
|
while num > 0:
|
|
result = alphabet[num % 36] + result
|
|
num //= 36
|
|
return result
|