113 lines
3.3 KiB
Python
113 lines
3.3 KiB
Python
"""Shared small utilities for the crawler package."""
|
|
|
|
import csv
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import re
|
|
import sys
|
|
from datetime import datetime, timedelta, timezone
|
|
from pathlib import Path
|
|
|
|
SAFE_CHAR_RE = re.compile(r'[<>:"/\\|?*\x00-\x1f]')
|
|
EM_TAG_RE = re.compile(r"</?em[^>]*>", re.IGNORECASE)
|
|
SPACE_RE = re.compile(r"\s+")
|
|
|
|
|
|
def clean_text(value) -> str:
|
|
if value is None:
|
|
return ""
|
|
if isinstance(value, list):
|
|
return " | ".join(t for item in value if (t := clean_text(item)))
|
|
text = EM_TAG_RE.sub("", str(value))
|
|
return SPACE_RE.sub(" ", text).strip()
|
|
|
|
|
|
def sanitize_filename(name: str, fallback: str = "unnamed") -> str:
|
|
name = SAFE_CHAR_RE.sub("_", name).strip().rstrip(".")
|
|
return name or fallback
|
|
|
|
|
|
def ensure_dir(path: Path) -> Path:
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
return path
|
|
|
|
|
|
def now_iso() -> str:
|
|
return datetime.now(timezone(timedelta(hours=8))).isoformat(timespec="seconds")
|
|
|
|
|
|
def sha256_of_file(path: Path) -> str:
|
|
if not path.exists():
|
|
return ""
|
|
h = hashlib.sha256()
|
|
with path.open("rb") as f:
|
|
for chunk in iter(lambda: f.read(65536), b""):
|
|
h.update(chunk)
|
|
return h.hexdigest()
|
|
|
|
|
|
def write_json(path: Path, payload) -> None:
|
|
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
|
|
|
|
def write_text(path: Path, content: str) -> None:
|
|
path.write_text(content, encoding="utf-8")
|
|
|
|
|
|
def write_jsonl(path: Path, rows: list) -> None:
|
|
with path.open("w", encoding="utf-8") as fh:
|
|
for row in rows:
|
|
fh.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
|
|
|
|
def append_jsonl(path: Path, rows: list) -> None:
|
|
with path.open("a", encoding="utf-8") as fh:
|
|
for row in rows:
|
|
fh.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
|
|
|
|
def read_jsonl(path: Path) -> list:
|
|
if not path.exists():
|
|
return []
|
|
rows, bad = [], 0
|
|
with path.open("r", encoding="utf-8", errors="replace") as fh:
|
|
for line in fh:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
rows.append(json.loads(line))
|
|
except json.JSONDecodeError:
|
|
bad += 1 # tolerate partial lines from interrupted appends
|
|
if bad:
|
|
print(f"[warn] skipped {bad} corrupt line(s) in {path}", file=sys.stderr)
|
|
return rows
|
|
|
|
|
|
def csv_value(v):
|
|
return json.dumps(v, ensure_ascii=False) if isinstance(v, (list, dict)) else v
|
|
|
|
|
|
def write_csv(path: Path, rows: list, columns: list) -> None:
|
|
with path.open("w", newline="", encoding="utf-8-sig") as fh:
|
|
writer = csv.DictWriter(fh, fieldnames=columns, extrasaction="ignore")
|
|
writer.writeheader()
|
|
for row in rows:
|
|
writer.writerow({k: csv_value(row.get(k, "")) for k in columns})
|
|
|
|
|
|
def setup_logger(output_root: Path, name: str) -> logging.Logger:
|
|
ensure_dir(output_root / "logs")
|
|
logger = logging.getLogger(name)
|
|
logger.setLevel(logging.INFO)
|
|
logger.handlers = []
|
|
fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s", "%Y-%m-%d %H:%M:%S")
|
|
sh = logging.StreamHandler(sys.stdout)
|
|
sh.setFormatter(fmt)
|
|
fh = logging.FileHandler(output_root / "logs" / "run.log", encoding="utf-8")
|
|
fh.setFormatter(fmt)
|
|
logger.addHandler(sh)
|
|
logger.addHandler(fh)
|
|
return logger
|