# -*- coding: utf-8 -*-
"""
下載青空文庫 ruby 文字檔 → 轉 UTF-8 → 依難度分 5 級 → 存到 grade 資料夾。
並產出 catalog.csv 索引（書名/作者/字數/級別/URL）。
"""
import csv, os, re, io, zipfile, time, sys, threading
from concurrent.futures import ThreadPoolExecutor, as_completed
from collections import Counter

BASE = "."
DL = os.path.join(BASE, "_download")
GRADE = os.path.join(BASE, "graded")
MANIFEST = os.path.join(BASE, "manifest.csv")
CATALOG = os.path.join(BASE, "catalog.csv")

os.makedirs(DL, exist_ok=True)
for g in range(1, 6):
    os.makedirs(os.path.join(GRADE, f"grade{g}"), exist_ok=True)

rows = list(csv.reader(open(MANIFEST, encoding="utf-8")))
H = rows[0]; idx = {n: i for i, n in enumerate(H)}
books = rows[1:]
print("總候選書數:", len(books))

# 取前 1000 本
books = books[:1000]
print("下載目標:", len(books))

# 清空舊的 grade 資料夾內容
for g in range(1, 6):
    d = os.path.join(GRADE, f"grade{g}")
    for f in os.listdir(d):
        os.remove(os.path.join(d, f))

def clean_zip_name(txt_name):
    # zip 內 txt 可能含怪字元，直接固定輸出檔名
    return None

def download_one(book):
    wid = book[idx["作品ID"]]
    title = book[idx["作品名"]].strip()
    author = (book[idx["姓"]].strip() + " " + book[idx["名"]].strip()).strip()
    url = book[idx["テキストファイルURL"]].strip()
    kana = book[idx["文字遣い種別"]].strip()
    cat = book[idx["分類番号"]].strip()
    is_child = "K913" in cat
    zpath = os.path.join(DL, f"{wid}.zip")
    try:
        import urllib.request
        if not os.path.exists(zpath):
            req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0"})
            with urllib.request.urlopen(req, timeout=60) as resp:
                data = resp.read()
            with open(zpath, "wb") as f:
                f.write(data)
        # 解壓
        with zipfile.ZipFile(zpath) as z:
            names = [n for n in z.namelist() if n.endswith(".txt")]
            if not names:
                return (wid, title, author, None, "no-txt")
            txt = z.read(names[0]).decode("shift_jis", errors="replace")
        # 清理：去注記區、去 ruby/外字標記，算正文字數
        n_chars = count_body(txt)
        # 分級
        tier = classify(n_chars, is_child, kana)
        # 寫出乾淨 UTF-8 檔
        out_name = f"{wid}_{author}_{title}.txt"
        out_name = sanitize(out_name)
        with open(os.path.join(GRADE, f"grade{tier}", out_name), "w", encoding="utf-8") as f:
            f.write(txt)
        return (wid, title, author, tier, n_chars)
    except Exception as e:
        return (wid, title, author, None, f"err:{e}")

def sanitize(s):
    s = re.sub(r'[\/:*?"<>|]', "_", s)
    return s[:150]

def count_body(txt):
    # 去掉開頭說明區（到第一個空行後的正文）
    # 青空文庫檔開頭是「作品名\n作者名\n---\n【記号説明】...」，正文從某處開始
    lines = txt.split("\n")
    # 找正文起點：跳過前幾行標題/作者/分隔線/記號說明
    start = 0
    for i, ln in enumerate(lines):
        if ln.strip() == "" :
            continue
        # 記號說明區通常以「【」開頭段落
        if "【" in ln and "】" in ln and "記号" in ln:
            continue
        # 分隔線
        if set(ln.strip()) <= set("-"):
            continue
        start = i
        break
    body = "\n".join(lines[start:])
    # 移除 ruby 與外字標記
    body = re.sub(r'《[^》]*》', '', body)          # ruby
    body = re.sub(r'｜', '', body)                  # ruby 起點
    body = re.sub(r'［＃[^］]*］', '', body)        # 入力者注
    body = re.sub(r'\s', '', body)
    # 只算日文漢字/假名/標點
    return len(body)

def classify(n, is_child, kana):
    old = kana in ("旧字旧仮名", "旧字新仮名")
    if is_child:
        if n < 8000: return 1
        elif n < 25000: return 2
        else: return 3
    else:
        if old:
            if n < 20000: return 4
            else: return 5
        else:
            if n < 10000: return 2
            elif n < 30000: return 3
            elif n < 80000: return 4
            else: return 5

# 平行下載
results = []
lock = threading.Lock()
done = 0
errs = []

def worker(b):
    global done
    r = download_one(b)
    with lock:
        done += 1
        if done % 50 == 0:
            print(f"  進度 {done}/{len(books)}", flush=True)
    return r

with ThreadPoolExecutor(max_workers=6) as ex:
    futs = [ex.submit(worker, b) for b in books]
    for f in as_completed(futs):
        results.append(f.result())

ok = [r for r in results if r[3] is not None]
bad = [r for r in results if r[3] is None]
print("成功:", len(ok), "失敗:", len(bad))
for r in bad[:20]:
    print("  FAIL:", r[0], r[1], r[4])

# 寫 catalog
tier_cnt = Counter(r[3] for r in ok)
print("各級數量:", dict(sorted(tier_cnt.items())))

with open(CATALOG, "w", encoding="utf-8", newline="") as f:
    w = csv.writer(f)
    w.writerow(["級別","作品ID","作品名","作者","字數","假名種別","児童","URL"])
    for r in sorted(ok, key=lambda x:(x[3], x[1])):
        b = next(b for b in books if b[idx["作品ID"]]==r[0])
        w.writerow([f"grade{r[3]}", r[0], r[1], r[2], r[4],
                    b[idx["文字遣い種別"]], "Y" if "K913" in b[idx["分類番号"]] else "N",
                    b[idx["テキストファイルURL"]]])

print("catalog 寫入:", CATALOG)
print("DONE")