# -*- coding: utf-8 -*-
"""手写多头注意力 · 完整可运行训练脚本
================================================================

这是在上一期《手写最小 Transformer》的 TinyGPT 基础上做的对照实验：
模型结构、训练循环、采样逻辑全部沿用上一期，只把注意力模块换掉——

    impl="head"   上一期单头注意力（d_k = 64）
    impl="multi"  4 个独立的头，nn.ModuleList 循环        （视频第二部分）
    impl="batch"  四维张量 + 一次批矩阵乘（工业界写法）   （视频第四部分）

三种实现用同一份语料、同一个随机种子、同一批训练数据，各训一遍，对比：
    · 参数量
    · 训练/验证 loss
    · 同温度下的生成样例

直接运行：
    python train_multihead.py

只跑某一种：
    python train_multihead.py --impl multi

依赖：torch（CPU 即可）。默认 600 步是"快速对照"；
想复现上一期的完整训练就把 STEPS 改成 2200（CPU 上约十几分钟）。

作者备注：本文件只读上一期的代码结构，不改动上一期的 train_tiny.py。
"""

import argparse
import json
import math
import os
import random

import torch
import torch.nn as nn
import torch.nn.functional as F

# ======================================================================
# 配置（与上一期一致；只把步数调小方便快速对照）
# ======================================================================
BLOCK = 32          # 序列窗口：一次看 32 个字符
N_EMBD = 64         # 嵌入维度 d_model
N_HEAD = 4          # 多头个数（本期主题）
N_LAYER = 1         # 1 层 Transformer Block（最小模型）
DROPOUT = 0.0
BATCH = 16
LR = 2e-3
STEPS = 600         # 快速对照；改成 2200 即上一期的完整训练
EVAL_EVERY = 100
SEED = 1337

# 语料：默认用本期视频的文案；找不到就依次回退
CORPUS_CANDIDATES = [
    r"D:\AI视频\2026-11-18 手写多头注意力\视频文案.txt",
    r"D:\AI视频\2026-11-15 手写最小Transformer\视频文案.txt",
]

BASE = os.path.dirname(os.path.abspath(__file__))
OUT_DIR = os.path.join(BASE, "check")


# ======================================================================
# 语料与字符级 tokenizer（与上一期相同：去 markdown、去公式行）
# ======================================================================
def load_corpus():
    for p in CORPUS_CANDIDATES:
        if os.path.exists(p):
            raw = open(p, encoding="utf-8-sig").read()
            text = "\n".join(
                line for line in raw.splitlines()
                if line.strip() and not line.startswith("#") and not line.startswith("$$")
            )
            print("语料:", p)
            return text
    raise SystemExit("找不到语料文件，请修改 CORPUS_CANDIDATES")


text = load_corpus()
chars = sorted(set(text))
V = len(chars)
stoi = {ch: i for i, ch in enumerate(chars)}
itos = {i: ch for i, ch in enumerate(chars)}
data = torch.tensor([stoi[c] for c in text], dtype=torch.long)
n_train = int(0.9 * len(data))
train_data, val_data = data[:n_train], data[n_train:]
print("词表 %d 字，语料 %d 字，训练 %d / 验证 %d" % (V, len(data), len(train_data), len(val_data)))

# 固定数据批次生成器：三种模型拿到完全相同的训练样本，对比才公平
_GEN = torch.Generator().manual_seed(20261118)


def get_batch(split, bs=BATCH):
    d = train_data if split == "train" else val_data
    ix = torch.randint(len(d) - BLOCK - 1, (bs,), generator=_GEN)
    x = torch.stack([d[i:i + BLOCK] for i in ix])
    y = torch.stack([d[i + 1:i + BLOCK + 1] for i in ix])
    return x, y


# ======================================================================
# 注意力模块
# ======================================================================
class Head(nn.Module):
    """单个注意力头。

    注意：输入永远是完整的 N_EMBD 维（不是把输入劈成四段），
    用 64→head_size 的线性层投影出自己的 Q/K/V。
    单头时 head_size=64；多头时 head_size=16。
    """

    def __init__(self, n_embd=N_EMBD, head_size=None):
        super().__init__()
        head_size = head_size or n_embd
        self.key = nn.Linear(n_embd, head_size, bias=False)
        self.query = nn.Linear(n_embd, head_size, bias=False)
        self.value = nn.Linear(n_embd, head_size, bias=False)
        self.register_buffer("tril", torch.tril(torch.ones(BLOCK, BLOCK)))

    def forward(self, x):
        B, T, C = x.shape
        q, k, v = self.query(x), self.key(x), self.value(x)      # 各 (B,T,head_size)
        # 缩放：除以根号 d_k。单头 d_k=64（除 8），多头 d_k=16（除 4）
        att = q @ k.transpose(-2, -1) * (1.0 / math.sqrt(k.shape[-1]))
        att = att.masked_fill(self.tril[:T, :T] == 0, float("-inf"))
        att = F.softmax(att, dim=-1)
        return att @ v


class MultiHeadAttention(nn.Module):
    """4 个独立的头：先各算各的，再在最后一维拼接，最后过输出投影 W_O。"""

    def __init__(self, n_embd=N_EMBD, n_head=N_HEAD):
        super().__init__()
        assert n_embd % n_head == 0, "嵌入维度必须能被头数整除"
        self.heads = nn.ModuleList([Head(n_embd, n_embd // n_head) for _ in range(n_head)])
        self.proj = nn.Linear(n_embd, n_embd, bias=False)        # W_O

    def forward(self, x):
        out = [h(x) for h in self.heads]      # 4 × (B, T, 16)
        out = torch.cat(out, dim=-1)          # (B, T, 64)  最后一维首尾相连
        return self.proj(out)                 # 融合回统一的表示空间


class MultiHeadAttentionBatch(nn.Module):
    """四维张量 + 一次批矩阵乘：工业界真正的写法。

    形状路径：
        x           (B, T, 64)
        q/k/v       (B, T, 64)        三个 64→64 线性层一次算出
        view        (B, T, 4, 16)     把最后一维拆成 头数 × 头维度
        transpose   (B, 4, T, 16)     交换序列长度与头数，让最后两维成为 5×16 矩阵
        Q @ K^T     (B, 4, T, T)      前两维是批处理，16 张 5×5 一次算完
        softmax @V  (B, 4, T, 16)
        transpose   (B, T, 4, 16)
        contiguous  (B, T, 4, 16)     转置不动内存，必须重排才能 view
        view        (B, T, 64)
        proj        (B, T, 64)
    """

    def __init__(self, n_embd=N_EMBD, n_head=N_HEAD):
        super().__init__()
        assert n_embd % n_head == 0
        self.n_head = n_head
        self.head_size = n_embd // n_head
        # 4 个头各自的 64×16 权重在列方向拼起来，就是一个 64×64 的大矩阵
        self.q_proj = nn.Linear(n_embd, n_embd, bias=False)
        self.k_proj = nn.Linear(n_embd, n_embd, bias=False)
        self.v_proj = nn.Linear(n_embd, n_embd, bias=False)
        self.proj = nn.Linear(n_embd, n_embd, bias=False)        # W_O
        self.register_buffer("tril", torch.tril(torch.ones(BLOCK, BLOCK)))

    def forward(self, x):
        B, T, C = x.shape
        h, d = self.n_head, self.head_size

        q = self.q_proj(x).view(B, T, h, d).transpose(1, 2)      # (B, 4, T, 16)
        k = self.k_proj(x).view(B, T, h, d).transpose(1, 2)
        v = self.v_proj(x).view(B, T, h, d).transpose(1, 2)

        att = q @ k.transpose(-2, -1) * (1.0 / math.sqrt(d))     # (B, 4, T, T)
        att = att.masked_fill(self.tril[:T, :T] == 0, float("-inf"))
        att = F.softmax(att, dim=-1)
        out = att @ v                                            # (B, 4, T, 16)

        out = out.transpose(1, 2).contiguous().view(B, T, C)     # (B, T, 64)
        return self.proj(out)


def build_attention(impl, n_embd=N_EMBD, n_head=N_HEAD):
    if impl == "head":
        return Head(n_embd, n_embd)          # 单头：head_size = 64
    if impl == "multi":
        return MultiHeadAttention(n_embd, n_head)
    if impl == "batch":
        return MultiHeadAttentionBatch(n_embd, n_head)
    raise ValueError("impl 只能是 head / multi / batch")


# ======================================================================
# 模型：Block 与 TinyGPT（其余结构一行都没改）
# ======================================================================
class FFN(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(N_EMBD, 4 * N_EMBD), nn.ReLU(), nn.Dropout(DROPOUT),
            nn.Linear(4 * N_EMBD, N_EMBD), nn.Dropout(DROPOUT),
        )

    def forward(self, x):
        return self.net(x)


class Block(nn.Module):
    """残差 + 前置 LayerNorm，唯一的改动就是注意力那一行。"""

    def __init__(self, impl="multi"):
        super().__init__()
        self.ln1 = nn.LayerNorm(N_EMBD)
        self.sa = build_attention(impl)      # ← 只换这里
        self.ln2 = nn.LayerNorm(N_EMBD)
        self.ff = FFN()

    def forward(self, x):
        x = x + self.sa(self.ln1(x))
        x = x + self.ff(self.ln2(x))
        return x


class TinyGPT(nn.Module):
    def __init__(self, impl="multi"):
        super().__init__()
        self.tok = nn.Embedding(V, N_EMBD)
        self.pos = nn.Embedding(BLOCK, N_EMBD)
        self.blocks = nn.Sequential(*[Block(impl) for _ in range(N_LAYER)])
        self.ln_f = nn.LayerNorm(N_EMBD)
        self.head = nn.Linear(N_EMBD, V)     # 输出层：64 → 词表大小

    def forward(self, idx, targets=None):
        B, T = idx.shape
        x = self.tok(idx) + self.pos(torch.arange(T))
        x = self.blocks(x)
        logits = self.head(self.ln_f(x))
        loss = None
        if targets is not None:
            loss = F.cross_entropy(logits.view(B * T, V), targets.view(B * T))
        return logits, loss

    @torch.no_grad()
    def generate(self, idx, n_new, temperature=1.0, greedy=False):
        for _ in range(n_new):
            logits, _ = self(idx[:, -BLOCK:])
            logits = logits[:, -1, :] / max(temperature, 1e-6)
            if greedy:
                nxt = logits.argmax(dim=-1, keepdim=True)
            else:
                nxt = torch.multinomial(F.softmax(logits, dim=-1), 1)
            idx = torch.cat([idx, nxt], dim=1)
        return idx


# ======================================================================
# 训练与采样
# ======================================================================
def sample_text(model, prompt, n_new=26, temperature=0.9):
    ctx = torch.tensor([[stoi.get(c, 0) for c in prompt]])
    out = model.generate(ctx, n_new, temperature=temperature)[0].tolist()
    return "".join(itos[i] for i in out)


def evaluate(model):
    model.eval()
    with torch.no_grad():
        tl = torch.stack([model(*get_batch("train"))[1] for _ in range(4)]).mean().item()
        vl = torch.stack([model(*get_batch("val"))[1] for _ in range(4)]).mean().item()
    model.train()
    return tl, vl


def train_one(impl, steps=STEPS, seed=SEED, verbose=True):
    """训练一种注意力实现，返回参数量、loss 历史与生成样例。"""
    torch.manual_seed(seed)
    random.seed(seed)
    model = TinyGPT(impl)
    nparam = sum(p.numel() for p in model.parameters())
    opt = torch.optim.AdamW(model.parameters(), lr=LR)

    if verbose:
        print("\n===== impl = %-5s  params = %d =====" % (impl, nparam), flush=True)

    hist = []
    snapshots = {}
    for step in range(steps + 1):
        if step % EVAL_EVERY == 0 or step == steps:
            tl, vl = evaluate(model)
            hist.append({"step": step, "train": round(tl, 4), "val": round(vl, 4)})
            if verbose:
                print("step %5d  train %.4f  val %.4f" % (step, tl, vl), flush=True)
            # 采样快照：看表达能力怎么随训练变化
            if step in (0, 200, 600, 1200, steps):
                snap = {}
                for T_ in (0.6, 1.0, 1.3):
                    snap[str(T_)] = sample_text(model, "天", 26, T_)
                ctx = torch.tensor([[stoi.get(c, 0) for c in "天"]])
                snap["greedy"] = "".join(
                    itos[i] for i in model.generate(ctx, 26, greedy=True)[0].tolist())
                snapshots[step] = snap
        x, y = get_batch("train")
        _, loss = model(x, y)
        opt.zero_grad(set_to_none=True)
        loss.backward()
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        opt.step()

    long_sample = sample_text(model, "它生成", 40, 0.9)
    return {
        "impl": impl, "params": nparam, "steps": steps,
        "final_train": hist[-1]["train"], "final_val": hist[-1]["val"],
        "history": hist, "snapshots": snapshots, "long_sample": long_sample,
    }


# ======================================================================
# 主程序：三组对照
# ======================================================================
def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--impl", default="all", choices=["all", "head", "multi", "batch"],
                    help="训练哪一种实现，默认三种都跑")
    ap.add_argument("--steps", type=int, default=STEPS)
    ap.add_argument("--out", default=os.path.join(OUT_DIR, "train_compare.json"))
    args = ap.parse_args()

    impls = ["head", "multi", "batch"] if args.impl == "all" else [args.impl]
    results = [train_one(impl, args.steps) for impl in impls]

    # ---- 汇总表 ----
    print("\n" + "=" * 68)
    print("%-8s %10s %10s %10s %10s" % ("实现", "参数量", "训练loss", "验证loss", "相对单头"))
    base = results[0]["params"] if results[0]["impl"] == "head" else None
    for r in results:
        rel = ("+%d" % (r["params"] - base)) if base else "-"
        print("%-8s %10d %10.4f %10.4f %10s" % (
            r["impl"], r["params"], r["final_train"], r["final_val"], rel))
    print("=" * 68)

    # ---- 生成样例对比 ----
    for r in results:
        print("\n[%s] 温度 0.9 生成：" % r["impl"])
        print("  " + r["long_sample"])

    os.makedirs(OUT_DIR, exist_ok=True)
    with open(args.out, "w", encoding="utf-8") as f:
        json.dump({
            "corpus": CORPUS_CANDIDATES[0], "vocab": V, "chars": len(data),
            "block": BLOCK, "n_embd": N_EMBD, "n_head": N_HEAD, "n_layer": N_LAYER,
            "batch": BATCH, "lr": LR, "steps": args.steps, "seed": SEED,
            "results": results,
        }, f, ensure_ascii=False, indent=1)
    print("\n结果已写入:", args.out)


if __name__ == "__main__":
    main()
