# -*- coding: utf-8 -*-
"""Train the minimal GPT exactly as described, on the episode script, CPU only.
Outputs: loss history, samples, temperature samples -> check/train.json
"""
import json, math, os, random
import torch
import torch.nn as nn
import torch.nn.functional as F

BASE = os.path.dirname(os.path.abspath(__file__))
torch.manual_seed(1337)
random.seed(1337)

text = open(r"D:\AI视频\2026-11-15 手写最小Transformer\视频文案.txt", encoding="utf-8-sig").read()
# strip markdown scaffolding so the corpus reads like prose
text = "\n".join(l for l in text.splitlines()
                 if l.strip() and not l.startswith("#") and not l.startswith("$$"))
chars = sorted(set(text))
V = len(chars)
stoi = {ch: i for i, ch in enumerate(chars)}
itos = {i: ch for i, ch in enumerate(chars)}
data = torch.tensor([stoi[c] for c in text], dtype=torch.long)
n = int(0.9 * len(data))
train, val = data[:n], data[n:]
print("vocab", V, "chars", len(data), "train", len(train), "val", len(val))

BLOCK = 32
N_EMBD = 64
N_HEAD = 1
N_LAYER = 1
DROPOUT = 0.0
BATCH = 16
LR = 2e-3
STEPS = 2200
EVAL_EVERY = 100


def get_batch(split, bs=BATCH):
    d = train if split == "train" else val
    ix = torch.randint(len(d) - BLOCK - 1, (bs,))
    x = torch.stack([d[i:i + BLOCK] for i in ix])
    y = torch.stack([d[i + 1:i + BLOCK + 1] for i in ix])
    return x, y


class Head(nn.Module):
    def __init__(self):
        super().__init__()
        self.key = nn.Linear(N_EMBD, N_EMBD, bias=False)
        self.query = nn.Linear(N_EMBD, N_EMBD, bias=False)
        self.value = nn.Linear(N_EMBD, N_EMBD, bias=False)
        self.register_buffer("tril", torch.tril(torch.ones(BLOCK, BLOCK)))

    def forward(self, x):
        B, T, C = x.shape
        k, q, v = self.key(x), self.query(x), self.value(x)
        wei = q @ k.transpose(-2, -1) * (1.0 / math.sqrt(k.shape[-1]))
        wei = wei.masked_fill(self.tril[:T, :T] == 0, float("-inf"))
        wei = F.softmax(wei, dim=-1)
        return wei @ v


class FFN(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(nn.Linear(N_EMBD, 4 * N_EMBD), nn.ReLU(),
                                 nn.Linear(4 * N_EMBD, N_EMBD))

    def forward(self, x):
        return self.net(x)


class Block(nn.Module):
    def __init__(self):
        super().__init__()
        self.ln1 = nn.LayerNorm(N_EMBD)
        self.sa = Head()
        self.ln2 = nn.LayerNorm(N_EMBD)
        self.ff = FFN()

    def forward(self, x):
        x = x + self.sa(self.ln1(x))
        x = x + self.ff(self.ln2(x))
        return x


class TinyGPT(nn.Module):
    def __init__(self):
        super().__init__()
        self.tok = nn.Embedding(V, N_EMBD)
        self.pos = nn.Embedding(BLOCK, N_EMBD)
        self.block = Block()
        self.ln_f = nn.LayerNorm(N_EMBD)
        self.head = nn.Linear(N_EMBD, V)

    def forward(self, idx, targets=None):
        B, T = idx.shape
        x = self.tok(idx) + self.pos(torch.arange(T))
        x = self.block(x)
        logits = self.head(self.ln_f(x))
        loss = None
        if targets is not None:
            loss = F.cross_entropy(logits.view(B * T, V), targets.view(B * T))
        return logits, loss

    @torch.no_grad()
    def generate(self, idx, n_new, temperature=1.0, top_k=None, greedy=False):
        for _ in range(n_new):
            idx_cond = idx[:, -BLOCK:]
            logits, _ = self(idx_cond)
            logits = logits[:, -1, :] / max(temperature, 1e-6)
            if greedy:
                nxt = logits.argmax(dim=-1, keepdim=True)
            else:
                probs = F.softmax(logits, dim=-1)
                nxt = torch.multinomial(probs, 1)
            idx = torch.cat([idx, nxt], dim=1)
        return idx


model = TinyGPT()
nparam = sum(p.numel() for p in model.parameters())
print("params", nparam)
opt = torch.optim.AdamW(model.parameters(), lr=LR)

hist = []
snapshot = {}
for step in range(STEPS + 1):
    if step % EVAL_EVERY == 0 or step == STEPS:
        model.eval()
        with torch.no_grad():
            tl = torch.stack([model(*get_batch("train"))[1] for _ in range(4)]).mean().item()
            vl = torch.stack([model(*get_batch("val"))[1] for _ in range(4)]).mean().item()
        hist.append({"step": step, "train": tl, "val": vl})
        print("step %5d  train %.4f  val %.4f" % (step, tl, vl), flush=True)
        if step in (0, 200, 600, 1200, STEPS):
            snap = {}
            for T_ in (0.6, 1.0, 1.3):
                ctx = torch.tensor([[stoi[c] for c in "今天"]])
                out = model.generate(ctx, 26, temperature=T_)[0].tolist()
                snap[str(T_)] = "".join(itos[i] for i in out)
            ctx = torch.tensor([[stoi[c] for c in "今天"]])
            snap["greedy"] = "".join(itos[i] for i in model.generate(ctx, 26, greedy=True)[0].tolist())
            snapshot[step] = snap
        model.train()
    x, y = get_batch("train")
    _, loss = model(x, y)
    opt.zero_grad(set_to_none=True)
    loss.backward()
    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
    opt.step()

# a final sample with a longer prompt
ctx = torch.tensor([[stoi[c] for c in "它生成"]])
long_sample = "".join(itos[i] for i in model.generate(ctx, 40, temperature=0.9)[0].tolist())

out = {
    "vocab": V, "chars": len(data), "train_chars": int(n), "val_chars": int(len(data) - n),
    "params": nparam, "block": BLOCK, "n_embd": N_EMBD, "steps": STEPS, "lr": LR,
    "history": hist, "snapshots": snapshot, "long_sample": long_sample,
    "final_loss": hist[-1]["train"], "final_val": hist[-1]["val"],
}
json.dump(out, open(os.path.join(BASE, "check", "train.json"), "w", encoding="utf-8"),
          ensure_ascii=False, indent=1)
print("done", json.dumps({"final": hist[-1], "params": nparam}, ensure_ascii=False))
print("sample:", long_sample)
