"""Keras LSTM English→Urdu lesson: python keras_seq2seq_urdu.py

Install: pip install tensorflow datasets numpy scikit-learn matplotlib sacrebleu
Default: stream 2,000 real pairs from the Hugging Face Shaistagi translation config.
Offline demo: python keras_seq2seq_urdu.py --source starter
Own data: python keras_seq2seq_urdu.py --pairs pairs.csv
CSV headers: english,urdu (UTF-8). Small data is for learning the pipeline,
not for a useful open-domain translator.
"""
# %% 1 · IMPORTS AND REPRODUCIBILITY
import argparse
import csv
import json
import random
import re
from collections import Counter
from pathlib import Path

import matplotlib
matplotlib.use("Agg")  # save a plot without requiring a desktop window
import matplotlib.pyplot as plt
import numpy as np
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras import layers
from datasets import load_dataset
from sacrebleu.metrics import CHRF
from sklearn.metrics import ConfusionMatrixDisplay, classification_report, confusion_matrix

# One seed controls Python, NumPy, and TensorFlow so students can compare runs.
# Exact numerical results can still vary across hardware and library versions.
SEED = 42
random.seed(SEED)
np.random.seed(SEED)
tf.random.set_seed(SEED)
SOURCE_LEN = 18              # source sentences are filtered to at most 15 words
TARGET_LEN = 20              # target words + startseq + endseq + spare padding
MAX_VOCAB = 10000
EMBED_DIM = 64
HIDDEN = 128

# %% 2 · COLLECT PARALLEL TEXT
# The dataset in the supplied snippet has title/content poetry fields, not
# English/Urdu parallel sentences. This configuration has english/urdu fields.
def load_hf_pairs(limit):
    # Streaming reads rows as needed; it does not load the full dataset first.
    stream = load_dataset(
        "ReySajju742/shaistagi_clean", "english_urdu_translation",
        split="train", streaming=True)
    stream = stream.shuffle(seed=SEED, buffer_size=3000)
    pairs = []
    seen = set()
    for row in stream:
        # This dataset configuration names its columns "english" and "urdu".
        english = str(row.get("english") or "").strip()
        urdu = str(row.get("urdu") or "").strip()
        key = (english.casefold(), urdu)
        # Keep this first classroom run small and avoid empty/non-Urdu rows.
        if (3 <= len(english.split()) <= 15
                and 1 <= len(urdu.split()) <= 16
                and re.search(r"[\u0600-\u06FF]", urdu)
                and key not in seen):
            pairs.append((english, urdu))
            seen.add(key)
        if len(pairs) >= limit:
            break
    if len(pairs) < 30:
        raise RuntimeError("Not enough valid English–Urdu pairs were streamed")
    return pairs


def starter_pairs():
    """Small compositional phrasebook for an offline pipeline rehearsal."""
    subjects = {"I": "مجھے", "We": "ہمیں", "You": "تمہیں",
                "He": "اسے", "She": "اسے", "They": "انہیں"}
    objects = {"cats": "بلیاں", "dogs": "کتے", "books": "کتابیں",
               "birds": "پرندے", "flowers": "پھول", "apples": "سیب",
               "mangoes": "آم", "stories": "کہانیاں"}
    pairs = []
    for en_subject, ur_subject in subjects.items():
        singular = en_subject in {"He", "She"}
        for en_object, ur_object in objects.items():
            verb = "likes" if singular else "like"
            pairs += [
                (f"{en_subject} {verb} {en_object}",
                 f"{ur_subject} {ur_object} پسند ہیں"),
                (f"{en_subject} really {verb} {en_object}",
                 f"{ur_subject} {ur_object} بہت پسند ہیں"),
                (f"{en_subject} {'does' if singular else 'do'} not like {en_object}",
                 f"{ur_subject} {ur_object} پسند نہیں ہیں"),
            ]
    return pairs


def read_csv_pairs(path):
    with path.open(encoding="utf-8-sig", newline="") as file:
        reader = csv.DictReader(file)
        if not {"english", "urdu"}.issubset(reader.fieldnames or []):
            raise ValueError("CSV needs english,urdu header columns")
        return [(r["english"].strip(), r["urdu"].strip()) for r in reader
                if r["english"].strip() and r["urdu"].strip()]

# %% 3 · SPLIT FIRST, THEN LEARN VOCABULARIES
# Teacher-forcing leakage can begin before fit(): do not adapt tokenizers on test.
def split_pairs(pairs):
    pairs = list(dict.fromkeys(pairs))  # deduplicate BEFORE the split
    if len(pairs) < 30:
        raise ValueError("At least 30 unique pairs are needed")
    # Keep alternate Urdu translations of one English source in ONE partition.
    by_source = {}
    for english, urdu in pairs:
        by_source.setdefault(english.casefold(), []).append((english, urdu))
    sources = list(by_source)
    if len(sources) < 30:
        raise ValueError("At least 30 distinct English sources are needed")
    random.Random(SEED).shuffle(sources)
    # Choose source groups, then flatten them. This prevents one English sentence
    # with two Urdu translations from appearing in both train and test.
    test_end = max(1, round(len(sources) * .15))
    val_end = test_end + max(1, round(len(sources) * .15))
    def collect(keys):
        return [pair for key in keys for pair in by_source[key]]
    return (collect(sources[val_end:]), collect(sources[test_end:val_end]),
            collect(sources[:test_end]))


def prepare(pairs, english_vectorizer, urdu_vectorizer):
    # startseq/endseq are the diagram's <BOS>/<EOS>. Plain words survive
    # TextVectorization's whitespace splitting and standardize=None.
    english = [en.lower() for en, _ in pairs]
    urdu = [f"startseq {ur} endseq" for _, ur in pairs]
    source_ids = english_vectorizer(english).numpy().astype("int32")
    full_target = urdu_vectorizer(urdu).numpy().astype("int32")
    # Teacher forcing is a ONE-STEP SHIFT of the SAME Urdu sentence:
    # sentence:       startseq مجھے بلیاں پسند ہیں endseq
    # decoder input:  startseq مجھے بلیاں پسند ہیں
    # expected label: مجھے     بلیاں پسند ہیں endseq
    # During training the decoder sees the correct previous Urdu word, even
    # if its own previous prediction would have been wrong.
    decoder_input = full_target[:, :-1]  # [BOS, y1, ..., y(m-1)]
    labels = full_target[:, 1:]          # [y1, ..., ym, EOS]
    weights = (labels != 0).astype("float32")  # ignore every PAD label
    return source_ids, decoder_input, labels, weights

# %% 4 · BUILD ONE SHARED LSTM ENCODER–DECODER
# Encoder LSTM hands BOTH h (working output) and c (long memory) to decoder.
def build_models(source_vocab_size, target_vocab_size):
    # Input shapes are token IDs, not one-hot vectors. The Embedding layers
    # convert each ID to a learned EMBED_DIM-wide row before either LSTM runs.
    source_ids = keras.Input((SOURCE_LEN,), dtype="int32", name="english_ids")
    prefix_ids = keras.Input((TARGET_LEN - 1,), dtype="int32", name="urdu_prefix")
    source_embedding = layers.Embedding(source_vocab_size, EMBED_DIM,
                                        mask_zero=True, name="english_embedding")
    target_embedding = layers.Embedding(target_vocab_size, EMBED_DIM,
                                        mask_zero=True, name="urdu_embedding")
    encoder_lstm = layers.LSTM(HIDDEN, return_state=True, name="encoder_lstm")
    decoder_lstm = layers.LSTM(HIDDEN, return_sequences=True,
                               return_state=True, name="decoder_lstm")
    to_vocabulary = layers.Dense(target_vocab_size, name="next_urdu_logits")

    # return_state=True exposes the final hidden state h and cell state c.
    # The decoder begins from exactly these two encoder states.
    _, h, c = encoder_lstm(source_embedding(source_ids))
    decoder_sequence, _, _ = decoder_lstm(target_embedding(prefix_ids),
                                          initial_state=[h, c])
    # At EACH target step, score all Urdu vocabulary entries. The loss will
    # compare these logits with the next gold token ID; no softmax is needed
    # here because from_logits=True is set when compiling.
    logits = to_vocabulary(decoder_sequence)  # [batch, target_steps, vocab]
    train_model = keras.Model([source_ids, prefix_ids], logits)

    # Reuse THE SAME trained layers at inference: no new random decoder.
    encoder_model = keras.Model(source_ids, [h, c])
    one_token = keras.Input((1,), dtype="int32", name="previous_token")
    old_h = keras.Input((HIDDEN,), name="previous_h")
    old_c = keras.Input((HIDDEN,), name="previous_c")
    one_output, new_h, new_c = decoder_lstm(target_embedding(one_token),
                                            initial_state=[old_h, old_c])
    step_logits = to_vocabulary(one_output)
    step_model = keras.Model([one_token, old_h, old_c],
                             [step_logits, new_h, new_c])
    return train_model, encoder_model, step_model

# %% 5 · INFERENCE USES ITS OWN PREVIOUS PREDICTION
# Unlike teacher forcing, no true Urdu prefix is available at test time.
def translate(english, english_vectorizer, encoder_model, step_model,
              urdu_vocabulary):
    word_to_id = {word: index for index, word in enumerate(urdu_vocabulary)}
    source = english_vectorizer([english.lower()]).numpy().astype("int32")
    h, c = encoder_model.predict(source, verbose=0)
    # At inference no reference translation is available. Feed BOS first.
    token = np.array([[word_to_id["startseq"]]], dtype="int32")
    output_words = []
    for _ in range(TARGET_LEN - 2):  # hard stop protects against missing EOS
        scores, h, c = step_model.predict([token, h, c], verbose=0)
        next_id = int(np.argmax(scores[0, -1]))  # greedy choice
        if next_id in (0, 1, word_to_id["endseq"]):
            break
        output_words.append(urdu_vocabulary[next_id])
        # This is the key difference from teacher forcing: a wrong choice is
        # now fed back and may affect every later word.
        token[0, 0] = next_id  # prediction, not a ground-truth token
    return " ".join(output_words)

# %% 6 · TRAIN WITH TEACHER FORCING AND MASKED LOSS
# The full shifted gold prefix is visible during training ONLY. The loss at
# position t asks: after gold y(t-1), did we predict gold y(t)?
def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--source", choices=["hf", "starter"], default="hf")
    parser.add_argument("--pairs", type=Path, help="optional english,urdu CSV")
    parser.add_argument("--samples", type=int, default=2000)
    parser.add_argument("--epochs", type=int, default=15)
    args = parser.parse_args()
    if args.pairs:
        raw = read_csv_pairs(args.pairs)
        source_name = str(args.pairs)
    elif args.source == "starter":
        raw = starter_pairs()
        source_name = "controlled starter phrasebook"
    else:
        raw = load_hf_pairs(args.samples)
        source_name = "Shaistagi English–Urdu translation stream"
    # A long CSV sentence would otherwise be truncated, possibly removing EOS.
    raw = [(en, ur) for en, ur in raw
           if 1 <= len(en.split()) <= SOURCE_LEN
           and 1 <= len(ur.split()) <= TARGET_LEN - 2]
    train, validation, test = split_pairs(raw)
    print(f"Source: {source_name}; train={len(train)}, "
          f"validation={len(validation)}, test={len(test)}")
    print("Example:", train[0])

    english_vectorizer = layers.TextVectorization(
        max_tokens=MAX_VOCAB, standardize=None,
        output_mode="int", output_sequence_length=SOURCE_LEN)
    urdu_vectorizer = layers.TextVectorization(
        max_tokens=MAX_VOCAB, standardize=None,
        output_mode="int", output_sequence_length=TARGET_LEN)
    # Learn vocabulary only from TRAIN: adapting on held-out text leaks data.
    # TextVectorization reserves ID 0 for padding and ID 1 for unknown words.
    english_vectorizer.adapt([en.lower() for en, _ in train])
    urdu_vectorizer.adapt([f"startseq {ur} endseq" for _, ur in train])
    urdu_vocabulary = urdu_vectorizer.get_vocabulary()
    train_source, train_prefix, train_labels, train_weights = prepare(
        train, english_vectorizer, urdu_vectorizer)
    val_source, val_prefix, val_labels, val_weights = prepare(
        validation, english_vectorizer, urdu_vectorizer)
    test_source, test_prefix, test_labels, test_weights = prepare(
        test, english_vectorizer, urdu_vectorizer)

    # Show teacher forcing with a REAL vectorized training row.
    lookup = urdu_vocabulary
    def words(ids):
        return [lookup[int(i)] for i in ids if int(i) != 0]
    print("TEACHER INPUT:", words(train_prefix[0]))
    print("NEXT-TOKEN LABELS:", words(train_labels[0]))
    print("Same target sentence, shifted one position.")

    train_model, encoder_model, step_model = build_models(
        len(english_vectorizer.get_vocabulary()), len(urdu_vocabulary))
    train_model.compile(
        optimizer=keras.optimizers.Adam(learning_rate=0.003),
        loss=keras.losses.SparseCategoricalCrossentropy(from_logits=True),
        # Weighted accuracy, like weighted loss, ignores PAD positions.
        weighted_metrics=[keras.metrics.SparseCategoricalAccuracy(
            name="masked_next_token_accuracy")])
    train_model.summary()
    train_model.fit(
        [train_source, train_prefix], train_labels,
        # train_weights is 0 on PAD labels, 1 on real next-word labels.
        sample_weight=train_weights, batch_size=32, epochs=args.epochs,
        validation_data=([val_source, val_prefix], val_labels, val_weights),
        callbacks=[keras.callbacks.EarlyStopping(
            monitor="val_loss", patience=3, restore_best_weights=True)],
        verbose=2)

    # %% 7 · TEST THE GENERATED SENTENCE; DIAGNOSE TOKENS SEPARATELY
    # Free-running chrF/exact match are the honest translation metrics.
    references = [urdu for _, urdu in test]
    predictions = [translate(en, english_vectorizer, encoder_model,
                             step_model, urdu_vocabulary) for en, _ in test]
    # Exact match is strict; chrF gives partial credit for character overlap.
    # Both use free-running outputs from translate(), like actual deployment.
    exact_match = np.mean([p == r for p, r in zip(predictions, references)])
    chrf = CHRF().corpus_score(predictions, [references]).score
    print(f"FREE-RUN test exact match={exact_match:.3f}; chrF={chrf:.2f}")
    for (en, ref), pred in list(zip(test, predictions))[:5]:
        print(f"EN: {en}\nUR reference: {ref}\nUR prediction: {pred}\n")

    # Teacher-forced accuracy and confusion matrix answer a DIFFERENT question:
    # given the correct previous word, which next token was predicted?
    print("Teacher-forced test:", train_model.evaluate(
        [test_source, test_prefix], test_labels,
        sample_weight=test_weights, return_dict=True, verbose=0))
    token_pred = train_model.predict([test_source, test_prefix],
                                    verbose=0).argmax(axis=-1)
    valid = test_labels != 0
    truth, guess = test_labels[valid], token_pred[valid]
    report = classification_report(truth, guess, output_dict=True,
                                   zero_division=0)
    print("Teacher-forced token macro F1:",
          round(report["macro avg"]["f1-score"], 3))
    # A full-vocabulary confusion matrix would be unreadable. Keep the ten
    # most frequent true tokens and group all remaining IDs as OTHER.
    top = [token_id for token_id, _ in Counter(truth).most_common(10)]
    mapped_true = [i if i in top else -1 for i in truth]
    mapped_pred = [i if i in top else -1 for i in guess]
    labels = top + [-1]
    print("Confusion plot token IDs:", {i: urdu_vocabulary[i] for i in top})
    matrix = confusion_matrix(mapped_true, mapped_pred, labels=labels)
    ConfusionMatrixDisplay(matrix, display_labels=[str(i) for i in top]
                           + ["OTHER"]).plot(xticks_rotation=45, cmap="Blues",
                                             colorbar=False)
    plt.title("Held-out next-token errors · gold prefix supplied")
    plt.tight_layout()
    plt.savefig("seq2seq_token_confusion.png", dpi=160)
    plt.close()

    train_model.save("seq2seq_urdu.keras")
    Path("seq2seq_vocab.json").write_text(json.dumps({
        "english": english_vectorizer.get_vocabulary(),
        "urdu": urdu_vocabulary,
        "source_length": SOURCE_LEN, "target_length": TARGET_LEN,
    }, ensure_ascii=False, indent=2), encoding="utf-8")
    print("Saved model, vocabularies, and token confusion plot.")
    if args.source == "starter" and not args.pairs:
        print("Starter scores only describe the tiny phrasebook grammar.")


if __name__ == "__main__":
    main()
