# --- NumPy (all of ML is matrix math) ---
import numpy as np

# Arrays are the fundamental data structure
X = np.array([[1, 2], [3, 4], [5, 6]])  # 3x2 matrix
print(X.shape)  # (3, 2)

# Vectorized operations (AVOID for loops!)
a = np.array([1, 2, 3])
b = np.array([4, 5, 6])
result = a * b  # element-wise: [4, 10, 18]
dot_product = a @ b  # 1*4 + 2*5 + 3*6 = 32

# Broadcasting: apply operations across shapes
X_mean = X.mean(axis=0)  # mean of each column
X_centered = X - X_mean  # subtracts mean from each row

# --- Pandas (data manipulation) ---
import pandas as pd

df = pd.read_csv('data.csv')
df.describe()              # statistical summary
df.isnull().sum()          # find missing values
df.groupby('category').mean()  # group operations

# --- Matplotlib (visualization) ---
import matplotlib.pyplot as plt

plt.scatter(X[:, 0], X[:, 1], c=y)
plt.xlabel('Feature 1')
plt.ylabel('Feature 2')
plt.show()

# Core ML stack
pip install numpy pandas matplotlib scikit-learn  # foundation
pip install jupyter notebook                       # interactive coding
pip install torch torchvision                      # PyTorch (deep learning)
pip install transformers datasets                  # Hugging Face (LLMs)
pip install tensorboard                            # visualization

import numpy as np
import matplotlib.pyplot as plt

# Generate synthetic data: y = 2x + 1 + noise
np.random.seed(42)
X = np.random.rand(100, 1) * 10  # 100 data points, 1 feature
y = 2 * X + 1 + np.random.randn(100, 1) * 0.5  # y = 2x + 1 + noise

# --- Linear Regression from Scratch ---
# Model: y_pred = w * X + b
# Loss: Mean Squared Error = 1/n * sum((y - y_pred)^2)
# Gradient descent: w -= lr * d(Loss)/dw

w = np.random.randn(1)  # random initial weight
b = np.random.randn(1)  # random initial bias
lr = 0.01               # learning rate
epochs = 1000

loss_history = []

for epoch in range(epochs):
    # Forward pass: make prediction
    y_pred = w * X + b

    # Compute loss
    loss = np.mean((y_pred - y) ** 2)
    loss_history.append(loss)

    # Backward pass: compute gradients
    dw = np.mean(2 * (y_pred - y) * X)  # derivative w.r.t weight
    db = np.mean(2 * (y_pred - y))       # derivative w.r.t bias

    # Update parameters
    w -= lr * dw
    b -= lr * db

    if epoch % 100 == 0:
        print(f"Epoch {epoch}: Loss = {loss:.4f}")

print(f"\nLearned: y = {w.item():.3f}x + {b.item():.3f}")
print(f"Actual:  y = 2x + 1")

# Plot
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(12, 4))
ax1.scatter(X, y, alpha=0.5, label='Data')
ax1.plot(X, w*X + b, 'r-', linewidth=2, label='Model')
ax1.set_xlabel('X')
ax1.set_ylabel('y')
ax1.legend()
ax1.set_title('Linear Regression Fit')

ax2.plot(loss_history)
ax2.set_xlabel('Epoch')
ax2.set_ylabel('Loss (MSE)')
ax2.set_title('Learning Curve')
ax2.set_yscale('log')
plt.tight_layout()
plt.show()

from sklearn.cluster import KMeans
import numpy as np

# Generate 3 clusters
np.random.seed(42)
cluster1 = np.random.randn(50, 2) + [0, 0]
cluster2 = np.random.randn(50, 2) + [3, 3]
cluster3 = np.random.randn(50, 2) + [0, 5]
X = np.vstack([cluster1, cluster2, cluster3])

# K-means finds the clusters automatically
kmeans = KMeans(n_clusters=3, random_state=42)
labels = kmeans.fit_predict(X)

print(f"Cluster centers:\n{kmeans.cluster_centers_}")
print(f"Cluster 0 has {(labels == 0).sum()} points")

from sklearn.decomposition import PCA
import matplotlib.pyplot as plt

# PCA on real data
pca = PCA(n_components=2)
X_reduced = pca.fit_transform(X)

print(f"Explained variance ratio: {pca.explained_variance_ratio_}")
print(f"First 2 components capture {sum(pca.explained_variance_ratio_):.1%} of variance")

plt.scatter(X_reduced[:, 0], X_reduced[:, 1], c=labels, cmap='viridis')
plt.title('Data projected to 2D with PCA')
plt.show()

from sklearn.model_selection import train_test_split

# ALWAYS split before any analysis
X_train, X_test, y_train, y_test = train_test_split(
    X, y, test_size=0.2, random_state=42
)

# X_train : y_train — what your model learns from (80%)
# X_test  : y_test  — what you finally test on (20%)
# Validation : a subset of training data you check during development

# During training, track BOTH training and validation loss
train_losses = []
val_losses = []

for epoch in range(epochs):
    # Train
    model.train()
    train_pred = model(X_train)
    train_loss = loss_fn(train_pred, y_train)
    train_losses.append(train_loss.item())

    # Validate (no gradient!)
    model.eval()
    with torch.no_grad():
        val_pred = model(X_val)
        val_loss = loss_fn(val_pred, y_val)
        val_losses.append(val_loss.item())

# Plot both — the gap tells you everything
import matplotlib.pyplot as plt
plt.plot(train_losses, label='Training Loss')
plt.plot(val_losses, label='Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.axvline(x=best_epoch, color='g', linestyle='--', label='Best Model')

# Visual intuition
import numpy as np
import matplotlib.pyplot as plt

# Generate many different training sets
predictions = []
for i in range(100):
    # Sample different training data
    X_sample = X_train[np.random.choice(len(X_train), 50)]
    y_sample = y_train[np.random.choice(len(y_train), 50)]

    # Train model on this sample
    model = SimpleModel()
    model.fit(X_sample, y_sample)

    # Predict on fixed test point
    pred = model.predict(X_test_point)
    predictions.append(pred)

# High variance = predictions are all over the place
# High bias = predictions are consistently wrong
print(f"Variance: {np.var(predictions):.4f}")
print(f"Bias: {np.mean(predictions) - true_value:.4f}")

from sklearn.linear_model import LinearRegression, LogisticRegression
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.svm import SVC
from sklearn.neighbors import KNeighborsClassifier
from sklearn.cluster import KMeans, DBSCAN
from sklearn.decomposition import PCA
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline

# The universal workflow
pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('classifier', RandomForestClassifier(n_estimators=100))
])

pipeline.fit(X_train, y_train)
accuracy = pipeline.score(X_test, y_test)

# 1. LOAD DATA
import pandas as pd
import numpy as np
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import mean_absolute_error, r2_score
import matplotlib.pyplot as plt

# Generate synthetic student data
np.random.seed(42)
n = 1000
data = pd.DataFrame({
    'hours_studied': np.random.uniform(0, 20, n),
    'previous_grade': np.random.uniform(50, 100, n),
    'attendance': np.random.uniform(60, 100, n),
    'extracurricular': np.random.randint(0, 2, n),
    'sleep_hours': np.random.uniform(4, 10, n),
})

# Create target with realistic relationships
true_score = (
    0.3 * data['hours_studied'] * 5 +
    0.4 * data['previous_grade'] +
    0.2 * data['attendance'] * 0.5 +
    0.1 * np.random.randn(n) * 5
)
data['final_score'] = true_score.clip(0, 100)

# 2. SPLIT
X = data.drop('final_score', axis=1)
y = data['final_score']
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2, random_state=42)

# 3. PREPROCESS
scaler = StandardScaler()
X_train_scaled = scaler.fit_transform(X_train)
X_test_scaled = scaler.transform(X_test)

# 4. TRAIN
model = RandomForestRegressor(n_estimators=100, random_state=42)
model.fit(X_train_scaled, y_train)

# 5. EVALUATE
y_pred = model.predict(X_test_scaled)
print(f"MAE: {mean_absolute_error(y_test, y_pred):.2f} points")
print(f"R²: {r2_score(y_test, y_pred):.3f}")

# 6. INTERPRET
feature_importance = pd.DataFrame({
    'feature': X.columns,
    'importance': model.feature_importances_
}).sort_values('importance', ascending=False)

print("\nWhat matters most for student performance:")
print(feature_importance)

# 7. VISUALIZE
plt.figure(figsize=(12, 4))

plt.subplot(1, 2, 1)
plt.scatter(y_test, y_pred, alpha=0.5)
plt.plot([50, 100], [50, 100], 'r--')
plt.xlabel('Actual Score')
plt.ylabel('Predicted Score')
plt.title('Predictions vs Reality')

plt.subplot(1, 2, 2)
plt.bar(feature_importance['feature'], feature_importance['importance'])
plt.xticks(rotation=45)
plt.title('Feature Importance')
plt.tight_layout()
plt.show()

import numpy as np
import matplotlib.pyplot as plt

x = np.linspace(-5, 5, 100)

# Sigmoid: squashes to (0, 1) — for probabilities
sigmoid = 1 / (1 + np.exp(-x))

# ReLU: max(0, x) — the default for hidden layers
relu = np.maximum(0, x)

# Tanh: squashes to (-1, 1)
tanh = np.tanh(x)

# GELU: modern alternative to ReLU (used in GPT, BERT)
gelu = 0.5 * x * (1 + np.tanh(np.sqrt(2/np.pi) * (x + 0.044715 * x**3)))

plt.figure(figsize=(12, 8))
activations = [
    (sigmoid, 'Sigmoid', 'Used for binary classification output'),
    (tanh, 'Tanh', 'Used in RNNs'),
    (relu, 'ReLU', 'Default for hidden layers'),
    (gelu, 'GELU', 'Used in Transformers/GPT')
]

for i, (act, name, desc) in enumerate(activations, 1):
    plt.subplot(2, 2, i)
    plt.plot(x, act, linewidth=2)
    plt.grid(True, alpha=0.3)
    plt.title(f'{name} — {desc}')
    plt.axhline(y=0, color='gray', linestyle='--', alpha=0.5)
    plt.axvline(x=0, color='gray', linestyle='--', alpha=0.5)

plt.tight_layout()
plt.show()

# Manual backprop for a tiny 2-layer network
import numpy as np

# Input
x = np.array([1.0, 2.0])
y_true = np.array([1.0])

# Initialize random weights
w1 = np.random.randn(2, 3)  # hidden layer weights
b1 = np.random.randn(3)     # hidden layer bias
w2 = np.random.randn(3, 1)  # output layer weights
b2 = np.random.randn(1)     # output layer bias

# --- Forward pass ---
z1 = x @ w1 + b1            # hidden layer
h = np.maximum(0, z1)       # ReLU activation
z2 = h @ w2 + b2            # output layer
y_pred = z2                 # linear output (regression)

loss = 0.5 * (y_pred - y_true) ** 2  # MSE loss

print(f"Prediction: {y_pred.item():.4f}, Truth: {y_true.item():.4f}")
print(f"Loss: {loss.item():.4f}")

# --- Backward pass (manual, no autograd) ---
dloss_dy = y_pred - y_true  # derivative of MSE

# Output layer
dloss_dw2 = h.T @ dloss_dy  # chain rule
dloss_db2 = dloss_dy

# Hidden layer (backprop through ReLU)
dloss_dh = dloss_dy @ w2.T
dloss_dz1 = dloss_dh * (z1 > 0)  # ReLU derivative: 1 if z1 > 0 else 0
dloss_dw1 = np.outer(x, dloss_dz1).reshape(w1.shape)
dloss_db1 = dloss_dz1

print(f"\nGradient for w1:\n{dloss_dw1}")
print(f"Gradient for w2:\n{dloss_dw2}")

import torch
import torch.nn as nn
import torch.optim as optim
import matplotlib.pyplot as plt
import numpy as np

# --- 1. Generate data ---
# Make a non-linear function for the network to learn
X = torch.linspace(-3, 3, 300).reshape(-1, 1)
y = torch.sin(X) + 0.1 * torch.randn_like(X)  # sin wave with noise

# --- 2. Define the network ---
class SimpleNet(nn.Module):
    def __init__(self):
        super().__init__()
        self.net = nn.Sequential(
            nn.Linear(1, 32),    # input → 32 hidden neurons
            nn.ReLU(),           # activation
            nn.Linear(32, 32),   # hidden → hidden
            nn.ReLU(),
            nn.Linear(32, 1),    # hidden → output
        )

    def forward(self, x):
        return self.net(x)

model = SimpleNet()

# --- 3. Set up training ---
optimizer = optim.Adam(model.parameters(), lr=0.01)
loss_fn = nn.MSELoss()

# --- 4. Train ---
losses = []
for epoch in range(3000):
    # Forward
    y_pred = model(X)
    loss = loss_fn(y_pred, y)

    # Backward
    optimizer.zero_grad()
    loss.backward()
    optimizer.step()

    losses.append(loss.item())

    if epoch % 500 == 0:
        print(f"Epoch {epoch}: Loss = {loss.item():.6f}")

# --- 5. Visualize ---
with torch.no_grad():
    X_sorted, _ = X.sort(0)
    y_pred = model(X_sorted)

plt.figure(figsize=(10, 4))
plt.subplot(1, 2, 1)
plt.scatter(X, y, alpha=0.3, label='Data')
plt.plot(X_sorted, y_pred, 'r-', linewidth=2, label='Neural Net')
plt.legend()
plt.title('Neural Network Learning sin(x)')

plt.subplot(1, 2, 2)
plt.plot(losses)
plt.yscale('log')
plt.title('Loss Over Time')
plt.xlabel('Epoch')
plt.tight_layout()
plt.show()

print("✅ Your neural network learned to approximate sin(x)!")

import torch
import torch.nn as nn
import torchvision
import torchvision.transforms as transforms

# --- Load MNIST (handwritten digits) ---
transform = transforms.ToTensor()
trainset = torchvision.datasets.MNIST(root='./data', train=True, download=True, transform=transform)
trainloader = torch.utils.data.DataLoader(trainset, batch_size=64, shuffle=True)

# --- CNN for digit classification ---
class DigitCNN(nn.Module):
    def __init__(self):
        super().__init__()
        self.conv_stack = nn.Sequential(
            # Conv layer 1: 1 → 32 channels
            nn.Conv2d(1, 32, kernel_size=3, padding=1),  # input: 1x28x28 → output: 32x28x28
            nn.ReLU(),
            nn.MaxPool2d(2, 2),  # 32x14x14

            # Conv layer 2: 32 → 64 channels
            nn.Conv2d(32, 64, kernel_size=3, padding=1),  # 64x14x14
            nn.ReLU(),
            nn.MaxPool2d(2, 2),  # 64x7x7
        )

        self.classifier = nn.Sequential(
            nn.Flatten(),
            nn.Linear(64 * 7 * 7, 128),
            nn.ReLU(),
            nn.Dropout(0.5),  # prevent overfitting
            nn.Linear(128, 10),  # 10 digits (0-9)
        )

    def forward(self, x):
        features = self.conv_stack(x)
        return self.classifier(features)

model = DigitCNN()

# --- Train one batch to see it work ---
images, labels = next(iter(trainloader))
output = model(images)
print(f"Input shape:  {images.shape}")   # [64, 1, 28, 28]
print(f"Output shape: {output.shape}")   # [64, 10]
print(f"Predicted:    {output.argmax(1)[:5]}")
print(f"Actual:       {labels[:5]}")

total_params = sum(p.numel() for p in model.parameters())
print(f"Total parameters: {total_params:,}")
# That's ~420K — compared to 196M for fully-connected!

import torch.nn as nn

# PyTorch makes this trivial
lstm = nn.LSTM(
    input_size=100,    # embedding dimension
    hidden_size=256,   # hidden state size
    num_layers=2,      # stacked LSTMs
    batch_first=True
)

# Input: (batch, sequence_length, input_size)
x = torch.randn(32, 50, 100)  # batch of 32, sequence of 50 tokens
output, (h_n, c_n) = lstm(x)
print(f"Output shape: {output.shape}")  # [32, 50, 256]

# During training: randomly drop 50% of neurons
layer = nn.Dropout(0.5)
x = torch.randn(8, 64)
dropped = layer(x)
print(f"Before: {x[0, :5]}")      # all values present
print(f"After:  {dropped[0, :5]}")  # ~50% are zero, rest scaled up

# Keeps activations in a healthy range for each mini-batch
nn.BatchNorm1d(256)  # for 1D features
nn.BatchNorm2d(64)   # for 2D features (images)

def train_model(model, train_loader, val_loader, epochs=100):
    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-3, weight_decay=1e-2)
    scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=epochs)
    best_val_loss = float('inf')

    for epoch in range(epochs):
        # Training
        model.train()
        train_loss = 0
        for X, y in train_loader:
            optimizer.zero_grad()
            loss = nn.CrossEntropyLoss()(model(X), y)
            loss.backward()
            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)  # clip gradients
            optimizer.step()
            train_loss += loss.item()

        # Validation
        model.eval()
        val_loss = 0
        with torch.no_grad():
            for X, y in val_loader:
                val_loss += nn.CrossEntropyLoss()(model(X), y).item()

        scheduler.step()

        # Early stopping
        if val_loss < best_val_loss:
            best_val_loss = val_loss
            torch.save(model.state_dict(), 'best_model.pt')

        if epoch % 10 == 0:
            print(f"Epoch {epoch}: train={train_loss:.4f}, val={val_loss:.4f}")

import torch
import torch.nn.functional as F

def attention(Q, K, V):
    """Simple scaled dot-product attention"""
    # Q, K, V: (batch, seq_len, dim)
    scores = Q @ K.transpose(-2, -1)  # similarity matrix: (batch, seq_len, seq_len)
    scores = scores / (K.size(-1) ** 0.5)  # scale to prevent softmax saturation
    weights = F.softmax(scores, dim=-1)    # attention weights sum to 1
    output = weights @ V  # weighted sum of values
    return output, weights

# Example: attending to a 5-word sequence
Q = K = V = torch.randn(1, 5, 64)  # same for self-attention
output, weights = attention(Q, K, V)
print(f"Attention weights shape: {weights.shape}")  # [1, 5, 5]
print("Each row shows how much each position attends to others:")
print(weights[0].round(decimals=2))

import torch
import torch.nn as nn

class MultiHeadAttention(nn.Module):
    def __init__(self, d_model, n_heads):
        super().__init__()
        self.n_heads = n_heads
        self.d_head = d_model // n_heads

        self.W_q = nn.Linear(d_model, d_model)
        self.W_k = nn.Linear(d_model, d_model)
        self.W_v = nn.Linear(d_model, d_model)
        self.W_o = nn.Linear(d_model, d_model)

    def forward(self, x, mask=None):
        B, L, D = x.shape

        # Linear projections → split into heads
        Q = self.W_q(x).view(B, L, self.n_heads, self.d_head).transpose(1, 2)
        K = self.W_k(x).view(B, L, self.n_heads, self.d_head).transpose(1, 2)
        V = self.W_v(x).view(B, L, self.n_heads, self.d_head).transpose(1, 2)

        # Scaled dot-product attention
        scores = Q @ K.transpose(-2, -1) / (self.d_head ** 0.5)
        if mask is not None:
            scores = scores.masked_fill(mask == 0, -1e9)
        weights = torch.softmax(scores, dim=-1)
        output = weights @ V  # (B, n_heads, L, d_head)

        # Concatenate heads
        output = output.transpose(1, 2).contiguous().view(B, L, D)
        return self.W_o(output)

class TransformerBlock(nn.Module):
    def __init__(self, d_model, n_heads, d_ff):
        super().__init__()
        self.attention = MultiHeadAttention(d_model, n_heads)
        self.norm1 = nn.LayerNorm(d_model)
        self.ff = nn.Sequential(
            nn.Linear(d_model, d_ff),
            nn.ReLU(),
            nn.Linear(d_ff, d_model)
        )
        self.norm2 = nn.LayerNorm(d_model)

    def forward(self, x, mask=None):
        # Self-attention with residual connection
        x = x + self.attention(self.norm1(x), mask)
        # Feed-forward with residual connection
        x = x + self.ff(self.norm2(x))
        return x

# Usage: 512-dim model with 8 heads, 2048-dim FF
block = TransformerBlock(d_model=512, n_heads=8, d_ff=2048)
x = torch.randn(2, 50, 512)  # batch=2, seq_len=50
output = block(x)
print(f"Output shape: {output.shape}")  # [2, 50, 512]

# Sinusoidal position encoding (original transformer)
def sinusoidal_pe(seq_len, d_model):
    pe = torch.zeros(seq_len, d_model)
    position = torch.arange(0, seq_len).unsqueeze(1)
    div_term = torch.exp(torch.arange(0, d_model, 2) * -(torch.log(10000.0) / d_model))
    pe[:, 0::2] = torch.sin(position * div_term)
    pe[:, 1::2] = torch.cos(position * div_term)
    return pe

# RoPE (Rotary Position Embedding) — used in Llama, GPT-NeoX
# Instead of adding position info to the input,
# RoPE rotates the query and key vectors by a position-dependent angle.
# This naturally captures relative position information.

import torch
import torch.nn as nn
import torch.nn.functional as F

class SimpleGNNLayer(nn.Module):
    def __init__(self, in_dim, out_dim):
        super().__init__()
        self.self_weight = nn.Linear(in_dim, out_dim)
        self.neighbor_weight = nn.Linear(in_dim, out_dim)

    def forward(self, x, edge_index, edge_weight=None):
        # x: [num_nodes, in_dim] — node features
        # edge_index: [2, num_edges] — source, target pairs

        src, dst = edge_index  # source → destination edges

        # Gather neighbor messages
        neighbor_msgs = x[src]  # features of source nodes

        # Aggregate (mean over incoming neighbors)
        agg = torch.zeros_like(x)
        agg.index_add_(0, dst, neighbor_msgs)  # sum neighbors per node
        deg = torch.bincount(dst, minlength=x.size(0)).unsqueeze(-1).float()
        agg = agg / deg.clamp(min=1)  # mean

        # Update: self + neighbor
        return F.relu(self.self_weight(x) + self.neighbor_weight(agg))

# At its core, an LLM learns to predict the next token
import torch
import torch.nn as nn

# Given: "The capital of France is ___"
tokens = ["The", "capital", "of", "France", "is"]
# Target: "Paris"

# The model computes:
# P(next_token | "The") = distribution over all 50K tokens
# P(next_token | "The", "capital") = better distribution
# P(next_token | "The", "capital", "of", "France", "is") = peaks at "Paris"

# Loss: cross-entropy between predicted distribution and actual next token
# This is the ONLY training signal for 15 trillion tokens

# Simplified auto-regressive generation
import torch
import torch.nn.functional as F

def generate(model, tokenizer, prompt, max_length=100, temperature=1.0):
    """
    Generate text token by token.

    Temperature controls randomness:
    - temp → 0: always picks most likely token (deterministic)
    - temp = 1: normal sampling
    - temp > 1: more random (creative)
    """
    input_ids = tokenizer.encode(prompt, return_tensors='pt')

    for _ in range(max_length):
        # Forward pass through the entire model
        with torch.no_grad():
            logits = model(input_ids)

        # Get prediction for the NEXT token only
        next_token_logits = logits[0, -1, :] / temperature

        # Convert to probabilities
        probs = F.softmax(next_token_logits, dim=-1)

        # Sample from the distribution
        next_token = torch.multinomial(probs, num_samples=1)

        # Append to sequence
        input_ids = torch.cat([input_ids, next_token.unsqueeze(0)], dim=-1)

        # Optional: stop at end-of-sequence token
        if next_token.item() == tokenizer.eos_token_id:
            break

    return tokenizer.decode(input_ids[0])

# Greedy: always pick the most likely token
# Problem: repetitive, boring, "I love pizza pizza pizza pizza..."

# Temperature sampling: controls sharpness of distribution
# temp = 0.7: good default for creative text
# temp = 0.1: good for factual answers

# Top-k sampling: only consider the k most likely tokens
# top_k = 50: common default

# Top-p (nucleus) sampling: only consider tokens that make up p probability mass
# top_p = 0.9: common default

# Recommended: temperature + top_p together
import torch
import torch.nn.functional as F

def sample_with_top_k_top_p(logits, top_k=50, top_p=0.9, temperature=1.0):
    logits = logits / temperature

    # Top-k filtering
    if top_k > 0:
        indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None]
        logits[indices_to_remove] = -float('Inf')

    # Top-p filtering
    if top_p < 1.0:
        sorted_logits, sorted_indices = torch.sort(logits, descending=True)
        cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)

        sorted_indices_to_remove = cumulative_probs > top_p
        sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
        sorted_indices_to_remove[..., 0] = 0

        indices_to_remove = sorted_indices[sorted_indices_to_remove]
        logits[..., indices_to_remove] = -float('Inf')

    probs = F.softmax(logits, dim=-1)
    return torch.multinomial(probs, 1)

# Zero-shot: just describe the task
prompt_zero = 'Translate to French: "Hello, how are you?"'

# Few-shot: give examples in the prompt  
prompt_few = """
English: "Hello"
French: "Bonjour"

English: "Good morning"
French: "Bonjour"

English: "Thank you"
French: "Merci"

English: "How are you?"
French: 
"""

from diffusers import StableDiffusionPipeline
import torch

# Load the model (download once)
pipe = StableDiffusionPipeline.from_pretrained(
    "runwayml/stable-diffusion-v1-5",
    torch_dtype=torch.float16
)
pipe = pipe.to("cuda")

# Generate an image
prompt = "a photo of a cat wearing a space suit, trending on artstation"
image = pipe(
    prompt,
    num_inference_steps=50,     # more steps = better quality
    guidance_scale=7.5,         # how closely to follow prompt
    negative_prompt="blurry, low quality"  # what to avoid
).images[0]

image.save("astronaut_cat.png")

# Training: match correct image-text pairs
pairs = [
    ("a dog playing fetch", img_dog),
    ("a cat sleeping", img_cat),
    ("a car on a road", img_car),
]

# For each pair: compute image embedding, text embedding
# Maximize cosine similarity for matching pairs
# Minimize for non-matching pairs

# Result: CLIP can tell you if any image matches any text
def classify_with_clip(clip_model, image, class_names):
    """Zero-shot classification using CLIP"""
    image_emb = clip_model.encode_image(image)
    text_embs = clip_model.encode_text(class_names)

    similarities = image_emb @ text_embs.T
    predicted_class = class_names[similarities.argmax()]
    return predicted_class

class LLaVAStyleModel(nn.Module):
    """Simplified multi-modal architecture"""
    def __init__(self, vision_encoder, llm, projector):
        super().__init__()
        self.vision_encoder = vision_encoder  # CLIP ViT
        self.projector = projector             # Linear projection
        self.llm = llm                         # Any LLM

    def forward(self, images, text_tokens):
        # Encode images
        image_features = self.vision_encoder(images)
        image_tokens = self.projector(image_features)

        # Concatenate: [image_tokens, text_tokens]
        combined = torch.cat([image_tokens, text_tokens], dim=1)

        # Process with LLM
        return self.llm(combined)

def agent_loop(model, tools, task, max_steps=10):
    """A simplified agent loop"""
    messages = [{"role": "user", "content": task}]

    for step in range(max_steps):
        # 1. LLM thinks about what to do next
        response = model.generate(messages)

        # 2. Parse action from response
        if "FINAL_ANSWER:" in response:
            return response.split("FINAL_ANSWER:")[-1].strip()

        # 3. Extract tool call (simplified)
        tool_name = extract_tool_name(response)
        tool_args = extract_tool_args(response)

        # 4. Execute tool
        if tool_name in tools:
            result = tools[tool_name](**tool_args)

        # 5. Add result to conversation
        messages.append({"role": "assistant", "content": response})
        messages.append({"role": "tool", "content": str(result)})

    return "Max steps reached"

import torch
import torch.nn as nn
import torch.nn.functional as F

class SparseMoE(nn.Module):
    def __init__(self, d_model, n_experts=8, top_k=2):
        super().__init__()
        self.n_experts = n_experts
        self.top_k = top_k

        # Each expert is a feed-forward network
        self.experts = nn.ModuleList([
            nn.Sequential(
                nn.Linear(d_model, d_model * 4),
                nn.ReLU(),
                nn.Linear(d_model * 4, d_model)
            ) for _ in range(n_experts)
        ])

        # Router: learns which expert to use for each token
        self.router = nn.Linear(d_model, n_experts)

    def forward(self, x):
        B, L, D = x.shape

        # Compute routing probabilities
        routing = self.router(x)  # [B, L, n_experts]
        routing_weights = F.softmax(routing, dim=-1)

        # Top-k routing
        top_k_weights, top_k_indices = torch.topk(routing_weights, self.top_k, dim=-1)
        top_k_weights = top_k_weights / top_k_weights.sum(dim=-1, keepdim=True)

        # Combine expert outputs
        output = torch.zeros_like(x)
        for i in range(self.n_experts):
            # Find which tokens use this expert
            mask = (top_k_indices == i).any(dim=-1)
            if mask.any():
                expert_output = self.experts[i](x[mask])
                # Weight by routing probability
                weights = top_k_weights[top_k_indices == i].unsqueeze(-1)
                output[mask] += expert_output * weights

        return output

# A sparse autoencoder learns to decompose activations into interpretable features
class SparseAutoencoder(nn.Module):
    def __init__(self, activation_dim, feature_dim):
        super().__init__()
        self.encoder = nn.Linear(activation_dim, feature_dim)
        self.decoder = nn.Linear(feature_dim, activation_dim)

    def forward(self, x):
        # Encode: compress to sparse features
        features = torch.relu(self.encoder(x))  # sparsity!

        # Decode: reconstruct activation
        reconstruction = self.decoder(features)

        return reconstruction, features

    def loss(self, x, reconstruction, features, sparsity_coeff=0.001):
        # Reconstruction loss
        recon_loss = ((x - reconstruction) ** 2).mean()
        # Sparsity penalty
        sparsity = sparsity_coeff * torch.abs(features).mean()
        return recon_loss + sparsity

Operation               Notation     Code (NumPy)
─────────────────────────────────────────────────
Vector dot product      a · b        np.dot(a, b) or a @ b
Matrix multiply         A × B        A @ B
Transpose               A^T          A.T
Matrix inverse          A^{-1}       np.linalg.inv(A)
Eigenvalues             λ            np.linalg.eigvals(A)
Singular Value Dec.     UΣV^T        np.linalg.svd(A)
Trace                   tr(A)        np.trace(A)
Determinant             |A|          np.linalg.det(A)
Norm (L2)               ||x||₂       np.linalg.norm(x)

Concept                 Shape        ML Where Used
─────────────────────────────────────────────────
Derivative              f'(x)        Gradients, optimization
Partial derivative      ∂f/∂xᵢ      Gradients for each weight
Gradient                ∇f           The direction of steepest descent
Chain rule              dz/dx =      Backpropagation spine
                        dz/dy · dy/dx
Jacobian                ∂f/∂x matrix Multiple outputs, multiple inputs
Hessian                 ∇²f          Second-order optimization (Adam)
Softmax                 e^{x_i}/Σ    Multi-class probabilities
LogSoftmax              log(softmax) Numerically stable classification

Concept                 Formula              ML Application
────────────────────────────────────────────────────────────
Bayes' Rule            P(A|B) =              Classification, generative models
                       P(B|A)P(A)/P(B)
Expectation            E[X] = ΣxP(x)         Mean prediction
Variance               Var(X)=E[(X-μ)²]      Model uncertainty
Entropy                H(X) = -Σp·log p      Information, loss functions
KL Divergence          KL(P||Q) = Σp·log p/q  Distribution matching (VAEs)
Cross-Entropy          H(P,Q) = -Σp·log q    Classification loss
MLE                    max ΠP(xᵢ|θ)          Training objective
MAP                    max P(θ)ΠP(xᵢ|θ)      Regularized training

Term                    Simple Definition
────────────────────────────────────────────────────────────────
Attention               Mechanism that lets a model focus on relevant parts of input
Autoregressive          Predicting the next token given previous ones
Backpropagation         Algorithm for computing gradients through a neural network
Chain-of-Thought        Prompting technique: "think step by step" for better reasoning
Diffusion Model         Generates data by learning to reverse a noise-addition process
Embedding               Converting discrete tokens to continuous vectors
Fine-tuning             Taking a pretrained model and training it on a specific task
Foundation Model        Large model trained on broad data, adaptable to many tasks
Gradient Descent        Iterative optimization by following the negative gradient
In-Context Learning     Learning from examples in the prompt without parameter updates
KL Divergence           Measure of how one probability distribution differs from another
Logits                  Raw (unnormalized) output scores before softmax
MoE (Mixture of Experts)Architecture where different "experts" activate for different inputs
NTK (Neural Tangent K.) Theory showing wide networks behave like kernel methods
Overfitting             Model memorizes training data, fails on new data
RLHF                    Training AI using human preferences as rewards
Scaling Laws            Empirical relationship between model/data/compute and performance
Self-Attention          Attention where queries, keys, and values all come from the same sequence
Softmax                 Converts scores into a probability distribution
Superposition           When a single neuron represents multiple concepts
Token                   The atomic unit of text for LLMs (word or subword)
Transformer             Architecture based on attention that dominates modern AI
Underfitting            Model is too simple to capture the underlying pattern
VC Dimension            Measure of model capacity — how many points it can shatter