🚫 COPIA BLOCCATA
# REGOLARIZZAZIONE IN PRATICA
import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_moons
from sklearn.model_selection import train_test_split
print("🎯 DEMOSTRAZIONE OVERFITTING E REGOLARIZZAZIONE")
print("=" * 70)
# Creiamo dataset complesso (non linearmente separabile)
X, y = make_moons(n_samples=300, noise=0.3, random_state=42)
print(f"\n📊 DATASET MOONS:")
print(f"• Campioni: {X.shape[0]}")
print(f"• Features: {X.shape[1]}")
print(f"• Classi: {len(np.unique(y))}")
# Split
X_train, X_test, y_train, y_test = train_test_split(
X, y, test_size=0.3, random_state=42
)
X_train, X_val, y_train, y_val = train_test_split(
X_train, y_train, test_size=0.2, random_state=42
)
print(f"\n📊 SPLIT DATI:")
print(f"• Training: {len(X_train)}")
print(f"• Validation: {len(X_val)}")
print(f"• Test: {len(X_test)}")
# Versione della nostra rete neurale con dropout
class RegularizedNeuralNetwork:
"""Rete neurale con tecniche di regolarizzazione"""
def __init__(self, input_size, hidden_size, output_size,
dropout_rate=0.0, l2_lambda=0.0):
self.W1 = np.random.randn(input_size, hidden_size) * np.sqrt(2.0 / input_size)
self.b1 = np.zeros((1, hidden_size))
self.W2 = np.random.randn(hidden_size, output_size) * np.sqrt(2.0 / hidden_size)
self.b2 = np.zeros((1, output_size))
self.dropout_rate = dropout_rate
self.l2_lambda = l2_lambda
self.train_loss = []
self.val_loss = []
self.train_acc = []
self.val_acc = []
def relu(self, x):
return np.maximum(0, x)
def dropout_mask(self, size, dropout_rate):
"""Crea maschera dropout"""
mask = np.random.binomial(1, 1-dropout_rate, size=size) / (1-dropout_rate)
return mask
def forward(self, X, training=False):
"""Forward pass con dropout (solo in training)"""
# Layer 1
self.z1 = np.dot(X, self.W1) + self.b1
self.a1 = self.relu(self.z1)
# Dropout layer 1
if training and self.dropout_rate > 0:
self.dropout_mask1 = self.dropout_mask(self.a1.shape, self.dropout_rate)
self.a1 *= self.dropout_mask1
# Layer 2
self.z2 = np.dot(self.a1, self.W2) + self.b2
# Output (sigmoid per binario)
self.output = 1 / (1 + np.exp(-self.z2))
return self.output
def compute_loss(self, y_true, y_pred):
"""Loss con regolarizzazione L2"""
n_samples = y_true.shape[0]
# Binary cross-entropy
epsilon = 1e-15
y_pred_clipped = np.clip(y_pred, epsilon, 1 - epsilon)
data_loss = -np.mean(y_true * np.log(y_pred_clipped) +
(1 - y_true) * np.log(1 - y_pred_clipped))
# L2 regularization
if self.l2_lambda > 0:
l2_loss = (self.l2_lambda/2) * (np.sum(self.W1**2) + np.sum(self.W2**2))
total_loss = data_loss + l2_loss
else:
total_loss = data_loss
return total_loss, data_loss
def backward(self, X, y_true, y_pred):
"""Backward pass con dropout"""
n_samples = X.shape[0]
# Gradiente output
dz2 = y_pred - y_true.reshape(-1, 1)
# Applica maschera dropout ai gradienti
if hasattr(self, 'dropout_mask1'):
da1 = np.dot(dz2, self.W2.T) * self.dropout_mask1
else:
da1 = np.dot(dz2, self.W2.T)
# Gradiente ReLU
dz1 = da1 * (self.z1 > 0)
# Gradienti con regolarizzazione L2
dW2 = np.dot(self.a1.T, dz2) / n_samples
db2 = np.sum(dz2, axis=0, keepdims=True) / n_samples
dW1 = np.dot(X.T, dz1) / n_samples
db1 = np.sum(dz1, axis=0, keepdims=True) / n_samples
# Aggiungi regolarizzazione L2
if self.l2_lambda > 0:
dW2 += self.l2_lambda * self.W2
dW1 += self.l2_lambda * self.W1
return {'dW1': dW1, 'db1': db1, 'dW2': dW2, 'db2': db2}
def update_parameters(self, gradients, learning_rate):
"""Aggiorna pesi"""
self.W1 -= learning_rate * gradients['dW1']
self.b1 -= learning_rate * gradients['db1']
self.W2 -= learning_rate * gradients['dW2']
self.b2 -= learning_rate * gradients['db2']
def train(self, X_train, y_train, X_val, y_val,
learning_rate=0.01, epochs=1000, early_stopping=True, patience=20):
"""Training con early stopping"""
best_val_loss = float('inf')
patience_counter = 0
best_weights = None
for epoch in range(epochs):
# Forward (training mode)
y_train_pred = self.forward(X_train, training=True)
# Calcola loss training
train_loss, _ = self.compute_loss(y_train, y_train_pred)
self.train_loss.append(train_loss)
# Accuracy training
train_pred_classes = (y_train_pred > 0.5).astype(int).flatten()
train_acc = np.mean(train_pred_classes == y_train)
self.train_acc.append(train_acc)
# Forward validation (no dropout)
y_val_pred = self.forward(X_val, training=False)
val_loss, _ = self.compute_loss(y_val, y_val_pred)
self.val_loss.append(val_loss)
# Accuracy validation
val_pred_classes = (y_val_pred > 0.5).astype(int).flatten()
val_acc = np.mean(val_pred_classes == y_val)
self.val_acc.append(val_acc)
# Backward e update
gradients = self.backward(X_train, y_train, y_train_pred)
self.update_parameters(gradients, learning_rate)
# Early stopping
if early_stopping:
if val_loss < best_val_loss:
best_val_loss = val_loss
patience_counter = 0
# Salva migliori pesi
best_weights = {
'W1': self.W1.copy(), 'b1': self.b1.copy(),
'W2': self.W2.copy(), 'b2': self.b2.copy()
}
else:
patience_counter += 1
if patience_counter >= patience:
print(f"🛑 Early stopping at epoch {epoch+1}")
# Ripristina migliori pesi
if best_weights:
self.W1 = best_weights['W1']
self.b1 = best_weights['b1']
self.W2 = best_weights['W2']
self.b2 = best_weights['b2']
break
# Stampa progresso
if (epoch + 1) % 100 == 0 or epoch == 0:
print(f"Epoca {epoch+1:4d} | "
f"Train Loss: {train_loss:.4f}, Acc: {train_acc:.3f} | "
f"Val Loss: {val_loss:.4f}, Acc: {val_acc:.3f}")
# TESTIAMO DIVERSI LIVELLI DI REGOLARIZZAZIONE
print("\n🔬 CONFRONTO MODELLI CON DIVERSI LIVELLI DI REGOLARIZZAZIONE")
print("=" * 70)
models = {
"Senza regolarizzazione": {
"dropout": 0.0,
"l2_lambda": 0.0,
"early_stopping": False
},
"Con Dropout (0.3)": {
"dropout": 0.3,
"l2_lambda": 0.0,
"early_stopping": False
},
"Con L2 (0.01)": {
"dropout": 0.0,
"l2_lambda": 0.01,
"early_stopping": False
},
"Con Early Stopping": {
"dropout": 0.0,
"l2_lambda": 0.0,
"early_stopping": True
},
"Combinazione (Dropout + L2 + ES)": {
"dropout": 0.2,
"l2_lambda": 0.005,
"early_stopping": True
}
}
results = {}
for name, params in models.items():
print(f"\n🎯 ALLENAMENTO: {name}")
print("-" * 40)
model = RegularizedNeuralNetwork(
input_size=2,
hidden_size=20, # Rete grande per enfatizzare overfitting
output_size=1,
dropout_rate=params["dropout"],
l2_lambda=params["l2_lambda"]
)
model.train(
X_train, y_train, X_val, y_val,
learning_rate=0.1,
epochs=1000,
early_stopping=params["early_stopping"],
patience=30
)
# Valutazione su test set
y_test_pred = model.forward(X_test, training=False)
test_pred_classes = (y_test_pred > 0.5).astype(int).flatten()
test_acc = np.mean(test_pred_classes == y_test)
results[name] = {
"train_acc": model.train_acc[-1],
"val_acc": model.val_acc[-1],
"test_acc": test_acc,
"train_loss": model.train_loss[-1],
"val_loss": model.val_loss[-1]
}
print(f"📊 Test Accuracy: {test_acc:.3f}")
# Mostra risultati comparativi
print("\n📊 RISULTATI COMPARATIVI:")
print("=" * 70)
print(f"{'Modello':30s} {'Train Acc':>10s} {'Val Acc':>10s} {'Test Acc':>10s} {'Gap':>10s}")
print("-" * 70)
for name, result in results.items():
train_val_gap = result["train_acc"] - result["val_acc"]
print(f"{name:30s} {result['train_acc']:10.3f} {result['val_acc']:10.3f} "
f"{result['test_acc']:10.3f} {train_val_gap:10.3f}")
print("\n💡 CONCLUSIONI:")
print("1. Il modello senza regolarizzazione ha il gap più grande (overfitting)")
print("2. Dropout riduce il gap ma può ridurre accuracy training")
print("3. L2 regularization rende il modello più 'semplice'")
print("4. Early stopping previene over-training")
print("5. La combinazione di tecniche dà i migliori risultati")
# Visualizzazione (scommenta su Colab)
"""
fig, axes = plt.subplots(2, 3, figsize=(15, 10))
# Re-allena per visualizzazione
for idx, (name, params) in enumerate(list(models.items())[:5]):
model = RegularizedNeuralNetwork(2, 20, 1, params["dropout"], params["l2_lambda"])
model.train(X_train, y_train, X_val, y_val,
learning_rate=0.1, epochs=1000,
early_stopping=params["early_stopping"], patience=30)
row, col = divmod(idx, 3)
# Plot decision boundary
h = 0.02
x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
xx, yy = np.meshgrid(np.arange(x_min, x_max, h),
np.arange(y_min, y_max, h))
Z = model.forward(np.c_[xx.ravel(), yy.ravel()], training=False)
Z = (Z > 0.5).astype(int).reshape(xx.shape)
axes[row, col].contourf(xx, yy, Z, alpha=0.8, cmap=plt.cm.RdYlBu)
axes[row, col].scatter(X_train[:, 0], X_train[:, 1], c=y_train,
edgecolors='k', cmap=plt.cm.RdYlBu, alpha=0.6)
axes[row, col].set_title(f'{name}\nTest Acc: {results[name]["test_acc"]:.3f}')
axes[row, col].set_xlim(xx.min(), xx.max())
axes[row, col].set_ylim(yy.min(), yy.max())
# Plot loss curves per l'ultimo modello
axes[1, 2].plot(model.train_loss, label='Train Loss')
axes[1, 2].plot(model.val_loss, label='Val Loss')
axes[1, 2].set_title('Loss durante training\n(Combinazione tecniche)')
axes[1, 2].set_xlabel('Epoca')
axes[1, 2].set_ylabel('Loss')
axes[1, 2].legend()
axes[1, 2].grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
"""