# Split FIRST, then fit only on training
X_train, X_test, y_train, y_test = train_test_split(X, y)

scaler = StandardScaler()
X_train = scaler.fit_transform(X_train)
X_test = scaler.transform(X_test)  # Only transform!

# Use appropriate metrics
from sklearn.metrics import classification_report, roc_auc_score

print(classification_report(y_test, y_pred))
print(f"ROC-AUC: {roc_auc_score(y_test, y_prob):.4f}")
print(f"F1: {f1_score(y_test, y_pred):.4f}")

# Temporal split - train on past, test on future
split_date = '2024-01-01'
train_mask = df['date'] < split_date

X_train = X[train_mask]
X_test = X[~train_mask]

# Or use TimeSeriesSplit for CV
from sklearn.model_selection import TimeSeriesSplit
tscv = TimeSeriesSplit(n_splits=5)

from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import make_pipeline

svm = make_pipeline(StandardScaler(), SVC())
svm.fit(X_train, y_train)

# Calculate on training data only
train_means = X_train.groupby('customer_type')['purchase'].mean()
X_train['avg_purchase_type'] = X_train['customer_type'].map(train_means)
X_test['avg_purchase_type'] = X_test['customer_type'].map(train_means)

# Strategy 1: Impute
from sklearn.impute import SimpleImputer
imputer = SimpleImputer(strategy='median')

# Strategy 2: Create missing indicator
df['feature_missing'] = df['feature'].isna().astype(int)
df['feature'] = df['feature'].fillna(df['feature'].median())

# Strategy 3: Drop only if too many missing
threshold = 0.5
cols_to_drop = df.columns[df.isna().mean() > threshold]

# Use nested cross-validation for honest estimate
from sklearn.model_selection import cross_val_score, GridSearchCV

# Inner loop: hyperparameter tuning
# Outer loop: performance estimation
outer_scores = cross_val_score(
    GridSearchCV(model, param_grid, cv=3),
    X, y, cv=5
)
print(f"Honest estimate: {outer_scores.mean():.4f}")

# Monitor distribution shifts
def check_drift(reference_data, new_data, threshold=0.1):
    for col in reference_data.columns:
        ref_mean = reference_data[col].mean()
        new_mean = new_data[col].mean()
        shift = abs(ref_mean - new_mean) / (ref_mean + 1e-10)
        
        if shift > threshold:
            print(f"⚠️ Drift detected in {col}: {shift:.1%}")

# Monitor predictions
def monitor_predictions(model, X_new):
    probs = model.predict_proba(X_new)[:, 1]
    if probs.mean() > historical_mean + 0.1:
        alert("Prediction distribution has shifted!")

# Strategy 1: Target encoding
city_means = df.groupby('city')['target'].mean()
df['city_encoded'] = df['city'].map(city_means)

# Strategy 2: Frequency encoding
city_freq = df['city'].value_counts(normalize=True)
df['city_freq'] = df['city'].map(city_freq)

# Strategy 3: Group rare categories
top_cities = df['city'].value_counts().head(50).index
df['city_grouped'] = df['city'].where(df['city'].isin(top_cities), 'Other')

from sklearn.model_selection import StratifiedKFold

# Stratified CV preserves class ratios in each fold
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
scores = cross_val_score(model, X, y, cv=cv)

import numpy as np

# Set seeds everywhere
RANDOM_STATE = 42

np.random.seed(RANDOM_STATE)

model = RandomForestClassifier(random_state=RANDOM_STATE)
X_train, X_test = train_test_split(X, y, random_state=RANDOM_STATE)

# Feature selection inside cross-validation or on train only
from sklearn.pipeline import Pipeline

pipeline = Pipeline([
    ('selector', SelectKBest(k=10)),
    ('classifier', RandomForestClassifier())
])

# Selector fit only on training folds
scores = cross_val_score(pipeline, X, y, cv=5)

# Use median for skewed distributions
df['income'].fillna(df['income'].median())

# Or use log transform first
import numpy as np
df['log_income'] = np.log1p(df['income'])
df['log_income'].fillna(df['log_income'].median())

from sklearn.model_selection import GridSearchCV

param_grid = {
    'n_estimators': [50, 100, 200],
    'max_depth': [3, 5, 7, 10],
    'learning_rate': [0.01, 0.1, 0.2]
}

grid = GridSearchCV(
    GradientBoostingClassifier(),
    param_grid,
    cv=5,
    scoring='roc_auc'
)
grid.fit(X_train, y_train)
print(f"Best params: {grid.best_params_}")

# Always compare to baselines -- this takes 5 minutes and saves weeks
from sklearn.dummy import DummyClassifier

# Baseline 1: Random guessing (the absolute floor)
dummy = DummyClassifier(strategy='stratified')
print(f"Random baseline: {cross_val_score(dummy, X, y).mean():.4f}")

# Baseline 2: Simple model (the "is ML even needed?" check)
lr = LogisticRegression()
print(f"Logistic Regression: {cross_val_score(lr, X, y).mean():.4f}")

# Now try complex model
complex_model = GradientBoostingClassifier(n_estimators=200)
print(f"Complex model: {cross_val_score(complex_model, X, y).mean():.4f}")

# The question is not "is the complex model good?"
# It's "is the improvement over the simple model worth the extra complexity?"