End-to-End Data Science Project

import pandas as pd
import numpy as np
from scipy import stats
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.model_selection import train_test_split
from sklearn.linear_model import LogisticRegression
from sklearn.metrics import (classification_report, roc_auc_score,
                              confusion_matrix, precision_recall_fscore_support)

# ── Raw dataset — intentional data quality issues ─────────────────────────────
raw = pd.DataFrame({
    "app_id":       [f"A{i:03d}" for i in range(1, 26)],
    "age":          [34,28,51,np.nan,44,39,np.nan,62,29,47,
                     33,55,41,36,48,27,59,43,31,52,
                     38,46,np.nan,37,50],
    "income":       [42000,31000,np.nan,58000,290000,47000,35000,np.nan,29000,61000,
                     np.nan,72000,38000,44000,55000,27000,81000,39000,33000,68000,
                     41000,52000,36000,np.nan,63000],
    "loan_amount":  [8000,12000,5000,15000,9000,11000,7000,18000,6000,14000,
                     10000,20000,8500,9500,13000,5500,16000,7500,6500,12500,
                     9000,11500,8000,7000,15000],
    "credit_score": [680,610,720,np.nan,740,695,620,np.nan,590,730,
                     660,np.nan,700,675,715,580,760,690,625,745,
                     685,710,635,670,720],
    "employment":   ["full_time","part_time","full_time","self_employed","full_time",
                     "full_time","part_time","full_time",None,"self_employed",
                     "full_time","full_time","part_time","full_time","self_employed",
                     "unemployed","full_time","full_time","part_time","full_time",
                     "full_time","self_employed","part_time","full_time","full_time"],
    "dependents":   [1,2,0,3,1,0,2,1,3,0,
                     2,1,0,1,2,3,0,1,2,0,
                     1,2,1,0,2],
    "late_payments":[0,1,0,2,0,0,1,0,3,0,
                     1,0,0,0,1,2,0,0,1,0,
                     0,1,2,0,0],
    "default":      [0,1,0,1,0,0,1,0,1,0,
                     1,0,0,0,1,1,0,0,1,0,
                     0,1,1,0,0],
})

# ── Data quality audit ────────────────────────────────────────────────────────
numeric_cols     = ["age","income","loan_amount","credit_score","dependents","late_payments"]
categorical_cols = ["employment"]

print("DATA QUALITY AUDIT — ORBIT LENDING BROKER FEED")
print(f"Shape: {raw.shape[0]} rows × {raw.shape[1]} columns\n")
print(f"{'Column':<16} {'dtype':<10} {'Nulls':>6} {'Null%':>7} "
      f"{'Unique':>7} {'Min':>10} {'Max':>10} {'Mean':>10}")
print("─" * 82)

for col in raw.columns:
    if col == "app_id":
        continue
    nulls   = raw[col].isna().sum()
    null_pct= nulls / len(raw) * 100
    unique  = raw[col].nunique(dropna=False)
    dtype   = str(raw[col].dtype)
    if raw[col].dtype in [np.float64, np.int64]:
        mn  = f"{raw[col].min():.0f}"
        mx  = f"{raw[col].max():.0f}"
        mu  = f"{raw[col].mean():.0f}"
    else:
        mn, mx, mu = "—", "—", "—"
    flag = " ⚠" if nulls > 0 else ""
    print(f"  {col:<14} {dtype:<10} {nulls:>6} {null_pct:>6.1f}% "
          f"{unique:>7} {mn:>10} {mx:>10} {mu:>10}{flag}")

print(f"\nTarget class balance:")
vc = raw["default"].value_counts()
print(f"  Non-default (0): {vc[0]}  ({vc[0]/len(raw)*100:.1f}%)")
print(f"  Default     (1): {vc[1]}  ({vc[1]/len(raw)*100:.1f}%)")

df = raw.copy()

# ── Documented imputation strategy ───────────────────────────────────────────
imputation_log = {}

# age: median imputation (skewed right from older applicants)
age_median = df["age"].median()
df["age"].fillna(age_median, inplace=True)
imputation_log["age"] = f"median imputation ({age_median:.0f})"

# income: cap outlier at 99th pct first, then median-impute missing
income_p99 = df["income"].quantile(0.99)
df["income"] = df["income"].clip(upper=income_p99)
income_median = df["income"].median()
df["income"].fillna(income_median, inplace=True)
imputation_log["income"] = (f"capped at 99th pct (£{income_p99:,.0f}), "
                             f"then median-imputed (£{income_median:,.0f})")

# credit_score: median imputation
cs_median = df["credit_score"].median()
df["credit_score"].fillna(cs_median, inplace=True)
imputation_log["credit_score"] = f"median imputation ({cs_median:.0f})"

# employment: mode imputation for single missing row
emp_mode = df["employment"].mode()[0]
df["employment"].fillna(emp_mode, inplace=True)
imputation_log["employment"] = f"mode imputation ('{emp_mode}')"

# ── Print cleaning log ────────────────────────────────────────────────────────
print("Cleaning and Imputation Log:")
for col, strategy in imputation_log.items():
    print(f"  {col:<16}  →  {strategy}")

# ── Post-cleaning audit ───────────────────────────────────────────────────────
print(f"\nPost-cleaning null check:")
nulls_after = df[["age","income","credit_score","employment"]].isna().sum()
for col, n in nulls_after.items():
    status = "✓ clean" if n == 0 else f"⚠ {n} nulls remain"
    print(f"  {col:<16}  {status}")

print(f"\nIncome after capping: min=£{df['income'].min():,.0f}  "
      f"max=£{df['income'].max():,.0f}  mean=£{df['income'].mean():,.0f}")
print(f"(Was: max=£290,000  mean=£51,682 — outlier removed)")

# ── Default rate by employment type ──────────────────────────────────────────
print("Default rate by employment type:")
emp_dr = (df.groupby("employment")["default"]
            .agg(applicants="count", defaults="sum")
            .assign(default_rate=lambda x: (x["defaults"]/x["applicants"]*100).round(1))
            .sort_values("default_rate", ascending=False))
print(emp_dr.to_string())

# ── Default rate by credit score band ────────────────────────────────────────
df["cs_band"] = pd.cut(df["credit_score"],
                        bins=[0,620,660,700,740,999],
                        labels=["<620","620–659","660–699","700–739","740+"])
print("\nDefault rate by credit score band:")
cs_dr = (df.groupby("cs_band", observed=True)["default"]
           .agg(applicants="count", defaults="sum")
           .assign(default_rate=lambda x: (x["defaults"]/x["applicants"]*100).round(1)))
print(cs_dr.to_string())

# ── Default rate by late payment count ───────────────────────────────────────
print("\nDefault rate by late payment count:")
lp_dr = (df.groupby("late_payments")["default"]
           .agg(applicants="count", defaults="sum")
           .assign(default_rate=lambda x: (x["defaults"]/x["applicants"]*100).round(1)))
print(lp_dr.to_string())

# ── Univariate correlation with default (point-biserial = Pearson for binary) ─
print("\nUnivariate correlation with default (|r|, ranked):")
num_feats = ["credit_score","late_payments","income","age","loan_amount","dependents"]
corr_rows = []
for feat in num_feats:
    r, p = stats.pearsonr(df[feat], df["default"])
    corr_rows.append({"feature": feat, "r": round(r, 4), "abs_r": abs(r), "p": round(p,4)})
corr_df = pd.DataFrame(corr_rows).sort_values("abs_r", ascending=False).reset_index(drop=True)
for _, row in corr_df.iterrows():
    direction = "↑ default risk" if row["r"] > 0 else "↓ default risk"
    sig = "***" if row["p"]<0.001 else "**" if row["p"]<0.01 else "*" if row["p"]<0.05 else "ns"
    print(f"  {row['feature']:<16} r={row['r']:+.4f}  {sig}  {direction}")

LOAN_TERM_MONTHS = 36       # assumed loan term for repayment calculation
ANNUAL_RATE      = 0.12     # assumed annual interest rate

# ── Monthly repayment: standard annuity formula ───────────────────────────────
# PMT = P × r(1+r)^n / ((1+r)^n − 1)
r_monthly = ANNUAL_RATE / 12
n         = LOAN_TERM_MONTHS
df["monthly_repayment"] = (df["loan_amount"] * r_monthly
                           * (1 + r_monthly)**n
                           / ((1 + r_monthly)**n - 1)).round(2)

# ── Engineered features ───────────────────────────────────────────────────────
df["debt_to_income"]       = (df["loan_amount"] / df["income"]).round(4)
df["loan_to_income"]       = (df["loan_amount"] / df["income"]).round(4)  # alias here
df["repayment_to_income"]  = (df["monthly_repayment"] / (df["income"]/12)).round(4)
df["income_per_dependent"] = np.where(
    df["dependents"] > 0,
    (df["income"] / df["dependents"]).round(0),
    df["income"]   # no dependents — full income available
)
df["has_late_payment"]     = (df["late_payments"] > 0).astype(int)

engineered = ["debt_to_income","repayment_to_income",
              "income_per_dependent","has_late_payment"]

# ── Correlation check: engineered vs default ──────────────────────────────────
print("Engineered feature correlations with default:")
print(f"{'Feature':<24} {'r':>8} {'p':>8} {'Signal'}")
print("─" * 56)
for feat in engineered:
    r, p = stats.pearsonr(df[feat], df["default"])
    sig = "***" if p<0.001 else "**" if p<0.01 else "*" if p<0.05 else "ns"
    direction = "↑" if r > 0 else "↓"
    print(f"  {feat:<22} {r:>+8.4f} {p:>8.4f}  {sig} {direction}")

# ── Preview engineered features ───────────────────────────────────────────────
print(f"\nEngineered feature preview (first 6 rows):")
preview_cols = ["app_id","debt_to_income","repayment_to_income",
                "income_per_dependent","has_late_payment","default"]
print(df[preview_cols].head(6).to_string(index=False))

# ── Final feature selection ───────────────────────────────────────────────────
feature_cols = ["credit_score","late_payments","income","age","loan_amount",
                "dependents","employment",
                "debt_to_income","repayment_to_income",
                "income_per_dependent","has_late_payment"]

TARGET = "default"
X_raw  = df[feature_cols].copy()
y      = df[TARGET].copy()

# ── One-hot encode employment (drop_first to avoid multicollinearity) ─────────
X_enc = pd.get_dummies(X_raw, columns=["employment"], drop_first=True)

feature_names = X_enc.columns.tolist()

# ── StandardScaler on all features ───────────────────────────────────────────
scaler  = StandardScaler()
X_scaled = scaler.fit_transform(X_enc)
X_scaled = pd.DataFrame(X_scaled, columns=feature_names)

# ── Stratified train/test split — preserve class balance ─────────────────────
X_train, X_test, y_train, y_test = train_test_split(
    X_scaled, y,
    test_size=0.30,
    random_state=42,
    stratify=y         # ensures same 68/32 split in both sets
)

print("Feature Matrix Summary:")
print(f"  Total features:      {len(feature_names)}")
print(f"  After OHE encoding:  {X_scaled.shape[1]} columns (employment expanded)")
print(f"\n  Features: {feature_names}")

print(f"\nTrain/Test Split (stratified, 70/30):")
print(f"  Train:  {len(X_train)} rows  — "
      f"default rate: {y_train.mean()*100:.1f}%")
print(f"  Test:   {len(X_test)}  rows  — "
      f"default rate: {y_test.mean()*100:.1f}%")
print(f"  Class balance preserved: "
      f"{'YES' if abs(y_train.mean()-y_test.mean()) < 0.05 else 'NO'}")

# ── Logistic regression baseline ─────────────────────────────────────────────
lr = LogisticRegression(random_state=42, max_iter=1000, C=0.5)
lr.fit(X_train, y_train)

y_pred      = lr.predict(X_test)
y_pred_prob = lr.predict_proba(X_test)[:,1]   # probability of default

# ── Metrics ───────────────────────────────────────────────────────────────────
roc_auc = roc_auc_score(y_test, y_pred_prob)
gini    = 2 * roc_auc - 1   # Gini = 2 × AUC − 1 (industry standard in credit risk)
cm      = confusion_matrix(y_test, y_pred)
prec, rec, f1, _ = precision_recall_fscore_support(y_test, y_pred, average="binary")

print("Baseline Logistic Regression — Test Set Performance:")
print(f"  ROC-AUC:   {roc_auc:.4f}  (random = 0.500)")
print(f"  Gini:      {gini:.4f}   (industry threshold ≥ 0.30 for credit models)")
print(f"  Precision: {prec:.4f}")
print(f"  Recall:    {rec:.4f}")
print(f"  F1-score:  {f1:.4f}")

print(f"\nConfusion Matrix (test set, n=8):")
print(f"                  Predicted 0  Predicted 1")
print(f"  Actual 0 (TN/FP):    {cm[0,0]:>4}         {cm[0,1]:>4}")
print(f"  Actual 1 (FN/TP):    {cm[1,0]:>4}         {cm[1,1]:>4}")
print(f"\n  False Negatives (missed defaults): {cm[1,0]}")
print(f"  False Positives (wrongly declined): {cm[0,1]}")

# ── Feature coefficients (log-odds — positive = higher default probability) ───
coef_df = (pd.DataFrame({"feature": feature_names, "coef": lr.coef_[0]})
             .reindex(pd.Series(lr.coef_[0]).abs().sort_values(ascending=False).index)
             .reset_index(drop=True))
coef_df = coef_df.sort_values("coef", key=abs, ascending=False).head(6)

print(f"\nTop 6 feature coefficients (log-odds scale):")
for _, row in coef_df.iterrows():
    direction = "↑ default" if row["coef"]>0 else "↓ default"
    print(f"  {row['feature']:<26}  {row['coef']:>+7.4f}  {direction}")