Feature Engineering for Fraud Detection

import pandas as pd
import numpy as np
from scipy import stats

# ── Raw transaction data ─────────────────────────────────────────────────────
df = pd.DataFrame({
    "txn_id":      [f"T{i:03d}" for i in range(1, 21)],
    "account_id":  ["ACC001","ACC002","ACC001","ACC003","ACC001",
                    "ACC002","ACC003","ACC004","ACC001","ACC002",
                    "ACC003","ACC005","ACC004","ACC002","ACC003",
                    "ACC005","ACC004","ACC001","ACC005","ACC004"],
    "timestamp":   ["2024-03-15 02:14","2024-03-15 09:32","2024-03-15 02:47",
                    "2024-03-15 14:22","2024-03-15 03:01","2024-03-15 10:15",
                    "2024-03-15 16:44","2024-03-15 11:30","2024-03-15 03:22",
                    "2024-03-16 08:45","2024-03-16 13:10","2024-03-16 09:20",
                    "2024-03-16 14:55","2024-03-16 11:05","2024-03-16 15:30",
                    "2024-03-16 10:40","2024-03-16 16:20","2024-03-17 01:50",
                    "2024-03-17 11:15","2024-03-17 13:45"],
    "amount":      [847.50, 42.80, 623.20, 128.60, 991.00, 38.40, 94.20,
                    215.30, 775.60, 51.20, 142.80, 88.60, 198.40, 44.10,
                    118.70, 92.30, 187.60, 654.90, 78.40, 221.50],
    "merchant_cat":["Electronics","Grocery","Electronics","Restaurant","Jewellery",
                    "Grocery","Pharmacy","Clothing","Electronics","Grocery",
                    "Restaurant","Fuel","Clothing","Grocery","Restaurant",
                    "Fuel","Clothing","Electronics","Entertainment","Clothing"],
    "channel":     ["Online","In-store","Online","In-store","Online",
                    "In-store","In-store","Online","Online","In-store",
                    "In-store","In-store","Online","In-store","In-store",
                    "In-store","Online","Online","Online","Online"],
    "is_fraud":    [1,0,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0]
})

# ── Step 1: Parse datetime and extract temporal features ─────────────────────
df["timestamp"]   = pd.to_datetime(df["timestamp"])
df["hour"]        = df["timestamp"].dt.hour
df["day_of_week"] = df["timestamp"].dt.dayofweek        # 0=Mon … 6=Sun
df["is_weekend"]  = (df["day_of_week"] >= 5).astype(int)
df["is_night"]    = ((df["hour"] >= 22) | (df["hour"] < 6)).astype(int)
df["is_online"]   = (df["channel"] == "Online").astype(int)

# Validate: temporal separation between fraud and legit
print("Temporal feature means — Fraud vs Legitimate:")
for feat in ["hour","is_weekend","is_night","is_online"]:
    fraud_mean = df[df["is_fraud"]==1][feat].mean()
    legit_mean = df[df["is_fraud"]==0][feat].mean()
    print(f"  {feat:<14}  Fraud: {fraud_mean:.2f}  Legit: {legit_mean:.2f}")

print(f"\nTransaction count by hour block:")
df["hour_block"] = pd.cut(df["hour"], bins=[0,6,12,18,24],
                          labels=["Night(0-6)","Morning(6-12)","Afternoon(12-18)","Evening(18-24)"],
                          right=False)
print(df.groupby("hour_block", observed=True)["is_fraud"].agg(
    transactions="count", fraud_count="sum", fraud_rate="mean"
).round(2).to_string())

# ── groupby + transform: broadcast account statistics to every row ────────────
df["acct_mean_amount"] = df.groupby("account_id")["amount"].transform("mean").round(2)
df["acct_std_amount"]  = df.groupby("account_id")["amount"].transform("std").round(2)
df["acct_txn_count"]   = df.groupby("account_id")["txn_id"].transform("count")
df["acct_max_amount"]  = df.groupby("account_id")["amount"].transform("max").round(2)

# ── Amount deviation: z-score relative to account baseline ───────────────────
df["amount_zscore"] = (
    (df["amount"] - df["acct_mean_amount"]) /
    df["acct_std_amount"].replace(0, np.nan)   # avoid division by zero
).round(3)

# ── Amount ratio: amount / account mean ──────────────────────────────────────
df["amount_ratio"] = (df["amount"] / df["acct_mean_amount"]).round(3)

# ── Log-transform of amount (stabilises right-skewed distribution) ───────────
df["log_amount"] = np.log1p(df["amount"]).round(4)

print("Account baseline features (sample — 8 rows):")
cols = ["txn_id","account_id","amount","acct_mean_amount","acct_std_amount",
        "amount_zscore","amount_ratio","is_fraud"]
print(df[cols].head(8).to_string(index=False))

# Validate: z-score separation between fraud and legitimate
print("\nAmount z-score — Fraud vs Legitimate:")
fraud_z = df[df["is_fraud"]==1]["amount_zscore"]
legit_z = df[df["is_fraud"]==0]["amount_zscore"]
print(f"  Fraud mean z-score:  {fraud_z.mean():+.3f}  (std: {fraud_z.std():.3f})")
print(f"  Legit mean z-score:  {legit_z.mean():+.3f}  (std: {legit_z.std():.3f})")
t_stat, p_val = stats.ttest_ind(fraud_z.dropna(), legit_z.dropna())
print(f"  t-test: t={t_stat:.3f}  p={p_val:.4f}")

# Sort by account and time — required for time-window calculations
df = df.sort_values(["account_id","timestamp"]).reset_index(drop=True)

# Velocity: count of transactions per account in preceding N hours
def velocity_in_window(group, hours):
    """For each row in group, count transactions within the preceding N hours."""
    counts = []
    timestamps = group["timestamp"].values
    for i, ts in enumerate(timestamps):
        window_start = ts - pd.Timedelta(hours=hours)
        # Count transactions strictly before this one (not including itself)
        count = sum(1 for j, t in enumerate(timestamps) if j < i and t >= window_start)
        counts.append(count)
    return counts

# Apply velocity function per account group
df["velocity_2h"]  = df.groupby("account_id", group_keys=False).apply(
    lambda g: pd.Series(velocity_in_window(g, hours=2), index=g.index)
)
df["velocity_24h"] = df.groupby("account_id", group_keys=False).apply(
    lambda g: pd.Series(velocity_in_window(g, hours=24), index=g.index)
)

print("Velocity features — Fraud vs Legitimate:")
for feat in ["velocity_2h","velocity_24h"]:
    f_mean = df[df["is_fraud"]==1][feat].mean()
    l_mean = df[df["is_fraud"]==0][feat].mean()
    print(f"  {feat:<14}  Fraud mean: {f_mean:.2f}  Legit mean: {l_mean:.2f}")

print("\nVelocity detail for ACC001 (fraudulent account):")
acc1 = df[df["account_id"]=="ACC001"][
    ["txn_id","timestamp","amount","velocity_2h","velocity_24h","is_fraud"]
]
print(acc1.to_string(index=False))

# Statistical test: velocity_2h
u, p = stats.mannwhitneyu(
    df[df["is_fraud"]==1]["velocity_2h"],
    df[df["is_fraud"]==0]["velocity_2h"],
    alternative="two-sided"
)
print(f"\nMann-Whitney U — velocity_2h fraud vs legit: U={u:.0f}  p={p:.4f}")

# ── Merchant category novelty: appears only once per account ─────────────────
cat_counts = df.groupby(["account_id","merchant_cat"])["txn_id"].transform("count")
df["cat_is_novel"] = (cat_counts == 1).astype(int)

# ── Account online transaction rate ──────────────────────────────────────────
df["acct_online_rate"] = df.groupby("account_id")["is_online"].transform("mean").round(3)

# ── Account fraud rate (ground truth — for validation only, never a model feature) ──
df["acct_fraud_rate"] = df.groupby("account_id")["is_fraud"].transform("mean").round(3)

# ── Composite risk score: weighted combination of engineered features ─────────
# Weights derived from correlation with is_fraud (Step 5 previews this)
df["risk_score"] = (
    df["is_night"]       * 0.35 +
    df["is_online"]      * 0.25 +
    df["velocity_2h"].clip(upper=5) / 5 * 0.25 +   # cap at 5, normalise 0-1
    df["amount_zscore"].clip(lower=0, upper=3) / 3 * 0.15   # positive z only
).round(4)

print("Novelty and composite features — sample:")
cols2 = ["txn_id","account_id","merchant_cat","cat_is_novel",
         "acct_online_rate","velocity_2h","risk_score","is_fraud"]
print(df[cols2].to_string(index=False))

# Risk score separation
print(f"\nComposite risk score — Fraud vs Legitimate:")
print(f"  Fraud mean:  {df[df['is_fraud']==1]['risk_score'].mean():.3f}")
print(f"  Legit mean:  {df[df['is_fraud']==0]['risk_score'].mean():.3f}")

# At threshold 0.5 — precision and recall
tp = ((df["risk_score"] >= 0.5) & (df["is_fraud"]==1)).sum()
fp = ((df["risk_score"] >= 0.5) & (df["is_fraud"]==0)).sum()
fn = ((df["risk_score"] <  0.5) & (df["is_fraud"]==1)).sum()
precision = tp / (tp + fp) if (tp+fp) > 0 else 0
recall    = tp / (tp + fn) if (tp+fn) > 0 else 0
print(f"\nAt threshold 0.5: Precision={precision:.2f}  Recall={recall:.2f}  TP={tp} FP={fp} FN={fn}")

# ── Feature correlation with is_fraud ────────────────────────────────────────
feature_cols = [
    "is_night","is_online","velocity_2h","velocity_24h",
    "amount_zscore","amount_ratio","log_amount","cat_is_novel",
    "acct_online_rate","acct_txn_count","hour","is_weekend"
]

print("Feature correlations with is_fraud (ranked by absolute value):")
corr_series = df[feature_cols + ["is_fraud"]].corr()["is_fraud"].drop("is_fraud")
corr_ranked = corr_series.abs().sort_values(ascending=False)
for feat in corr_ranked.index:
    r = corr_series[feat]
    bar = "█" * int(abs(r) * 20)
    print(f"  {feat:<20} r = {r:+.3f}  {bar}")

# ── Final feature matrix ──────────────────────────────────────────────────────
feature_matrix = df[["txn_id"] + feature_cols + ["is_fraud"]].copy()

# Fill NaN z-scores (single-transaction accounts) with 0
feature_matrix["amount_zscore"] = feature_matrix["amount_zscore"].fillna(0)

print(f"\nFinal feature matrix: {feature_matrix.shape[0]} rows × {len(feature_cols)} features")
print(f"Fraud transactions: {feature_matrix['is_fraud'].sum()} of {len(feature_matrix)}")
print(f"\nFeature matrix (fraud transactions only):")
print(feature_matrix[feature_matrix["is_fraud"]==1][
    ["txn_id","is_night","is_online","velocity_2h","amount_zscore","risk_score"]
].to_string(index=False))

# ── Data quality checks ───────────────────────────────────────────────────────
print("Feature matrix quality checks:")
print(f"  Shape:          {feature_matrix.shape}")
print(f"  Missing values: {feature_matrix.isnull().sum().sum()}")
print(f"  Duplicate rows: {feature_matrix.duplicated().sum()}")

# ── Class imbalance ───────────────────────────────────────────────────────────
n_fraud = feature_matrix["is_fraud"].sum()
n_total = len(feature_matrix)
imbalance_ratio = (n_total - n_fraud) / n_fraud
print(f"\nClass distribution:")
print(f"  Legitimate: {n_total - n_fraud} ({(n_total-n_fraud)/n_total:.1%})")
print(f"  Fraud:      {n_fraud} ({n_fraud/n_total:.1%})")
print(f"  Imbalance ratio: {imbalance_ratio:.1f}:1 — recommend SMOTE or class_weight='balanced'")

# ── Feature statistics by class ───────────────────────────────────────────────
top_features = ["is_night","is_online","velocity_2h","amount_zscore"]
print("\nTop feature means — Fraud vs Legitimate:")
for feat in top_features:
    f = feature_matrix[feature_matrix["is_fraud"]==1][feat].mean()
    l = feature_matrix[feature_matrix["is_fraud"]==0][feat].mean()
    print(f"  {feat:<20}  Fraud: {f:+.3f}   Legit: {l:+.3f}   Delta: {f-l:+.3f}")

# ── Feature engineering summary for model team ────────────────────────────────
print("\nFeature engineering summary:")
print(f"  Total features engineered: {len(feature_cols)}")
print(f"  Recommended for training:  is_night, is_online, velocity_2h,")
print(f"                             velocity_24h, amount_zscore, log_amount,")
print(f"                             acct_online_rate, acct_txn_count, hour")
print(f"  Drop (zero correlation):   cat_is_novel, is_weekend")
print(f"  Imputation applied:        amount_zscore NaN → 0 (single-txn accounts)")