Feature Selection

# Load the ecommerce dataset to demonstrate feature selection
import pandas as pd
import numpy as np
from sklearn.feature_selection import SelectKBest, f_regression, chi2
from sklearn.preprocessing import LabelEncoder

# Load data - simulating Swiggy's delivery prediction scenario
df = pd.read_csv('dataplexa_ecommerce.csv')
print(f"Dataset shape: {df.shape}")
print(f"Columns: {df.columns.tolist()}")

# Prepare features for selection - encode categorical variables
# We'll predict revenue using other features
le = LabelEncoder()

# Create feature matrix X and target y
df_encoded = df.copy()
df_encoded['gender_encoded'] = le.fit_transform(df['gender'])
df_encoded['city_encoded'] = le.fit_transform(df['city'])
df_encoded['category_encoded'] = le.fit_transform(df['product_category'])

# Select numerical features for our analysis
feature_cols = ['customer_age', 'gender_encoded', 'city_encoded', 
               'category_encoded', 'quantity', 'unit_price', 'rating']
X = df_encoded[feature_cols]
y = df_encoded['revenue']  # Target variable

print("Feature matrix shape:", X.shape)
print("Features:", feature_cols)
print("Target (revenue) range:", y.min(), "to", y.max())

# Univariate feature selection using F-test
# f_regression tests linear relationship between each feature and continuous target
selector = SelectKBest(score_func=f_regression, k=5)  # Select top 5 features
X_selected = selector.fit_transform(X, y)

# Get feature scores and selected features
feature_scores = selector.scores_
selected_features = selector.get_support()

# Create results dataframe for better visualization
results_df = pd.DataFrame({
    'feature': feature_cols,
    'f_score': feature_scores,
    'selected': selected_features
}).sort_values('f_score', ascending=False)

print("Feature Selection Results (F-test):")
print(results_df)

# Model-based feature selection using Random Forest
from sklearn.ensemble import RandomForestRegressor
from sklearn.feature_selection import SelectFromModel

# Train Random Forest to get feature importances
# Random Forest measures how much each feature decreases impurity
rf_model = RandomForestRegressor(n_estimators=100, random_state=42)
rf_model.fit(X, y)

# Get feature importances from the trained model
feature_importance = rf_model.feature_importances_

print("Random Forest Feature Importance:")
for i, (feature, importance) in enumerate(zip(feature_cols, feature_importance)):
    print(f"{feature}: {importance:.4f}")

# Select features above median importance automatically
# This removes features that contribute less than average
selector_model = SelectFromModel(rf_model, threshold='median')
X_model_selected = selector_model.fit_transform(X, y)

# See which features were selected
selected_features_model = selector_model.get_support()
selected_feature_names = [feature_cols[i] for i, selected in enumerate(selected_features_model) if selected]

print(f"Features selected by model (above median importance): {len(selected_feature_names)}")
print("Selected features:", selected_feature_names)
print(f"Reduced from {X.shape[1]} to {X_model_selected.shape[1]} features")

# Recursive Feature Elimination with cross-validation
from sklearn.feature_selection import RFE, RFECV
from sklearn.linear_model import LinearRegression
from sklearn.model_selection import cross_val_score

# RFE with exact number of features (let's select 4 features)
lr_model = LinearRegression()
rfe = RFE(estimator=lr_model, n_features_to_select=4)
X_rfe = rfe.fit_transform(X, y)

# Get selected features and their ranking
selected_features_rfe = rfe.support_
feature_ranking = rfe.ranking_

print("RFE Results (selecting 4 features):")
for feature, selected, rank in zip(feature_cols, selected_features_rfe, feature_ranking):
    status = "SELECTED" if selected else f"Rank {rank}"
    print(f"{feature}: {status}")

# RFECV finds optimal number of features automatically
# Uses cross-validation to test different feature counts
rfecv = RFECV(estimator=lr_model, cv=5, scoring='neg_mean_squared_error')
X_rfecv = rfecv.fit_transform(X, y)

# Results of automatic feature selection
optimal_features = rfecv.n_features_
selected_features_rfecv = rfecv.support_
cv_scores = rfecv.grid_scores_

print(f"Optimal number of features: {optimal_features}")
print("Features selected by RFECV:")
for feature, selected in zip(feature_cols, selected_features_rfecv):
    if selected:
        print(f"  ✓ {feature}")
    else:
        print(f"  ✗ {feature}")

print(f"\nCross-validation scores by feature count:")
for i, score in enumerate(cv_scores, 1):
    print(f"  {i} features: {-score:.2f} MSE")