Types of Data

# Identify numerical columns automatically — saves time when datasets have 50+ columns
import pandas as pd
import numpy as np

# Load the ecommerce dataset
df = pd.read_csv('dataplexa_ecommerce.csv')

# Get numerical columns only — excludes strings/objects automatically
numerical_cols = df.select_dtypes(include=[np.number]).columns.tolist()
print("Numerical columns:", numerical_cols)
print("\nData types:")
print(df[numerical_cols].dtypes)

# Examine unique values to understand discrete vs continuous nature
print("Quantity unique values (first 10):", sorted(df['quantity'].unique())[:10])
print("Rating unique values:", sorted(df['rating'].unique()))
print("\nRevenue sample — notice the decimal precision:")
print(df['revenue'].head(10).tolist())

# Analyze categorical columns — understanding distribution is crucial for business insights
categorical_cols = df.select_dtypes(include=['object']).columns.tolist()
print("Categorical columns:", categorical_cols)

# Value counts show business patterns, not just data types
print("\n=== CITY DISTRIBUTION ===")
city_counts = df['city'].value_counts()
print(city_counts)
print(f"Mumbai dominates: {city_counts['Mumbai']/len(df)*100:.1f}% of all orders")

# Handle dates properly — they're ordinal but need special treatment
print("Date as string:", df['date'].dtype)
print("Unique dates:", df['date'].nunique())
print("Date range:", df['date'].min(), "to", df['date'].max())

# Convert to proper datetime for time-series analysis
df['date'] = pd.to_datetime(df['date'])
print("\nAfter conversion:", df['date'].dtype)

# Now we can extract ordinal components
df['month'] = df['date'].dt.month
df['day_of_week'] = df['date'].dt.day_name()
print("Month distribution:")
print(df['month'].value_counts().sort_index())

# Wrong approach: treating rating as truly continuous and cuisine as if it has no meaning
# This is statistically questionable but commonly done

# Right approach: acknowledge data type constraints
print("=== RATING ANALYSIS BY DATA TYPE ===")

# Rating is ordinal (1 < 2 < 3 < 4 < 5) — use median and percentiles, not just mean
rating_stats = df['rating'].describe()
print("Rating distribution:")
print(rating_stats)

# For ordinal data, look at distribution across the scale
rating_counts = df['rating'].value_counts().sort_index()
print("\nRating frequency (shows skew better than mean):")
for rating, count in rating_counts.items():
    if rating in [1.0, 2.0, 3.0, 4.0, 5.0]:  # Show only whole number ratings for clarity
        print(f"{rating:.0f} stars: {count:,} orders ({count/len(df)*100:.1f}%)")

# Convert continuous age to ordinal age groups — enables different business insights
df['age_group'] = pd.cut(df['customer_age'], 
                        bins=[0, 25, 35, 45, 55, 100], 
                        labels=['18-25', '26-35', '36-45', '46-55', '55+'],
                        right=False)

print("Age group distribution:")
age_dist = df['age_group'].value_counts().sort_index()
print(age_dist)

# Convert continuous revenue to ordinal spending tiers
# Use quantiles to ensure balanced groups for statistical power
revenue_quantiles = df['revenue'].quantile([0.33, 0.67])
print(f"\nRevenue quantiles: 33rd = ₹{revenue_quantiles[0.33]:,.0f}, 67th = ₹{revenue_quantiles[0.67]:,.0f}")

df['spending_tier'] = pd.cut(df['revenue'], 
                            bins=[0, revenue_quantiles[0.33], revenue_quantiles[0.67], float('inf')],
                            labels=['Low', 'Medium', 'High'])

print("\nSpending tier distribution:")
print(df['spending_tier'].value_counts())

# Complete data type audit — run this on every new dataset
def data_type_audit(df):
    """Comprehensive data type analysis for business decision making"""
    
    print("=== DATA TYPE AUDIT REPORT ===\n")
    
    # Basic info
    print(f"Dataset shape: {df.shape[0]:,} rows × {df.shape[1]} columns")
    print(f"Memory usage: {df.memory_usage(deep=True).sum() / 1024**2:.1f} MB\n")
    
    # Categorize columns by data type and business meaning
    numerical_cols = df.select_dtypes(include=[np.number]).columns.tolist()
    categorical_cols = df.select_dtypes(include=['object']).columns.tolist()
    datetime_cols = df.select_dtypes(include=['datetime64']).columns.tolist()
    boolean_cols = df.select_dtypes(include=['bool']).columns.tolist()
    
    print("COLUMN CLASSIFICATION:")
    print(f"Numerical ({len(numerical_cols)}): {numerical_cols}")
    print(f"Categorical ({len(categorical_cols)}): {categorical_cols}")
    print(f"DateTime ({len(datetime_cols)}): {datetime_cols}")
    print(f"Boolean ({len(boolean_cols)}): {boolean_cols}\n")
    
    # Check for potential issues
    print("POTENTIAL ISSUES:")
    for col in numerical_cols:
        unique_count = df[col].nunique()
        if unique_count < 10:
            print(f"⚠️  {col}: Only {unique_count} unique values — might be categorical")
    
    for col in categorical_cols:
        unique_count = df[col].nunique()
        if unique_count > 50:
            print(f"⚠️  {col}: {unique_count} unique values — high cardinality")
            
    return {
        'numerical': numerical_cols,
        'categorical': categorical_cols, 
        'datetime': datetime_cols,
        'boolean': boolean_cols
    }

# Run the audit
audit_results = data_type_audit(df)