Categorical Features

import pandas as pd

housing_df = pd.DataFrame({
    'property_type': ['flat','house','bungalow','house','flat',
                      'house','flat','bungalow','house','flat'],
    'sale_price':    [245000,410000,310000,560000,198000,
                      348000,230000,285000,495000,215000]
})

# Step 1 — check distribution before encoding
# A category appearing only once will be all-0 or all-1 after a split — no signal
print("Category distribution:\n")
counts = housing_df['property_type'].value_counts()
pcts   = housing_df['property_type'].value_counts(normalize=True).mul(100).round(1)
for val in counts.index:
    print(f"  {val:<12}  {counts[val]} rows  ({pcts[val]}%)")

# Step 2 — pd.get_dummies() creates one binary column per category
# prefix='type' keeps column names readable
# drop_first=True removes the 'bungalow' column — it's implied when flat=0 and house=0
encoded = pd.get_dummies(housing_df['property_type'], prefix='type', drop_first=True)
result  = pd.concat([encoded, housing_df['sale_price']], axis=1)

print("\nAfter one-hot encoding:\n")
print(result.to_string(index=False))

# Step 3 — validate each binary column against the target
print("\nCorrelation with sale_price:\n")
for col in ['type_flat', 'type_house']:
    corr = result[col].corr(result['sale_price'])
    print(f"  {col:<15}  {corr:+.4f}")

import pandas as pd

housing_df = pd.DataFrame({
    'condition': ['good','fair','excellent','poor','good',
                  'excellent','fair','poor','excellent','good'],
    'sale_price':[295000,220000,480000,165000,310000,
                  510000,240000,180000,495000,330000]
})

# .map() applies a dictionary to each value in the column
# The integer values must reflect the true ranking — you set this deliberately
ordinal_map = {'poor': 0, 'fair': 1, 'good': 2, 'excellent': 3}
housing_df['condition_enc'] = housing_df['condition'].map(ordinal_map)

# Validate monotonicity — mean price should increase with condition rank
# If it doesn't, the ordering may not hold in this specific dataset
print("Mean sale price by condition rank:\n")
summary = (housing_df.groupby(['condition_enc','condition'])['sale_price']
           .agg(['mean','count']).round(0).reset_index())
summary.columns = ['rank','condition','mean_price','count']
print(summary.to_string(index=False))

# Correlation — ordinal encoding should show strong positive relationship
corr = housing_df['condition_enc'].corr(housing_df['sale_price'])
print(f"\nCorrelation of condition_enc with sale_price: {corr:+.4f}")

import pandas as pd

housing_df = pd.DataFrame({
    'postcode':   ['SW1','E1','SW1','N1','E1','SW1','N1','W1','E1','W1'],
    'sale_price': [480000,245000,510000,310000,260000,495000,295000,410000,255000,420000]
})

# Show how many unique values this column has — the first cardinality check
print(f"Unique postcodes: {housing_df['postcode'].nunique()}")
print(f"Rows: {len(housing_df)}")
print(f"\nOne-hot encoding would create {housing_df['postcode'].nunique() - 1} new columns\n")

# FREQUENCY ENCODING — replace each category with its proportion in the dataset
# .value_counts(normalize=True) gives proportions (0 to 1) instead of raw counts
freq_map = housing_df['postcode'].value_counts(normalize=True)
housing_df['postcode_freq'] = housing_df['postcode'].map(freq_map)

print("Frequency map:\n")
for postcode, freq in freq_map.items():
    print(f"  {postcode:<6}  {freq:.2f}  ({freq*100:.0f}% of rows)")

print("\nFrequency encoded dataset:\n")
print(housing_df[['postcode','postcode_freq','sale_price']].to_string(index=False))

corr = housing_df['postcode_freq'].corr(housing_df['sale_price'])
print(f"\nCorrelation of postcode_freq with sale_price: {corr:+.4f}")

import pandas as pd
import numpy as np

housing_df = pd.DataFrame({
    'agent_code': ['AG1','AG2','AG3','AG4','AG5','AG6','AG7','AG8',
                   'AG1','AG3'],
    'sale_price': [480000,245000,510000,310000,260000,495000,295000,410000,
                   520000,490000]
})

# Step 1: Label encode — assign an integer to each unique category
# pd.factorize() returns (integer_codes, unique_labels)
# We use it here just to get a consistent integer per category
codes, uniques = pd.factorize(housing_df['agent_code'])
housing_df['agent_int'] = codes

print("Integer mapping:")
for i, label in enumerate(uniques):
    # format(i, '03b') converts integer i to a 3-digit binary string
    print(f"  {label}  →  int={i}  →  binary={format(i, '03b')}")

# Step 2: convert each integer to binary columns
# n_bits = number of bits needed to represent all categories
n_bits = int(np.ceil(np.log2(housing_df['agent_code'].nunique())))

for bit in range(n_bits):
    # Right-shift by bit position and AND with 1 extracts each individual bit
    housing_df[f'agent_b{bit}'] = (housing_df['agent_int'] >> bit) & 1

binary_cols = [f'agent_b{i}' for i in range(n_bits)]

print(f"\nOne-hot would need {housing_df['agent_code'].nunique() - 1} columns")
print(f"Binary encoding needs {n_bits} columns\n")

# Validate — do any binary columns correlate with sale price?
print("Correlation of binary columns with sale_price:\n")
for col in binary_cols:
    corr = housing_df[col].corr(housing_df['sale_price'])
    print(f"  {col}  {corr:+.4f}")

print("\nSample output:\n")
print(housing_df[['agent_code','agent_int'] + binary_cols + ['sale_price']].to_string(index=False))