Text Features Basics

# pandas — core data library, always imported as pd
import pandas as pd

# re — Python's built-in regular expression library
# Used here to count punctuation patterns in text
import re

# Customer review dataset — 10 rows with sentiment labels
reviews_df = pd.DataFrame({
    'review_id': ['R01','R02','R03','R04','R05',
                  'R06','R07','R08','R09','R10'],
    'text': [
        'Absolutely love this product! Works perfectly every time!',
        'Terrible quality. Broke after two days. Complete waste of money.',
        'Great value for the price, highly recommend to everyone',
        'Awful. Does not work as advertised. Very disappointed.',
        'Fantastic! Best purchase I have made this year, so happy!',
        'Poor build quality, feels cheap and stopped working quickly',
        'Really pleased with this item, arrived fast and well packaged',
        'Horrible experience, customer service was rude and unhelpful',
        'Outstanding product, exceeded my expectations completely',
        'Dreadful quality, returned immediately, do not buy this'
    ],
    'sentiment': [1,0,1,0,1,0,1,0,1,0]  # 1=positive, 0=negative
})

# Word count — split on whitespace and count the resulting list
reviews_df['word_count'] = reviews_df['text'].str.split().str.len()

# Character count — total length of the string including spaces
reviews_df['char_count'] = reviews_df['text'].str.len()

# Exclamation marks — count how many ! appear in each review
# re.findall returns a list of all matches; len() counts them
reviews_df['exclamation_count'] = reviews_df['text'].apply(
    lambda x: len(re.findall(r'!', x)))

# Average word length — total characters in words divided by word count
# re.findall(r'\b\w+\b') extracts all words without punctuation
reviews_df['avg_word_len'] = reviews_df['text'].apply(
    lambda x: sum(len(w) for w in re.findall(r'\b\w+\b', x))
              / max(len(re.findall(r'\b\w+\b', x)), 1))

# Validate each feature against the target
print("Correlation with sentiment:\n")
for col in ['word_count', 'char_count', 'exclamation_count', 'avg_word_len']:
    corr = reviews_df[col].corr(reviews_df['sentiment'])
    print(f"  {col:<22}  {corr:+.4f}")

print("\nSurface features sample:\n")
print(reviews_df[['review_id','word_count','char_count',
                  'exclamation_count','sentiment']].to_string(index=False))

import pandas as pd

reviews_df = pd.DataFrame({
    'text': [
        'Absolutely love this product! Works perfectly every time!',
        'Terrible quality. Broke after two days. Complete waste of money.',
        'Great value for the price, highly recommend to everyone',
        'Awful. Does not work as advertised. Very disappointed.',
        'Fantastic! Best purchase I have made this year, so happy!',
        'Poor build quality, feels cheap and stopped working quickly',
        'Really pleased with this item, arrived fast and well packaged',
        'Horrible experience, customer service was rude and unhelpful',
        'Outstanding product, exceeded my expectations completely',
        'Dreadful quality, returned immediately, do not buy this'
    ],
    'sentiment': [1,0,1,0,1,0,1,0,1,0]
})

# Positive signal words — domain knowledge says these predict good reviews
# case=False makes the check case-insensitive
pos_pattern = r'love|great|fantastic|excellent|outstanding|pleased|recommend|best'
reviews_df['has_positive_word'] = (
    reviews_df['text'].str.contains(pos_pattern, case=False)).astype(int)

# Negative signal words — words associated with dissatisfied customers
neg_pattern = r'terrible|awful|horrible|dreadful|poor|waste|broke|disappointed'
reviews_df['has_negative_word'] = (
    reviews_df['text'].str.contains(neg_pattern, case=False)).astype(int)

# Quality words — appears in both positive and negative reviews — signal ambiguous
reviews_df['mentions_quality'] = (
    reviews_df['text'].str.contains(r'quality', case=False)).astype(int)

# Validate all three flags against sentiment
print("Keyword flag correlations with sentiment:\n")
for col in ['has_positive_word', 'has_negative_word', 'mentions_quality']:
    corr = reviews_df[col].corr(reviews_df['sentiment'])
    dist = reviews_df[col].value_counts().to_dict()
    print(f"  {col:<25}  {corr:+.4f}   distribution: {dist}")

print("\nKeyword flags sample:\n")
print(reviews_df[['text','has_positive_word',
                  'has_negative_word','sentiment']].to_string(index=False))

import pandas as pd
import re

reviews_df = pd.DataFrame({
    'text': [
        'Absolutely love this product! Works perfectly every time!',
        'Terrible quality. Broke after two days. Complete waste of money.',
        'Great value for the price, highly recommend to everyone',
        'Awful. Does not work as advertised. Very disappointed.',
        'Fantastic! Best purchase I have made this year, so happy!',
        'Poor build quality, feels cheap and stopped working quickly',
        'Really pleased with this item, arrived fast and well packaged',
        'Horrible experience, customer service was rude and unhelpful',
        'Outstanding product, exceeded my expectations completely',
        'Dreadful quality, returned immediately, do not buy this'
    ],
    'sentiment': [1,0,1,0,1,0,1,0,1,0]
})

# Stop words — common English words that carry no sentiment signal
# In production you would use nltk.corpus.stopwords.words('english')
STOP_WORDS = {'the','a','an','this','is','it','and','to','of','in','for',
              'not','my','i','with','was','so','very','does','do','after',
              'have','made','really','every','time','well','buy','has'}

def clean_text(text):
    # Step 1: lowercase — so "Love" and "love" are treated as the same word
    text = text.lower()
    # Step 2: remove punctuation and numbers — keep only letters and spaces
    text = re.sub(r'[^a-z\s]', '', text)
    # Step 3: remove stop words — split, filter, rejoin
    words = [w for w in text.split() if w not in STOP_WORDS]
    return ' '.join(words)

# Apply the cleaning function to every review
reviews_df['cleaned'] = reviews_df['text'].apply(clean_text)

# Word count after cleaning — compare to raw to see how much was stripped
reviews_df['clean_word_count'] = reviews_df['cleaned'].str.split().str.len()
reviews_df['raw_word_count']   = reviews_df['text'].str.split().str.len()

print("Raw vs cleaned text:\n")
for _, row in reviews_df.iterrows():
    print(f"  RAW:     {row['text']}")
    print(f"  CLEANED: {row['cleaned']}")
    print(f"  Words:   {row['raw_word_count']} → {row['clean_word_count']}\n")

import pandas as pd
import re
from sklearn.feature_extraction.text import TfidfVectorizer

# Cleaned reviews (output from the previous step)
cleaned_reviews = [
    'absolutely love product works perfectly',
    'terrible quality broke two days complete waste money',
    'great value price highly recommend everyone',
    'awful work advertised disappointed',
    'fantastic best purchase year happy',
    'poor build quality feels cheap stopped working quickly',
    'pleased item arrived fast packaged',
    'horrible experience customer service rude unhelpful',
    'outstanding product exceeded expectations completely',
    'dreadful quality returned immediately'
]
sentiment = [1,0,1,0,1,0,1,0,1,0]

# TfidfVectorizer converts a list of text documents into a TF-IDF feature matrix
# max_features limits vocabulary to the top N most informative words
# ngram_range=(1,2) includes single words AND two-word phrases (bigrams)
vectorizer = TfidfVectorizer(max_features=15, ngram_range=(1, 2))

# fit_transform learns the vocabulary and scores, then transforms in one step
# The result is a sparse matrix — each row is a review, each column is a word/phrase
tfidf_matrix = vectorizer.fit_transform(cleaned_reviews)

# Convert to a readable DataFrame so we can see the feature names and values
feature_names = vectorizer.get_feature_names_out()
tfidf_df = pd.DataFrame(tfidf_matrix.toarray(),
                        columns=feature_names).round(3)

print("Top 15 TF-IDF features (first 5 reviews):\n")
print(tfidf_df.head(5).to_string())

# Show which vocabulary the vectoriser learned
print(f"\nVocabulary ({len(feature_names)} terms):")
print(list(feature_names))