NLP Sentiment Insights

import pandas as pd
import numpy as np
import re
from scipy import stats
from collections import Counter

# ── Dataset ──────────────────────────────────────────────────────────────────
df = pd.DataFrame({
    "review_id":   [f"R{i:03d}" for i in range(1, 21)],
    "review_text": [
        "Battery dies after 3 hours. Completely unacceptable for a premium product. Very disappointed.",
        "Sound quality is absolutely incredible. Best headphones I have ever owned. Crystal clear audio.",
        "Connectivity keeps dropping every few minutes. Very frustrating experience. Would not recommend.",
        "Comfortable to wear all day. Great noise cancellation and clear audio. Excellent product.",
        "Terrible battery life and awful sound after firmware update. Returning immediately.",
        "Amazing sound and superb comfort. Perfect for long listening sessions. Highly recommend.",
        "Battery drains too fast. Connection drops constantly. Extremely poor quality for the price.",
        "Outstanding noise cancellation. Fantastic audio clarity. Very comfortable fit. Love them.",
        "Firmware v2.1 ruined everything. Battery now terrible. Sound quality degraded badly.",
        "Wonderful product overall. Sound is rich and warm. Comfortable even after hours of use.",
        "Keeps disconnecting from phone. Bluetooth pairing is unreliable and very frustrating.",
        "Brilliant headphones. Exceptional clarity and deep bass. Comfortable and stylish design.",
        "Battery life is absolutely dreadful. Dies within 2 hours. Completely useless on commute.",
        "Great value for money. Solid sound quality and good battery life. Would recommend to friend.",
        "Awful connectivity issues since firmware update. Sound cuts out randomly. Very annoying.",
        "Superb audio performance. Rich sound stage. Comfortable padded earcups. Very happy purchase.",
        "Battery barely lasts 4 hours. Connectivity unstable. Not worth the premium price at all.",
        "Incredible sound quality. Deep bass and crisp highs. Comfortable all-day wear. Love these.",
        "Terrible experience overall. Battery, sound, and connectivity all failed after update.",
        "Perfect headphones. Exceptional sound. Long battery life. Very comfortable. Highly recommend."
    ],
    "star_rating": [1,5,2,4,1,5,1,5,1,4,2,5,1,4,2,5,2,5,1,5],
    "variant":     ["Pro","Standard","Pro","Standard","Pro","Standard","Pro","Standard",
                    "Pro","Standard","Pro","Standard","Pro","Standard","Pro","Standard",
                    "Pro","Standard","Pro","Standard"],
    "firmware":    ["v2.1","v1.8","v2.1","v1.8","v2.1","v1.8","v2.1","v1.8",
                    "v2.1","v1.8","v2.1","v1.8","v2.1","v1.8","v2.1","v1.8",
                    "v2.1","v1.8","v2.1","v1.8"],
    "verified":    [1,1,0,1,1,1,1,1,1,1,0,1,1,1,0,1,1,1,1,1]
})

# ── Stop words ───────────────────────────────────────────────────────────────
STOP_WORDS = {
    "the","a","an","and","or","but","in","on","at","to","for","of","with",
    "is","are","was","were","be","been","being","have","has","had","do",
    "does","did","will","would","could","should","may","might","this","that",
    "i","my","me","we","you","your","it","its","they","them","their","all",
    "after","from","not","very","so","too","also","even","just","would","there",
    "about","than","more","would","these","those","which","what","how","when",
    "who","if","up","out","no","into","over","such","own"
}

def clean_and_tokenise(text):
    """Lower-case, strip punctuation/digits, split, remove stop words."""
    text  = text.lower()
    text  = re.sub(r"[^a-z\s]", "", text)      # keep only letters and spaces
    text  = re.sub(r"\s+", " ", text).strip()   # collapse multiple spaces
    tokens = text.split()
    return [t for t in tokens if t not in STOP_WORDS and len(t) > 2]

df["tokens"]      = df["review_text"].apply(clean_and_tokenise)
df["token_count"] = df["tokens"].apply(len)

print("Text cleaning complete:")
print(f"  Reviews processed: {len(df)}")
print(f"  Mean tokens/review (after cleaning): {df['token_count'].mean():.1f}")
print(f"\nExample — R001 tokens:")
print(f"  {df.loc[0,'tokens']}")
print(f"\nExample — R002 tokens:")
print(f"  {df.loc[1,'tokens']}")

# Domain-specific sentiment lexicon — consumer electronics tuned
POSITIVE_WORDS = {
    "incredible","amazing","excellent","outstanding","superb","fantastic",
    "brilliant","wonderful","perfect","great","good","clear","crisp","rich",
    "comfortable","exceptional","love","best","happy","recommend","solid",
    "impressive","superb","warm","deep","stylish","long","worthy","value"
}
NEGATIVE_WORDS = {
    "terrible","awful","dreadful","poor","bad","horrible","worst","useless",
    "disappointed","frustrating","annoying","unreliable","unacceptable","broken",
    "degraded","ruined","unstable","constantly","drops","dropping","dies",
    "barely","drains","fails","failed","issues","cuts","randomly","returning"
}

def score_sentiment(tokens):
    """Net sentiment score normalised by token count."""
    pos = sum(1 for t in tokens if t in POSITIVE_WORDS)
    neg = sum(1 for t in tokens if t in NEGATIVE_WORDS)
    n   = max(len(tokens), 1)          # avoid division by zero
    return round((pos - neg) / n, 4)

def sentiment_label(score):
    if score > 0.05:  return "Positive"
    if score < -0.05: return "Negative"
    return "Neutral"

df["pos_count"]  = df["tokens"].apply(lambda t: sum(1 for w in t if w in POSITIVE_WORDS))
df["neg_count"]  = df["tokens"].apply(lambda t: sum(1 for w in t if w in NEGATIVE_WORDS))
df["sent_score"] = df["tokens"].apply(score_sentiment)
df["sentiment"]  = df["sent_score"].apply(sentiment_label)

print("Sentiment scoring complete:")
print(df[["review_id","star_rating","pos_count","neg_count","sent_score","sentiment"]].to_string(index=False))

# Validate: correlate sent_score with star_rating
r, p = stats.pearsonr(df["sent_score"], df["star_rating"])
print(f"\nLexicon validation — sent_score vs star_rating:")
print(f"  Pearson r = {r:.3f}  p = {p:.4f}  {'***' if p<0.001 else '**' if p<0.01 else '*'}")

# Mean sentiment by star rating
print("\nMean sentiment score by star rating:")
print(df.groupby("star_rating")["sent_score"].mean().round(3).to_string())

# Split corpus into negative and positive reviews
neg_reviews = df[df["star_rating"] <= 2]
pos_reviews = df[df["star_rating"] >= 4]

# Flatten token lists and count
def token_freq(review_subset):
    all_tokens = [t for tokens in review_subset["tokens"] for t in tokens]
    return Counter(all_tokens)

neg_freq = token_freq(neg_reviews)
pos_freq = token_freq(pos_reviews)

# Top terms in negative reviews (excluding sentiment words — focus on topics)
SENTIMENT_WORDS = POSITIVE_WORDS | NEGATIVE_WORDS
neg_topic_freq = {k:v for k,v in neg_freq.items() if k not in SENTIMENT_WORDS}
pos_topic_freq = {k:v for k,v in pos_freq.items() if k not in SENTIMENT_WORDS}

# Build comparison DataFrame
all_topic_words = set(neg_topic_freq) | set(pos_topic_freq)
topic_df = pd.DataFrame({
    "word":     list(all_topic_words),
    "neg_freq": [neg_topic_freq.get(w, 0) for w in all_topic_words],
    "pos_freq": [pos_topic_freq.get(w, 0) for w in all_topic_words]
})
topic_df["neg_ratio"] = (topic_df["neg_freq"] / (topic_df["pos_freq"] + 0.5)).round(2)
topic_df = topic_df[topic_df["neg_freq"] >= 2].sort_values("neg_ratio", ascending=False)

print("Top complaint-signal terms (neg_freq >= 2, ranked by neg/pos ratio):")
print(topic_df.head(12).to_string(index=False))

# Also: top 10 raw negative terms
print("\nTop 10 most frequent tokens in negative reviews:")
for word, count in neg_freq.most_common(15):
    if word not in SENTIMENT_WORDS:
        print(f"  {word:<20} {count}")

# Feature mention flags — using str.contains on cleaned text
df["text_clean"] = df["review_text"].str.lower()

features = {
    "battery":      r"battery|battery life|charge|charging",
    "sound":        r"sound|audio|bass|clarity|crisp|clear",
    "connectivity": r"connect|bluetooth|pairing|dropping|drops|disconn",
    "comfort":      r"comfort|comfortable|fit|wear|earcup|cushion"
}

for feat, pattern in features.items():
    df[f"feat_{feat}"] = df["text_clean"].str.contains(pattern, regex=True).astype(int)

# Sentiment profile per feature
print("Feature-level sentiment attribution:")
for feat in features:
    col      = f"feat_{feat}"
    mentions = df[df[col]==1]
    neg_ment = mentions[mentions["star_rating"] <= 2]
    if len(mentions) == 0:
        continue
    mean_sent   = mentions["sent_score"].mean()
    neg_rate    = len(neg_ment) / len(mentions)
    mean_rating = mentions["star_rating"].mean()
    print(f"\n  Feature: {feat.upper()}")
    print(f"    Mentions:        {len(mentions)} reviews")
    print(f"    Mean sentiment:  {mean_sent:+.3f}")
    print(f"    Mean star rating:{mean_rating:.1f}")
    print(f"    Negative mention rate: {neg_rate:.1%}")

# Cross-tab: feature mentions by firmware version
print("\nFeature mention counts by firmware version:")
fw_feat = df.groupby("firmware")[[f"feat_{f}" for f in features]].sum()
fw_feat.columns = list(features.keys())
print(fw_feat.to_string())

from scipy.stats import mannwhitneyu, kruskal

# Firmware sentiment comparison
fw_v18 = df[df["firmware"]=="v1.8"]["sent_score"]
fw_v21 = df[df["firmware"]=="v2.1"]["sent_score"]

u_stat, p_fw = mannwhitneyu(fw_v18, fw_v21, alternative="two-sided")
print("Mann-Whitney U — firmware v1.8 vs v2.1:")
print(f"  v1.8 mean sentiment: {fw_v18.mean():+.3f}  (n={len(fw_v18)})")
print(f"  v2.1 mean sentiment: {fw_v21.mean():+.3f}  (n={len(fw_v21)})")
print(f"  U statistic: {u_stat:.0f}  p-value: {p_fw:.4f}")
print(f"  Result: {'Significant *** (p<0.001)' if p_fw<0.001 else 'Significant ** (p<0.01)' if p_fw<0.01 else 'Significant * (p<0.05)' if p_fw<0.05 else 'Not significant'}")

# Variant sentiment comparison
var_pro = df[df["variant"]=="Pro"]["sent_score"]
var_std = df[df["variant"]=="Standard"]["sent_score"]

u_stat2, p_var = mannwhitneyu(var_pro, var_std, alternative="two-sided")
print("\nMann-Whitney U — Pro vs Standard:")
print(f"  Pro mean sentiment:      {var_pro.mean():+.3f}  (n={len(var_pro)})")
print(f"  Standard mean sentiment: {var_std.mean():+.3f}  (n={len(var_std)})")
print(f"  U statistic: {u_stat2:.0f}  p-value: {p_var:.4f}")
print(f"  Result: {'Significant *** (p<0.001)' if p_var<0.001 else 'Significant ** (p<0.01)' if p_var<0.01 else 'Significant * (p<0.05)' if p_var<0.05 else 'Not significant'}")

# Verified vs unverified sentiment
ver  = df[df["verified"]==1]["sent_score"]
unver= df[df["verified"]==0]["sent_score"]
u3, p_ver = mannwhitneyu(ver, unver, alternative="two-sided")
print(f"\nVerified vs Unverified reviews:")
print(f"  Verified mean:   {ver.mean():+.3f}  (n={len(ver)})")
print(f"  Unverified mean: {unver.mean():+.3f}  (n={len(unver)})")
print(f"  p-value: {p_ver:.4f} — {'Significant' if p_ver<0.05 else 'Not significant'}")

# Full sentiment summary by key dimensions
dimensions = {
    "firmware":  "firmware",
    "variant":   "variant",
    "sentiment_label": "sentiment"
}
print("Sentiment summary by key dimensions:")
for label, col in dimensions.items():
    grp = df.groupby(col)["sent_score"].agg(
        n     = "count",
        mean  = "mean",
        std   = "std",
        min   = "min",
        max   = "max"
    ).round(3)
    print(f"\n  By {label}:")
    print(grp.to_string())

# Margin of error at 95% confidence for mean sentiment
n  = len(df)
se = df["sent_score"].std() / np.sqrt(n)
moe = 1.96 * se
print(f"\nPortfolio-level sentiment:")
print(f"  Mean sentiment score: {df['sent_score'].mean():+.3f}")
print(f"  95% CI:               [{df['sent_score'].mean()-moe:+.3f}, {df['sent_score'].mean()+moe:+.3f}]")
print(f"  n = {n} reviews")

# Sample size for ±0.05 precision at 95% confidence
target_moe = 0.05
required_n = int(np.ceil((1.96 * df["sent_score"].std() / target_moe) ** 2))
print(f"\nRequired n for ±0.05 margin of error at 95% confidence: {required_n} reviews")