utils/cluster_n_sentiments.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "358a4cb0-2420-48c1-ad8f-063ba67f07c1",
   "metadata": {},
   "source": [
    "1. Load CSV with tweets (or Instagram posts/TikTok comments, etc)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "00887620-2da2-4b2a-b97b-6326c7046d8e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "tweets = pd.read_csv(\"Bellingcat 2023.csv\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9c89dfd2-0cfe-47a1-b856-8dd587dc9b30",
   "metadata": {},
   "source": [
    "2. Filter out quotes and replies (only for Twitter data, you may need another filtering for another sources)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3a9cc778-775f-409a-8a4f-bd26a5de2dd4",
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets = tweets[tweets[\"type\"] == \"Post\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf9d68a1-6346-4aa5-8dd6-47741c4a83a3",
   "metadata": {},
   "source": [
    "3. Detect language of each tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "db42b32e-05cc-4746-805a-d47479754032",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langdetect import detect, LangDetectException\n",
    "\n",
    "def detect_no_fail(text):\n",
    "    try:\n",
    "        return detect(text)\n",
    "    except LangDetectException:\n",
    "        return None\n",
    "\n",
    "tweets[\"lang\"] = tweets.apply(lambda x: detect_no_fail(x[\"text\"]), axis=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ebcda578-4f88-4686-a052-efbbf5480742",
   "metadata": {},
   "source": [
    "4. Filter out non english tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66269d89-3ad1-4004-8f47-90c071939966",
   "metadata": {},
   "outputs": [],
   "source": [
    "tweets = tweets[tweets[\"lang\"] == \"en\"]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3bb63a11-ec8d-4a88-99cb-e16dcd5dcc3d",
   "metadata": {},
   "source": [
    "5. Vectorize each tweet "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bcd9ee9b-dd90-469d-9211-dfa403c3102e",
   "metadata": {},
   "outputs": [],
   "source": [
    "from InstructorEmbedding import INSTRUCTOR\n",
    "model = INSTRUCTOR('hkunlp/instructor-large')\n",
    "instructions = [\n",
    "    ['Represent the tweet for clustering: ', tweet]\n",
    "    for tweet in tweets[\"text\"].to_list()\n",
    "]\n",
    "embeddings = model.encode(instructions)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "adb8454b-2b92-488c-96bf-0b89c5112cbe",
   "metadata": {},
   "source": [
    "6. Use MiniBatchKMeans algorythm to claterize tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e222007-3b92-45a1-968c-8c7be0f96a3c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sklearn.cluster\n",
    "clustering_model = sklearn.cluster.MiniBatchKMeans(n_clusters=15)\n",
    "clustering_model.fit(embeddings)\n",
    "cluster_assignment = clustering_model.labels_"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "37c1268e-e6d5-404f-ab64-a4ae54f15861",
   "metadata": {},
   "source": [
    "7. Find 10 most representative (10 nearest to cluster center) tweets fir each cluster"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "19930de1-00bc-450a-bc43-a4601bce3636",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "# Assuming 'embeddings' is the array of tweet embeddings and 'clustering_model' is your MiniBatchKMeans model\n",
    "centroids = clustering_model.cluster_centers_\n",
    "\n",
    "representative_tweets = {}\n",
    "for cluster_num in range(15):\n",
    "    # Get the indices of tweets in this cluster\n",
    "    indices = np.where(cluster_assignment == cluster_num)[0]\n",
    "    \n",
    "    # Calculate distances of tweets in this cluster to the centroid\n",
    "    distances = np.linalg.norm(embeddings[indices] - centroids[cluster_num], axis=1)\n",
    "    \n",
    "    # Get indices of tweets with the shortest distances\n",
    "    representative_idx = np.argsort(distances)[:10]  # Adjust the number 10 as needed\n",
    "    \n",
    "    # Store the indices or the tweets themselves\n",
    "    representative_tweets[cluster_num] = indices[representative_idx]\n",
    "\n",
    "# Now 'representative_tweets' holds the indices of the most representative tweets for each cluster"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4bceb55-d014-4fff-a794-e3364117bd51",
   "metadata": {},
   "source": [
    "8. Use GPT-4-Turbo API to generate topics using respresentative tweets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8d082fe2-f19d-4ad3-91a0-ef6366bf27ac",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "import time\n",
    "from tqdm import tqdm\n",
    "\n",
    "def get_cluster_name(tweets):\n",
    "    GENERAL_TOPIC = \"Bellingcat\"\n",
    "\n",
    "    openai.api_key = \"\" # Your API key here\n",
    "    \n",
    "    prompt = f\"Review the tweets contained in the previous messages. Identify and provide a single, overarching topic that unites these tweets, using a maximum of three words. Ensure the response contains only this topic, with no additional comments or information. If no common topic can be determined, respond with 'Undefined'. General topic of this tweets is `{GENERAL_TOPIC}`, don't use it, try to find more specific topic\"\n",
    "    messages = [{\"role\": \"system\", \"content\": prompt}]\n",
    "    messages.extend([{\"role\": \"user\", \"content\": tweet} for tweet in tweets])\n",
    "    messages.append({\"role\": \"user\", \"content\": prompt})\n",
    "\n",
    "    try: \n",
    "        response = openai.ChatCompletion.create(model=\"gpt-4-1106-preview\", messages=messages)\n",
    "    except openai.error.RateLimitError:\n",
    "        time.sleep(60)\n",
    "        response = openai.ChatCompletion.create(model=\"gpt-4-1106-preview\", messages=messages)\n",
    "\n",
    "    return response.choices[0].message['content']\n",
    "\n",
    "for cluster in tqdm(tweets[\"cluster\"].unique()):\n",
    "    representative_ids = representative_tweets[cluster]\n",
    "    representative_tweets_texts = [\n",
    "        tweets[\"text\"].iloc[i]\n",
    "        for i in representative_ids\n",
    "    ]\n",
    "    \n",
    "    cluster_name = get_cluster_name(representative_tweets_texts)\n",
    "    tweets.loc[tweets[\"cluster\"] == cluster, \"cluster_name\"] = cluster_name"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "424decbc-4d93-4e9e-99bb-a3d679750295",
   "metadata": {},
   "source": [
    "9. Estimate sentiment of each tweet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ee50f208-7e1b-4445-88fa-1e3a87c4f023",
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
    "import torch\n",
    "\n",
    "# Load model and tokenizer\n",
    "model_name = \"cardiffnlp/twitter-roberta-base-sentiment-latest\"\n",
    "model = AutoModelForSequenceClassification.from_pretrained(model_name)\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name)\n",
    "\n",
    "def preprocess_tweet(tweet):\n",
    "    return tweet\n",
    "\n",
    "def analyze_sentiment(tweet):\n",
    "    inputs = tokenizer(tweet, return_tensors=\"pt\", padding=True, truncation=True)\n",
    "    outputs = model(**inputs)\n",
    "    scores = outputs.logits\n",
    "    return torch.softmax(scores, dim=1).detach().numpy()[0]\n",
    "\n",
    "def scale_score(probabilities):\n",
    "    # Assuming probabilities order: [Negative, Neutral, Positive]\n",
    "    neg, neu, pos = probabilities\n",
    "    # Weighted score: Negative scores contribute negatively, positive scores contribute positively\n",
    "    scaled_score = neg * (-10) + neu * 0 + pos * 10\n",
    "    return scaled_score\n",
    "\n",
    "\n",
    "# Example usage\n",
    "def get_sentiment(tweet):\n",
    "    preprocessed_tweet = preprocess_tweet(tweet)\n",
    "    model_score = analyze_sentiment(preprocessed_tweet)\n",
    "    scaled_score = scale_score(model_score)\n",
    "    return scaled_score\n",
    "\n",
    "tweets[\"sentiment\"] = tweets.apply(lambda x: get_sentiment(x[\"text\"]), axis=1)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}