categorical_free_text.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a0de5042",
   "metadata": {},
   "source": [
    "# Categorical free text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e3ed4fc",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "# Import spreadsheet cleaning loader\n",
    "from load_survey_data import load_data, headers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7c18897c",
   "metadata": {},
   "outputs": [],
   "source": [
    "sheet = \"results.xlsx\"\n",
    "datetime_format = \"%Y-%m-%d %H:%M:%S %Z\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c261b9a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df = load_data(sheet, datetime_format)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "720f7084",
   "metadata": {},
   "outputs": [],
   "source": [
    "from analysis import categories, categories_per_answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "170f8b33",
   "metadata": {},
   "outputs": [],
   "source": [
    "from tabulate import tabulate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88a29d05-d097-40d0-b40e-efa046c9a516",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Set to True to review the text that is dropped\n",
    "show_dropped_rows = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5fb78d07-03b0-4855-b283-5cfa45b582b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_categorised = pd.DataFrame(index=df.index.copy())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ae89d5c4",
   "metadata": {},
   "source": [
    "## 7.a. Which non-desktop interfaces are important to you?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca3b7aa3",
   "metadata": {},
   "outputs": [],
   "source": [
    "question = '7.a. Which non-desktop interfaces are important to you?'\n",
    "answers=df[question]\n",
    "keep_words=[\n",
    "    'beam',\n",
    "    'cassandra',\n",
    "    'commandline',\n",
    "    'docker',\n",
    "    'flink',\n",
    "    'geode',\n",
    "    'git',\n",
    "    'hadoop',\n",
    "    'impala',\n",
    "    'jupyter',\n",
    "    'kafka',\n",
    "    'linux',\n",
    "    'nifi',\n",
    "    'omero',\n",
    "    'phoenix',\n",
    "    'rstudio',\n",
    "    'solr',\n",
    "    'spark',\n",
    "    'tinkerpop',\n",
    "    'vscode',\n",
    "    'xnat',\n",
    "    'zepplin',\n",
    "]\n",
    "synonyms={\n",
    "    'commandline': [r'command\\s*line', 'cli'],\n",
    "    'git': ['github', 'gitlab', 'git'],\n",
    "    'jupyter': [r'jupyter[\\S]*'],\n",
    "    'rstudio': ['rserver'],\n",
    "}\n",
    "drop_rows=[8,11,14,18,20,23,29,34,35,36,46,64,71,75,92,93,94,97,100,104]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b902e88f",
   "metadata": {},
   "outputs": [],
   "source": [
    "categories(\n",
    "    answers=answers,\n",
    "    keep_words=keep_words,\n",
    "    synonyms=synonyms,\n",
    "    drop_rows=drop_rows\n",
    ").round(decimals=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef4ad4f2-ce8c-4bef-aa1b-e74f080b596e",
   "metadata": {},
   "outputs": [],
   "source": [
    "if show_dropped_rows:\n",
    "    print('\\n'.join(answers[drop_rows]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "daeef932",
   "metadata": {},
   "outputs": [],
   "source": [
    "dfc = categories_per_answer(\n",
    "    answers=answers,\n",
    "    keep_words=keep_words,\n",
    "    synonyms=synonyms,\n",
    "    drop_rows=drop_rows\n",
    ")\n",
    "print(tabulate(\n",
    "    dfc,\n",
    "    ['id', 'interfaces'],\n",
    "    tablefmt='github'\n",
    "))\n",
    "df_categorised[question] = dfc"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "405cdb06",
   "metadata": {},
   "source": [
    "## 8.a. Which programming languages are important to you?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30e764ca",
   "metadata": {},
   "outputs": [],
   "source": [
    "question = '8.a. Which programming languages are important to you?'\n",
    "answers=df[question]\n",
    "keep_words=['python', 'r', 'stata', 'julia', 'javascript', 'c#', 'java', 'c++', 'sql', 'html', 'shell', 'spark', 'pyspark', 'matlab', 'cuda', 'go', 'spss', 'terraform', 'c', 'groovy']\n",
    "synonyms={'shell': ['bash', 'zsh'], 'sql': ['mysql', 'postgres', 'postgresql'], 'spark': [r'[\\S]*spark']}\n",
    "drop_rows=[8,34,47,67,70,94,101,104]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "580a39ad",
   "metadata": {},
   "outputs": [],
   "source": [
    "categories(\n",
    "    answers=answers,\n",
    "    keep_words=keep_words,\n",
    "    synonyms=synonyms,\n",
    "    drop_rows=drop_rows\n",
    ").round(decimals=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d724f0f7-12cc-4146-9f73-16ce1945c5b4",
   "metadata": {},
   "outputs": [],
   "source": [
    "if show_dropped_rows:\n",
    "    print('\\n'.join(answers[drop_rows]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1e4472f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "dfc = categories_per_answer(\n",
    "    answers=answers,\n",
    "    keep_words=keep_words,\n",
    "    synonyms=synonyms,\n",
    "    drop_rows=drop_rows\n",
    ")\n",
    "print(tabulate(\n",
    "    dfc,\n",
    "    ['id', 'interfaces'],\n",
    "    tablefmt='github'\n",
    "))\n",
    "df_categorised[question] = dfc"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "958147d3",
   "metadata": {},
   "source": [
    "## 9.a. Which repositories are important to you?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b0430da6",
   "metadata": {},
   "outputs": [],
   "source": [
    "question = '9.a. Which repositories are important to you?'\n",
    "answers=df[question]\n",
    "keep_words=['cran', 'pypi', 'anaconda', 'kaggle', 'github', 'dockerhub', 'quay', 'conda-forge', 'bioconductor', 'bioconda', 'distribution']\n",
    "synonyms={'cran': ['\\br\\b'], 'pypi': ['python'], 'anaconda': [r'\\*conda', r'\\bconda(?!-)'], 'github': ['git'], 'dockerhub': ['docker hub'], 'distribution': ['apt', 'linux']}\n",
    "drop_rows=[3,8,12,22,28,31,35,36,41,47,54,58,64,67,70,71,72,88,91,92,94,99,101,104]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "30c76b7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "categories(\n",
    "    answers=answers,\n",
    "    keep_words=keep_words,\n",
    "    synonyms=synonyms,\n",
    "    drop_rows=drop_rows\n",
    ").round(decimals=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a4f0c2af-1ddd-4196-a3b5-a843d6c79067",
   "metadata": {},
   "outputs": [],
   "source": [
    "if show_dropped_rows:\n",
    "    print('\\n'.join(answers[drop_rows]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "795ef54a",
   "metadata": {},
   "outputs": [],
   "source": [
    "dfc = categories_per_answer(\n",
    "    answers=answers,\n",
    "    keep_words=keep_words,\n",
    "    synonyms=synonyms,\n",
    "    drop_rows=drop_rows\n",
    ")\n",
    "print(tabulate(\n",
    "    dfc,\n",
    "    ['id', 'interfaces'],\n",
    "    tablefmt='github'\n",
    "))\n",
    "df_categorised[question] = dfc"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "700c433e",
   "metadata": {},
   "source": [
    "## 10.a. Which commercially licenced software is important to you?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ef0696df",
   "metadata": {},
   "outputs": [],
   "source": [
    "question = '10.a. Which commercially licenced software is important to you?'\n",
    "answers=df[question]\n",
    "keep_words=['sas', 'office', 'stata', 'spss', 'powerbi', 'tableau', 'matlab', 'jasp', 'arcgis', 'nvivo', 'gitkraken', 'mathematica', 'maple', 'photoshop']\n",
    "synonyms={'powerbi': ['bi'], 'office': ['excel']}\n",
    "drop_rows=[8,19,22,31,36,46,64,70,71,94,99,104]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "479f12c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "categories(\n",
    "    answers=answers,\n",
    "    keep_words=keep_words,\n",
    "    synonyms=synonyms,\n",
    "    drop_rows=drop_rows\n",
    ").round(decimals=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9846334f-9605-48f8-a129-da6beb7b0b58",
   "metadata": {},
   "outputs": [],
   "source": [
    "if show_dropped_rows:\n",
    "    print('\\n'.join(answers[drop_rows]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "06124b72",
   "metadata": {},
   "outputs": [],
   "source": [
    "dfc = categories_per_answer(\n",
    "    answers=answers,\n",
    "    keep_words=keep_words,\n",
    "    synonyms=synonyms,\n",
    "    drop_rows=drop_rows\n",
    ")\n",
    "print(tabulate(\n",
    "    dfc,\n",
    "    ['id', 'interfaces'],\n",
    "    tablefmt='github'\n",
    "))\n",
    "df_categorised[question] = dfc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7fc08042",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_categorised.columns = [f'{c} (converted)' for c in df_categorised.columns]\n",
    "df_categorised.to_csv('categorised.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}