notebooks/ProjectAnalysis/TestAnalysis/01-CreateResume.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "# To import python scripts from other folders\n",
    "sys.path.append('../')\n",
    "import matplotlib.pyplot as plt\n",
    "import json\n",
    "from ProjectTestAnalysis import ProjectTestAnalysis\n",
    "import os\n",
    "import pandas as pd\n",
    "import concurrent\n",
    "from statistics import median, mean\n",
    "root=\"/home/jovyan/work\"\n",
    "results_path=root+\"/results/\"\n",
    "procesed_results_path=root+\"/notebooks/ProjectAnalysis/TestAnalysis/results/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "ignored_android_projects = [\n",
    "     'ActionBarSherlock',\n",
    "     'roboguice',\n",
    "     'android-Ultra-Pull-To-Refresh',\n",
    "     'ViewPagerIndicator',\n",
    "     'SlidingMenu',\n",
    "     'NineOldAndroids',\n",
    "     'ListViewAnimations',\n",
    "     'Android-PullToRefresh',\n",
    "     'ActiveAndroid',\n",
    "     'android-common',\n",
    "     'drag-sort-listview',\n",
    "]\n",
    "ignored_non_runnable_projects = [\n",
    "    \"guice\",\n",
    "    \"Essentials\",\n",
    "    \"neo4j\",\n",
    "    \"spring-cloud-microservice-example\",\n",
    "    \"canal\",\n",
    "    \"hive\",\n",
    "    \"spring-boot\",\n",
    "    \"YCSB\",\n",
    "    \"wildfly\",\n",
    "    \"gephi\",\n",
    "    \"deeplearning4j\",\n",
    "    \"DataX\",\n",
    "    \"netty\",\n",
    "    \"hbase\",\n",
    "    \"zheng\",\n",
    "    \"openhab\",\n",
    "    \"jstorm\",\n",
    "    \"clojure\",\n",
    "    \"learning-spark\",\n",
    "    \"Mycat-Server\" \n",
    "]\n",
    "ignored_projects = ignored_android_projects + ignored_non_runnable_projects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def mean_f(x): return mean(x) if len(x) > 0 else 0.0\n",
    "def median_f(x): return median(x) if len(x) > 0 else 0.0\n",
    "def div_zero_f(x, y):\n",
    "    if x == 0 or y == 0: return 0\n",
    "    return x / y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getProjects(dataset):\n",
    "    path = root+\"/configFiles/%sProjects/\"%dataset\n",
    "    projects = []\n",
    "    for configFile in os.listdir(path):\n",
    "        with open(path+configFile) as f:\n",
    "            project_info = json.load(f)\n",
    "            project_name = project_info[\"project\"]\n",
    "            if os.path.isdir(procesed_results_path+project_name) and project_name not in ignored_projects:\n",
    "                projects.append((dataset, project_name))\n",
    "    return projects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_project(dataset, project_name):\n",
    "            \n",
    "    # Retrive data\n",
    "    try:\n",
    "        pa = ProjectTestAnalysis(project_name,  2, root=root, forceGenerate=False)\n",
    "    except Exception as e:\n",
    "        print(project_name)\n",
    "    results_df = pa.getSummary()\n",
    "    results_dict = results_df.set_index('commit').to_dict('index')\n",
    "\n",
    "    # Show charts\n",
    "    \n",
    "    # pa.generateAndSavePlot()\n",
    "    \n",
    "    # Generate table\n",
    "\n",
    "    tests_per_commit = results_df[\"n_test\"].tolist()\n",
    "    \n",
    "    report_df = pa.getReport()\n",
    "\n",
    "    total_commits = report_df['id'].count()\n",
    "    total_buildable = report_df[(report_df['build'] == 'SUCCESS')]['id'].count()\n",
    "\n",
    "    total_buildable_test_w_test = 0\n",
    "    total_success_test = 0\n",
    "    total_failures = 0 \n",
    "    total_errors = 0  \n",
    "    \n",
    "    testable_rate_all_commits = []\n",
    "    testable_rate_buildable_commits = []\n",
    "    testable_rate_test_buildable_commits = []\n",
    "    \n",
    "    for _, commit in report_df.iterrows():\n",
    "\n",
    "        c_hash = commit['commit']\n",
    "        \n",
    "        if commit['build'] == 'SUCCESS':\n",
    "            \n",
    "            test_results = results_dict[c_hash]\n",
    "            \n",
    "            if commit['test_build'] == 'SUCCESS' and test_results['n_test'] > 0:\n",
    "\n",
    "                total_buildable_test_w_test += 1\n",
    "\n",
    "                testable_rate_all_commits.append(test_results['testable_rate'])\n",
    "                testable_rate_buildable_commits.append(test_results['testable_rate'])\n",
    "                testable_rate_test_buildable_commits.append(test_results['testable_rate'])\n",
    "\n",
    "                if commit['test'] == 'SUCCESS': \n",
    "                    total_success_test += 1\n",
    "\n",
    "                if commit['test'] == 'FAIL':\n",
    "                    # At least 1 failure (no errors)\n",
    "                    if test_results['n_failures'] > 0 and test_results['n_errors'] == 0:\n",
    "                        total_failures += 1\n",
    "                    # At least 1 error\n",
    "                    else:\n",
    "                        total_errors += 1\n",
    "            else:\n",
    "                testable_rate_all_commits.append(0.0)\n",
    "                testable_rate_buildable_commits.append(0.0)\n",
    "        else:\n",
    "            testable_rate_all_commits.append(0.0)\n",
    "    \n",
    "    mean_consecutive_fails, meadian_consecutive_fails = pa.getMeanAndMedianOfConsecutiveFails()\n",
    "\n",
    "    # TestCases\n",
    "    \n",
    "    test_case_df = pa.getTestCasesRank()\n",
    "\n",
    "    different_tests    = 0\n",
    "    always_success     = 0\n",
    "    success_percent    = 0\n",
    "    never_success      = 0\n",
    "    always_error       = 0\n",
    "    always_fail        = 0\n",
    "    always_skipped     = 0\n",
    "\n",
    "    success_percent_per_test = []\n",
    "    \n",
    "    if test_case_df is not None: \n",
    "\n",
    "        different_tests = len(test_case_df.index)\n",
    "\n",
    "        for index, row in test_case_df.iterrows():\n",
    "\n",
    "            if row['commits'] == row['success']: always_success += 1\n",
    "            if row['success'] == 0: never_success += 1\n",
    "            if row['commits'] == row['failures']: always_fail += 1\n",
    "            if row['commits'] == row['errors']: always_error += 1\n",
    "            if row['commits'] == row['skipped']: always_skipped += 1\n",
    "\n",
    "            success_percent_per_test.append(row['success'] / row['commits'])\n",
    "    \n",
    "    testability_all_commits = div_zero_f(total_success_test, total_commits)\n",
    "    testability_buildable_commits = div_zero_f(total_success_test, total_buildable)\n",
    "    testability_test_buildable_commits = div_zero_f(total_success_test, total_buildable_test_w_test)\n",
    "    \n",
    "    buildability = div_zero_f(total_buildable, total_commits)\n",
    "    \n",
    "    test_buildability_a = div_zero_f(total_buildable_test_w_test, total_commits)\n",
    "    test_buildability_s = div_zero_f(total_buildable_test_w_test, total_buildable)\n",
    "    \n",
    "    total_failures_percent = div_zero_f(total_failures, total_buildable_test_w_test)\n",
    "    total_errors_percent = div_zero_f(total_errors, total_buildable_test_w_test)\n",
    "\n",
    "    testability_rate_all_commits = mean_f(testable_rate_all_commits)\n",
    "    testability_rate_all_commits_std = pd.Series(testable_rate_all_commits).std()\n",
    "    \n",
    "    \n",
    "    testability_rate_buildable_commits = mean_f(testable_rate_buildable_commits)\n",
    "    testability_rate_test_buildable_commits = mean_f(testable_rate_test_buildable_commits)\n",
    "    testability_rate_test_buildable_commits_std = pd.Series(testable_rate_test_buildable_commits).std()\n",
    "\n",
    "    \n",
    "    \n",
    "    ordered_snapshots = results_df.sort_values(by=['n_days'], ascending=False)\n",
    "    oldest = ordered_snapshots.iloc[0]['n_days']\n",
    "    newest = ordered_snapshots.iloc[-1]['n_days']\n",
    "    \n",
    "    loc = pa.getLoCReport().iloc[-1]['loc']\n",
    "    \n",
    "    return ([\n",
    "        pa.project,                                    \n",
    "        dataset,                                       \n",
    "        oldest - newest,\n",
    "        loc,\n",
    "        total_commits,                                 \n",
    "        total_buildable,                               \n",
    "        buildability,              \n",
    "     \n",
    "        total_buildable_test_w_test,                          \n",
    "        test_buildability_a, \n",
    "        test_buildability_s,\n",
    "        \n",
    "        total_success_test,                                   \n",
    "        testability_all_commits,                              \n",
    "        #testability_buildable_commits,\n",
    "        testability_test_buildable_commits,\n",
    "        \n",
    "        testability_rate_all_commits,\n",
    "        testability_rate_all_commits_std,\n",
    "        #testability_rate_buildable_commits,\n",
    "        testability_rate_test_buildable_commits,\n",
    "        testability_rate_test_buildable_commits_std\n",
    "    ], results_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "columns = [\n",
    "        'Project', \n",
    "        'Dataset',\n",
    "        'Age',\n",
    "        'LoC',\n",
    "        'Total Commits', \n",
    "        'Source buildable commits', \n",
    "        'Source buildability', \n",
    "        'Test buildable commits',\n",
    "        'Test buildability_A',\n",
    "        'Test buildability_S',\n",
    "        \n",
    "        'Fully Testable commits',\n",
    "        'FullyTestability_A',\n",
    "        'FullyTestability_T',\n",
    "        \n",
    "        'TestabilityRate_A',\n",
    "        'TestabilityRate_A_std',\n",
    "        'TestabilityRate_T',\n",
    "        'TestabilityRate_T_std',\n",
    "    ]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_projects_resume(projects):\n",
    "    \n",
    "    future_results = []\n",
    "    project_results = []\n",
    "    snapshots_results = []\n",
    "    with concurrent.futures.ThreadPoolExecutor(max_workers=32) as executor:\n",
    "        for dataset, project_name in projects:\n",
    "            future_results.append(executor.submit(process_project, dataset, project_name))\n",
    "            \n",
    "        for f_result in concurrent.futures.as_completed(future_results):\n",
    "            project_result, snapshots_result = f_result.result()\n",
    "            project_results.append(project_result)\n",
    "            snapshots_results.append(snapshots_result)\n",
    "\n",
    "    df_projects = pd.DataFrame(project_results, columns = columns)\n",
    "    \n",
    "    print(\"Projects: %d\"%df_projects['Project'].count())\n",
    "    df_projects = df_projects.sort_values(by=['Project'])\n",
    "    return df_projects.sort_values(by=['Dataset']), pd.concat(snapshots_results)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "many4j = getProjects(\"ManySStub4J\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# ManySStub4J\n",
    "#many4j_projects, many4j_snapshots = get_projects_resume(many4j)\n",
    "#many4j_projects.to_csv(results_path+'Many4JResults.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def showInfoProject(name):\n",
    "    p_data = process_project(\"ManySStub4J\",name)\n",
    "    return pd.DataFrame([p_data[0]], columns = columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Project</th>\n",
       "      <th>Dataset</th>\n",
       "      <th>Age</th>\n",
       "      <th>LoC</th>\n",
       "      <th>Total Commits</th>\n",
       "      <th>Source buildable commits</th>\n",
       "      <th>Source buildability</th>\n",
       "      <th>Test buildable commits</th>\n",
       "      <th>Test buildability_A</th>\n",
       "      <th>Test buildability_S</th>\n",
       "      <th>Fully Testable commits</th>\n",
       "      <th>FullyTestability_A</th>\n",
       "      <th>FullyTestability_T</th>\n",
       "      <th>TestabilityRate_A</th>\n",
       "      <th>TestabilityRate_A_std</th>\n",
       "      <th>TestabilityRate_T</th>\n",
       "      <th>TestabilityRate_T_std</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>camel</td>\n",
       "      <td>ManySStub4J</td>\n",
       "      <td>5211</td>\n",
       "      <td>NaN</td>\n",
       "      <td>53286</td>\n",
       "      <td>21</td>\n",
       "      <td>0.000394</td>\n",
       "      <td>20</td>\n",
       "      <td>0.000375</td>\n",
       "      <td>0.952381</td>\n",
       "      <td>20</td>\n",
       "      <td>0.000375</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.000375</td>\n",
       "      <td>0.01937</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Project      Dataset   Age  LoC  Total Commits  Source buildable commits  \\\n",
       "0   camel  ManySStub4J  5211  NaN          53286                        21   \n",
       "\n",
       "   Source buildability  Test buildable commits  Test buildability_A  \\\n",
       "0             0.000394                      20             0.000375   \n",
       "\n",
       "   Test buildability_S  Fully Testable commits  FullyTestability_A  \\\n",
       "0             0.952381                      20            0.000375   \n",
       "\n",
       "   FullyTestability_T  TestabilityRate_A  TestabilityRate_A_std  \\\n",
       "0                 1.0           0.000375                0.01937   \n",
       "\n",
       "   TestabilityRate_T  TestabilityRate_T_std  \n",
       "0                1.0                    0.0  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "showInfoProject(\"camel\")#[['Test buildability_A', 'Test buildability_S', 'FullyTestability_A', 'FullyTestability_T', 'TestabilityRate_A', 'TestabilityRate_T' ]]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}