notebooks/exploration/explore_malicious.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# CONSTANTS\n",
    "\n",
    "# path to folder with records.json and config.yml\n",
    "# or path to folder of folders with records.json and config.yml\n",
    "# or path to single json\n",
    "INPUT_RECORDS_PATH = \"/var/tmp/xmacko1/bachelor_thesis/repos/AndroidMalwareCrypto/data/merged_malicious.json\"\n",
    "# /var/tmp/xmacko1/bachelor_thesis/experiments/2012_malicious/records.json \n",
    "# /var/tmp/xmacko1/bachelor_thesis/experiments/2013_malicious/records.json \n",
    "# /var/tmp/xmacko1/bachelor_thesis/experiments/2014_malicious/records.json \n",
    "# /var/tmp/xmacko1/bachelor_thesis/experiments/2015_malicious/records.json\n",
    "# /var/tmp/xmacko1/bachelor_thesis/experiments/2016_malicious/records.json\n",
    "# /var/tmp/xmacko1/bachelor_thesis/experiments/2017_malicious/records.json\n",
    "# /var/tmp/xmacko1/bachelor_thesis/experiments/2018_malicious/records.json\n",
    "# /var/tmp/xmacko1/bachelor_thesis/experiments/2019_malicious/records.json\n",
    "# /var/tmp/xmacko1/bachelor_thesis/experiments/2020_malicious/records.json\n",
    "# /var/tmp/xmacko1/bachelor_thesis/experiments/2020_malicious_missing/records.json\n",
    "\n",
    "# path to config.yml or dict - ignored if INPUT_JSON_PATH is path to directory\n",
    "CONFIG = \"/var/tmp/xmacko1/bachelor_thesis/repos/AndroidMalwareCrypto/data/config.yml\" # path or dict\n",
    "\n",
    "API_DEFINITION = \"/var/tmp/xmacko1/bachelor_thesis/repos/AndroidMalwareCrypto/data/api_definition.yml\" # path or dict\n",
    "NEW_TRIGGERS = \"/var/tmp/xmacko1/bachelor_thesis/repos/AndroidMalwareCrypto/data/new_triggers.yml\" # path or dict\n",
    "REPORT_PATH = \"/var/tmp/xmacko1/bachelor_thesis/repos/AndroidMalwareCrypto/results/exploration/malicious_report\" # optional path\n",
    "\n",
    "# config that does not clean labels\n",
    "CLEANING_CONFIG = {\n",
    "    \"remove_third_party_crypto_from_packages\": True,\n",
    "    \"remove_similar_records_classes\": True,\n",
    "    \"crypto_api_calls\": {\n",
    "        \"remove_when_imports_empty\": True,\n",
    "        \"remove_duplicates\": True,\n",
    "        \"remove_false_positives\": True,\n",
    "        \"calls_to_filter\": [\"Cipher\", \"PRIVATE_KEY\", \"SECRET_KEY\", \"PUBLIC_KEY\", \"ENCRYPT_MODE\"]\n",
    "    }\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "import pprint\n",
    "from copy import deepcopy\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import yaml\n",
    "import re\n",
    "import logging\n",
    "import copy\n",
    "import matplotlib\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "from dataclasses import dataclass, field, InitVar\n",
    "from pathlib import Path\n",
    "import yaml\n",
    "import re\n",
    "import copy\n",
    "import sys\n",
    "import csv\n",
    "import math\n",
    "from typing import Any, Optional, Union, Dict\n",
    "import matplotlib.ticker as mtick\n",
    "\n",
    "sys.path.insert(0, os.path.abspath(\"../..\"))\n",
    "from androidcrypto.analysis.evaluator import Evaluator, EvaluatorInput\n",
    "from androidcrypto.analysis.preparation.config import CleaningConfig\n",
    "from androidcrypto.analysis.preparation.loading import load_json_data\n",
    "from androidcrypto.analysis.preparation.cleaning import clean_loaded_df_based_on_config\n",
    "\n",
    "# Pandas display options\n",
    "pd.set_option(\"display.max_colwidth\", 102)\n",
    "pd.set_option(\"display.max_rows\", 102)\n",
    "\n",
    "sns.set_style(\"whitegrid\", {\n",
    "    \"ytick.major.size\": 0.1,\n",
    "    \"ytick.minor.size\": 0.05,\n",
    "    'grid.linestyle': '--'\n",
    " })\n",
    "\n",
    "def get_category_triggers(api_definition, category):\n",
    "    obfuscated_trigger = None\n",
    "    complex_cat = False\n",
    "    \n",
    "    if len(api_definition[category].keys()) > 2:\n",
    "        triggers = {}\n",
    "        complex_cat = True\n",
    "    else:\n",
    "        triggers = []\n",
    "    \n",
    "    for key, subcat_triggers in api_definition[category].items():\n",
    "        if key == 'obfuscated':\n",
    "            obfuscated_trigger = subcat_triggers\n",
    "            if complex_cat == True:\n",
    "                triggers[key] = subcat_triggers\n",
    "        else:\n",
    "            if complex_cat == True:\n",
    "                triggers[key] = subcat_triggers\n",
    "            else:\n",
    "                triggers.extend(subcat_triggers)\n",
    "                \n",
    "    return obfuscated_trigger, triggers, complex_cat\n",
    "\n",
    "def get_triggers(inp):\n",
    "    if isinstance(inp, dict):\n",
    "        triggers = []\n",
    "        for k in inp.keys():\n",
    "            if k != 'obfuscated':\n",
    "                triggers.extend(inp[k])\n",
    "        return inp['obfuscated'], triggers\n",
    "    if isinstance(inp, list):\n",
    "        return None, inp\n",
    "    \n",
    "def strip_triggers(triggers):\n",
    "    dct = {}\n",
    "    for t in triggers:\n",
    "        if '\"' in t:\n",
    "            dct[t] = t.split('\"')[1]\n",
    "        else:\n",
    "            dct[t] = 'obfuscated'\n",
    "    return dct"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cleaning: Missing values.\n",
      "Cleaning: Third party packages.\n",
      "Cleaning: Duplicate (similar) crypto API records keys (classes).\n",
      "Cleaning: Crypto API records calls.\n",
      "Cleaning: Removing crypto API calls.\n",
      "Cleaning: Removing crypto API duplicates.\n",
      "Cleaning: Removing crypto API false positives.\n"
     ]
    }
   ],
   "source": [
    "def read_records_and_config(records_path: str,\n",
    "                            config_path: Optional[str]=None\n",
    "                           ) -> Union[EvaluatorInput, str]:\n",
    "    \n",
    "    if os.path.isdir(records_path):\n",
    "        return records_path\n",
    "    \n",
    "    records = load_json_data(records_path).to_dict(orient=\"index\")\n",
    "    \n",
    "    config = {}\n",
    "    if config_path:\n",
    "        with open(config_path, 'r') as handle:\n",
    "            config = yaml.safe_load(handle)\n",
    "    return EvaluatorInput(\n",
    "        records,\n",
    "        config\n",
    "    )\n",
    "\n",
    "def clean_evaluator_input(evaluator_input: EvaluatorInput, raw_cleaning_config: Dict[str, Any]) -> EvaluatorInput:\n",
    "    \n",
    "    df = pd.DataFrame.from_dict(evaluator_input.records, orient=\"index\")\n",
    "    cleaning_config = CleaningConfig(raw_cleaning_config)\n",
    "    cleaned_df = clean_loaded_df_based_on_config(df, cleaning_config)\n",
    "    evaluator_input.records = cleaned_df.to_dict(orient=\"index\")\n",
    "    return evaluator_input\n",
    "\n",
    "evaluator_input = read_records_and_config(INPUT_RECORDS_PATH, CONFIG)\n",
    "if isinstance(evaluator_input, EvaluatorInput):\n",
    "    evaluator_input = clean_evaluator_input(evaluator_input, CLEANING_CONFIG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Statistics about third-party packages\n",
      "218397 out of 301898 (72.34%) apks contain at least one third party package\n",
      "In total, the dataset contains 2042919 unique third-party package names.\n",
      "On average, each apk contains: 6.766917965670524 third-party packages.\n",
      "--------------------------------------------------------------------------------\n",
      "In total, the dataset contains 2042919 unique third-party package names.\n",
      "General statistics about dataset\n",
      "Number of analyzed samples: 301,898\n",
      "How many of analyzed samples use some crypto...\n",
      "|   year |   all_samples |   containing_crypto |   empty_samples |   percentage_containing_crypto |\n",
      "|-------:|--------------:|--------------------:|----------------:|-------------------------------:|\n",
      "|   2012 |         39767 |               21188 |           18579 |                       53.2804  |\n",
      "|   2013 |         39674 |               26325 |           13349 |                       66.3533  |\n",
      "|   2014 |         39794 |               22207 |           17587 |                       55.8049  |\n",
      "|   2015 |         39745 |               19316 |           20429 |                       48.5998  |\n",
      "|   2016 |         39325 |               16317 |           23008 |                       41.4927  |\n",
      "|   2017 |         34386 |                8177 |           26209 |                       23.78    |\n",
      "|   2018 |         19305 |                5098 |           14207 |                       26.4077  |\n",
      "|   2019 |         39863 |                6592 |           33271 |                       16.5366  |\n",
      "|   2020 |         10039 |                 593 |            9446 |                        5.90696 |\n",
      "--------------------------------------------------------------------------------\n",
      "Usage of third party cryptographic libraries\n",
      "Java libraries:\n",
      "year  lib_name                              \n",
      "2012  Lorg/keyczar                              39\n",
      "      Lnet/sqlcipher                             8\n",
      "2013  Lorg/keyczar                              42\n",
      "      Lnet/sqlcipher                             1\n",
      "2014  Lorg/keyczar                              33\n",
      "      Lnet/sqlcipher                            16\n",
      "2015  Lnet/sqlcipher                            27\n",
      "      Lorg/keyczar                               5\n",
      "2016  Lnet/sqlcipher                            15\n",
      "      Lorg/keyczar                               5\n",
      "      Linfo/guardianproject/netcipher/client     1\n",
      "      Linfo/guardianproject/netcipher/proxy      1\n",
      "2017  Lnet/sqlcipher                             3\n",
      "2018  Lnet/sqlcipher                             2\n",
      "      Lgnu/crypto/assembly                       1\n",
      "      Lgnu/crypto/cipher                         1\n",
      "      Lgnu/crypto/jce/cipher                     1\n",
      "      Lgnu/crypto/key/rsa                        1\n",
      "      Lgnu/crypto/keyring                        1\n",
      "      Lgnu/crypto/sasl/crammd5                   1\n",
      "      Lgnu/crypto/sasl/plain                     1\n",
      "      Lgnu/crypto/sig/dss                        1\n",
      "Name: lib_name, dtype: int64\n",
      "--------------------------------------------------------------------------------\n",
      "Prevalence of java libraries per year:\n",
      "year\n",
      "2012    47\n",
      "2013    43\n",
      "2014    49\n",
      "2015    32\n",
      "2016    21\n",
      "2017     3\n",
      "2018     3\n",
      "Name: sha256, dtype: int64\n",
      "--------------------------------------------------------------------------------\n",
      "Overall popularity of java libraries:\n",
      "Lorg/keyczar                              124\n",
      "Lnet/sqlcipher                             72\n",
      "Lgnu/crypto/jce/cipher                      1\n",
      "Lgnu/crypto/assembly                        1\n",
      "Lgnu/crypto/sasl/plain                      1\n",
      "Linfo/guardianproject/netcipher/client      1\n",
      "Lgnu/crypto/sasl/crammd5                    1\n",
      "Lgnu/crypto/cipher                          1\n",
      "Linfo/guardianproject/netcipher/proxy       1\n",
      "Lgnu/crypto/keyring                         1\n",
      "Lgnu/crypto/key/rsa                         1\n",
      "Lgnu/crypto/sig/dss                         1\n",
      "Name: lib_name, dtype: int64\n",
      "Native libraries:\n",
      "No native cryptographic libraries were found\n",
      "--------------------------------------------------------------------------------\n",
      "Number of crypto API imports per sample:\n",
      "|       |   crypto_imports_sum |\n",
      "|:------|---------------------:|\n",
      "| count |          125813      |\n",
      "| mean  |              13.9052 |\n",
      "| std   |              27.1782 |\n",
      "| min   |               0      |\n",
      "| 25%   |               2      |\n",
      "| 50%   |               7      |\n",
      "| 75%   |              15      |\n",
      "| max   |            2462      |\n",
      "Out of 86 analyzed classes, the following number is actually imported:\n",
      "year\n",
      "2012    57\n",
      "2013    60\n",
      "2014    74\n",
      "2015    75\n",
      "2016    70\n",
      "2017    63\n",
      "2018    73\n",
      "2019    59\n",
      "2020    51\n",
      "Name: api_class, dtype: int64\n",
      "--------------------------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "# Load evaluator object and evaluate basic stuff\n",
    "\n",
    "evaluator = Evaluator(\n",
    "    evaluator_input, # input - either folder path or EvaluatorInput\n",
    "    report_path=REPORT_PATH,\n",
    "    api_definition=API_DEFINITION,\n",
    "    new_triggers=NEW_TRIGGERS,\n",
    ")\n",
    "evaluator.evaluate()\n",
    "#df_cipher = evaluator.plot_ciphers(evaluator.df_records, '', deepcopy(evaluator.exp_config['evaluate']['categories']['Cipher']))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluation of categories"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def map_norm_factor_multiindex(x):\n",
    "    return evaluator.norm_factor[x[0]]\n",
    "\n",
    "def get_all_triggers(obj):\n",
    "    triggers = []\n",
    "    \n",
    "    if isinstance(obj, dict):\n",
    "        if len(obj.keys()) == 2:\n",
    "            for key in obj.keys():\n",
    "                if key != 'obfuscated':\n",
    "                    triggers.extend(get_all_triggers(obj[key]))\n",
    "                else:\n",
    "                    triggers.append(obj[key])\n",
    "        elif len(obj.keys()) > 2:\n",
    "            for key in obj.keys():\n",
    "                triggers.extend(get_all_triggers(obj[key]))\n",
    "    elif isinstance(obj, list):\n",
    "        triggers = obj\n",
    "    \n",
    "    return triggers\n",
    "\n",
    "@dataclass\n",
    "class Comparison:\n",
    "    name: str\n",
    "    categories: Any\n",
    "    path: str\n",
    "    obfuscated_trigger: str\n",
    "        \n",
    "    def __repr__(self):\n",
    "        return f'name: {self.name}, path: {self.path}, obf: {self.obfuscated_trigger}'       \n",
    "\n",
    "def get_all_comps(obj, path, name, obfuscated_trigger):\n",
    "    comps = []\n",
    "    \n",
    "    if isinstance(obj, list):\n",
    "        cats = {x.split('\"')[1]: [x] for x in obj}\n",
    "        comps.append(Comparison(name, cats, path, obfuscated_trigger))\n",
    "    \n",
    "    if isinstance(obj, dict):\n",
    "        if len(obj.keys()) > 2:\n",
    "            comps.append(Comparison(name, {key: get_all_triggers(val) for key, val in obj.items()}, os.path.join(path), obfuscated_trigger))\n",
    "            \n",
    "            for key, val in obj.items():\n",
    "                comps.extend(get_all_comps(val, os.path.join(path, key), key, None))\n",
    "            \n",
    "        if len(obj.keys()) == 2:\n",
    "            comps.extend(get_all_comps(obj['values'], path, name, obj['obfuscated']))\n",
    "    \n",
    "    return comps\n",
    "\n",
    "def get_df_temp(category_name, category_triggers, df_eval, sum_temp_n_call_sites):\n",
    "    df = df_eval.loc[df_eval.trigger.isin(category_triggers)].groupby('year')\n",
    "    \n",
    "    temp_n_call_sites = df.trigger.count()\n",
    "    temp_n_call_sites.name = 'n_call_sites'\n",
    "\n",
    "    temp_norm_n_call_sites = temp_n_call_sites / temp_n_call_sites.index.map(evaluator.norm_factor)\n",
    "    temp_norm_n_call_sites = temp_norm_n_call_sites.map(math.ceil)\n",
    "    temp_norm_n_call_sites.name = 'norm_n_call_sites'\n",
    "\n",
    "    temp_n_apks = df.sha256.nunique()\n",
    "    temp_n_apks.name = 'n_apks'\n",
    "\n",
    "    df_to_return = pd.concat([temp_n_call_sites, temp_norm_n_call_sites, temp_n_apks], axis=1)\n",
    "    df_to_return['category'] = category_name\n",
    "    df_to_return = df_to_return.set_index('category', append=True)\n",
    "    df_to_return['norm_n_apks'] = df_to_return.n_apks / df_to_return.index.map(map_norm_factor_multiindex)\n",
    "    df_to_return.norm_n_apks = df_to_return.norm_n_apks.astype('int64')\n",
    "\n",
    "    df_to_return['norm_n_apks_ratio'] = 100 * (df_to_return.norm_n_apks / 10000)\n",
    "    df_to_return['n_call_sites_ratio'] = 100 * (df_to_return.n_call_sites / sum_temp_n_call_sites)\n",
    "    \n",
    "    return df_to_return\n",
    "\n",
    "def get_df_summary(df_eval, df_temp, dataset_size, all_triggers):\n",
    "    df = df_temp.drop(columns=['norm_n_call_sites', 'norm_n_apks', 'norm_n_apks_ratio', 'n_call_sites_ratio']).sum(level='category')\n",
    "    sum_n_call_sites = df.n_call_sites.sum()\n",
    "    sum_n_apks = df_eval.loc[df_eval.trigger.isin(all_triggers)].sha256.nunique()\n",
    "\n",
    "    df['call_sites_ratio'] = 100 * df.n_call_sites / sum_n_call_sites\n",
    "    df['n_apks_ratio'] = 100 * df.n_apks / dataset_size\n",
    "\n",
    "    sum_row = pd.Series([sum_n_call_sites, sum_n_apks, 1, 100 * sum_n_apks / dataset_size], index=['n_call_sites', 'n_apks', 'call_sites_ratio', 'n_apks_ratio'], name='Total')\n",
    "    df = df.append(sum_row)\n",
    "    \n",
    "    return df\n",
    "\n",
    "def evaluate_comparison(comparison):\n",
    "    if not os.path.exists(comparison.path):\n",
    "        os.makedirs(comparison.path)\n",
    "    \n",
    "    if comparison.obfuscated_trigger is not None:\n",
    "        comparison.categories['obfuscated'] = [comparison.obfuscated_trigger]\n",
    "    \n",
    "    all_triggers = get_all_triggers(comparison.categories)\n",
    "    sum_temp_n_call_sites  = evaluator.df_records.loc[evaluator.df_records.trigger.isin(all_triggers)].groupby('year').trigger.count()\n",
    "    temps = []\n",
    "\n",
    "    for cat_name, cat_triggers in comparison.categories.items():\n",
    "        temps.append(get_df_temp(cat_name, cat_triggers, evaluator.df_records, sum_temp_n_call_sites))\n",
    "\n",
    "    df_temp = pd.concat(temps)\n",
    "    \n",
    "    if comparison.name == 'all':\n",
    "        all_n_apks = evaluator.df_records.loc[evaluator.df_records.trigger.isin(all_triggers)].groupby('year').sha256.nunique()\n",
    "        all_apks_df = all_n_apks.to_frame(name='n_apks')\n",
    "        all_apks_df['norm_n_apks'] = all_apks_df.n_apks / all_apks_df.index.map(evaluator.norm_factor)\n",
    "        all_apks_df['norm_n_apks'] = all_apks_df['norm_n_apks'].map(math.ceil)\n",
    "        all_apks_df['norm_n_apks_ratio'] = 100 * all_apks_df.norm_n_apks / 10000\n",
    "        all_apks_df['category'] = 'all'\n",
    "        all_apks_df.set_index('category', append=True, inplace=True)\n",
    "        \n",
    "\n",
    "        df_temp = df_temp.append(all_apks_df)\n",
    "    \n",
    "    \n",
    "    df_temp.to_html(os.path.join(comparison.path, 'temporal_evolution.html'))\n",
    "    \n",
    "    \n",
    "    \n",
    "    if not df_temp.empty:\n",
    "        plot_temporal_evolution(df_temp, os.path.join(comparison.path, 'temporal_evolution.png'), comparison.name)\n",
    "        plot_boxplot(df_temp, os.path.join(comparison.path, 'boxplot.png'), comparison.name)\n",
    "    \n",
    "    df_sum = get_df_summary(evaluator.df_records, df_temp, evaluator.dset_size, all_triggers)\n",
    "    df_sum.to_html(os.path.join(comparison.path, 'summary.html'))\n",
    "    \n",
    "    appendix_csv = df_sum.drop(columns=['n_apks', 'n_apks_ratio', 'call_sites_ratio'])\n",
    "    appendix_csv.n_call_sites = appendix_csv.n_call_sites.astype('int64')\n",
    "    appendix_csv.to_csv(os.path.join(comparison.path, 'summary.csv'), sep='&')\n",
    "\n",
    "    return df_temp\n",
    "    \n",
    "def plot_temporal_evolution(df_temp, path, title, drop_obfuscated=True):\n",
    "    \n",
    "    df = df_temp.reset_index()\n",
    "    \n",
    "    if drop_obfuscated is True:\n",
    "        df = df.loc[df.category != 'obfuscated']\n",
    "    \n",
    "    df = df.sort_values(by='norm_n_apks_ratio', ascending=False)\n",
    "    \n",
    "    ax = sns.lineplot(x='year', y='norm_n_apks_ratio', hue='category', data=df, marker='o')\n",
    "    #ax.yaxis.set_major_locator(mtick.MultipleLocator(5)) # density of grid lines, 5% now\n",
    "    ax.xaxis.set_major_locator(mtick.MultipleLocator(1))\n",
    "    ax.yaxis.set_major_formatter(mtick.PercentFormatter())\n",
    "    ax.set(xlabel='Year', ylabel='% of all samples',\n",
    "           title=title)\n",
    "    ax.legend(bbox_to_anchor=(1.05, 1), loc=2, borderaxespad=0.)\n",
    "    ax.figure.savefig(path, dpi=300, format='png', bbox_inches='tight', pad_inches=0.1)\n",
    "    plt.close()\n",
    "\n",
    "def plot_boxplot(df_temp, path, title, drop_obfuscated=True):\n",
    "    df = df_temp.reset_index()\n",
    "    \n",
    "    if drop_obfuscated is True:\n",
    "        df = df.loc[df.category != 'obfuscated']\n",
    "    \n",
    "    df = df.sort_values(by='norm_n_apks_ratio', ascending=False)\n",
    "    \n",
    "    df = df.sort_values(by=['norm_n_apks'], ascending=False)\n",
    "    ax = sns.boxplot(x='category', y='norm_n_apks', data=df)\n",
    "    plt.xticks(rotation=90)\n",
    "    ax.set(xlabel=f'category', ylabel='Number of APKs per 10k APKs', title=f'{title} frequency')\n",
    "    ax.figure.savefig(path, dpi=300, format='png',\n",
    "                      bbox_inches='tight', pad_inches=0.1)\n",
    "    plt.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Ratio of samples containing cryptographic API, considered as the samples that contain any of the analyzed imports."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "year\n",
       "2012    21148\n",
       "2013    26323\n",
       "2014    22194\n",
       "2015    19307\n",
       "2016    16311\n",
       "2017     8177\n",
       "2018     5098\n",
       "2019     6592\n",
       "2020      593\n",
       "Name: sha256, dtype: int64"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.df_imports.groupby('year').sha256.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "81"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "evaluator.df_imports.api_class.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "comparisons = get_all_comps(evaluator.api_definition, '/var/tmp/xmacko1/bachelor_thesis/repos/AndroidMalwareCrypto/results/exploration/malicious_report', 'all', None)\n",
    "for cmp in comparisons:\n",
    "    df_temp = evaluate_comparison(cmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Number of call sites per positive APK (increased in 2018)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_triggs = get_all_triggers(evaluator.api_definition)\n",
    "df = evaluator.df_records.loc[evaluator.df_records.trigger.isin(all_triggs)]\n",
    "years = df.groupby('year').trigger.count()\n",
    "n_apks = df.groupby('year').sha256.nunique()\n",
    "call_sites_per_positive = years / n_apks\n",
    "twelve_seventeen = call_sites_per_positive.loc[call_sites_per_positive.index < 2018]\n",
    "print(f'Number of call sites per positive APK, average 2012-2017: {twelve_seventeen.mean()}')\n",
    "print(f'Number of call sites per positive APK, 2018: {call_sites_per_positive[2018]}')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Number of third-party packages 2012-2017 vs 2018 (hypothesis about increased call sites per APK)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df_2012_2017 = evaluator.df.loc[evaluator.df.year != 2018]\n",
    "df_2018 = evaluator.df.loc[evaluator.df.year == 2018]\n",
    "print(f'Average number of detected third-party packages 2012-2017: {df_2012_2017.n_third_party.mean()}')\n",
    "print(f'Average number of detected third-party packages 2018: {df_2018.n_third_party.mean()}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rsa_triggers = get_all_triggers(evaluator.api_definition['encryption']['values']['RSA'])\n",
    "sum_temp_n_call_sites  = evaluator.df_records.loc[evaluator.df_records.trigger.isin(rsa_triggers)].groupby('year').trigger.count()\n",
    "obfuscated_trigger = 'Cipher.getInstance('\n",
    "\n",
    "rsa_temp = get_df_temp('RSA', rsa_triggers, evaluator.df_records, sum_temp_n_call_sites)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rsa_temp"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Manual comparison with ASIACCS 2018 benign dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def apply_rsa_correction(cs_aes, cs_des, cs_3des, cs_rc4, cs_blowfish, cs_obf, cs_rsa, cs_pbe):\n",
    "    \"\"\"\n",
    "    Because RSA has the same interface as symmetric ciphers, it skews the results. For instance, sum of ratio of symmetric ciphers does not sum up to 100%. At the same time\n",
    "    some obfuscated statements belong to RSA but are interpreted as symmetric. This why we apply the following corrections.\n",
    "    The input to this function is number of call sites for all ciphersuites and it prints the ratio of symmetric ciphers to the output, as well as the total number of symmetric call sites (as well as RSA)\n",
    "    \"\"\"\n",
    "    clean_all = cs_rsa + cs_aes + cs_des + cs_3des + cs_rc4 + cs_blowfish + cs_pbe\n",
    "    cs_all = clean_all + cs_obf\n",
    "    \n",
    "    cs_symmetric = cs_all - cs_rsa - ((cs_rsa / clean_all) * cs_obf)\n",
    "    cs_obfuscated = cs_obf * (1 - (cs_rsa / clean_all))\n",
    "    \n",
    "    rsa_obf = (cs_rsa / clean_all) * cs_obf\n",
    "    rsa_all = rsa_obf + cs_rsa\n",
    "\n",
    "    print(f'After correction:')\n",
    "    print(f'Sum symmetric: {cs_symmetric}')\n",
    "    print(f'Call sites obfuscated: {cs_obfuscated}')\n",
    "    print(f'Ratio AES: {100 * cs_aes / cs_symmetric}')\n",
    "    print(f'Ratio DES: {100 * cs_des / cs_symmetric}')\n",
    "    print(f'Ratio 3DES: {100 * cs_3des / cs_symmetric}')\n",
    "    print(f'Ratio RC4: {100 * cs_rc4 / cs_symmetric}')\n",
    "    print(f'Ratio Blowfish: {100 * cs_blowfish / cs_symmetric}')\n",
    "    print(f'Ratio PBE: {100 * cs_pbe / cs_symmetric}')\n",
    "    print(f'Ratio obfuscated: {100 * cs_obfuscated / cs_symmetric}')\n",
    "    \n",
    "\n",
    "    print(f'\\nRSA call sites obfuscated: {(cs_rsa / clean_all) * cs_obf}')\n",
    "    print(f'RSA ratio of obfuscated: {100 * rsa_obf / rsa_all}')\n",
    "    \n",
    "# Correction for M_all\n",
    "apply_rsa_correction(61516, 70522, 7954, 245, 413, 83295, 15708, 2108)\n",
    "\n",
    "# Correction for M16\n",
    "#apply_rsa_correction(15236, 7141, 907, 116, 90, 13259, 6926, 227)\n",
    "\n",
    "# Correction for M12\n",
    "#apply_rsa_correction(241761, 61516, 70522, 7954, 245, 413, 83295, 15708, 2108)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### benign_n_samples = 115683\n",
    "benign_n_call_sites = 251021\n",
    "downscale_factor = evaluator.dataset_sizes[2016] / benign_n_samples\n",
    "third_party_lib_factor = 9.3 / 100\n",
    "print(f'Benign call sites downscaled from {benign_n_call_sites}: {downscale_factor * third_party_lib_factor * benign_n_call_sites}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_n_calls(year, triggers):\n",
    "    return evaluator.df_records.loc[(evaluator.df_records.year == year) & (evaluator.df_records.trigger.isin(triggers))].shape[0]\n",
    "\n",
    "def analyze_cipher_distribution(year, categories):\n",
    "    obf_triggers = ['Cipher.getInstance(']\n",
    "    obf_calls = get_n_calls(year, obf_triggers)\n",
    "    flatten = lambda l: [item for sublist in l for item in sublist]\n",
    "    \n",
    "    all_triggers = flatten([evaluator.api_definition['encryption'][cat] for cat in categories]) + obf_triggers\n",
    "    all_calls = get_n_calls(year, all_triggers)\n",
    "    print(f'all calls: {all_calls} ({(100 * all_calls / all_calls):.2f}%)')\n",
    "    print(f'obfuscated calls: {obf_calls} ({(100 * obf_calls / all_calls):.2f}%)')\n",
    "    \n",
    "    for cat in categories:\n",
    "        cat_triggers = evaluator.api_definition['encryption'][cat]\n",
    "        cat_calls = get_n_calls(year, cat_triggers)\n",
    "        print(f'{cat} calls: {cat_calls} ({(100 * cat_calls / all_calls):.2f}%)')\n",
    "\n",
    "# todo: place RC4 below instead of ARC4\n",
    "ciphers = ['AES', 'DES', '3DES', 'ARC4', 'BLOWFISH', 'CHACHA']\n",
    "\n",
    "analyze_cipher_distribution(2016, ciphers)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Manual comparison with CCS 2013 benign dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Some marginal functions for processing df_cipher\n",
    "def strip_get_instance(x):\n",
    "    prefix = 'Cipher.getInstance('\n",
    "    if x == prefix:\n",
    "        return 'obfuscated'\n",
    "\n",
    "    if str(x).startswith(prefix):\n",
    "        return str(x)[len(prefix) + 1:]\n",
    "    else:\n",
    "        return x\n",
    "\n",
    "def div_norm(x):\n",
    "    return int((11748 / 21649) * x)\n",
    "\n",
    "def rename_star_notation(x):\n",
    "    if x == 'AES':\n",
    "        return 'AES *'\n",
    "    if x == 'DES':\n",
    "        return 'DES *'\n",
    "    if x == 'DESede':\n",
    "        return 'DESede *'\n",
    "    return x\n",
    "\n",
    "def get_cat(string):\n",
    "    if 'AES' in string:\n",
    "        return 'AES'\n",
    "    if 'DESede' in string:\n",
    "        return '3DES'\n",
    "    if 'DES' in string and ('DESede' not in string):\n",
    "        return 'DES'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "triggers = evaluator.api_definition['encryption']['values']['AES'] + evaluator.api_definition['encryption']['values']['DES'] + evaluator.api_definition['encryption']['values']['3DES']\n",
    "obfuscated_trigger = 'Cipher.getInstance('\n",
    "rsa_popularity = 0.0649 # red from tables\n",
    "symmetric_fraction_of_obfuscated = 1 - rsa_popularity\n",
    "df_cipher_clean = evaluator.df_records.loc[(evaluator.df_records.trigger.isin(triggers)) & (evaluator.df_records.year == 2012)]\n",
    "df_cipher_obfuscated = evaluator.df_records.loc[(evaluator.df_records.trigger == obfuscated_trigger) & (evaluator.df_records.year == 2012)]\n",
    "print(f'Number of clean call sites: {df_cipher_clean.shape[0]}')\n",
    "print(f'Number of obfuscated call sites {int(symmetric_fraction_of_obfuscated * df_cipher_obfuscated.shape[0])}')\n",
    "\n",
    "df_cipher_clean.trigger = df_cipher_clean.trigger.apply(strip_get_instance)\n",
    "df_cipher_clean.trigger = df_cipher_clean.trigger.apply(rename_star_notation)\n",
    "val_counts = df_cipher_clean.trigger.value_counts()\n",
    "val_counts = val_counts.apply(div_norm)\n",
    "\n",
    "malicious_df = val_counts.to_frame(name='malicious_frequency')\n",
    "malicious_df.malicious_frequency = malicious_df.malicious_frequency.astype('int64')\n",
    "malicious_df.index.name = 'Encryption scheme'\n",
    "\n",
    "# benign_dict taken from Table 4 in CryptoLint paper\n",
    "benign_dict = {'AES/CBC/PKCS5Padding': 5878, 'AES *': 4803, 'DES/ECB/NoPadding': 1151, 'DES *': 741, 'DESede *': 501, 'DESede/ECB/PKCS5Padding': 473, 'AES/CBC/NoPadding': 468, 'AES/ECB/PKCS5Padding': 443, 'AES/CBC/PKCS7Padding': 235, 'DES/ECB/PKCS5Padding': 221, 'AES/ECB/NoPadding': 220, 'DES/CBC/PKCS5Padding': 205, 'AES/ECB/PKCS7Padding': 155, 'AES/CFB8/NoPadding': 104}\n",
    "benign_df = pd.Series(benign_dict).to_frame(name='benign_frequency')\n",
    "benign_df.index.name = 'Encryption scheme'\n",
    "\n",
    "result_df = malicious_df.merge(benign_df, how='outer', on='Encryption scheme').fillna(0).sort_values(by=['malicious_frequency', 'benign_frequency'], ascending=False)\n",
    "\n",
    "sum_aes_malicious = result_df.malicious_frequency.loc[(result_df.index.map(get_cat) == 'AES') & (result_df.malicious_frequency > 100)].sum()\n",
    "sum_des_malicious = result_df.malicious_frequency.loc[(result_df.index.map(get_cat) == 'DES') & (result_df.malicious_frequency > 100)].sum()\n",
    "sum_desede_malicious = result_df.malicious_frequency.loc[(result_df.index.map(get_cat) == '3DES') & (result_df.malicious_frequency > 100)].sum()\n",
    "sum_malicious = sum_aes_malicious + sum_des_malicious + sum_desede_malicious\n",
    "\n",
    "sum_aes_benign = result_df.benign_frequency.loc[(result_df.index.map(get_cat) == 'AES') & (result_df.benign_frequency > 100)].sum()\n",
    "sum_des_benign = result_df.benign_frequency.loc[(result_df.index.map(get_cat) == 'DES') & (result_df.benign_frequency > 100)].sum()\n",
    "sum_desede_benign = result_df.benign_frequency.loc[(result_df.index.map(get_cat) == '3DES') & (result_df.benign_frequency > 100)].sum()\n",
    "sum_benign = sum_aes_benign + sum_des_benign + sum_desede_benign\n",
    "\n",
    "result_df = result_df.append(pd.Series([sum_aes_malicious, sum_aes_benign], index=['malicious_frequency', 'benign_frequency'], name='AES sum with freq. > 100'))\n",
    "result_df = result_df.append(pd.Series([100 * sum_aes_malicious / sum_malicious, 100 * sum_aes_benign / sum_benign], index=['malicious_frequency', 'benign_frequency'], name='AES ratio (%) with freq. > 100'))\n",
    "\n",
    "result_df = result_df.append(pd.Series([sum_des_malicious, sum_des_benign], index=['malicious_frequency', 'benign_frequency'], name='DES sum with freq. > 100'))\n",
    "result_df = result_df.append(pd.Series([100 * sum_des_malicious / sum_malicious, 100 * sum_des_benign / sum_benign], index=['malicious_frequency', 'benign_frequency'], name='DES ratio (%) with freq. > 100'))\n",
    "\n",
    "result_df = result_df.append(pd.Series([sum_desede_malicious, sum_desede_benign], index=['malicious_frequency', 'benign_frequency'], name='DESede sum with freq. > 100'))\n",
    "result_df = result_df.append(pd.Series([100 * sum_desede_malicious / sum_malicious, 100 * sum_desede_benign / sum_benign], index=['malicious_frequency', 'benign_frequency'], name='DESede ratio (%) with freq. > 100'))\n",
    "\n",
    "result_df = result_df.append(pd.Series([sum_malicious, sum_benign], index=['malicious_frequency', 'benign_frequency'], name='Total sum with freq. > 100'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('/Users/adam/phd/projects/CryptoMalware/experiments/cagliari_2017/results/2017.json', 'r') as json_handle:\n",
    "    data = json.load(json_handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "package_names = set()\n",
    "packages = set()\n",
    "\n",
    "for key in data.keys():\n",
    "    packages.update(data[key]['standard_third_party_packages'])\n",
    "    package_names.update(data[key]['third_party_packages'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Manual analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dct = {'md5': ['e8d0fd1ffda4c5e5ffa548e4dfe1592a51afcb34a762cc8434950d7da4ecef3b',\n",
    "               'c93a6b4149a5641cc436f37373200ec5a9da4989d063394ab961bb7141d18e5c',\n",
    "               '9f23427f4adf2d3fe5bad394e77f1b5c2288069ee9c3bb48b8cbf33a4bdcab9d',\n",
    "               'f26a759ff6b136ba291494b0cbf42f06b240092662ab78d2f7d6369d685b5a4a',\n",
    "               '46547374d21a79b9f01b12e7b7361de09eeef1e74cb34187bbd1c493315e98ff'\n",
    "              ],\n",
    "      'sha256': ['78628a017231e52e8bdfe382b3d48cac4355bda5fbf9a3cdc58122fa84e7fd6a',\n",
    "                 '3d44791ccd1d5b871a1e4d4fcfc90b667f219c7979ca02d220e442552ec2b357'\n",
    "                ],\n",
    "      'sha1': ['9fb6230d41fa2a9c7b89d61ab61e77079dd415b481810f320aa267758ced59e9',\n",
    "              'fcd823ed368dc7ac957c038e228fdb624292a9380032ba30b69a4130a208e62d',\n",
    "              '2f34b37bc01283c1aa63b5ef8fde2212d523c405eff185af4c79f5624189c390'\n",
    "              ],\n",
    "      'rsa': ['0d9e99e79d9a04e41da65fed037b8560612c12d55958166427557f78bc887585',\n",
    "              '99c6624fe43caea607c7597c84bc47c6141ba954664afde16d8f4bcaf2db33ec',\n",
    "              'a3f29b5242227abc8f2df9590b5d97289e3597f7cb787f0deef3b885677466ef',\n",
    "              '514247f42dc2f7f20552529f44a8e405d8b27586430514bbbfd6f4ba55831f5b',\n",
    "              '784a3698b008d3abeaa842337ec9140436ba71da1ac20687a0f419b1d05f6d37',\n",
    "              'cfc0607ebc8f274504a96cb4cf209d9a3c04954154f7ef24d8d201f82f67d53d',\n",
    "              '1bf69d660f575c554630c93931e8dce39122d453ccd0983a88add544153ee7ca',\n",
    "              'b6dfcac06b2ae667b1e56d23494587193678264b650df964463ae52c2be791eb',\n",
    "              'cfc0607ebc8f274504a96cb4cf209d9a3c04954154f7ef24d8d201f82f67d53d',\n",
    "              'b34c5f84b80b2a2f8014b7314edb977c890d97cb1fc6b0d5297105015cccc2ef'\n",
    "             ],\n",
    "      'aes': ['022a27f0cfe06ae493b9dc1ed02f70ea524d2e41d8f0be399008e6355dc75277',\n",
    "              '9086598c4e0b7ef4d7a43255f2c43350afefecc23db7c275fc06a2033b3420d9',\n",
    "              'a002fecefcd5e26b14e4221c1c59a2e60751034e8f9cd2a36c4e2a4267d03521'\n",
    "             ],\n",
    "      'des': ['5e9fddc8e93e6bd69a660f7efca65461726db129bd9ad9f6b0a028032eb38a5e',\n",
    "              '875750f03cd870e4c144f57de7422e3dd7f623006ac2688a421b69b630f3a259',\n",
    "              'b789fef22167cb9af1932c9ea23e1b51335e4ae603f0a782e7e8ea776648ccbd',\n",
    "              '26d8dc757490956b091e7d8ad1a93f883a1797b6c45883250d2888d1153bf28a',\n",
    "              'd9e5c58dced69e209b5fc8721cc249ef3cd8b9f425469a3d8e3a1c829b0932a9'\n",
    "             ],\n",
    "      'blowfish': ['a5513cc8ed6bac8ec3252a51b550ee38c1b3a463e5f21303a1af0b7d83e0fb7a',\n",
    "                   'f848c3bf8b9713a6415ec493605eb27eb41557d24e093bb8e4876cab46b015e9'\n",
    "            ],\n",
    "      'dh': ['b0d973b1ebda12a28beffc865fabc747ccade09eaf0ae554a9f576bee3d1bb01',\n",
    "             'ee4444e10c057122d3d97fdd1c9d46f8863addf7b60db2a2654905daccb23f00',\n",
    "             'abbb73e18017d4901e9809cd9d7de502c4abe3551f49166e655177fe5ee4ef68',\n",
    "             'd9bd95d05d5e0790748bace25bd11883e8994d3333b55328d804854522c3a6af',\n",
    "             '7d5d2939fb8d450ca1bc5ea7d4d2b2239045493d0c0a35fbdd795a9985506ebe',\n",
    "             '16017a29d76089e35a1b9f87a7f4558e5bd9f621265fa4a91b43d11fc7eb99e8',\n",
    "             '8bca5ec199908ee43861f82c02e2f813eb415604633eb034c720340401234093',\n",
    "             '8bca5ec199908ee43861f82c02e2f813eb415604633eb034c720340401234093',\n",
    "             '16017a29d76089e35a1b9f87a7f4558e5bd9f621265fa4a91b43d11fc7eb99e8',\n",
    "             '63890df926f97eb10370a0fb5f667ce9610887128a8cc69e0e53c57ffe06c05d'],\n",
    "      'mac': ['315509ae39b059f308d2fce0cd635127c97b02353de6234e3499d6b08d692642',\n",
    "              '7287cfb31d279097b3a90af6826dc74674d2c0d7546b0db6a7412e91d9d4e37f',\n",
    "              'b8c0b290c066a8760344a7dad29868bafbe9611e5211a139b22cf8b1cf2c5443',\n",
    "              '181aa9b28012c556b21de54e9bcd85cd2fd35f94bc6e6cde63d08bc36eff33ae',\n",
    "              '360dc7e163a02436b260a177c586f449f399d4eab25e06ced8cdfb19c049c59f',\n",
    "              'dfc738beeddc25b30428b7d00e8769d2af2771d66411feaf42ca98836083edbc',\n",
    "              '57af0852f4f251f1a2190f769f05a75fa1685dbf7d17a446096a7598d7b5ea5b',\n",
    "              'cc795c8a898c381f9c59d286bc154781801becb9cddc72e215e62f45ad0d29df',\n",
    "              'b58a9abd12899906b88ff98ca31313029c58e08ad7a72e24b99178a850eeb34a',\n",
    "              '1ffee8a27098dcbffbc92d21bd7718b6b4a1e9efa35865500e2ea25dda6d73c0'],\n",
    "      'signature': ['9ae00a496ede07c7087f41579e2943126d49b033ed88948424806d4bf7ec5a61',\n",
    "                    '9ed543588a5e9e27f7a4684218f4fc56966b574c6bb16add3446d4902a5aa67c',\n",
    "                    '043b67f5c28433903d050ba443b47362acd6bdc4c52541f0388d0b95d884b6a7',\n",
    "                    'a68ce270206df711fa432ca8f4d707933236a6bacdac54ca5b959e2b49697652',\n",
    "                    'fccdc054aa8dafae9e29a3d59655879af45e7acf65e5e8a4d01deb47262ff7c6',\n",
    "                    '8864a6b13d96215e91dd84edf3ec95ce8e603c5c152588ebc68b7284801466a3',\n",
    "                    'bbba662a9d44492d6e57fc800342828eccc29176fa8823d379254dfdda382b6b',\n",
    "                    '58d75f26c56e9f91b06513cf9a990289222d85641d338be6d9cb3967583d3727',\n",
    "                    '8eef99dcc1181d4c7a53f1d9515acdfda76bc51d3fe1652bac7fcae7d0dffcd8',\n",
    "                    '5a775ad6e8dbf0073d2c65be82e311b62e63d610375d9ae81e00aabb6a536f5a'],\n",
    "      'prng': ['df2d6cfa74a5d50e0d5e8c6735f31d7c6d6e12676977227e42fe6b1cadfe5cf5',\n",
    "              '50de67b3ea3a3d420238a3d3993c24f652d195e5ca37df1a18641a4dd8450758',\n",
    "              '3bccdf503b3b0bab6fda1c02d1a540cc0204d962211c87e728ab71d88fc74de7',\n",
    "              'cd37472842f4354530296370b07d4961d35707ffc41551a1426d1210d528c2f7',\n",
    "              'd6ddc4111fd614930ffd3738dbbdb6a4f39c3ddf646ac0dc792a9cb7cbc0e5b0',\n",
    "              'f08d4acbf920fa6aed10da82822ec958e412afcf9935c48080ab32d0d8d2489d',\n",
    "              '570232b8f1b197fcf31a91a924f0fa2b0480a7b1cd98c2f0ab79b1110eb94e3c',\n",
    "              'd5492faf790cab01ae9b90b76e0e2967f0527736f7670ec8e6bc3214419d89f6',\n",
    "              '35d463488526625555c8350b5d1404efadf49a655a5307e826253657a8a7fad3',\n",
    "              '67765c08639ad4f3c9c3061a10fa05806ed60256f35cc4bfaddc5ec8447cceaa']\n",
    "      }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dct = {      'dh': ['b0d973b1ebda12a28beffc865fabc747ccade09eaf0ae554a9f576bee3d1bb01',\n",
    "             'ee4444e10c057122d3d97fdd1c9d46f8863addf7b60db2a2654905daccb23f00',\n",
    "             'abbb73e18017d4901e9809cd9d7de502c4abe3551f49166e655177fe5ee4ef68',\n",
    "             'd9bd95d05d5e0790748bace25bd11883e8994d3333b55328d804854522c3a6af',\n",
    "             '7d5d2939fb8d450ca1bc5ea7d4d2b2239045493d0c0a35fbdd795a9985506ebe',\n",
    "             '16017a29d76089e35a1b9f87a7f4558e5bd9f621265fa4a91b43d11fc7eb99e8',\n",
    "             '8bca5ec199908ee43861f82c02e2f813eb415604633eb034c720340401234093',\n",
    "             '8bca5ec199908ee43861f82c02e2f813eb415604633eb034c720340401234093',\n",
    "             '16017a29d76089e35a1b9f87a7f4558e5bd9f621265fa4a91b43d11fc7eb99e8',\n",
    "             '63890df926f97eb10370a0fb5f667ce9610887128a8cc69e0e53c57ffe06c05d'],\n",
    "      'mac': ['315509ae39b059f308d2fce0cd635127c97b02353de6234e3499d6b08d692642',\n",
    "              '7287cfb31d279097b3a90af6826dc74674d2c0d7546b0db6a7412e91d9d4e37f',\n",
    "              'b8c0b290c066a8760344a7dad29868bafbe9611e5211a139b22cf8b1cf2c5443',\n",
    "              '181aa9b28012c556b21de54e9bcd85cd2fd35f94bc6e6cde63d08bc36eff33ae',\n",
    "              '360dc7e163a02436b260a177c586f449f399d4eab25e06ced8cdfb19c049c59f',\n",
    "              'dfc738beeddc25b30428b7d00e8769d2af2771d66411feaf42ca98836083edbc',\n",
    "              '57af0852f4f251f1a2190f769f05a75fa1685dbf7d17a446096a7598d7b5ea5b',\n",
    "              'cc795c8a898c381f9c59d286bc154781801becb9cddc72e215e62f45ad0d29df',\n",
    "              'b58a9abd12899906b88ff98ca31313029c58e08ad7a72e24b99178a850eeb34a',\n",
    "              '1ffee8a27098dcbffbc92d21bd7718b6b4a1e9efa35865500e2ea25dda6d73c0'],\n",
    "      'signature': ['9ae00a496ede07c7087f41579e2943126d49b033ed88948424806d4bf7ec5a61',\n",
    "                    '9ed543588a5e9e27f7a4684218f4fc56966b574c6bb16add3446d4902a5aa67c',\n",
    "                    '043b67f5c28433903d050ba443b47362acd6bdc4c52541f0388d0b95d884b6a7',\n",
    "                    'a68ce270206df711fa432ca8f4d707933236a6bacdac54ca5b959e2b49697652',\n",
    "                    'fccdc054aa8dafae9e29a3d59655879af45e7acf65e5e8a4d01deb47262ff7c6',\n",
    "                    '8864a6b13d96215e91dd84edf3ec95ce8e603c5c152588ebc68b7284801466a3',\n",
    "                    'bbba662a9d44492d6e57fc800342828eccc29176fa8823d379254dfdda382b6b',\n",
    "                    '58d75f26c56e9f91b06513cf9a990289222d85641d338be6d9cb3967583d3727',\n",
    "                    '8eef99dcc1181d4c7a53f1d9515acdfda76bc51d3fe1652bac7fcae7d0dffcd8',\n",
    "                    '5a775ad6e8dbf0073d2c65be82e311b62e63d610375d9ae81e00aabb6a536f5a'],\n",
    "      'prng': ['df2d6cfa74a5d50e0d5e8c6735f31d7c6d6e12676977227e42fe6b1cadfe5cf5',\n",
    "              '50de67b3ea3a3d420238a3d3993c24f652d195e5ca37df1a18641a4dd8450758',\n",
    "              '3bccdf503b3b0bab6fda1c02d1a540cc0204d962211c87e728ab71d88fc74de7',\n",
    "              'cd37472842f4354530296370b07d4961d35707ffc41551a1426d1210d528c2f7',\n",
    "              'd6ddc4111fd614930ffd3738dbbdb6a4f39c3ddf646ac0dc792a9cb7cbc0e5b0',\n",
    "              'f08d4acbf920fa6aed10da82822ec958e412afcf9935c48080ab32d0d8d2489d',\n",
    "              '570232b8f1b197fcf31a91a924f0fa2b0480a7b1cd98c2f0ab79b1110eb94e3c',\n",
    "              'd5492faf790cab01ae9b90b76e0e2967f0527736f7670ec8e6bc3214419d89f6',\n",
    "              '35d463488526625555c8350b5d1404efadf49a655a5307e826253657a8a7fad3',\n",
    "              '67765c08639ad4f3c9c3061a10fa05806ed60256f35cc4bfaddc5ec8447cceaa']\n",
    "      }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "def get_androzoo_url(sha256):\n",
    "    return 'https://androzoo.uni.lu/api/download?apikey=' + '4538b69cb8c5f980232dfa314a35a272051ab7bfbb78aa62e13e5a1dc874dcbf' + '&sha256=' + sha256\n",
    "\n",
    "def download_file(category, sha256):\n",
    "    dir_path = os.path.join('/Users/adam/phd/projects/CryptoMalware/experiments/manual_analysis', category)\n",
    "    \n",
    "    if not os.path.exists(dir_path):\n",
    "        os.makedirs(dir_path)\n",
    "    \n",
    "    sample_path = os.path.join('/Users/adam/phd/projects/CryptoMalware/experiments/manual_analysis', category, sha256 + '.apk')\n",
    "    print(sample_path)\n",
    "    androzoo_url = get_androzoo_url(sha256)\n",
    "    \n",
    "    print(androzoo_url)\n",
    "    response = requests.get(androzoo_url)\n",
    "    \n",
    "    if response.status_code != 200:\n",
    "        print(f'Failed with status code: {response.status_code}')\n",
    "        \n",
    "    with open(sample_path, 'wb') as handle:\n",
    "        handle.write(response.content)\n",
    "\n",
    "df = evaluator.df_records"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for key, val in dct.items():\n",
    "    for x in val:\n",
    "        download_file(key, x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}