Interactive-Concepts-Visualization.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This notebook contains code that was used for preprocessing of data and other helper functions.\n",
    "\n",
    "### Collect random sample from ImageNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from random import randint\n",
    "import tensorflow as tf\n",
    "import os\n",
    "import urllib\n",
    "from urllib.parse import urlparse\n",
    "from PIL import Image\n",
    "import tensorflow as tf\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib\n",
    "import numpy as np\n",
    "import json\n",
    "import random\n",
    "from sklearn.decomposition import PCA\n",
    "import glob\n",
    "import os.path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def file_len(fname):\n",
    "    with open(fname, 'rb') as f:\n",
    "        for i, l in enumerate(f):\n",
    "            pass\n",
    "    return i + 1\n",
    "\n",
    "def sample_imagenet(ind, size=10):\n",
    "    rand_dir = 'random_' + str(ind)\n",
    "    os.makedirs(rand_dir)\n",
    "    fname = '../fall11_urls.txt'\n",
    "    images = open(fname,'rb')\n",
    "    whatlines = []\n",
    "    for line in range(size):\n",
    "        whatlines.append(randint(0,file_len(fname)))\n",
    "        \n",
    "    sample = [x.decode(\"utf-8\").split('\\t')[1] for i, x in enumerate(images) if i in whatlines]   \n",
    "    for item in sample:\n",
    "        a = urlparse(item)\n",
    "        name = os.path.basename(a.path)        \n",
    "        try:\n",
    "            urllib.request.urlretrieve(item, '/tmp/' + name)\n",
    "            im=Image.open('/tmp/' + name)\n",
    "            os.rename('/tmp/' + name, './'+rand_dir + '/' + name)\n",
    "\n",
    "        except:\n",
    "            pass\n",
    "        \n",
    "    return sample"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create random experiments datasets:\n",
    "number_expriments = 10\n",
    "for i in range(number_expriments):\n",
    "    _ = sample_imagenet(i,size=100)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reduce dimension and plot explained variance ratio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DimensionReducer():\n",
    "    def __init__(self, activations, pc_components=4):\n",
    "        self.flattened = self.flatten(activations)\n",
    "        print(np.shape(self.flattened))\n",
    "        self.activations = activations\n",
    "        self.pca = PCA(n_components=pc_components)\n",
    "        self.pca.fit(self.flattened)\n",
    "        self.save_params()\n",
    "\n",
    "    def save_params(self):\n",
    "        self.mu = np.mean(self.flattened, axis=0)\n",
    "        with open('pca_params','w') as f:\n",
    "            f.write(json.dumps({\"mean\":self.mu.tolist(),\"components\":self.pca.components_.tolist()}))\n",
    "\n",
    "    def flatten(self, activations):\n",
    "        return [x.flatten() for x in activations]\n",
    "    \n",
    "    def get_reduced(self, activations):\n",
    "        self.reduced = self.pca.transform(self.flatten(activations))\n",
    "        return self.reduced\n",
    "        \n",
    "    def get_random_concept(self, n=100): # n - number of random images to draw\n",
    "\n",
    "        mu = np.mean(self.flattened, axis=0)\n",
    "\n",
    "        b = max(self.reduced[:,0])\n",
    "        a = min(self.reduced[:,0])\n",
    "        b1 = max(self.reduced[:,1])\n",
    "        a1 = min(self.reduced[:,1])    \n",
    "        random_pca1 = (b - a) * np.random.random_sample(n) + a\n",
    "        random_pca2 = (b1 - a1) * np.random.random_sample(n) + a1\n",
    "        random_activations = np.stack((random_pca1, random_pca2), axis=-1)\n",
    "\n",
    "        Xhat = np.dot(random_activations, self.pca.components_)\n",
    "        Xhat += mu\n",
    "\n",
    "        return Xhat"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Get sample of activations from every class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_name = './acts_grads/'\n",
    "\n",
    "acts_files = [dir_name + f for i,f in enumerate(os.listdir(dir_name)) if 'acts' in f]\n",
    "grads_files = [dir_name + f for i,f in enumerate(os.listdir(dir_name)) if 'grads' in f]\n",
    "\n",
    "classes = None\n",
    "with open('./concept-vis/data/classes.json') as f:\n",
    "    classes = json.load(f)\n",
    "cls = {x['id']:x['class'] for x in classes[:50]}\n",
    "\n",
    "def id2class(x):\n",
    "    return cls[x].split(',')[0]\n",
    "\n",
    "def get_sample(name):\n",
    "    data = np.load(name)\n",
    "    dim = np.shape(data)[2:]\n",
    "    data = data.reshape((-1, *dim))\n",
    "    draw = random.choices(data,k=3)\n",
    "    return draw\n",
    "\n",
    "def get_sample_of_activations(acts_files):\n",
    "    sample = []\n",
    "    for z in acts_files:\n",
    "        print(z)\n",
    "        draw = get_sample(z)\n",
    "        if os.path.isfile('./backup.npy'):\n",
    "            sample = np.load('backup.npy')\n",
    "            sample = np.concatenate((sample,draw))\n",
    "        else:\n",
    "            sample = draw\n",
    "            \n",
    "        np.save('backup', sample)\n",
    "            \n",
    "        print(np.shape(sample))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = get_sample_of_activations(acts_files)\n",
    "print(np.shape(sample))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sample = np.load('../tcav_backup/backup.npy')\n",
    "dr = DimensionReducer(np.array(sample),pc_components=35)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Plot explained variance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "variance = dr.pca.explained_variance_ratio_\n",
    "plt.figure(figsize=(10,7))\n",
    "plt.bar(range(1,1+len(variance)),variance)\n",
    "plt.xlabel('principal component nr')\n",
    "plt.ylabel(\"explained variance ratio\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "matplotlib.rcParams.update({'font.size': 22})\n",
    "plt.figure(figsize=(10,5))\n",
    "plt.hist(variance)\n",
    "plt.xlabel(\"Explained variance ratio [%]\")\n",
    "plt.ylabel(\"Count\")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Reduce dimension of whole dataset and save components as a points.js file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for batch_of_acts, batch_of_grads in zip(acts_files, grads_files):\n",
    "    folder = batch_of_acts.split('/')[-1].split('_')[0]   \n",
    "    if folder in cls.keys():\n",
    "        names = None\n",
    "        with open(dir_name + folder + '_names') as f:\n",
    "            names = [ x.strip() for x in f.readlines()]\n",
    "\n",
    "        paths = ['https://0.0.0.0:8009/images/'+ folder + '/' + x for x in names]\n",
    "        data = np.load(batch_of_acts) \n",
    "        dim = np.shape(data)[2:]\n",
    "        reduced = dr.get_reduced(data.reshape((-1, *dim)))\n",
    "        data_grad = np.load(batch_of_grads) \n",
    "        dim = np.shape(data_grad)[2:]\n",
    "        reduced_grad = dr.get_reduced(data_grad.reshape((-1, *dim)))\n",
    "\n",
    "        points = []\n",
    "        for i, item in enumerate(zip(paths,reduced,reduced_grad)):\n",
    "            tmp = {'coords': [*item[1]], 'id': folder + '_' + str(i), 'patch_path': item[0], 'class':id2class(folder), 'class_id':folder, 'gradient':[*item[2]]}\n",
    "            points.append(tmp)   \n",
    "        with open(\"../points/\" + batch_of_acts.split('/')[-1] + \".json\",'w') as f:\n",
    "            json.dump(points, f)\n",
    "\n",
    "        \n",
    "json_dir = '../points/'\n",
    "data = []\n",
    "for x in os.listdir(json_dir):\n",
    "    if x.split('_')[0].startswith('n') and 'sorted' in x.split('_')[1]:\n",
    "\n",
    "        with open(json_dir + x) as f:\n",
    "            tmp = json.loads(f.read())           \n",
    "            data.extend(tmp)\n",
    "        \n",
    "with open(\"../concept-vis/data/points.js\",'w') as f:\n",
    "    f.write(\"var points_data = {}\".format(json.dumps(data)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create heatmap.js file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = None\n",
    "with open(\"data/points.js\") as f:\n",
    "    data = f.read()[18:]\n",
    "\n",
    "points = json.loads(data)\n",
    "\n",
    "pc0 = [x['coords'][0] for x in points]\n",
    "pc1 = [x['coords'][1] for x in points]\n",
    "\n",
    "g0 = [x['gradient'][0] for x in points]\n",
    "g1 = [x['gradient'][1] for x in points]\n",
    "print(\"var min_g0 = {};\".format(min(g0)))\n",
    "print(\"var min_g1 = {};\".format(min(g1)))\n",
    "print(\"var max_g0 = {};\".format(max(g0)))\n",
    "print(\"var max_g1 = {};\".format(max(g1)))\n",
    "\n",
    "\n",
    "\n",
    "print(\"var min_x = {};\".format(min(pc0)))\n",
    "print(\"var max_x = {};\".format(max(pc0)))\n",
    "print(\"var min_y = {};\".format(min(pc1)))\n",
    "print(\"var max_y = {};\".format(max(pc1)))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# simulate data for the background heatmap\n",
    "error_heatmap = []\n",
    "for x in np.linspace(min(pc0), max(pc0), 100):\n",
    "    for y in np.linspace(min(pc0), max(pc1), 100):\n",
    "        error_heatmap.append({\n",
    "            \"coords\": [x, y],\n",
    "            \"error\": 0.7\n",
    "        })\n",
    "\n",
    "with open(\"data/heatmap.js\", \"w\") as f:\n",
    "    f.write(\"var hm_data = {}\".format(json.dumps(error_heatmap)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Create classes_short.js (list of classes from data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('./data/points.js') as f:\n",
    "    data = json.loads(f.read()[18:])\n",
    "\n",
    "classes = []\n",
    "names = set()\n",
    "for x in data:\n",
    "    if x['class'] not in names:\n",
    "        classes.append({'class':x['class'],\"id\":x['class_id']})\n",
    "        names.add(x['class'])\n",
    "\n",
    "with open('classes_short.js', 'w') as f:\n",
    "    f.write('var classes_short ='+json.dumps(classes))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}