create_subt_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create SubT Dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os,shutil\n",
    "import random"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['scene000006',\n",
       " 'scene000019',\n",
       " 'scene000066',\n",
       " 'scene000033',\n",
       " 'scene000001',\n",
       " 'scene000016',\n",
       " 'scene000061',\n",
       " 'scene000058',\n",
       " 'scene000007',\n",
       " 'scene000069',\n",
       " 'scene000035',\n",
       " 'scene000032',\n",
       " 'scene000013',\n",
       " 'scene000054',\n",
       " 'scene000014',\n",
       " 'scene000011',\n",
       " 'scene000015',\n",
       " 'scene000071',\n",
       " 'scene000003',\n",
       " 'scene000017',\n",
       " 'scene000018',\n",
       " 'scene000049',\n",
       " 'scene000008',\n",
       " 'scene000037',\n",
       " 'scene000022',\n",
       " 'scene000068',\n",
       " 'scene000050',\n",
       " 'scene000072',\n",
       " 'scene000056',\n",
       " 'scene000045',\n",
       " 'scene000020',\n",
       " 'scene000074',\n",
       " 'scene000065',\n",
       " 'scene000012',\n",
       " 'scene000047',\n",
       " 'scene000048',\n",
       " 'scene000044',\n",
       " 'scene000073',\n",
       " 'scene000067',\n",
       " 'scene000027',\n",
       " 'scene000005',\n",
       " 'scene000043',\n",
       " 'scene000010',\n",
       " 'scene000059',\n",
       " 'scene000025',\n",
       " 'scene000076',\n",
       " 'scene000004',\n",
       " 'scene000038',\n",
       " 'scene000028',\n",
       " 'scene000029',\n",
       " 'scene000070',\n",
       " 'scene000077',\n",
       " 'scene000053',\n",
       " 'scene000021',\n",
       " 'scene000036',\n",
       " 'scene000041',\n",
       " 'scene000075',\n",
       " 'scene000009',\n",
       " 'scene000046',\n",
       " 'scene000055',\n",
       " 'scene000064',\n",
       " 'scene000024',\n",
       " 'scene000034',\n",
       " 'scene000023',\n",
       " 'scene000042',\n",
       " 'scene000002',\n",
       " 'scene000040',\n",
       " 'scene000051',\n",
       " 'scene000063',\n",
       " 'scene000060',\n",
       " 'scene000039',\n",
       " 'scene000052',\n",
       " 'scene000026',\n",
       " 'scene000031',\n",
       " 'scene000057',\n",
       " 'scene000062',\n",
       " 'scene000030']"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "path = \"/home/alex/Desktop/3_cls_xml_image_depth/image/backpack\"\n",
    "root = \"/home/alex/Desktop/new\"\n",
    "obj_list = os.listdir(path)\n",
    "obj_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: '/home/alex/Desktop/new/JPEGImages/scene000006_10.jpg'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-5-978ab1d3b615>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m         \u001b[0mnew_img_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mroot\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'/JPEGImages/'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'_'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mimg_name\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0mnew_ann_path\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mroot\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'/Annotations/'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mobj\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m'_'\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mann_name\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m         \u001b[0mshutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopyfile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mimg_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_img_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m         \u001b[0mshutil\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcopyfile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mann_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_ann_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/usr/lib/python3.6/shutil.py\u001b[0m in \u001b[0;36mcopyfile\u001b[0;34m(src, dst, follow_symlinks)\u001b[0m\n\u001b[1;32m    119\u001b[0m     \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mfsrc\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 121\u001b[0;31m             \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdst\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'wb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mfdst\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    122\u001b[0m                 \u001b[0mcopyfileobj\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfsrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfdst\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    123\u001b[0m     \u001b[0;32mreturn\u001b[0m \u001b[0mdst\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/home/alex/Desktop/new/JPEGImages/scene000006_10.jpg'"
     ]
    }
   ],
   "source": [
    "for obj in obj_list:\n",
    "    ann_list = os.listdir(path + '/' + obj )\n",
    "    for ann in ann_list:\n",
    "        ann_name = ann.split('.')[0]+'.xml'\n",
    "        img_name = ann.split('.')[0]+'.jpg'\n",
    "        img_path = path + '/' + obj+ '/' + img_name\n",
    "        ann_path = path + '/' + obj+ '/' + ann_name\n",
    "        new_img_path = root + '/JPEGImages/' + obj + '_' + img_name\n",
    "        new_ann_path = root + '/Annotations/' + obj + '_' + ann_name\n",
    "        shutil.copyfile(img_path, new_img_path)\n",
    "        shutil.copyfile(ann_path, new_ann_path)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create train.txt and test.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "file_path = '/home/alex/subt_ssd_training/data/subt_artifact/ImageSets/Main/'\n",
    "train_file = open(file_path + \"train.txt\", \"w\")\n",
    "test_file = open(file_path + \"test.txt\", \"w\")\n",
    "img_path = \"/home/alex/subt_ssd_training/data/subt_artifact/JPEGImages/\"\n",
    "img_list = os.listdir(img_path)\n",
    "ratio = 0.9\n",
    "counter = 0\n",
    "data_num = len(img_list)\n",
    "pick_list = []\n",
    "while counter < data_num:\n",
    "    idx = random.randint(0, data_num - 1)\n",
    "    if idx not in pick_list:\n",
    "        name = img_list[idx].split('.')[0] + '\\n'\n",
    "        if counter < data_num * ratio:\n",
    "            train_file.write(name)\n",
    "        else:\n",
    "            test_file.write(name)\n",
    "        counter = counter + 1\n",
    "train_file.close()\n",
    "test_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Create specific object txt file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "file_path = '/home/alex/subt_ssd_training/data/subt_artifact/ImageSets/Main'\n",
    "ext_file = open(file_path + \"extinguisher.txt\", \"w\")\n",
    "img_path = \"/home/alex/subt_ssd_training/data/subt_artifact/JPEGImages\"\n",
    "img_list = os.listdir(img_path)\n",
    "ratio = 0.9\n",
    "counter = 0\n",
    "data_num = len(img_list)\n",
    "pick_list = []\n",
    "for img in img_list:\n",
    "    data_name = img.split('.')[0]\n",
    "    ext_cha = img.replace('_', '.').split('.')[0]\n",
    "    if ext_cha == \"extinguisher\":\n",
    "        name = data_name + '\\n'\n",
    "        ext_file.write(name)\n",
    "ext_file.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Move file from one folder to another"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def move_folder(orig_path, new_path):\n",
    "    data_list = os.listdir(orig_path)\n",
    "    for data in data_list:\n",
    "        orig_data = orig_path + \"/\" + data\n",
    "        new_data = new_path + \"/\" + data\n",
    "        os.rename(orig_data, new_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: '/media/arg_ws3/5E703E3A703E18EB/data/MM_sem/train/A'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-20-6f99eec7a13f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmove_folder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"/media/arg_ws3/5E703E3A703E18EB/data/MM_sem/train/A\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"/media/arg_ws3/5E703E3A703E18EB/data/MM_sem/A\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-19-9881dddb5c43>\u001b[0m in \u001b[0;36mmove_folder\u001b[0;34m(orig_path, new_path)\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mmove_folder\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0morig_path\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnew_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m     \u001b[0mdata_list\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mlistdir\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0morig_path\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mdata_list\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0morig_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0morig_path\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"/\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0mnew_data\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mnew_path\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m\"/\"\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: '/media/arg_ws3/5E703E3A703E18EB/data/MM_sem/train/A'"
     ]
    }
   ],
   "source": [
    "move_folder(\"/media/arg_ws3/5E703E3A703E18EB/data/MM_sem/train/A\", \"/media/arg_ws3/5E703E3A703E18EB/data/MM_sem/A\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Read annotation file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/home/biomotion/subt_docker/data/extinguisher/frame000057.jpg\n"
     ]
    }
   ],
   "source": [
    "import xml.etree.ElementTree as ET\n",
    "import os.path as osp\n",
    "f_path = '/media/arg_ws3/5E703E3A703E18EB/data/subt_artifact/ImageSets/Main/ext.txt'\n",
    "annopath = osp.join('/media/arg_ws3/5E703E3A703E18EB/data/subt_artifact/Annotations', '%s.xml')\n",
    "ids_list = []\n",
    "for line in open(osp.join(f_path)):\n",
    "    ids_list.append(line.strip())\n",
    "for ids in ids_list:\n",
    "    target = ET.parse(annopath % ids).getroot()\n",
    "    name = target.find('object').find('name').text\n",
    "    if name != \"extinguisher\":\n",
    "        print(target.find('path').text)\n",
    "#target = ET.parse(annopath % id).getroot()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}