prepare_data.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import shutil\n",
    "import numpy as np\n",
    "import pandas\n",
    "from matplotlib import pyplot as plt\n",
    "from scipy.misc import imresize,imsave\n",
    "from multiprocessing import Pool\n",
    "from functools import partial\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def resize_all(id,namelist,path2):\n",
    "    im = plt.imread(namelist[id])\n",
    "    resizeim = imresize(im,(299,299))\n",
    "    if len(resizeim.shape) == 2:\n",
    "        resizeim = np.array([resizeim]*3).transpose([1,2,0])\n",
    "    imsave(os.path.join(path2,os.path.basename(namelist[id]).split('.')[0]+'.png'),resizeim)\n",
    "    \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# generate the training set, set your imagenet_path\n",
    "\n",
    "dataset1: the default dataset\n",
    "\n",
    "dataset2: only 750 classes in the training set, and 250 classes in the testset, designed to evaluate the transferability\n",
    "\n",
    "the training and validation sets are extracted from the training set of the imagenet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "imagenet_path = '/work/imagenet/train/'\n",
    "path2 = './Originset/'\n",
    "n_per_class = 4 # train\n",
    "n_per_class_test = [10,40] # test\n",
    "\n",
    "n_train = int(n_per_class*0.75 )\n",
    "subdirs = os.listdir(imagenet_path)\n",
    "subdirs = np.sort(subdirs)\n",
    "label_mapping={}\n",
    "example = pandas.read_csv('./sample_dev_dataset.csv')\n",
    "\n",
    "for id,name in enumerate(subdirs):\n",
    "    label_mapping[name] = id+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class1 = np.load('utils/dataset2_trainclass.npy')\n",
    "class2 = np.load('utils/dataset2_valclass.npy')\n",
    "class1 = [label_mapping[name] for name in class1]\n",
    "class2 = [label_mapping[name] for name in class2]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "n_repeat = n_per_class\n",
    "info_list = np.zeros([n_repeat*1000,12]).astype('str')\n",
    "trainset_d1 = np.array([])\n",
    "valset_d1 = np.array([])\n",
    "trainset_d2 = np.array([])\n",
    "valset_d2 = np.array([])\n",
    "namelist = np.array([])\n",
    "\n",
    "i_cum =0\n",
    "\n",
    "for i_dir,dir in enumerate(subdirs):\n",
    "    fullpath = os.path.join(imagenet_path,dir)\n",
    "    filelist = os.listdir(fullpath)\n",
    "    randid = np.random.permutation(len(filelist))[:n_repeat]\n",
    "\n",
    "    chosen_im = np.array(filelist)[randid]\n",
    "    rename_im = np.array([n.split('.')[0]+'png' for n in chosen_im])\n",
    "    trainset_d1 = np.concatenate([trainset_d1,rename_im[:n_train]])\n",
    "    valset_d1 = np.concatenate([valset_d1,rename_im[n_train:]])\n",
    "    \n",
    "    fullimpath = [os.path.join(fullpath,f) for f in chosen_im]\n",
    "    namelist = np.concatenate([namelist,fullimpath])\n",
    "\n",
    "    labels = label_mapping[dir]\n",
    "    if labels in class1:\n",
    "        trainset_d2 = np.concatenate([trainset_d2,rename_im])\n",
    "        valset_d2 = np.concatenate([valset_d2,rename_im])\n",
    "\n",
    "    for i in range(n_repeat):\n",
    "        target_class = labels\n",
    "        while target_class==labels:\n",
    "            target_class = np.random.randint(1000)\n",
    "#         info_list[i].append([chosen_im[i].split('.')[0],0,0,0,1,1,labels,target_class,0,0,0,0])\n",
    "        info_list[i_cum] = np.array([chosen_im[i].split('.')[0],0,0,0,1,1,labels,target_class,0,0,0,0])\n",
    "        \n",
    "        i_cum += 1\n",
    "newpd = pandas.DataFrame(info_list)\n",
    "newpd.columns = example.columns\n",
    "newpd.to_csv('dev_dataset.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "pool = Pool()\n",
    "resize_partial = partial(resize_all,namelist=namelist,path2=path2)\n",
    "_ = pool.map(resize_partial,range(len(namelist)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "np.save('./utils/dataset1_train_split.npy',trainset_d1)\n",
    "np.save('./utils/dataset1_val_split.npy',valset_d1)\n",
    "np.save('./utils/dataset2_train_split.npy',trainset_d2)\n",
    "np.save('./utils/dataset2_val_split.npy',valset_d2)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "# generate the testing set\n",
    "\n",
    "the test set is extracted from the validation set of the imagenet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "imagenet_path = '/work/imagenet/val/'\n",
    "path2 = './Originset_test/'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open('/work/imagenet/meta/val.txt') as f:\n",
    "    tmp = f.readlines()\n",
    "label_val = {}\n",
    "for line in tmp:\n",
    "    label_val[line.split(' ')[0]] = int(line.split(' ')[1].split('\\n')[0])+1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "example = pandas.read_csv('/work/adv/toolkit/dataset/dev_dataset.csv')\n",
    "keys = np.array(label_val.keys())\n",
    "values = np.array(label_val.values())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "i_cum =0\n",
    "namelist = []\n",
    "# info_list = np.zeros([n_repeat*1000,12]).astype('str')\n",
    "info_list = []\n",
    "\n",
    "for i_class in range(1,1001):\n",
    "    if i_class in class1:\n",
    "        n_repeat = n_per_class_test[0]\n",
    "    else:\n",
    "        n_repeat = n_per_class_test[1]\n",
    "    filelist = keys[values == i_class]\n",
    "    randid = np.random.permutation(len(filelist))[:n_repeat]\n",
    "    chosen_im = np.array(filelist)[randid]\n",
    "    fullimpath = [os.path.join(imagenet_path,f) for f in chosen_im]\n",
    "    labels = i_class\n",
    "    for i in range(n_repeat):\n",
    "        target_class = labels\n",
    "        while target_class==labels:\n",
    "            target_class = np.random.randint(1000)\n",
    "        info_list.append([chosen_im[i].split('.')[0],0,0,0,1,1,labels,target_class,0,0,0,0])\n",
    "#         info_list[i_cum] = np.array([chosen_im[i].split('.')[0],0,0,0,1,1,labels,target_class,0,0,0,0])\n",
    "        namelist.append(fullimpath[i])\n",
    "        i_cum += 1\n",
    "newpd = pandas.DataFrame(info_list)\n",
    "newpd.columns = example.columns\n",
    "newpd.to_csv('dev_dataset_test.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "label1 = pandas.read_csv('dev_dataset.csv')\n",
    "label1 = np.array([label1['ImageId'],label1['TrueLabel']])\n",
    "label2 = pandas.read_csv('dev_dataset_test.csv')\n",
    "label2 = np.array([label2['ImageId'],label2['TrueLabel']])\n",
    "\n",
    "tmp = np.concatenate([label1,label2],1).T\n",
    "labels = {}\n",
    "for key,value in tmp:\n",
    "    labels[key] = value\n",
    "np.save('utils/labels.npy',labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "names = label2[0]\n",
    "values = label2[1]\n",
    "allnames = []\n",
    "for i in range(1,1001):\n",
    "    class_names = names[values==i][:10]\n",
    "    allnames.append(class_names)\n",
    "allnames = np.concatenate(allnames)\n",
    "np.save('utils/dataset1_test_split.npy',allnames)\n",
    "\n",
    "allnames = []\n",
    "class2 = np.load('utils/dataset2_valclass.npy')\n",
    "class2 = [label_mapping[name] for name in class2]\n",
    "for i in class2:\n",
    "    class_names = names[values==i]\n",
    "    allnames.append(class_names)\n",
    "allnames = np.concatenate(allnames)\n",
    "np.save('utils/dataset2_test_split.npy',allnames)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "resize_partial = partial(resize_all,namelist=namelist,path2=path2)\n",
    "_ = pool.map(resize_partial,range(len(namelist)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}