src/08-holidays-siamese-finetune.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from __future__ import division, print_function\n",
    "from keras import backend as K\n",
    "from keras.applications import inception_v3\n",
    "from keras.callbacks import ModelCheckpoint\n",
    "from keras.layers import Input, merge\n",
    "from keras.layers.core import Activation, Dense, Dropout, Lambda\n",
    "from keras.models import Model, load_model\n",
    "from keras.preprocessing.image import ImageDataGenerator\n",
    "from keras.utils import np_utils\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.metrics import accuracy_score, confusion_matrix\n",
    "from random import shuffle\n",
    "from scipy.misc import imresize\n",
    "import itertools\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import os\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "DATA_DIR = \"../data\"\n",
    "IMAGE_DIR = os.path.join(DATA_DIR, \"holiday-photos\")\n",
    "\n",
    "HOLIDAY_FILE_IDS = os.path.join(DATA_DIR, \"holiday-ids.txt\")\n",
    "HOLIDAY_VECS_FILE = os.path.join(DATA_DIR, \"holiday-vecs.npy\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Prepare Data\n",
    "\n",
    "We use the naming convention to identify as a positive pair any two images that are part of the same group, and negative as a random pair of images in different groups."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4144\n",
      "[('149600.jpg', '149604.jpg', 1), ('133602.jpg', '133600.jpg', 1), ('104003.jpg', '117201.jpg', 0), ('148200.jpg', '148201.jpg', 1), ('124302.jpg', '143903.jpg', 0)]\n"
     ]
    }
   ],
   "source": [
    "def get_random_image(img_groups, group_names, gid):\n",
    "    gname = group_names[gid]\n",
    "    photos = img_groups[gname]\n",
    "    pid = np.random.choice(np.arange(len(photos)), size=1)[0]\n",
    "    pname = photos[pid]\n",
    "    return gname + pname + \".jpg\"\n",
    "    \n",
    "def create_triples(image_dir):\n",
    "    img_groups = {}\n",
    "    for img_file in os.listdir(image_dir):\n",
    "        prefix, suffix = img_file.split(\".\")\n",
    "        gid, pid = prefix[0:4], prefix[4:]\n",
    "        if img_groups.has_key(gid):\n",
    "            img_groups[gid].append(pid)\n",
    "        else:\n",
    "            img_groups[gid] = [pid]\n",
    "    pos_triples, neg_triples = [], []\n",
    "    # positive pairs are any combination of images in same group\n",
    "    for key in img_groups.keys():\n",
    "        triples = [(key + x[0] + \".jpg\", key + x[1] + \".jpg\", 1) \n",
    "                 for x in itertools.combinations(img_groups[key], 2)]\n",
    "        pos_triples.extend(triples)\n",
    "    # need equal number of negative examples\n",
    "    group_names = list(img_groups.keys())\n",
    "    for i in range(len(pos_triples)):\n",
    "        g1, g2 = np.random.choice(np.arange(len(group_names)), size=2, replace=False)\n",
    "        left = get_random_image(img_groups, group_names, g1)\n",
    "        right = get_random_image(img_groups, group_names, g2)\n",
    "        neg_triples.append((left, right, 0))\n",
    "    pos_triples.extend(neg_triples)\n",
    "    shuffle(pos_triples)\n",
    "    return pos_triples\n",
    "\n",
    "triples_data = create_triples(IMAGE_DIR)\n",
    "\n",
    "print(len(triples_data))\n",
    "print(triples_data[0:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using the triples, we create a cache keyed by image filename and whose value is the numpy vector represented by the image. Since we plan on using pretrained InceptionV3 network, our images are resized to (299, 299, 3)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "images from 0/4144 pairs loaded to cache\n",
      "images from 1000/4144 pairs loaded to cache\n",
      "images from 2000/4144 pairs loaded to cache\n",
      "images from 3000/4144 pairs loaded to cache\n",
      "images from 4000/4144 pairs loaded to cache\n",
      "images from 4143/4144 pairs loaded to cache, COMPLETE\n"
     ]
    }
   ],
   "source": [
    "def load_image_cache(image_cache, image_filename):\n",
    "    image = plt.imread(os.path.join(IMAGE_DIR, image_filename))\n",
    "    image = imresize(image, (299, 299))\n",
    "    image = image.astype(\"float32\")\n",
    "    image = inception_v3.preprocess_input(image)\n",
    "    image_cache[image_filename] = image\n",
    "    \n",
    "image_cache = {}\n",
    "num_pairs = len(triples_data)\n",
    "for i, (image_filename_l, image_filename_r, _) in enumerate(triples_data):\n",
    "    if i % 1000 == 0:\n",
    "        print(\"images from {:d}/{:d} pairs loaded to cache\".format(i, num_pairs))\n",
    "    if not image_cache.has_key(image_filename_l):\n",
    "        load_image_cache(image_cache, image_filename_l)\n",
    "    if not image_cache.has_key(image_filename_r):\n",
    "        load_image_cache(image_cache, image_filename_r)\n",
    "print(\"images from {:d}/{:d} pairs loaded to cache, COMPLETE\".format(i, num_pairs))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(32, 299, 299, 3) (32, 299, 299, 3) (32, 2)\n"
     ]
    }
   ],
   "source": [
    "def pair_generator(triples, image_cache, datagens, batch_size=32):\n",
    "    while True:\n",
    "        # shuffle once per batch\n",
    "        indices = np.random.permutation(np.arange(len(triples)))\n",
    "        num_batches = len(triples) // batch_size\n",
    "        for bid in range(num_batches):\n",
    "            batch_indices = indices[bid * batch_size : (bid + 1) * batch_size]\n",
    "            batch = [triples[i] for i in batch_indices]\n",
    "            X1 = np.zeros((batch_size, 299, 299, 3))\n",
    "            X2 = np.zeros((batch_size, 299, 299, 3))\n",
    "            Y = np.zeros((batch_size, 2))\n",
    "            for i, (image_filename_l, image_filename_r, label) in enumerate(batch):\n",
    "                if datagens is None or len(datagens) == 0:\n",
    "                    X1[i] = image_cache[image_filename_l]\n",
    "                    X2[i] = image_cache[image_filename_r]\n",
    "                else:\n",
    "                    X1[i] = datagens[0].random_transform(image_cache[image_filename_l])\n",
    "                    X2[i] = datagens[1].random_transform(image_cache[image_filename_r])\n",
    "                Y[i] = [1, 0] if label == 0 else [0, 1]\n",
    "            yield [X1, X2], Y\n",
    "\n",
    "\n",
    "datagen_args = dict(rotation_range=10,\n",
    "                    width_shift_range=0.2,\n",
    "                    height_shift_range=0.2,\n",
    "                    zoom_range=0.2)\n",
    "datagens = [ImageDataGenerator(**datagen_args),\n",
    "            ImageDataGenerator(**datagen_args)]\n",
    "pair_gen = pair_generator(triples_data, image_cache, datagens, 32)\n",
    "[X1, X2], Y = pair_gen.next()\n",
    "print(X1.shape, X2.shape, Y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Define Model\n",
    "\n",
    "The model is composed of two pretrained Inception V3 networks without their last prediction layer, connected to a merge layer that computes element-wise dot product of the two (2048,) sized vectors produced by the Inception V3. This is then fed into a 3 layer fully connected network that produces the similar / not similar prediction.\n",
    "\n",
    "The Inception V3 network weights are frozen, and the Fully Connected network weights are loaded from one trained using pre-computed image vectors and allowed to be fine-tuned."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10,) (10,)\n",
      "(10,)\n"
     ]
    }
   ],
   "source": [
    "# distance measure\n",
    "def cosine_distance(vecs, normalize=False):\n",
    "    x, y = vecs\n",
    "    if normalize:\n",
    "        x = K.l2_normalize(x, axis=0)\n",
    "        y = K.l2_normalize(x, axis=0)\n",
    "    return K.prod(K.stack([x, y], axis=1), axis=1)\n",
    "\n",
    "def cosine_distance_output_shape(shapes):\n",
    "    return shapes[0]\n",
    "\n",
    "vecs = [np.random.random((10,)), np.random.random((10,))]\n",
    "print(vecs[0].shape, vecs[1].shape)\n",
    "s = cosine_distance(vecs)\n",
    "print(s.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# load 2 copies of the inception model\n",
    "# in case of a siamese network, the same instance of the network will be trained,\n",
    "# but in our case the network is untrainable, so we can have 2 copies\n",
    "inception_1 = inception_v3.InceptionV3(weights=\"imagenet\", include_top=True)\n",
    "inception_2 = inception_v3.InceptionV3(weights=\"imagenet\", include_top=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Here the last two layers are avg_pool and prediction as shown below:\n",
    "#    avg_pool (None, 8, 8, 2048) (None, 2048)\n",
    "#    predictions (None, 2048) (None, 1000)\n",
    "#\n",
    "# for layer in inception_1.layers:\n",
    "#     print(layer.name, layer.input_shape, layer.output_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# freeze weights on the inception network and give each layer a unique name\n",
    "# since we will combine them into a single network\n",
    "for layer in inception_1.layers:\n",
    "    layer.trainable = False\n",
    "    layer.name = layer.name + \"_1\"\n",
    "for layer in inception_2.layers:\n",
    "    layer.trainable = False\n",
    "    layer.name = layer.name + \"_2\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# outputs of the inception networks, these will be connected to our head FCN\n",
    "vector_1 = inception_1.get_layer(\"avg_pool_1\").output\n",
    "vector_2 = inception_2.get_layer(\"avg_pool_2\").output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "input_3 (None, 2048) (None, 2048)\n",
      "input_4 (None, 2048) (None, 2048)\n",
      "lambda_1 [(None, 2048), (None, 2048)] (None, 2048)\n",
      "dense_4 (None, 2048) (None, 512)\n",
      "dropout_3 (None, 512) (None, 512)\n",
      "activation_4 (None, 512) (None, 512)\n",
      "dense_5 (None, 512) (None, 128)\n",
      "dropout_4 (None, 128) (None, 128)\n",
      "activation_5 (None, 128) (None, 128)\n",
      "dense_6 (None, 128) (None, 2)\n",
      "activation_6 (None, 2) (None, 2)\n"
     ]
    }
   ],
   "source": [
    "# load the pretrained similarity head network. This has been trained to predict similar\n",
    "# images using image vectors\n",
    "sim_head = load_model(os.path.join(DATA_DIR, \"models\", \"inceptionv3-dot-best.h5\"))\n",
    "for layer in sim_head.layers:\n",
    "    print(layer.name, layer.input_shape, layer.output_shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# attach output of the inception networks to the similarity head\n",
    "# output is a prediction tensor\n",
    "prediction = sim_head([vector_1, vector_2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# declare a model that takes image inputs on its truncated Inception subnetworks\n",
    "# and returns the prediction as the output. Inputs are Input(shape=(299, 299, 3))\n",
    "model = Model(inputs=[inception_1.input, inception_2.input], outputs=prediction)\n",
    "\n",
    "model.compile(optimizer=\"adam\", loss=\"categorical_crossentropy\", metrics=[\"accuracy\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Train Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "BATCH_SIZE = 32\n",
    "NUM_EPOCHS = 10\n",
    "BEST_MODEL_FILE = os.path.join(DATA_DIR, \"models\", \"inception-ft-best.h5\")\n",
    "FINAL_MODEL_FILE = os.path.join(DATA_DIR, \"models\", \"inception-ft-final.h5\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2983 332 829\n"
     ]
    }
   ],
   "source": [
    "triples_data_trainval, triples_data_test = train_test_split(triples_data, train_size=0.8)\n",
    "triples_data_train, triples_data_val = train_test_split(triples_data_trainval, train_size=0.9)\n",
    "print(len(triples_data_train), len(triples_data_val), len(triples_data_test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "datagen_args = dict(rotation_range=10,\n",
    "                    width_shift_range=0.2,\n",
    "                    height_shift_range=0.2,\n",
    "                    zoom_range=0.2)\n",
    "datagens = [ImageDataGenerator(**datagen_args),\n",
    "            ImageDataGenerator(**datagen_args)]\n",
    "train_pair_gen = pair_generator(triples_data_train, image_cache, datagens, BATCH_SIZE)\n",
    "val_pair_gen = pair_generator(triples_data_val, image_cache, None, BATCH_SIZE)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_train_steps = len(triples_data_train) // BATCH_SIZE\n",
    "num_val_steps = len(triples_data_val) // BATCH_SIZE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/10\n",
      "93/93 [==============================] - 150s - loss: 0.8327 - acc: 0.5759 - val_loss: 0.5848 - val_acc: 0.5687\n",
      "Epoch 2/10\n",
      "93/93 [==============================] - 123s - loss: 0.5829 - acc: 0.6784 - val_loss: 0.5199 - val_acc: 0.7312\n",
      "Epoch 3/10\n",
      "93/93 [==============================] - 126s - loss: 0.4994 - acc: 0.7688 - val_loss: 0.4864 - val_acc: 0.8063\n",
      "Epoch 4/10\n",
      "93/93 [==============================] - 128s - loss: 0.4420 - acc: 0.7957 - val_loss: 0.4454 - val_acc: 0.8063\n",
      "Epoch 5/10\n",
      "93/93 [==============================] - 129s - loss: 0.3982 - acc: 0.8296 - val_loss: 0.4148 - val_acc: 0.8281\n",
      "Epoch 6/10\n",
      "93/93 [==============================] - 109s - loss: 0.3725 - acc: 0.8417 - val_loss: 0.4169 - val_acc: 0.8219\n",
      "Epoch 7/10\n",
      "93/93 [==============================] - 109s - loss: 0.3316 - acc: 0.8585 - val_loss: 0.4290 - val_acc: 0.8313\n",
      "Epoch 8/10\n",
      "93/93 [==============================] - 124s - loss: 0.3313 - acc: 0.8542 - val_loss: 0.3393 - val_acc: 0.8531\n",
      "Epoch 9/10\n",
      "93/93 [==============================] - 109s - loss: 0.2883 - acc: 0.8827 - val_loss: 0.3516 - val_acc: 0.8500\n",
      "Epoch 10/10\n",
      "93/93 [==============================] - 125s - loss: 0.2635 - acc: 0.8935 - val_loss: 0.3013 - val_acc: 0.8750\n"
     ]
    }
   ],
   "source": [
    "checkpoint = ModelCheckpoint(filepath=BEST_MODEL_FILE, save_best_only=True)\n",
    "history = model.fit_generator(train_pair_gen, \n",
    "                             steps_per_epoch=num_train_steps,\n",
    "                             epochs=NUM_EPOCHS,\n",
    "                             validation_data=val_pair_gen,\n",
    "                             validation_steps=num_val_steps,\n",
    "                             callbacks=[checkpoint])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAagAAAEYCAYAAAAJeGK1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8VOXZ//HPRQiEhC0EEEiARGUZCFuIiKICQhVFUUQF\nxLbYWp76c6/2Kdr+Wmu1tb8qxbYuVYtdBHkouNBWi/URVFxZBCSEfQ1r2CEkkOX6/XHPkMkekknO\nTHK9X6/zysyZMzNXRpxv7vvc575FVTHGGGPCTROvCzDGGGPKYwFljDEmLFlAGWOMCUsWUMYYY8KS\nBZQxxpiwZAFljDEmLFlAGWOMCUsWUMaUQ0SWiMgREWnudS3GNFYWUMaUIiLJwOWAAuPq8X2b1td7\nGRMJLKCMKetbwOfAn4FvB3aKSAsReUZEdojIMRFZKiIt/I9dJiKfishREdklIlP9+5eIyJ1BrzFV\nRJYG3VcRuVtENgGb/Pue9b/GcRFZISKXBx0fJSKPisgWETnhf7yriDwnIs8E/xIislBEHqyLD8iY\n+mABZUxZ3wJm+7erReQ8//6ngcHApUA74L+BIhHpDrwL/B7oAAwEVp3D+90IXAz08d9f5n+NdsAc\n4O8iEuN/7AfAZOBaoDXwHeAU8Bdgsog0ARCR9sBo//ONiUgWUMYEEZHLgO7APFVdAWwBbvN/8X8H\nuF9Vd6tqoap+qqqngduA91X1dVXNV9VDqnouAfUrVT2sqrkAqvqa/zUKVPUZoDnQy3/sncBPVHWD\nOqv9x34JHANG+Y+bBCxR1f21/EiM8YwFlDElfRt4T1UP+u/P8e9rD8TgAqu0rhXsr65dwXdE5GER\nyfR3Ix4F2vjfv6r3+gtwu//27cDfalGTMZ6zk7LG+PnPJ90KRInIPv/u5kBboDOQB1wArC711F3A\nkApeNgeIDbrfqZxjzi4p4D/f9N+4llCGqhaJyBFAgt7rAmBtOa/zGrBWRAYAPuCtCmoyJiJYC8qY\nYjcChbhzQQP9mw/4GHdeahYwQ0S6+AcrXOIfhj4bGC0it4pIUxFJEJGB/tdcBdwkIrEiciHw3Spq\naAUUANlAUxH5Ke5cU8ArwC9EpIc4/UUkAUBVs3Dnr/4GLAh0GRoTqSygjCn2beBVVd2pqvsCG/AH\nYAowHfgaFwKHgV8DTVR1J27QwkP+/auAAf7X/C1wBtiP64KbXUUNi4B/AxuBHbhWW3AX4AxgHvAe\ncBz4E9Ai6PG/AP2w7j3TAIgtWGhMwyEiV+C6+rqr/c9tIpy1oIxpIEQkGrgfeMXCyTQEFlDGNAAi\n4gOO4gZzzPS4HGNCwrr4jDHGhCVrQRljjAlL1boOSkTGAM8CUbj+7adKPd4dNwS3A24U0+3+Ia8V\nat++vSYnJ9ekZmOMMRFsxYoVB1W1Q1XHVRlQIhIFPAd8A8gClonIQlVdF3TY08BfVfUvInIl8Cvg\nm5W9bnJyMsuXL6/q7Y0xxjQwIrKjOsdVp4tvCLBZVbeq6hlgLnBDqWP6AB/4by8u53FjjDHmnFQn\noBIpeaFgln9fsNXATf7b44FWgavbg4nINBFZLiLLs7Oza1KvMcYYj+TmQl5e/b1fqObiexj4g38N\nnI+A3bgpY0pQ1ZeAlwDS09Nt+KAxxnigsBCOHIFDh0puhw+X3Rf8WG4uzJoFd9xRP3VWJ6B242ZQ\nDkjy7ztLVffgb0GJSEtggqoeDVWRxhhjylKFnJyKA6WisDlaybdzVBS0awcJCW5LTobBg4v3paXV\n269XrYBaBvQQkRRcME3CrX9zln9xtMOqWgQ8ghvRZ4wx5hyowvbtkJVVvbA5fBjOnKn49Vq3Lhk2\nF1xQfDt4f/DWujWIVPya9anKgFLVAhG5BzeJZRQwS1UzRORxYLmqLgRGAL8SEcV18d1dhzUbY0zE\nKyyETZtg5cqS27FjZY9t1qxkqPTsWTJUygubdu0gOrr+f69Q8mwmifT0dLVh5saYxiA/HzIzSwbR\nqlWuew6geXMYMMB1nw0aBCkpJcMmLi4MWjVFRbBzJ7RpA/HxtXopEVmhqulVHWcLFhpjTAjl5cHa\ntSXDaM0aOH3aPR4X50Lou991gZSWBr17h1FrRxX27XO/REaG+xm4ffIkvPKKK74eWEAZY0wN5eTA\n6tUlwygjAwoK3ONt27oAuvfe4jC68EI3ECEsHDpUNoTWrnUntwI6doTUVDd0LzUVRoyot/IsoIwx\nphqOHYOvvioZRhs2uJ4vgA4d3Gi3sWOLwyg5OQy65gBOnHDhExxGa9e6llJAmzYugG65Bfr2dbf7\n9nUB5RELKGOMKeXgQRdGK1YUh9GWLcWPJya6ALr1VhdKaWnQpUsYhFFuLqxfX7Z7bkfQzEItWrjg\nGTPGhVAgiBITw+AXKMkCyhhTK4WFsHevO3++a1fJn4cPuwEAzZtDTEzxz9rcLm9f01p8k+3dW3Yk\n3c6dxY+npLgA+s53igcxnHde7T+3WsnPd0MAg1tDGRmweXNxky462p3cuvRSmDatOIySk6FJZCxk\nYQFljKmQqjtNsWtX2fAJ/Nyzx4VUsNatoVs3aN8eTp1yQXX6tBtAkJdX8nbgfE1tREVVM+SaKV10\nN93yNlJ48Agbt0Vz4Gg0Z2hGAdF06taM233R9Li5Gb37ReMb0Iw27aPdOO/ooJ9F0fXzJV9YCNu2\nlW0RbdjgQgpcHT16uPCZNKk4iC68MIxGXtSMBZQxjVhOTvnBE3w7N7fkc5o1g65d3TZyZPHtbt2K\nb7dpU/0aCguLA6uiEDvX21EnjtLu4EY6HN7Aefs30vnERhJPbaRb7kZa6KmKi9np3xZVo/CoqLLB\nFfhZ3r7q/oyOdlfqrl0L69aV/A+QnOzC57rris8T9e7t0rcBsoAypoHKz3etm/JaPYEQCh6sBe4U\nROfOLmwGDHDfg4HgCfzs0CG0jYeoKIiNdds5OX3anRjauNG1KDZuLL4dPBl1kyaun+6intBzOPTq\n5a507dDBfUhnztTs57k+58SJ6r9Wp04ufL7//eJzRH36QKtWofvgI4AFlDERRtXNpZadDQcOuK28\nLri9e92xweLji4Nm2LCyrZ/ExDDrFSoqcq2J4PAJ3N6+vfh8C7gTQz17wrhx7mcgiM4/3/XzmYhj\nAWWMx4qK3MzS2dnFoRO4Xd6+gwfLP2/TokVx4Fx9ddlut65doWXL+v/9quXw4eLgCQ6iTZtKdnHF\nxbnQuegimDKlOIR69HAXHZkGxQLKmBArLHTft5WFTPC+Q4fKDjIIaNPG9UR17OgaAhdf7O4H9gV+\nJiW5KXHCbJRwSXl5bpRZea2hgweLj4uKcl1yvXrBqFElW0NhMZbb1BcLKGOq4cgRdz6nqtZNdrYL\nnOCep2Dx8cUB06OHGwEcuF86eNq3d+fNw1penvtwjh4t/hl8e9++4hDasaNkn2OnTi54xo8vGUIp\nKRHwi5v6YAFlTClapGz58hAbFm5g/8cbKVi3kbaH3VWap4glhzhyiAPiaB8bR8dWcTRtE0ez+Dia\nd48jrqPbWnWKo02XOOKT4kjoFkdC52bhdX4HXNPt2LGSoVJZ4JTeF5hgriKtWrkkvuQS+Pa3i4Oo\nRw83Ft2YSlhAmcbr1CnYvJn8jA3s/XAjJ5ZvpOnWDXQ8upEL9QgX+g/Ll2hOJCTTtHkUzQpyaHo6\nh6i8HOT0aTiF2/ZX4/2aNnXnUOLi3JC1wO2abMHPj4mB48crDpPKgub48cprjopy53YCW3y860+M\njy++H/xY6X02OMHUggWUadgKC13XUtD5jvyMDRSs20iL7F0ARAPdgCwS2dmiF7t7TKTFgJ4kXtmL\npCt7En1+Mu3Km6qgoMCFXE5O+VtljwVvx4+7IXelnxeqpXBatiwZGt27uzHkVYVLfLx7rp3zMR6x\ngDKRT9Wd/CnnehjdsgUJWnL0eJM2ZBb1YiPD2dykJ/kpvWg3tCcXjrmQoaNbcmmnc3jfpk1dN1Vd\ndFWpuvM71Qm4vDxXQ3mB07Zt7eYBMsZD9i/XRI6cHDfsODiEAqEUtAxpUXQzjiRcyJYmvVjW7HpW\nnunJBnqxt2VPeg7rwLDLhMsugwlDanBxaH0RcePGW7RwoyWMaYSqFVAiMgZ4Frfk+yuq+lSpx7sB\nfwHa+o+ZrqrvhLhW0xgUFLgLMMubHWD37pLHdu1KwYW92HvFFNae6clH+3vxjw09ycztTtG+KLp1\ng8vGuQtS77/MXYwfNuvwGGOqVGVAiUgU8BzwDSALWCYiC1V1XdBhPwHmqeoLItIHeAdIroN6TW2s\nXQs//zl89pnXlZQv0FUXmAQTXFdV0PUwh9r3Ytmxnry39UKWfBnL6g/dkO4mTdxplZHfhZ8MK54l\nwRgTuarTghoCbFbVrQAiMhe4AQgOKAUCHfFtgD2hLNLU0saN8NhjMHeuO+l9443he51J+/YukHr1\novCCnmTsS2DpJ8Inn8DSl4qXQYiLg6FD4Sc/gcsucxew2qhlYxqW6gRUIrAr6H4WcHGpYx4D3hOR\ne4E4YHRIqjO1s20bPP44/PWvbijyj34EDz/sphwIQzk5sGwZLF0Kn8yDTz8tHgXdubMLoh/8wP0c\nMMDO/RvT0IXqf/HJwJ9V9RkRuQT4m4ikqmqJ6+lFZBowDaBbt24hemtTxq5d8OST8Kc/uZMu99/v\nwsnzVdYcVTf/5+rVJbdNm4pHVqemwuTJLoyGDQujpbONMfWmOgG1GwjuzU/y7wv2XWAMgKp+JiIx\nQHvgQPBBqvoS8BJAenp6iC7yMGft3Qu/+hX88Y/um37aNHj0UTdFtUdOn3brrAUH0Zo1JZd5OP98\n1yK67TZIT3fT/8THe1ayMSZMVCeglgE9RCQFF0yTgNtKHbMTGAX8WUR8QAyQjakf2dnw//4fPPec\nW0vmjjvcyZnu3eu1jP37SwbRqlWwfn3xRKgtWkC/fjBhAgwc6EKpXz87d2SMKV+VAaWqBSJyD26N\nyShglqpmiMjjwHJVXQg8BLwsIg/iBkxMVQ3VZfCmQkeOwDPPwLPPutkHpkyBn/7ULfVch/Lz3ajv\n0l10+4Om+0lKcgF0ww3u54ABriwb5m2MqS7xKkfS09N1+fLlnrx3xDt+3IXSM8+4C1RvvdWN0vP5\nQv5Whw+XDaKMDNdQAzcYsE+f4hAaOBD69w/bcRjGmDAgIitUNb2q42wcVCTJyYE//MF15x0+7IaL\n//znLhFqqbDQrZ5dOox2BY3f7NjRhdB99xUHUu/eYbYCqzGmwbCAigR5efDii24AxIEDcM01bvh4\nepV/gJTrxAk3UCE4iL7+2vUSguuG69XLjaALnCsaMMAt32OMMfXFAiqcnTnjhoo/+aSb5ufKK+EX\nv3DD3Gpg61bX+vnXv4r3tW3rwufOO4uDqG9fd9mUMcZ4yQIqHBUUuItrH3/cLRVx2WXw2mswYkSN\nXu7MGXj6aZdtTZu6kedDh7ow6trVri8yxoQnC6hwUlgIr7/uzitt3gwXXeSuabrqqhqnyEcfwfe/\nD5mZbnj3zJluhJ0xxoS7Jl4XYHCznf797+6ioG9+0000t3AhfPEFXH11jcLp4EF3OdTw4ZCbC//8\nJ8yfb+FkjIkcFlBeUnVBlJbmhoqLuKBauRKuv75GwVRUBLNmuUEOr70G06e7YeFjx9ZB/cYYU4cs\noLygCosWuSm4b7jBDR9/7TU3tO7mm93aETWQkeFaTN/9rrs2adUqN/AvbBflM8aYSlhA1bclS+Dy\ny2HMGDdkfNYsd4JoypQaT7Nw6hQ88ogbEr5unRv49+GHbjSeMcZEKhskUV8+/RT+7/+FDz5wk7e+\n8AJ85zu1XpfpX/+Ce+5xi9BOnQq/+Y2tEG6MaRisBVXXli+Ha691a0ZkZLhhdJs3u6F1tQinrCzX\nG3jddW4S1iVL4NVXLZyMMQ2HBVRdWbPGTUV00UXw5Zfw61+7uYTuv79WV8EWFLhp+Hw+13r65S/d\nuabhw0NYuzHGhAHr4qsLM2bAQw9Bmzbu6tj77gvJmhJffukaXl995U5hPfecW0vJGGMaIguoUPvb\n31w4TZgAL78ckpX3jh2DH/8Ynn/eLX3+97+7l7cZIIwxDZkFVCgtWuQGPowcCbNnQ/PmtXo5Vfif\n/4EHH3QD/u691zXIbIE/Y0xjYAEVKsuXu2ZN377w5pu1DqfNm+Huu+G999yk5f/8JwweHKJajTEm\nAtggiVDYvNmN1OvQAd591517qqHTp10rKTUVPvsMfv97+PxzCydjTONjLaja2rfPzZcXmB2ic+ca\nv9TixXDXXW459YkT3ViLLl1CWKsxxkSQarWgRGSMiGwQkc0iMr2cx38rIqv820YRORr6UsPQiROu\n5bRvn+uD69mzRi9z4AB861tuuaf8fNcImzvXwskY07hV2YISkSjgOeAbQBawTEQWquq6wDGq+mDQ\n8fcCg+qg1vBy5gzcdJO73ukf/3Dz6p2joiI3LdGPfgQnT8JPfuLWamrRog7qNcaYCFOdFtQQYLOq\nblXVM8Bc4IZKjp8MvB6K4sJWUZFby+L9913CXHPNOb/EmjVuHcJp06B/f7fs+i9+YeFkjDEB1Qmo\nRGBX0P0s/74yRKQ7kAJ8UMHj00RkuYgsz87OPtdaw8cPfwhz5ripwr/97XN6ak6Oe3paGmzaBH/5\nizv35PPVUa3GGBOhQj1IYhIwX1ULy3tQVV8CXgJIT0/XEL93/XjmGTd64d57Xd/cOVi40D1t5064\n80546ilISKijOo0x5yw/P5+srCzy8vK8LqVBiImJISkpiejo6Bo9vzoBtRvoGnQ/yb+vPJOAu2tU\nSSSYPRseftgtLjhzZrWncti508129Pbbbvj40qVu7lhjTHjJysqiVatWJCcnIzZVS62oKocOHSIr\nK4uUlJQavUZ1uviWAT1EJEVEmuFCaGHpg0SkNxAPfFajSsLde++59SxGjIC//rVaiwrm57sGV58+\n7um//rVbLNfCyZjwlJeXR0JCgoVTCIgICQkJtWqNVtmCUtUCEbkHWAREAbNUNUNEHgeWq2ogrCYB\nc1U1MrvuKrNihZslok8feOutas0S8fnn8F//5QZDXHedu+A2ObnuSzXG1I6FU+jU9rOs1jkoVX0H\neKfUvp+Wuv9YrSoJV5s3u1F6CQnVmiUiN9cNFX/2Wbcu4RtvuFU37N+8McacG5vqqDL797t1LYqK\n3CwRVVw5u3q1W/5p5kz4P//HLb8+fryFkzGmeo4ePcrzzz9/zs+79tprOXq04c2PYAFVkRMnYOxY\n2LvXrQzYq1eFhxYWuqXWL7oIDh2Cf/8b/vAHaNWqHus1xkS8igKqoKCg0ue98847tG3btq7K8ozN\nxVeeM2fcOadVq9zQu0pmidi5001T9OGHbmKJP/7Rll03xtTM9OnT2bJlCwMHDiQ6OpqYmBji4+NZ\nv349Gzdu5MYbb2TXrl3k5eVx//33M23aNACSk5NZvnw5J0+e5JprruGyyy7j008/JTExkbfffpsW\nEToDgAVUaUVFbk2n//wHZs1yragKzJnjuvIKC92hU6dad54xDcUDD7i/UUNp4EB3CqAiTz31FGvX\nrmXVqlUsWbKEsWPHsnbt2rPDtGfNmkW7du3Izc3loosuYsKECSSUuphy06ZNvP7667z88svceuut\nLFiwgNtvvz20v0g9sS6+0n70I3e90y9/6aYzKseRIzB5MkyZ4pZ/Wr3aHWrhZIwJpSFDhpS4huh3\nv/sdAwYMYOjQoezatYtNmzaVeU5KSgoDBw4EYPDgwWzfvr2+yg05a0EFmzEDnn4a7rkHppeZtB2A\nDz5wsxvt2wdPPOHyrKl9isY0OJW1dOpLXFzc2dtLlizh/fff57PPPiM2NpYRI0aUe41R86DLYKKi\nosjNza2XWuuCtaAC5syBhx6Cm28ud5aI06fdJBKjRkFsrFtM8Mc/tnAyxoROq1atOHHiRLmPHTt2\njPj4eGJjY1m/fj2ff/55PVdX/+zrFdz5pqlTYfhw+NvfICqqxMNff+26877+2i0o+JvfQNAfNsYY\nExIJCQkMGzaM1NRUWrRowXnnnXf2sTFjxvDiiy/i8/no1asXQ4cO9bDS+iFeTfyQnp6uy5cv9+S9\nS1i50gXT+efDRx+VuBC3qMg1ph55BOLj3UCIa6/1sFZjTJ3KzMzEZ0sLhFR5n6mIrFDV9Kqe27hb\nUFu2uFki2rUrM0vErl2uUfXBB3DDDfDyy9Chg3elGmNMY9N4z0EdOOBmiSgoKDNLxNy5bhHBL76A\nV16BN9+0cDLGmPrWOAPq5EnXV7d7t5slondvAI4ehdtvd0PIe/d210B897s2fNwYY7zQ+AIqeJaI\nefPAf6JxyRLXapo7F37+c/j4Y7jwQm9LNcaYxqxxBVRRkWsSvfcevPQSXHcdp0+7a5muvNKtovHJ\nJ/DTn9rwcWOM8Vrj+hqePh1ee81dYfud75CR4YaPr14N06a5xQVbtvS6SGOMMdCYWlC//a27gOnu\nuyma/ijPPguDB8OePbBwoZvk1cLJGBNJWvq/tPbs2cPNN99c7jEjRoygqkt6Zs6cyalTp87eD5fl\nOxpHQM2dCz/4AUyYwO7/fpYx1wgPPADf+Ia7+Pb6670u0Bhjaq5Lly7Mnz+/xs8vHVDhsnxHtQJK\nRMaIyAYR2Swi5U5SJyK3isg6EckQkTmhLbMW/vd/3XoYV1zBGze9Rr+BUXzyCbz4oms5BV2obYwx\nnpo+fTrPPffc2fuPPfYYTzzxBKNGjSItLY1+/frx9ttvl3ne9u3bSU1NBSA3N5dJkybh8/kYP358\nibn47rrrLtLT0+nbty8/+9nPADcB7Z49exg5ciQjR44E3PIdBw8eBGDGjBmkpqaSmprKTP8Ehdu3\nb8fn8/G9732Pvn37ctVVV9XJnH9VnoMSkSjgOeAbQBawTEQWquq6oGN6AI8Aw1T1iIh0DHmlNfHV\nVzB+PIU9enFPl7d5cUoMQ4a42Yx69vS6OGNMWPNgvY2JEyfywAMPcPfddwMwb948Fi1axH333Ufr\n1q05ePAgQ4cOZdy4cUgF17+88MILxMbGkpmZyZo1a0hLSzv72JNPPkm7du0oLCxk1KhRrFmzhvvu\nu48ZM2awePFi2pdazG7FihW8+uqrfPHFF6gqF198McOHDyc+Pr5elvWoTgtqCLBZVbeq6hlgLnBD\nqWO+BzynqkcAVPVASKusia1b4ZpryIuNZ9jxf/PSvLb89KewdKmFkzEmPA0aNIgDBw6wZ88eVq9e\nTXx8PJ06deLRRx+lf//+jB49mt27d7N///4KX+Ojjz46GxT9+/enf//+Zx+bN28eaWlpDBo0iIyM\nDNatW1fRywCwdOlSxo8fT1xcHC1btuSmm27i448/BupnWY/qjOJLBHYF3c8CSi8x2xNARD4BooDH\nVPXfIamwJg4cQK+6mtxj+aTnLebMBYksXQqXXOJZRcaYSOPRehu33HIL8+fPZ9++fUycOJHZs2eT\nnZ3NihUriI6OJjk5udxlNqqybds2nn76aZYtW0Z8fDxTp06t0esE1MeyHqEaJNEU6AGMACYDL4tI\nmTNsIjJNRJaLyPLs7OwQvXUpJ0+SO/o68rbuZlTePxl2p49VqyycjDGRYeLEicydO5f58+dzyy23\ncOzYMTp27Eh0dDSLFy9mx44dlT7/iiuuYM4cNwxg7dq1rFmzBoDjx48TFxdHmzZt2L9/P+++++7Z\n51S0zMfll1/OW2+9xalTp8jJyeHNN9/k8ssvD+FvW7nqtKB2A12D7if59wXLAr5Q1Xxgm4hsxAXW\nsuCDVPUl4CVws5nXtOiK6Jl8dg65maTMFXyr1Vv86K+XcOONoX4XY4ypO3379uXEiRMkJibSuXNn\npkyZwvXXX0+/fv1IT0+nt39qtorcdddd3HHHHfh8Pnw+H4MHDwZgwIABDBo0iN69e9O1a1eGDRt2\n9jnTpk1jzJgxdOnShcWLF5/dn5aWxtSpUxkyZAgAd955J4MGDaq3VXqrXG5DRJoCG4FRuGBaBtym\nqhlBx4wBJqvqt0WkPfAVMFBVD1X0uqFebmPvHiXjom8zes/feDb1ZSb+5046dQrZyxtjGgFbbiP0\narPcRpVdfKpaANwDLAIygXmqmiEij4vIOP9hi4BDIrIOWAz8sLJwCrU33oB5FzzC6D1/44vrfsF9\nayycjDEm0lVrqiNVfQd4p9S+nwbdVuAH/q3enDgB998PrV59lmf5NUcm3cXFc34MNvu4McZEvIid\nSWL1andJQe6f/4ff8iBFN95E/Gu/t7UxjDG14tUq4w1RbT/LiA2oDh3gqqYfMDvqmzS5/DKavD4b\noqK8LssYE8FiYmI4dOiQhVQIqCqHDh0iJiamxq8RsbOZd8lezfN7b0R693JzFtXiQzDGGICkpCSy\nsrKos8tgGpmYmBiSkpJq/PyIDSg6dECGD4cXXoAwmNTQGBP5oqOjSUlJ8boM4xe5AdWlC/zjH15X\nYYwxpo5E7DkoY4wxDZsFlDHGmLBU5UwSdfbGItlA5ZNKVU974GAIXqcxsc+sZuxzO3f2mdVMQ//c\nuqtqh6oO8iygQkVElldnygxTzD6zmrHP7dzZZ1Yz9rk51sVnjDEmLFlAGWOMCUsNIaBe8rqACGSf\nWc3Y53bu7DOrGfvcaADnoIwxxjRMDaEFZYwxpgGygDLGGBOWIjagRGSMiGwQkc0iMt3reiKBiHQV\nkcUisk5EMkTkfq9rihQiEiUiX4nIP72uJVKISFsRmS8i60UkU0Qu8bqmcCciD/r/31wrIq+LSKOe\nBTsiA0pEooDngGuAPsBkEenjbVURoQB4SFX7AEOBu+1zq7b7cStKm+p7Fvi3qvYGBmCfX6VEJBG4\nD0hX1VQgCpjkbVXeisiAAoYAm1V1q6qeAeYCN3hcU9hT1b2qutJ/+wTuCyPR26rCn4gkAWOBV7yu\nJVKISBvgCuBPAKp6RlWPeltVRGgKtBCRpkAssMfjejwVqQGVCOwKup+FfdGeExFJBgYBX3hbSUSY\nCfw3UOR1IREkBcgGXvV3jb4iInFeFxXOVHU38DSwE9gLHFPV97ytyluRGlCmFkSkJbAAeEBVj3td\nTzgTkes7MgekAAAdtElEQVSAA6q6wutaIkxTIA14QVUHATmAnSuuhIjE43qCUoAuQJyI3O5tVd6K\n1IDaDXQNup/k32eqICLRuHCarapveF1PBBgGjBOR7biu5CtF5DVvS4oIWUCWqgZa6PNxgWUqNhrY\npqrZqpoPvAFc6nFNnorUgFoG9BCRFBFphjuRuNDjmsKeiAjunECmqs7wup5IoKqPqGqSqibj/p19\noKqN+q/a6lDVfcAuEenl3zUKWOdhSZFgJzBURGL9/6+OopEPLInIFXVVtUBE7gEW4Ua6zFLVDI/L\nigTDgG8CX4vIKv++R1X1HQ9rMg3XvcBs/x+RW4E7PK4nrKnqFyIyH1iJG3H7FY18yiOb6sgYY0xY\nitQuPmOMMQ2cBZQxxpiwZAFljDEmLFlAGWOMCUsWUMYYY8KSBZQxxpiwZAFljDEmLFlAGWOMCUsW\nUMYYY8KSBZQxxpiwZAFljDEmLFlAGWOMCUsWUMYYY8KSBZQxISAi20VktNd1GNOQWEAZY4wJSxZQ\nxtQhEfmeiGwWkcMislBEuvj3i4j8VkQOiMhxEflaRFL9j10rIutE5ISI7BaRh739LYzxhgWUMXVE\nRK4EfgXcCnQGdgBz/Q9fBVwB9ATa+I855H/sT8B/qWorIBX4oB7LNiZsROSS78ZEiCnALFVdCSAi\njwBHRCQZyAdaAb2BL1U1M+h5+UAfEVmtqkeAI/VatTFhwlpQxtSdLrhWEwCqehLXSkpU1Q+APwDP\nAQdE5CURae0/dAJwLbBDRD4UkUvquW5jwoIFlDF1Zw/QPXBHROKABGA3gKr+TlUHA31wXX0/9O9f\npqo3AB2Bt4B59Vy3MWHBAsqY0IkWkZjABrwO3CEiA0WkOfBL4AtV3S4iF4nIxSISDeQAeUCRiDQT\nkSki0kZV84HjQJFnv5ExHrKAMiZ03gFyg7YRwP8FFgB7gQuASf5jWwMv484v7cB1/f3G/9g3ge0i\nchz4Pu5cljGNjqiq1zUYY4wxZVgLyhhjTFiygDLGGBOWLKCMMcaEJQsoY4wxYcmzmSTat2+vycnJ\nXr29McYYj6xYseKgqnao6jjPAio5OZnly5d79fbGGGM8IiI7qj7KuviMMcaEqYgNqNOn4Wc/g1On\nvK7EGGNMXYjYgFq6FJ54Aq69Fk6e9LoaY4wxoRaxy22MGgWzZ8Ptt8PVV8M770CbNl5XZYyJZPn5\n+WRlZZGXl+d1KQ1CTEwMSUlJREdH1+j51QooERkDPAtEAa+o6lOlHu8G/AVo6z9muqq+U6OKzsGk\nSRAd7X5edRX8+98QH1/X72qMaaiysrJo1aoVycnJiIjX5UQ0VeXQoUNkZWWRkpJSo9eosotPRKJw\na9Zcg1sWYLKI9Cl12E+Aeao6CDcZ5vM1qqYGJkyABQtg1SrXqjp0qOrnGGNMefLy8khISLBwCgER\nISEhoVat0eqcgxoCbFbVrap6Brdk9Q2ljlHc7Mzglq/eU+OKamDcOHjrLVi3DkaOhAMH6vPdjTEN\niYVT6NT2s6xOQCUCu4LuZ/n3BXsMuF1EsnBLDtxb3guJyDQRWS4iy7Ozs2tQbsWuuQb++U/YvBlG\njIC9e0P68sYYY+pZqEbxTQb+rKpJuKWq/yYiZV5bVV9S1XRVTe/QocqLiM/Z6NHw7ruwc6cLqd27\nQ/4WxhhTZ44ePcrzz5/7GZJrr72Wo0eP1kFF3qpOQO0GugbdT/LvC/Zd/MtSq+pnQAzQPhQFnqvh\nw2HRIteCuuIK2FGt65WNMcZ7FQVUQUFBpc975513aNu2bV2V5ZnqBNQyoIeIpIhIM9wgiIWljtkJ\njAIQER8uoELbh3cOhg2D//zHDZgYPhy2bvWqEmOMqb7p06ezZcsWBg4cyEUXXcTll1/OuHHj6NPH\njUu78cYbGTx4MH379uWll146+7zk5GQOHjzI9u3b8fl8fO9736Nv375cddVV5ObmevXr1FqVw8xV\ntUBE7gEW4YaQz1LVDBF5HFiuqguBh4CXReRB3ICJqerxUr0XXwwffADf+IYLqQ8+gB49vKzIGBNJ\nHnjAjQ4OpYEDYebMih9/6qmnWLt2LatWrWLJkiWMHTuWtWvXnh2mPWvWLNq1a0dubi4XXXQREyZM\nICEhocRrbNq0iddff52XX36ZW2+9lQULFnD77beH9hepJ9W6Dsp/TdM7pfb9NOj2OmBYaEurvbQ0\nF0yjR7uQ+t//BZ/P66qMMaZ6hgwZUuIaot/97ne8+eabAOzatYtNmzaVCaiUlBQGDhwIwODBg9m+\nfXu91RtqETuTRHUNGABLlrhrpEaMcCGVmup1VcaYcFdZS6e+xMXFnb29ZMkS3n//fT777DNiY2MZ\nMWJEudcYNW/e/OztqKioiO7ii9i5+M5F377w4YfQtKkLqa++8roiY4wpq1WrVpw4caLcx44dO0Z8\nfDyxsbGsX7+ezz//vJ6rq3+NIqAAevVyIRUbC1deCcuWeV2RMcaUlJCQwLBhw0hNTeWHP/xhicfG\njBlDQUEBPp+P6dOnM3ToUI+qrD/i1ViG9PR09WLBwu3b3WwThw+7ufsuuaTeSzDGhKnMzEx8dqI6\npMr7TEVkhaqmV/XcRtOCCkhOho8+go4d3QSzH3/sdUXGGGPK0+gCCqBrV9fdl5QEY8a4kX7GGGPC\nS6MMKIAuXdzovpQUGDvWzT5hjDEmfDTagAI47zxYvNgNoBg3zk02a4wxJjw06oAC6NDBdfH16wc3\n3QT+a+CMMcZ4rNEHFEC7dvD++zB4MNxyC8yb53VFxhhjLKD82raF995zw84nT4bZs72uyBhjKtey\nZUsA9uzZw80331zuMSNGjKCqS3pmzpzJqVOnzt4Pl+U7LKCCtGrlro0aPhy++U149VWvKzLGmKp1\n6dKF+fPn1/j5pQMqXJbvsIAqJS7ODZYYPRq+8x0ImtHeGGPq1PTp03nuuefO3n/sscd44oknGDVq\nFGlpafTr14+33367zPO2b99Oqn+S0dzcXCZNmoTP52P8+PEl5uK76667SE9Pp2/fvvzsZz8D3AS0\ne/bsYeTIkYwcORIoXr4DYMaMGaSmppKamspM/wSF9bWsR4OfLLYmYmNh4UKYMAH+67/gzBm45x6v\nqzLG1CsP1tuYOHEiDzzwAHfffTcA8+bNY9GiRdx33320bt2agwcPMnToUMaNG4eIlPsaL7zwArGx\nsWRmZrJmzRrS0tLOPvbkk0/Srl07CgsLGTVqFGvWrOG+++5jxowZLF68mPbtS64zu2LFCl599VW+\n+OILVJWLL76Y4cOHEx8fXy/LelgLqgIxMfDGG3DDDXDvvfDMM15XZIxp6AYNGsSBAwfYs2cPq1ev\nJj4+nk6dOvHoo4/Sv39/Ro8eze7du9m/f3+Fr/HRRx+dDYr+/fvTv3//s4/NmzePtLQ0Bg0aREZG\nBuvWrau0nqVLlzJ+/Hji4uJo2bIlN910Ex/7p9+pj2U9rAVViebN4e9/hylT4OGHXUvqkUe8rsoY\nUy88Wm/jlltuYf78+ezbt4+JEycye/ZssrOzWbFiBdHR0SQnJ5e7zEZVtm3bxtNPP82yZcuIj49n\n6tSpNXqdgPpY1sNaUFWIjoY5c+C22+DRR+HnPwdv1wo2xjRkEydOZO7cucyfP59bbrmFY8eO0bFj\nR6Kjo1m8eDE7duyo9PlXXHEFc+bMAWDt2rWsWbMGgOPHjxMXF0ebNm3Yv38/77777tnnVLTMx+WX\nX85bb73FqVOnyMnJ4c033+Tyyy8P4W9bOWtBVUPTpvDXv0KzZvDYY64l9cQTUEEXsDHG1Fjfvn05\nceIEiYmJdO7cmSlTpnD99dfTr18/0tPT6d27d6XPv+uuu7jjjjvw+Xz4fD4GDx4MwIABAxg0aBC9\ne/ema9euDBtWvAj6tGnTGDNmDF26dGHx4sVn96elpTF16lSGDBkCwJ133smgQYPqbZXeRrfcRm0U\nFcH3vw8vvwwPPQS/+Y2FlDENiS23EXq1WW7DWlDnoEkTePFF15J65hnXknr2WQspY4ypCxZQ56hJ\nE/j9790AihkzXEg9/7zbb4wxJnSq9bUqImNEZIOIbBaR6RUcc6uIrBORDBGZE9oyw4sIPP00TJ8O\nf/wj3HknFBZ6XZUxJhS8Ou3RENX2s6yyBSUiUcBzwDeALGCZiCxU1XVBx/QAHgGGqeoREelYq6oi\ngAj88peuJfXzn7uW1J//7AZUGGMiU0xMDIcOHSIhIaHCC2FN9agqhw4dIiYmpsavUZ2v0yHAZlXd\nCiAic4EbgOArvL4HPKeqR/yFHahxRRFExI3qi46Gn/wE8vPhtdfcfWNM5ElKSiIrK4vs7GyvS2kQ\nYmJiSEpKqvHzqxNQicCuoPtZwMWljukJICKfAFHAY6r67xpXFWF+/GPXkvrhD11IzZ3rBlIYYyJL\ndHQ0KSkpXpdh/ELVIdUU6AGMAJKAj0Skn6qWmK9dRKYB0wC6detWu3dUdeO+o6Jq9zoh8vDDLpTu\nv98tfDh/vpsuyRhjTM1UZ5DEbqBr0P0k/75gWcBCVc1X1W3ARlxglaCqL6lquqqmd+jQoaY1O+vX\nQ/v2brK8mTNh9WoXWB667z544QX417/gmmvg3XfduSljjDHnrjoBtQzoISIpItIMmAQsLHXMW7jW\nEyLSHtfltzWEdZYVFeWmG1+7Fh580M0S3LGj2/f737v9HozG+f733TpSK1bAtde6kr71LTc7ei2m\nvTLGmEanWjNJiMi1wEzc+aVZqvqkiDwOLFfVheKGuzwDjAEKgSdVdW5lrxnSmSR27oTFi2HJEvcz\nMFdVhw4wYgSMHOm2Xr3q7aravDz4z39gwQJ4+204ehRatoTrrnMZes01bu0pY4xpbKo7k0TDnOpo\n2zYXVIFtt79HslOnkoF14YX1ElhnzrgyFiyAN9+EgwehRQsXUjffDGPHQuvWdV6GMcaEhcYdUMFU\nYcuWkoG1b597LDGxOKxGjoR6GL1TUAAff+wGUbzxhiulWTO4+mrXsho3DuLj67wMY4zxjAVURVRh\nw4bi7sDFiyFwzUP37iVbWLUdaViFoiL49FPXslqwAHbtchf6jhrlWlY33OB6KY0xpiGxgKouVVi3\nrjisliyBw4fdY+efXxxWI0a4FlcdlrFsmQuq+fNh61Y3v9/w4S6sxo+Hzp3r7O2NMabeWEDVVFER\nfP11cQvrww/dCAeAHj1KBlanTnVSgqobNT9/vts2bHCnyoYNc92AEyZA165Vv44xxoQjC6hQKSx0\naRFoXX30ERw/7h7z+YrDasSIOumPCzTwAi2rr792+4cMcS2rCRNcQ88YYyKFBVRdKSiAr74q7hL8\n+GPIyXGPpaYWt7Auu6xOAmvTpuKwWrHC7Rs0yAXVzTe7kfTGGBPOLKDqS36+S4pAYC1dCrm57rE2\nbVzzJiWl7M/u3Ws9F9K2bW4k4IIF8Nlnbl/fvsUtq9RUW0zRGBN+LKC8cuYMfPml27ZudSmydSts\n315yKgkR6NKlOLBKh1jnzue0CmJWlrvGav5816hThZ49i89ZpaVZWBljwoMFVLgpKnIXPQUCq/TP\n3btLTs3UvDkkJ5ff+jr/fNc6q8C+ffDWW65ltXixO42WnFzcDThkiK0AbIzxjgVUpDl92k3RVF6A\nbdsGR46UPD4+vvLuQ/96H4cOuamWFixwUy/l57tTY5deCpdc4rb0dIiN9eB3NsY0ShZQDc3Ro2XD\nK3B7+/aS06aLQFJSmeA60eF83tuUwj+WdeLTz4RNm9zhTZu6uXYvuaQ4uLp1sy5BY0zdsIBqTIqK\nYM+eirsP9+wpeXxcHPTuTd75PrbH+Fie4+O9XT7eXnsBx3PdcsCdOxeH1aWXunNYzZt78LsZYxoc\nCyhTLDe3ZPfhpk2Qmem2XcWLJWvTpuR17cGe1j6+LvDx4QEfH2b72EAvCprFkZZWsmuwDifWMMY0\nYBZQpnpOnHCLPwYCK7Bt2eJGV/gdbt2djVE+vjzu4+tCH5n4OJHow3dZwtlW1sCBEB3t4e9ijIkI\nFlCmds6cKdnS8m+6YQMSuM4LONikAxlFLrA2N/WhvX0kXOajz1VJXHKpcN55Hv4OxpiwZAFl6kZR\nkesuDAqt06szkcxMmuUUjzQ8QUvW05uslj7OXOCj9cU+uo/x0fOaC2ga09TDX8AY4zULKFO/VOHA\nAcjMJH9NJgc/zuTM6kxa7sokIW/32cPOEM2e2B6c7Oqj+UAfnUb6aDXE5+ZosrHuxjQKFlAmbOix\n4+xbsp4d767j5LJMmm3JpPOxTM5nK1EUAVCEcDKhO1xwIbG+7jS9oLsb697d/zMp6ey1XcaYyGYB\nZcLaqVOw4pM8Nr6ziUMfu67CbqcySWEb3dlBJ/aXOF79U0NJcGh1717yduvWHv02xphzYQFlIoqq\nGwG/ejWsXQsbVudxePUuirbtILFoJ93ZQbLspHfsDrrLTjrk7iSqML/ki7RpUza0goPsvPNsjqe6\noOr+4jh5sng7ccL9bNHCLWTW1M47mmIWUKZBOH0aNm50oRW8bd9axHnsoxs76RG9g8Edd9K35Q5S\nmuykY94OWh7aSZPjR0u+WLNmbqXH8lpf3bq5x2o5w3zYKypyy8OUFyZVbRUdl5NTch7J0jp0gFtv\nhdtucxfQ2RQljZ4FlGnQcnLcIMLSwbW7eDwGXVoeZ+T5OxjSaSf9Wu8gJWon553eQcyBnciOHW6G\njdL//jt1Ktv66tbNbc2buy/4oiJ3jVjgdvBW0f66fKywsGzoVBQop05V/0OOioJWraBly+Kt9P2q\ntn374H/+BxYudLP5d+8Okye7sOrXLzT/GEzECWlAicgY4FkgCnhFVZ+q4LgJwHzgIlWtNH0soExd\nOHoUMjJKhtbXX7tJcwMSEtxaWQN8ZxiSuJv+bXZwftOdxB3cATt3umH0O/y3g5dICWfNm59bcFQn\nbJo1C11r58QJN8X+66/De++5UO3b1wXV5MluzkjTaIQsoEQkCtgIfAPIApYBk1V1XanjWgH/ApoB\n91hAmXARGAFfOrjWrnXfmwFdurjgSk11352pfZW+HbOJO7TTTQmVn+/OYTVp4loXgdult5o+VtPn\nRkVF1jme7Gz4+99dWC1d6vYNHerC6tZbsau7G75QBtQlwGOqerX//iMAqvqrUsfNBP4D/BB42ALK\nhDtVt9Bj6dBat65kwyklxQVWcrKbRLdLF7cFbrdrZ6dVamzHDpg714XV6tUucEeNcq2qm26qdN0z\nE7lCGVA3A2NU9U7//W8CF6vqPUHHpAE/VtUJIrKECgJKRKYB0wC6des2eMeOHefwKxlTPwoL3by6\nwaGVkeHC7OjRssc3a1YyuILDK/h2fLwFWaXWrXNBNWeOG9LZvDmMHevCauxYNyLQNAj1FlAi0gT4\nAJiqqtsrC6hg1oIykSg3F/budeMrAlvw/cDtY8fKPrd584rDK/h227aNPMhU4csvXVjNnQv797vz\nZTfd5MJq1KjI6tI0ZdRbF5+ItAG2ACf9T+kEHAbGVRZSFlCmITt1qnpBdvx42efGxFQeZIH7bdo0\ngiArLITFi11YLVjgkr9jR7jlFhu2HsFCGVBNcYMkRgG7cYMkblPVjAqOX4K1oIyplpyc6gVZ8GCO\ngBYtXFglJkLPnuDzFW/dujXAa5JPn4Z333VdgP/4hztRmJwMkybZsPUIE+ph5tcCM3HDzGep6pMi\n8jiwXFUXljp2CRZQxoTUyZPlB9eePW6A4fr1cPBg8fEtWrj5d4NDy+eDHj0ayJSGx4/D22+7sPrP\nf1xLKzXVdQHasPWwZxfqGtPIHDxY/tqTwWORoqLgggtcWPXuXRxcvXtH8FSGgWHrc+bAJ5+4fTZs\nPaxZQBljANeNuGFD2fDatMld2hWQmFi2xeXzuVM+EXOaJzBsfc4cWLOmeNj6bbfB+PHeDFsvPb1U\n6Rk+ypvxo3NnSEtzW8eO9V9zHbOAMsZUKj/fjeYODq1AiJ08WXxcfHzJ1lZg697dtcjCVkaGG1zx\n+uslh63fdhtce235w9YDE99WFSKV3S/9WE5O9WuOjoa4uJLXMyQmwuDBxYGVluZOPkbMXw1lWUAZ\nY2pE1c1pWLqrMDPTzcgREBNT9jxX795uwEbz5t7VX0Zg2PqcOW5ewMCw9QEDyrZsqpr4NljwXIVV\nzVkYfL+yxwInCI8dg1WrYMUKWLnSbevXF9d23nnFYRUIr27dIia0LKCMMSF3+HD557m2by/+7mzS\nBM4/3wVWcnLJVlbp78/g+3XxWOn7UVpAyo4l9F/3Oued2EKz9q1oeV5LYju2RFqdQ5C0bOlSuD4D\n4eRJN9tGILBWrHAXNxcWusfbtSsZWGlp7j9EGA7ntIAyxtSbU6fcsijB3YSZmW6EYeArpvRXTfB9\nLx4LfK+Dy5s+fdyUVsFbUlKYN0pyc91syIHAWrnS3Q+cXGzTBgYNKtna6tHD875ZCyhjjKnE4cPu\nNFXwtm6d6wEMaN26/OAK61NAZ864+bkCLa2VK13LKzDBZFwcDBxYsrXl89Xr7BwWUMYYUwMHD7qg\nKh1e2dnFx7RtW35wdeoUpsGVn++atsEtrVWrigdwxMS4c3LBAzFSU+vsojkLKGOMCaHs7LKhlZFR\ncq2x+PiyodW3b5gO1S8sdNcaBA/EWLmyeP6t6Gg3O0dw92C/fiGZtNcCyhhj6ljwWmOltyNHio9L\nSCg/uDp08K72chUVuSH5wQMxVq50/aHgzl29+CLceWet3sYCyhhjPKLqVrsvL7iCZ7rv0KH84EpI\n8K72MlTd6tKBwLrpJteiqgULKGOMCTOqbv7E8oIreELgVq3c+azOnd0WfDv4fkJCGHYdVkN1A8oW\nVTHGmHoi4iaGSEyEq64q3h9Y3TkQVrt2uQmB9+51DZe9e0vO7hEQHe2u2S0vvIJvn3deZE4SbAFl\njDEeE4GuXd02Zkz5xwRmtN+3rzi8gm9v3w6ffVZytGGwhISqg6xzZ9d6CxcWUMYYEwFatnTX2Pbo\nUflx+flu4EYguMoLtQ0b3M8zZ8o+Py6u4vDq1MmNRu/cuW5+x9IsoIwxpgGJji7uRqyMqhtpWF6I\nBe6vWQOLFpVc+fmPf4Rp0+r2dwiwgDLGmEZIxE3f166dGzlYmVOnigOsPteCtIAyxhhTqdhYN+/s\n+efX7/uG3zS3xhhjDBZQxhhjwpRnF+qKSDawIwQv1R44GILXaUzsM6sZ+9zOnX1mNdPQP7fuqlrl\nRE+eBVSoiMjy6lyRbIrZZ1Yz9rmdO/vMasY+N8e6+IwxxoQlCyhjjDFhqSEE1EteFxCB7DOrGfvc\nzp19ZjVjnxsN4ByUMcaYhqkhtKCMMcY0QBZQxhhjwlLEBpSIjBGRDSKyWUSme11PJBCRriKyWETW\niUiGiNzvdU2RQkSiROQrEfmn17VEChFpKyLzRWS9iGSKyCVe1xTuRORB//+ba0XkdRGJ8bomL0Vk\nQIlIFPAccA3QB5gsIn28rSoiFAAPqWofYChwt31u1XY/kOl1ERHmWeDfqtobGIB9fpUSkUTgPiBd\nVVOBKGCSt1V5KyIDChgCbFbVrap6BpgL3OBxTWFPVfeq6kr/7RO4L4wqJuU3IpIEjAVe8bqWSCEi\nbYArgD8BqOoZVT3qbVURoSnQQkSaArHAHo/r8VSkBlQisCvofhb2RXtORCQZGAR84W0lEWEm8N9A\nkdeFRJAUIBt41d81+oqIxHldVDhT1d3A08BOYC9wTFXf87Yqb0VqQJlaEJGWwALgAVU9XtXxjZmI\nXAccUNUVXtcSYZoCacALqjoIyAHsXHElRCQe1xOUAnQB4kTkdm+r8lakBtRuoGvQ/ST/PlMFEYnG\nhdNsVX3D63oiwDBgnIhsx3UlXykir3lbUkTIArJUNdBCn48LLFOx0cA2Vc1W1XzgDeBSj2vyVKQG\n1DKgh4ikiEgz3InEhR7XFPZERHDnBDJVdYbX9UQCVX1EVZNUNRn37+wDVW3Uf9VWh6ruA3aJSC//\nrlHAOg9LigQ7gaEiEuv/f3UUjXxgSUSuqKuqBSJyD7AIN9JllqpmeFxWJBgGfBP4WkRW+fc9qqrv\neFiTabjuBWb7/4jcCtzhcT1hTVW/EJH5wErciNuvaORTHtlUR8YYY8JSpHbxGWOMaeAsoIwxxoQl\nCyhjjDFhyQLKGGNMWLKAMsYYE5YsoIwxxoQlCyhjjDFh6f8DXvu6D4vSEjkAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fced4824590>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.subplot(211)\n",
    "plt.title(\"Accuracy\")\n",
    "plt.plot(history.history[\"acc\"], color=\"blue\", label=\"train\")\n",
    "plt.plot(history.history[\"val_acc\"], color=\"red\", label=\"validation\")\n",
    "plt.legend(loc=\"best\")\n",
    "\n",
    "plt.subplot(212)\n",
    "plt.title(\"Loss\")\n",
    "plt.plot(history.history[\"loss\"], color=\"blue\", label=\"train\")\n",
    "plt.plot(history.history[\"val_loss\"], color=\"red\", label=\"validation\")\n",
    "plt.legend(loc=\"best\")\n",
    "\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.save(FINAL_MODEL_FILE, overwrite=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "==== Evaluation Results: final model on test set ====\n",
      "Accuracy Score: 0.881\n",
      "Confusion Matrix\n",
      "[[380  60]\n",
      " [ 39 353]]\n",
      "==== Evaluation Results: best model on test set ====\n",
      "Accuracy Score: 0.880\n",
      "Confusion Matrix\n",
      "[[372  62]\n",
      " [ 38 360]]\n"
     ]
    }
   ],
   "source": [
    "def evaluate_model(model):\n",
    "    ytest, ytest_ = [], []\n",
    "    test_pair_gen = pair_generator(triples_data_test, image_cache, None, BATCH_SIZE)\n",
    "    num_test_steps = len(triples_data_test) // BATCH_SIZE\n",
    "    curr_test_steps = 0\n",
    "    for [X1test, X2test], Ytest in test_pair_gen:\n",
    "        if curr_test_steps > num_test_steps:\n",
    "            break\n",
    "        Ytest_ = model.predict([X1test, X2test])\n",
    "        ytest.extend(np.argmax(Ytest, axis=1).tolist())\n",
    "        ytest_.extend(np.argmax(Ytest_, axis=1).tolist())\n",
    "        curr_test_steps += 1\n",
    "    acc = accuracy_score(ytest, ytest_)\n",
    "    cm = confusion_matrix(ytest, ytest_)\n",
    "    return acc, cm\n",
    "\n",
    "print(\"==== Evaluation Results: final model on test set ====\")\n",
    "final_model = load_model(FINAL_MODEL_FILE)\n",
    "acc, cm = evaluate_model(final_model)\n",
    "print(\"Accuracy Score: {:.3f}\".format(acc))\n",
    "print(\"Confusion Matrix\")\n",
    "print(cm)\n",
    "\n",
    "print(\"==== Evaluation Results: best model on test set ====\")\n",
    "best_model = load_model(BEST_MODEL_FILE)\n",
    "acc, cm = evaluate_model(best_model)\n",
    "print(\"Accuracy Score: {:.3f}\".format(acc))\n",
    "print(\"Confusion Matrix\")\n",
    "print(cm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}