{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "___\n",
    "\n",
    "<a href='http://www.pieriandata.com'> <img src='../Pierian_Data_Logo.png' /></a>\n",
    "___\n",
    "# Question and Answer Chat Bots"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----\n",
    "\n",
    "------"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading the Data\n",
    "\n",
    "We will be working with the Babi Data Set from Facebook Research.\n",
    "\n",
    "Full Details: https://research.fb.com/downloads/babi/\n",
    "\n",
    "- Jason Weston, Antoine Bordes, Sumit Chopra, Tomas Mikolov, Alexander M. Rush,\n",
    "  \"Towards AI-Complete Question Answering: A Set of Prerequisite Toy Tasks\",\n",
    "  http://arxiv.org/abs/1502.05698\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pickle\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(\"train_qa.txt\", \"rb\") as fp:   # Unpickling\n",
    "    train_data =  pickle.load(fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "with open(\"test_qa.txt\", \"rb\") as fp:   # Unpickling\n",
    "    test_data =  pickle.load(fp)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "----"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exploring the Format of the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "list"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(['Mary',\n",
       "  'moved',\n",
       "  'to',\n",
       "  'the',\n",
       "  'bathroom',\n",
       "  '.',\n",
       "  'Sandra',\n",
       "  'journeyed',\n",
       "  'to',\n",
       "  'the',\n",
       "  'bedroom',\n",
       "  '.'],\n",
       " ['Is', 'Sandra', 'in', 'the', 'hallway', '?'],\n",
       " 'no')"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Mary moved to the bathroom . Sandra journeyed to the bedroom .'"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "' '.join(train_data[0][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Is Sandra in the hallway ?'"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "' '.join(train_data[0][1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'no'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data[0][2]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----\n",
    "\n",
    "## Setting up Vocabulary of All Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Create a set that holds the vocab words\n",
    "vocab = set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "all_data = test_data + train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for story, question , answer in all_data:\n",
    "    # In case you don't know what a union of sets is:\n",
    "    # https://www.programiz.com/python-programming/methods/set/union\n",
    "    vocab = vocab.union(set(story))\n",
    "    vocab = vocab.union(set(question))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab.add('no')\n",
    "vocab.add('yes')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'.',\n",
       " '?',\n",
       " 'Daniel',\n",
       " 'Is',\n",
       " 'John',\n",
       " 'Mary',\n",
       " 'Sandra',\n",
       " 'apple',\n",
       " 'back',\n",
       " 'bathroom',\n",
       " 'bedroom',\n",
       " 'discarded',\n",
       " 'down',\n",
       " 'dropped',\n",
       " 'football',\n",
       " 'garden',\n",
       " 'got',\n",
       " 'grabbed',\n",
       " 'hallway',\n",
       " 'in',\n",
       " 'journeyed',\n",
       " 'kitchen',\n",
       " 'left',\n",
       " 'milk',\n",
       " 'moved',\n",
       " 'no',\n",
       " 'office',\n",
       " 'picked',\n",
       " 'put',\n",
       " 'the',\n",
       " 'there',\n",
       " 'to',\n",
       " 'took',\n",
       " 'travelled',\n",
       " 'up',\n",
       " 'went',\n",
       " 'yes'}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "vocab_len = len(vocab) + 1 #we add an extra space to hold a 0 for Keras's pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "max_story_len = max([len(data[0]) for data in all_data])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "156"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_story_len"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "max_question_len = max([len(data[1]) for data in all_data])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "max_question_len"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Vectorizing the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'.',\n",
       " '?',\n",
       " 'Daniel',\n",
       " 'Is',\n",
       " 'John',\n",
       " 'Mary',\n",
       " 'Sandra',\n",
       " 'apple',\n",
       " 'back',\n",
       " 'bathroom',\n",
       " 'bedroom',\n",
       " 'discarded',\n",
       " 'down',\n",
       " 'dropped',\n",
       " 'football',\n",
       " 'garden',\n",
       " 'got',\n",
       " 'grabbed',\n",
       " 'hallway',\n",
       " 'in',\n",
       " 'journeyed',\n",
       " 'kitchen',\n",
       " 'left',\n",
       " 'milk',\n",
       " 'moved',\n",
       " 'no',\n",
       " 'office',\n",
       " 'picked',\n",
       " 'put',\n",
       " 'the',\n",
       " 'there',\n",
       " 'to',\n",
       " 'took',\n",
       " 'travelled',\n",
       " 'up',\n",
       " 'went',\n",
       " 'yes'}"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Reserve 0 for pad_sequences\n",
    "vocab_size = len(vocab) + 1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "-----------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.preprocessing.sequence import pad_sequences\n",
    "from keras.preprocessing.text import Tokenizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# integer encode sequences of words\n",
    "tokenizer = Tokenizer(filters=[])\n",
    "tokenizer.fit_on_texts(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'.': 13,\n",
       " '?': 12,\n",
       " 'apple': 3,\n",
       " 'back': 18,\n",
       " 'bathroom': 23,\n",
       " 'bedroom': 29,\n",
       " 'daniel': 21,\n",
       " 'discarded': 5,\n",
       " 'down': 22,\n",
       " 'dropped': 4,\n",
       " 'football': 37,\n",
       " 'garden': 36,\n",
       " 'got': 34,\n",
       " 'grabbed': 15,\n",
       " 'hallway': 27,\n",
       " 'in': 8,\n",
       " 'is': 30,\n",
       " 'john': 10,\n",
       " 'journeyed': 35,\n",
       " 'kitchen': 11,\n",
       " 'left': 31,\n",
       " 'mary': 25,\n",
       " 'milk': 7,\n",
       " 'moved': 28,\n",
       " 'no': 6,\n",
       " 'office': 24,\n",
       " 'picked': 1,\n",
       " 'put': 33,\n",
       " 'sandra': 2,\n",
       " 'the': 32,\n",
       " 'there': 16,\n",
       " 'to': 14,\n",
       " 'took': 9,\n",
       " 'travelled': 20,\n",
       " 'up': 17,\n",
       " 'went': 26,\n",
       " 'yes': 19}"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_story_text = []\n",
    "train_question_text = []\n",
    "train_answers = []\n",
    "\n",
    "for story,question,answer in train_data:\n",
    "    train_story_text.append(story)\n",
    "    train_question_text.append(question)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_story_seq = tokenizer.texts_to_sequences(train_story_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_story_text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "10000"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_story_seq)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# word_index = tokenizer.word_index"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Functionalize Vectorization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def vectorize_stories(data, word_index=tokenizer.word_index, max_story_len=max_story_len,max_question_len=max_question_len):\n",
    "    '''\n",
    "    INPUT: \n",
    "    \n",
    "    data: consisting of Stories,Queries,and Answers\n",
    "    word_index: word index dictionary from tokenizer\n",
    "    max_story_len: the length of the longest story (used for pad_sequences function)\n",
    "    max_question_len: length of the longest question (used for pad_sequences function)\n",
    "\n",
    "\n",
    "    OUTPUT:\n",
    "    \n",
    "    Vectorizes the stories,questions, and answers into padded sequences. We first loop for every story, query , and\n",
    "    answer in the data. Then we convert the raw words to an word index value. Then we append each set to their appropriate\n",
    "    output list. Then once we have converted the words to numbers, we pad the sequences so they are all of equal length.\n",
    "    \n",
    "    Returns this in the form of a tuple (X,Xq,Y) (padded based on max lengths)\n",
    "    '''\n",
    "    \n",
    "    \n",
    "    # X = STORIES\n",
    "    X = []\n",
    "    # Xq = QUERY/QUESTION\n",
    "    Xq = []\n",
    "    # Y = CORRECT ANSWER\n",
    "    Y = []\n",
    "    \n",
    "    \n",
    "    for story, query, answer in data:\n",
    "        \n",
    "        # Grab the word index for every word in story\n",
    "        x = [word_index[word.lower()] for word in story]\n",
    "        # Grab the word index for every word in query\n",
    "        xq = [word_index[word.lower()] for word in query]\n",
    "        \n",
    "        # Grab the Answers (either Yes/No so we don't need to use list comprehension here)\n",
    "        # Index 0 is reserved so we're going to use + 1\n",
    "        y = np.zeros(len(word_index) + 1)\n",
    "        \n",
    "        # Now that y is all zeros and we know its just Yes/No , we can use numpy logic to create this assignment\n",
    "        #\n",
    "        y[word_index[answer]] = 1\n",
    "        \n",
    "        # Append each set of story,query, and answer to their respective holding lists\n",
    "        X.append(x)\n",
    "        Xq.append(xq)\n",
    "        Y.append(y)\n",
    "        \n",
    "    # Finally, pad the sequences based on their max length so the RNN can be trained on uniformly long sequences.\n",
    "        \n",
    "    # RETURN TUPLE FOR UNPACKING\n",
    "    return (pad_sequences(X, maxlen=max_story_len),pad_sequences(Xq, maxlen=max_question_len), np.array(Y))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "inputs_train, queries_train, answers_train = vectorize_stories(train_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "inputs_test, queries_test, answers_test = vectorize_stories(test_data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0,  0,  0, ..., 32, 29, 13],\n",
       "       [ 0,  0,  0, ..., 32, 36, 13],\n",
       "       [ 0,  0,  0, ..., 32, 36, 13],\n",
       "       ...,\n",
       "       [ 0,  0,  0, ..., 32,  3, 13],\n",
       "       [ 0,  0,  0, ..., 32, 36, 13],\n",
       "       [ 0,  0,  0, ...,  3, 16, 13]])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inputs_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[30, 10,  8, 32, 11, 12],\n",
       "       [30, 10,  8, 32, 11, 12],\n",
       "       [30, 10,  8, 32, 36, 12],\n",
       "       ...,\n",
       "       [30, 25,  8, 32, 29, 12],\n",
       "       [30,  2,  8, 32, 36, 12],\n",
       "       [30, 25,  8, 32, 36, 12]])"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "queries_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       ...,\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.],\n",
       "       [0., 0., 0., ..., 0., 0., 0.]])"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answers_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  0.,   0.,   0.,   0.,   0.,   0., 503.,   0.,   0.,   0.,   0.,\n",
       "         0.,   0.,   0.,   0.,   0.,   0.,   0.,   0., 497.,   0.,   0.,\n",
       "         0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,   0.,\n",
       "         0.,   0.,   0.,   0.,   0.])"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sum(answers_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "19"
      ]
     },
     "execution_count": 39,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.word_index['yes']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenizer.word_index['no']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Creating the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from keras.models import Sequential, Model\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.layers import Input, Activation, Dense, Permute, Dropout\n",
    "from keras.layers import add, dot, concatenate\n",
    "from keras.layers import LSTM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Placeholders for Inputs\n",
    "\n",
    "Recall we technically have two inputs, stories and questions. So we need to use placeholders. `Input()` is used to instantiate a Keras tensor.\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "input_sequence = Input((max_story_len,))\n",
    "question = Input((max_question_len,))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### Building the Networks\n",
    "\n",
    "To understand why we chose this setup, make sure to read the paper we are using:\n",
    "\n",
    "* Sainbayar Sukhbaatar, Arthur Szlam, Jason Weston, Rob Fergus,\n",
    "  \"End-To-End Memory Networks\",\n",
    "  http://arxiv.org/abs/1503.08895"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Encoders\n",
    "\n",
    "### Input Encoder m"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Input gets embedded to a sequence of vectors\n",
    "input_encoder_m = Sequential()\n",
    "input_encoder_m.add(Embedding(input_dim=vocab_size,output_dim=64))\n",
    "input_encoder_m.add(Dropout(0.3))\n",
    "\n",
    "# This encoder will output:\n",
    "# (samples, story_maxlen, embedding_dim)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Input Encoder c"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# embed the input into a sequence of vectors of size query_maxlen\n",
    "input_encoder_c = Sequential()\n",
    "input_encoder_c.add(Embedding(input_dim=vocab_size,output_dim=max_question_len))\n",
    "input_encoder_c.add(Dropout(0.3))\n",
    "# output: (samples, story_maxlen, query_maxlen)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Question Encoder"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# embed the question into a sequence of vectors\n",
    "question_encoder = Sequential()\n",
    "question_encoder.add(Embedding(input_dim=vocab_size,\n",
    "                               output_dim=64,\n",
    "                               input_length=max_question_len))\n",
    "question_encoder.add(Dropout(0.3))\n",
    "# output: (samples, query_maxlen, embedding_dim)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Encode the Sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# encode input sequence and questions (which are indices)\n",
    "# to sequences of dense vectors\n",
    "input_encoded_m = input_encoder_m(input_sequence)\n",
    "input_encoded_c = input_encoder_c(input_sequence)\n",
    "question_encoded = question_encoder(question)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### Use dot product to compute the match between first input vector seq and the query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# shape: `(samples, story_maxlen, query_maxlen)`\n",
    "match = dot([input_encoded_m, question_encoded], axes=(2, 2))\n",
    "match = Activation('softmax')(match)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Add this match matrix with the second input vector sequence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# add the match matrix with the second input vector sequence\n",
    "response = add([match, input_encoded_c])  # (samples, story_maxlen, query_maxlen)\n",
    "response = Permute((2, 1))(response)  # (samples, query_maxlen, story_maxlen)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Concatenate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# concatenate the match matrix with the question vector sequence\n",
    "answer = concatenate([response, question_encoded])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<tf.Tensor 'concatenate_1/concat:0' shape=(?, 6, 220) dtype=float32>"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "answer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Reduce with RNN (LSTM)\n",
    "answer = LSTM(32)(answer)  # (samples, 32)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Regularization with Dropout\n",
    "answer = Dropout(0.5)(answer)\n",
    "answer = Dense(vocab_size)(answer)  # (samples, vocab_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# we output a probability distribution over the vocabulary\n",
    "answer = Activation('softmax')(answer)\n",
    "\n",
    "# build the final model\n",
    "model = Model([input_sequence, question], answer)\n",
    "model.compile(optimizer='rmsprop', loss='categorical_crossentropy',\n",
    "              metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "__________________________________________________________________________________________________\n",
      "Layer (type)                    Output Shape         Param #     Connected to                     \n",
      "==================================================================================================\n",
      "input_1 (InputLayer)            (None, 156)          0                                            \n",
      "__________________________________________________________________________________________________\n",
      "input_2 (InputLayer)            (None, 6)            0                                            \n",
      "__________________________________________________________________________________________________\n",
      "sequential_1 (Sequential)       multiple             2432        input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "sequential_3 (Sequential)       (None, 6, 64)        2432        input_2[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "dot_1 (Dot)                     (None, 156, 6)       0           sequential_1[1][0]               \n",
      "                                                                 sequential_3[1][0]               \n",
      "__________________________________________________________________________________________________\n",
      "activation_1 (Activation)       (None, 156, 6)       0           dot_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "sequential_2 (Sequential)       multiple             228         input_1[0][0]                    \n",
      "__________________________________________________________________________________________________\n",
      "add_1 (Add)                     (None, 156, 6)       0           activation_1[0][0]               \n",
      "                                                                 sequential_2[1][0]               \n",
      "__________________________________________________________________________________________________\n",
      "permute_1 (Permute)             (None, 6, 156)       0           add_1[0][0]                      \n",
      "__________________________________________________________________________________________________\n",
      "concatenate_1 (Concatenate)     (None, 6, 220)       0           permute_1[0][0]                  \n",
      "                                                                 sequential_3[1][0]               \n",
      "__________________________________________________________________________________________________\n",
      "lstm_1 (LSTM)                   (None, 32)           32384       concatenate_1[0][0]              \n",
      "__________________________________________________________________________________________________\n",
      "dropout_4 (Dropout)             (None, 32)           0           lstm_1[0][0]                     \n",
      "__________________________________________________________________________________________________\n",
      "dense_1 (Dense)                 (None, 38)           1254        dropout_4[0][0]                  \n",
      "__________________________________________________________________________________________________\n",
      "activation_2 (Activation)       (None, 38)           0           dense_1[0][0]                    \n",
      "==================================================================================================\n",
      "Total params: 38,730\n",
      "Trainable params: 38,730\n",
      "Non-trainable params: 0\n",
      "__________________________________________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 10000 samples, validate on 1000 samples\n",
      "Epoch 1/120\n",
      "10000/10000 [==============================] - 7s 701us/step - loss: 0.8846 - acc: 0.4966 - val_loss: 0.6938 - val_acc: 0.4970\n",
      "Epoch 2/120\n",
      "10000/10000 [==============================] - 4s 364us/step - loss: 0.7022 - acc: 0.4987 - val_loss: 0.6935 - val_acc: 0.5030\n",
      "Epoch 3/120\n",
      "10000/10000 [==============================] - 3s 347us/step - loss: 0.6958 - acc: 0.5042 - val_loss: 0.6937 - val_acc: 0.4970\n",
      "Epoch 4/120\n",
      "10000/10000 [==============================] - 3s 348us/step - loss: 0.6946 - acc: 0.5097 - val_loss: 0.6977 - val_acc: 0.4970\n",
      "Epoch 5/120\n",
      "10000/10000 [==============================] - 3s 345us/step - loss: 0.6943 - acc: 0.5073 - val_loss: 0.6932 - val_acc: 0.5030\n",
      "Epoch 6/120\n",
      "10000/10000 [==============================] - 3s 347us/step - loss: 0.6954 - acc: 0.4873 - val_loss: 0.6938 - val_acc: 0.4970\n",
      "Epoch 7/120\n",
      "10000/10000 [==============================] - 3s 345us/step - loss: 0.6946 - acc: 0.4970 - val_loss: 0.6953 - val_acc: 0.4970\n",
      "Epoch 8/120\n",
      "10000/10000 [==============================] - 3s 347us/step - loss: 0.6948 - acc: 0.4955 - val_loss: 0.6939 - val_acc: 0.4970\n",
      "Epoch 9/120\n",
      "10000/10000 [==============================] - 3s 346us/step - loss: 0.6944 - acc: 0.4937 - val_loss: 0.6933 - val_acc: 0.5030\n",
      "Epoch 10/120\n",
      "10000/10000 [==============================] - 4s 360us/step - loss: 0.6939 - acc: 0.5011 - val_loss: 0.6937 - val_acc: 0.4970\n",
      "Epoch 11/120\n",
      "10000/10000 [==============================] - 4s 365us/step - loss: 0.6941 - acc: 0.5051 - val_loss: 0.6934 - val_acc: 0.5030\n",
      "Epoch 12/120\n",
      "10000/10000 [==============================] - 4s 352us/step - loss: 0.6941 - acc: 0.5014 - val_loss: 0.6955 - val_acc: 0.4970\n",
      "Epoch 13/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.6936 - acc: 0.5104 - val_loss: 0.6943 - val_acc: 0.4940\n",
      "Epoch 14/120\n",
      "10000/10000 [==============================] - 3s 350us/step - loss: 0.6938 - acc: 0.5045 - val_loss: 0.6938 - val_acc: 0.4950\n",
      "Epoch 15/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.6914 - acc: 0.5216 - val_loss: 0.6944 - val_acc: 0.5030\n",
      "Epoch 16/120\n",
      "10000/10000 [==============================] - 4s 351us/step - loss: 0.6830 - acc: 0.5467 - val_loss: 0.6825 - val_acc: 0.5250\n",
      "Epoch 17/120\n",
      "10000/10000 [==============================] - 4s 356us/step - loss: 0.6663 - acc: 0.5840 - val_loss: 0.6656 - val_acc: 0.6020\n",
      "Epoch 18/120\n",
      "10000/10000 [==============================] - 4s 368us/step - loss: 0.6404 - acc: 0.6339 - val_loss: 0.6247 - val_acc: 0.6690\n",
      "Epoch 19/120\n",
      "10000/10000 [==============================] - 4s 364us/step - loss: 0.6049 - acc: 0.6829 - val_loss: 0.5708 - val_acc: 0.7210\n",
      "Epoch 20/120\n",
      "10000/10000 [==============================] - 4s 356us/step - loss: 0.5569 - acc: 0.7290 - val_loss: 0.5159 - val_acc: 0.7460\n",
      "Epoch 21/120\n",
      "10000/10000 [==============================] - 4s 358us/step - loss: 0.5180 - acc: 0.7549 - val_loss: 0.4775 - val_acc: 0.7870\n",
      "Epoch 22/120\n",
      "10000/10000 [==============================] - 4s 354us/step - loss: 0.4891 - acc: 0.7774 - val_loss: 0.4449 - val_acc: 0.7970\n",
      "Epoch 23/120\n",
      "10000/10000 [==============================] - 4s 354us/step - loss: 0.4528 - acc: 0.8020 - val_loss: 0.4142 - val_acc: 0.8190\n",
      "Epoch 24/120\n",
      "10000/10000 [==============================] - 4s 354us/step - loss: 0.4253 - acc: 0.8161 - val_loss: 0.4205 - val_acc: 0.8280\n",
      "Epoch 25/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.4009 - acc: 0.8354 - val_loss: 0.4094 - val_acc: 0.8280\n",
      "Epoch 26/120\n",
      "10000/10000 [==============================] - 3s 348us/step - loss: 0.3815 - acc: 0.8432 - val_loss: 0.3919 - val_acc: 0.8240\n",
      "Epoch 27/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.3653 - acc: 0.8496 - val_loss: 0.3926 - val_acc: 0.8450\n",
      "Epoch 28/120\n",
      "10000/10000 [==============================] - 4s 351us/step - loss: 0.3535 - acc: 0.8549 - val_loss: 0.3939 - val_acc: 0.8430\n",
      "Epoch 29/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.3435 - acc: 0.8581 - val_loss: 0.3716 - val_acc: 0.8320\n",
      "Epoch 30/120\n",
      "10000/10000 [==============================] - 4s 350us/step - loss: 0.3403 - acc: 0.8603 - val_loss: 0.3677 - val_acc: 0.8340\n",
      "Epoch 31/120\n",
      "10000/10000 [==============================] - 4s 351us/step - loss: 0.3302 - acc: 0.8570 - val_loss: 0.3681 - val_acc: 0.8430\n",
      "Epoch 32/120\n",
      "10000/10000 [==============================] - 3s 348us/step - loss: 0.3295 - acc: 0.8593 - val_loss: 0.3476 - val_acc: 0.8380\n",
      "Epoch 33/120\n",
      "10000/10000 [==============================] - 3s 347us/step - loss: 0.3239 - acc: 0.8628 - val_loss: 0.3521 - val_acc: 0.8430\n",
      "Epoch 34/120\n",
      "10000/10000 [==============================] - 4s 350us/step - loss: 0.3171 - acc: 0.8677 - val_loss: 0.3443 - val_acc: 0.8390\n",
      "Epoch 35/120\n",
      "10000/10000 [==============================] - 3s 347us/step - loss: 0.3168 - acc: 0.8629 - val_loss: 0.3507 - val_acc: 0.8340\n",
      "Epoch 36/120\n",
      "10000/10000 [==============================] - 4s 351us/step - loss: 0.3121 - acc: 0.8664 - val_loss: 0.3558 - val_acc: 0.8310\n",
      "Epoch 37/120\n",
      "10000/10000 [==============================] - 4s 351us/step - loss: 0.3107 - acc: 0.8662 - val_loss: 0.3411 - val_acc: 0.8430\n",
      "Epoch 38/120\n",
      "10000/10000 [==============================] - 4s 355us/step - loss: 0.3061 - acc: 0.8698 - val_loss: 0.3460 - val_acc: 0.8400\n",
      "Epoch 39/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.3065 - acc: 0.8671 - val_loss: 0.3493 - val_acc: 0.8400\n",
      "Epoch 40/120\n",
      "10000/10000 [==============================] - 4s 352us/step - loss: 0.3060 - acc: 0.8688 - val_loss: 0.3446 - val_acc: 0.8410\n",
      "Epoch 41/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.3000 - acc: 0.8696 - val_loss: 0.3542 - val_acc: 0.8450\n",
      "Epoch 42/120\n",
      "10000/10000 [==============================] - 4s 354us/step - loss: 0.3039 - acc: 0.8665 - val_loss: 0.3692 - val_acc: 0.8350\n",
      "Epoch 43/120\n",
      "10000/10000 [==============================] - 3s 348us/step - loss: 0.3015 - acc: 0.8695 - val_loss: 0.3513 - val_acc: 0.8400\n",
      "Epoch 44/120\n",
      "10000/10000 [==============================] - 3s 347us/step - loss: 0.2986 - acc: 0.8694 - val_loss: 0.3577 - val_acc: 0.8320\n",
      "Epoch 45/120\n",
      "10000/10000 [==============================] - 3s 346us/step - loss: 0.2952 - acc: 0.8730 - val_loss: 0.3496 - val_acc: 0.8400\n",
      "Epoch 46/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.2969 - acc: 0.8681 - val_loss: 0.3424 - val_acc: 0.8450\n",
      "Epoch 47/120\n",
      "10000/10000 [==============================] - 3s 347us/step - loss: 0.2923 - acc: 0.8721 - val_loss: 0.3549 - val_acc: 0.8280\n",
      "Epoch 48/120\n",
      "10000/10000 [==============================] - 3s 346us/step - loss: 0.2911 - acc: 0.8732 - val_loss: 0.4681 - val_acc: 0.8140\n",
      "Epoch 49/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.2917 - acc: 0.8703 - val_loss: 0.3502 - val_acc: 0.8390\n",
      "Epoch 50/120\n",
      "10000/10000 [==============================] - 3s 344us/step - loss: 0.2900 - acc: 0.8746 - val_loss: 0.3515 - val_acc: 0.8400\n",
      "Epoch 51/120\n",
      "10000/10000 [==============================] - 4s 352us/step - loss: 0.2855 - acc: 0.8757 - val_loss: 0.3499 - val_acc: 0.8360\n",
      "Epoch 52/120\n",
      "10000/10000 [==============================] - 4s 354us/step - loss: 0.2864 - acc: 0.8735 - val_loss: 0.3531 - val_acc: 0.8410\n",
      "Epoch 53/120\n",
      "10000/10000 [==============================] - 4s 356us/step - loss: 0.2864 - acc: 0.8772 - val_loss: 0.3905 - val_acc: 0.8270\n",
      "Epoch 54/120\n",
      "10000/10000 [==============================] - 3s 346us/step - loss: 0.2857 - acc: 0.8752 - val_loss: 0.3618 - val_acc: 0.8390\n",
      "Epoch 55/120\n",
      "10000/10000 [==============================] - 3s 346us/step - loss: 0.2819 - acc: 0.8742 - val_loss: 0.3501 - val_acc: 0.8380\n",
      "Epoch 56/120\n",
      "10000/10000 [==============================] - 3s 348us/step - loss: 0.2853 - acc: 0.8775 - val_loss: 0.3484 - val_acc: 0.8400\n",
      "Epoch 57/120\n",
      "10000/10000 [==============================] - 4s 355us/step - loss: 0.2767 - acc: 0.8804 - val_loss: 0.3463 - val_acc: 0.8410\n",
      "Epoch 58/120\n",
      "10000/10000 [==============================] - 4s 355us/step - loss: 0.2802 - acc: 0.8780 - val_loss: 0.3763 - val_acc: 0.8350\n",
      "Epoch 59/120\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 3s 350us/step - loss: 0.2844 - acc: 0.8777 - val_loss: 0.3483 - val_acc: 0.8420\n",
      "Epoch 60/120\n",
      "10000/10000 [==============================] - 4s 350us/step - loss: 0.2759 - acc: 0.8828 - val_loss: 0.3819 - val_acc: 0.8340\n",
      "Epoch 61/120\n",
      "10000/10000 [==============================] - 4s 351us/step - loss: 0.2722 - acc: 0.8799 - val_loss: 0.3596 - val_acc: 0.8400\n",
      "Epoch 62/120\n",
      "10000/10000 [==============================] - 4s 366us/step - loss: 0.2704 - acc: 0.8845 - val_loss: 0.3751 - val_acc: 0.8400\n",
      "Epoch 63/120\n",
      "10000/10000 [==============================] - 4s 372us/step - loss: 0.2691 - acc: 0.8854 - val_loss: 0.3745 - val_acc: 0.8430\n",
      "Epoch 64/120\n",
      "10000/10000 [==============================] - 4s 356us/step - loss: 0.2698 - acc: 0.8865 - val_loss: 0.3562 - val_acc: 0.8400\n",
      "Epoch 65/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.2674 - acc: 0.8875 - val_loss: 0.3534 - val_acc: 0.8400\n",
      "Epoch 66/120\n",
      "10000/10000 [==============================] - 4s 356us/step - loss: 0.2622 - acc: 0.8888 - val_loss: 0.3763 - val_acc: 0.8390\n",
      "Epoch 67/120\n",
      "10000/10000 [==============================] - 3s 347us/step - loss: 0.2593 - acc: 0.8885 - val_loss: 0.3670 - val_acc: 0.8470\n",
      "Epoch 68/120\n",
      "10000/10000 [==============================] - 3s 347us/step - loss: 0.2567 - acc: 0.8937 - val_loss: 0.3699 - val_acc: 0.8560\n",
      "Epoch 69/120\n",
      "10000/10000 [==============================] - 4s 361us/step - loss: 0.2573 - acc: 0.8951 - val_loss: 0.3676 - val_acc: 0.8430\n",
      "Epoch 70/120\n",
      "10000/10000 [==============================] - 4s 364us/step - loss: 0.2489 - acc: 0.8962 - val_loss: 0.3564 - val_acc: 0.8510\n",
      "Epoch 71/120\n",
      "10000/10000 [==============================] - 4s 363us/step - loss: 0.2479 - acc: 0.8961 - val_loss: 0.3605 - val_acc: 0.8460\n",
      "Epoch 72/120\n",
      "10000/10000 [==============================] - 4s 353us/step - loss: 0.2406 - acc: 0.9026 - val_loss: 0.3605 - val_acc: 0.8560\n",
      "Epoch 73/120\n",
      "10000/10000 [==============================] - 4s 354us/step - loss: 0.2404 - acc: 0.9020 - val_loss: 0.3490 - val_acc: 0.8510\n",
      "Epoch 74/120\n",
      "10000/10000 [==============================] - 4s 358us/step - loss: 0.2374 - acc: 0.9045 - val_loss: 0.3400 - val_acc: 0.8470\n",
      "Epoch 75/120\n",
      "10000/10000 [==============================] - 4s 381us/step - loss: 0.2299 - acc: 0.9060 - val_loss: 0.3453 - val_acc: 0.8490\n",
      "Epoch 76/120\n",
      "10000/10000 [==============================] - 4s 352us/step - loss: 0.2301 - acc: 0.9046 - val_loss: 0.3372 - val_acc: 0.8490\n",
      "Epoch 77/120\n",
      "10000/10000 [==============================] - 4s 353us/step - loss: 0.2250 - acc: 0.9076 - val_loss: 0.3354 - val_acc: 0.8510\n",
      "Epoch 78/120\n",
      "10000/10000 [==============================] - 3s 346us/step - loss: 0.2147 - acc: 0.9087 - val_loss: 0.3416 - val_acc: 0.8490\n",
      "Epoch 79/120\n",
      "10000/10000 [==============================] - 4s 351us/step - loss: 0.2111 - acc: 0.9119 - val_loss: 0.3774 - val_acc: 0.8520\n",
      "Epoch 80/120\n",
      "10000/10000 [==============================] - 4s 351us/step - loss: 0.2148 - acc: 0.9139 - val_loss: 0.3209 - val_acc: 0.8650\n",
      "Epoch 81/120\n",
      "10000/10000 [==============================] - 3s 347us/step - loss: 0.2045 - acc: 0.9158 - val_loss: 0.3157 - val_acc: 0.8650\n",
      "Epoch 82/120\n",
      "10000/10000 [==============================] - 3s 345us/step - loss: 0.1916 - acc: 0.9194 - val_loss: 0.3012 - val_acc: 0.8700\n",
      "Epoch 83/120\n",
      "10000/10000 [==============================] - 3s 350us/step - loss: 0.1881 - acc: 0.9228 - val_loss: 0.2922 - val_acc: 0.8670\n",
      "Epoch 84/120\n",
      "10000/10000 [==============================] - 3s 348us/step - loss: 0.1783 - acc: 0.9266 - val_loss: 0.2849 - val_acc: 0.8770\n",
      "Epoch 85/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.1933 - acc: 0.9209 - val_loss: 0.3006 - val_acc: 0.8730\n",
      "Epoch 86/120\n",
      "10000/10000 [==============================] - 4s 352us/step - loss: 0.1824 - acc: 0.9279 - val_loss: 0.2729 - val_acc: 0.8810\n",
      "Epoch 87/120\n",
      "10000/10000 [==============================] - 3s 350us/step - loss: 0.1779 - acc: 0.9282 - val_loss: 0.2774 - val_acc: 0.8840\n",
      "Epoch 88/120\n",
      "10000/10000 [==============================] - 3s 345us/step - loss: 0.1710 - acc: 0.9303 - val_loss: 0.2758 - val_acc: 0.8810\n",
      "Epoch 89/120\n",
      "10000/10000 [==============================] - 3s 348us/step - loss: 0.1658 - acc: 0.9345 - val_loss: 0.2854 - val_acc: 0.8880\n",
      "Epoch 90/120\n",
      "10000/10000 [==============================] - 4s 358us/step - loss: 0.1637 - acc: 0.9347 - val_loss: 0.2634 - val_acc: 0.8930\n",
      "Epoch 91/120\n",
      "10000/10000 [==============================] - 4s 351us/step - loss: 0.1577 - acc: 0.9358 - val_loss: 0.2546 - val_acc: 0.8910\n",
      "Epoch 92/120\n",
      "10000/10000 [==============================] - 4s 354us/step - loss: 0.1611 - acc: 0.9387 - val_loss: 0.2445 - val_acc: 0.9080\n",
      "Epoch 93/120\n",
      "10000/10000 [==============================] - 4s 368us/step - loss: 0.1560 - acc: 0.9381 - val_loss: 0.2369 - val_acc: 0.9040\n",
      "Epoch 94/120\n",
      "10000/10000 [==============================] - 4s 352us/step - loss: 0.1470 - acc: 0.9409 - val_loss: 0.2764 - val_acc: 0.8950\n",
      "Epoch 95/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.1475 - acc: 0.9428 - val_loss: 0.2634 - val_acc: 0.8980\n",
      "Epoch 96/120\n",
      "10000/10000 [==============================] - 3s 350us/step - loss: 0.1418 - acc: 0.9454 - val_loss: 0.2367 - val_acc: 0.9070\n",
      "Epoch 97/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.1436 - acc: 0.9453 - val_loss: 0.2460 - val_acc: 0.9120\n",
      "Epoch 98/120\n",
      "10000/10000 [==============================] - 4s 350us/step - loss: 0.1434 - acc: 0.9430 - val_loss: 0.2593 - val_acc: 0.9130\n",
      "Epoch 99/120\n",
      "10000/10000 [==============================] - 3s 348us/step - loss: 0.1348 - acc: 0.9465 - val_loss: 0.2851 - val_acc: 0.9000\n",
      "Epoch 100/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.1406 - acc: 0.9431 - val_loss: 0.2609 - val_acc: 0.9040\n",
      "Epoch 101/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.1342 - acc: 0.9478 - val_loss: 0.2705 - val_acc: 0.9050\n",
      "Epoch 102/120\n",
      "10000/10000 [==============================] - 3s 346us/step - loss: 0.1352 - acc: 0.9475 - val_loss: 0.2505 - val_acc: 0.9010\n",
      "Epoch 103/120\n",
      "10000/10000 [==============================] - 3s 345us/step - loss: 0.1291 - acc: 0.9502 - val_loss: 0.2708 - val_acc: 0.9080\n",
      "Epoch 104/120\n",
      "10000/10000 [==============================] - 4s 351us/step - loss: 0.1250 - acc: 0.9523 - val_loss: 0.2634 - val_acc: 0.9120\n",
      "Epoch 105/120\n",
      "10000/10000 [==============================] - 3s 347us/step - loss: 0.1203 - acc: 0.9519 - val_loss: 0.2725 - val_acc: 0.9070\n",
      "Epoch 106/120\n",
      "10000/10000 [==============================] - 4s 350us/step - loss: 0.1187 - acc: 0.9540 - val_loss: 0.2557 - val_acc: 0.9170\n",
      "Epoch 107/120\n",
      "10000/10000 [==============================] - 3s 347us/step - loss: 0.1182 - acc: 0.9531 - val_loss: 0.2664 - val_acc: 0.9090\n",
      "Epoch 108/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.1181 - acc: 0.9530 - val_loss: 0.2334 - val_acc: 0.9130\n",
      "Epoch 109/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.1158 - acc: 0.9554 - val_loss: 0.2899 - val_acc: 0.9120\n",
      "Epoch 110/120\n",
      "10000/10000 [==============================] - 3s 350us/step - loss: 0.1167 - acc: 0.9567 - val_loss: 0.2754 - val_acc: 0.9090\n",
      "Epoch 111/120\n",
      "10000/10000 [==============================] - 4s 350us/step - loss: 0.1120 - acc: 0.9561 - val_loss: 0.2898 - val_acc: 0.9100\n",
      "Epoch 112/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.1118 - acc: 0.9588 - val_loss: 0.2541 - val_acc: 0.9140\n",
      "Epoch 113/120\n",
      "10000/10000 [==============================] - 3s 346us/step - loss: 0.1083 - acc: 0.9583 - val_loss: 0.2511 - val_acc: 0.9110\n",
      "Epoch 114/120\n",
      "10000/10000 [==============================] - 4s 359us/step - loss: 0.1131 - acc: 0.9560 - val_loss: 0.2496 - val_acc: 0.9180\n",
      "Epoch 115/120\n",
      "10000/10000 [==============================] - 4s 364us/step - loss: 0.1050 - acc: 0.9599 - val_loss: 0.3021 - val_acc: 0.9170\n",
      "Epoch 116/120\n",
      "10000/10000 [==============================] - 3s 342us/step - loss: 0.1038 - acc: 0.9619 - val_loss: 0.2673 - val_acc: 0.9160\n",
      "Epoch 117/120\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 4s 350us/step - loss: 0.1100 - acc: 0.9599 - val_loss: 0.2855 - val_acc: 0.9160\n",
      "Epoch 118/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.1012 - acc: 0.9610 - val_loss: 0.2887 - val_acc: 0.9200\n",
      "Epoch 119/120\n",
      "10000/10000 [==============================] - 4s 352us/step - loss: 0.1040 - acc: 0.9618 - val_loss: 0.2774 - val_acc: 0.9120\n",
      "Epoch 120/120\n",
      "10000/10000 [==============================] - 3s 349us/step - loss: 0.1010 - acc: 0.9615 - val_loss: 0.3139 - val_acc: 0.9120\n"
     ]
    }
   ],
   "source": [
    "# train\n",
    "history = model.fit([inputs_train, queries_train], answers_train,batch_size=32,epochs=120,validation_data=([inputs_test, queries_test], answers_test))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Saving the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filename = 'chatbot_120_epochs.h5'\n",
    "model.save(filename)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating the Model\n",
    "\n",
    "### Plotting Out Training History"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dict_keys(['val_loss', 'val_acc', 'loss', 'acc'])\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd4VGX68PHvnUnvjUAIhN6LNBEL\nCigriIqIshZ27eiqu+ru2nbVdX23uPtz1bXi6rp2sSMqCoJdepUqhJpAII30PvO8fzxDmIQAA2Yy\nk+T+XFcuZs45c+Y+mXDueboYY1BKKaUAgvwdgFJKqcChSUEppVQdTQpKKaXqaFJQSilVR5OCUkqp\nOpoUlFJK1dGkoNoUEXlJRP7i5bE7ReQcX8ekVCDRpKCUUqqOJgWlWiARCfZ3DKp10qSgAo672uZO\nEflBRMpE5L8i0l5EPhWREhFZICIJHsdfKCIbRKRQRL4SkX4e+4aKyCr3694Cwhu81/kissb92kUi\nMtjLGCeJyGoRKRaRTBF5sMH+M9znK3Tvv9q9PUJE/iUiu0SkSES+c28bIyJZjfweznE/flBE3hWR\n10SkGLhaREaKyGL3e2SLyFMiEurx+gEi8rmIFIjIfhH5g4h0EJFyEUnyOG64iOSKSIg3165aN00K\nKlBNBcYDvYELgE+BPwDJ2L/b3wCISG/gTeB2oB0wF/hIRELdN8jZwKtAIvCO+7y4XzsMeBG4EUgC\nngPmiEiYF/GVAb8E4oFJwK9E5CL3edPd8T7pjmkIsMb9ukeA4cBp7pjuAlxe/k4mA++63/N1wAnc\n4f6dnAqcDdzsjiEGWAB8BnQEegILjTH7gK+AaR7nnQ7MMsbUeBmHasU0KahA9aQxZr8xZg/wLbDU\nGLPaGFMFfAAMdR/3c+ATY8zn7pvaI0AE9qY7CggBHjfG1Bhj3gWWe7zHDcBzxpilxhinMeZloMr9\nuqMyxnxljFlnjHEZY37AJqaz3LuvBBYYY950v2++MWaNiAQB1wK3GWP2uN9zkfuavLHYGDPb/Z4V\nxpiVxpglxphaY8xObFI7GMP5wD5jzL+MMZXGmBJjzFL3vpexiQARcQCXYxOnUpoUVMDa7/G4opHn\n0e7HHYFdB3cYY1xAJpDm3rfH1J/1cZfH4y7A79zVL4UiUgh0dr/uqETkFBH50l3tUgTchP3Gjvsc\n2xp5WTK2+qqxfd7IbBBDbxH5WET2uauU/uZFDAAfAv1FpDu2NFZkjFl2gjGpVkaTgmrp9mJv7gCI\niGBviHuAbCDNve2gdI/HmcBfjTHxHj+Rxpg3vXjfN4A5QGdjTBwwEzj4PplAj0ZekwdUHmFfGRDp\ncR0ObNWTp4ZTGj8LbAZ6GWNisdVrx4oBY0wl8Da2RPMLtJSgPGhSUC3d28AkETnb3VD6O2wV0CJg\nMVAL/EZEgkXkYmCkx2ufB25yf+sXEYlyNyDHePG+MUCBMaZSREYCV3jsex04R0Smud83SUSGuEsx\nLwKPikhHEXGIyKnuNowtQLj7/UOA+4BjtW3EAMVAqYj0BX7lse9joIOI3C4iYSISIyKneOx/Bbga\nuBB4zYvrVW2EJgXVohljfsTWjz+J/SZ+AXCBMabaGFMNXIy9+R3Atj+87/HaFdh2hafc+zPcx3rj\nZuAhESkBHsAmp4Pn3Q2ch01QBdhG5pPcu38PrMO2bRQA/wCCjDFF7nO+gC3llAH1eiM14vfYZFSC\nTXBvecRQgq0augDYB2wFxnrs/x7bwL3K3R6hFACii+wo1TaJyBfAG8aYF/wdiwocmhSUaoNE5GTg\nc2ybSIm/41GBQ6uPlGpjRORl7BiG2zUhqIa0pKCUUqqOlhSUUkrVaXGTaiUnJ5uuXbv6OwyllGpR\nVq5cmWeMaTj25TAtLil07dqVFStW+DsMpZRqUURk17GP0uojpZRSHjQpKKWUqqNJQSmlVJ0W16bQ\nmJqaGrKysqisrPR3KD4VHh5Op06dCAnRtVCUUr7RKpJCVlYWMTExdO3alfoTYrYexhjy8/PJysqi\nW7du/g5HKdVKtYrqo8rKSpKSklptQgAQEZKSklp9aUgp5V+tIikArTohHNQWrlEp5V+tovpIKaVa\nGqfLsHxnAev3FHFKtyQGpsUiIrhchj2FFezKLyfzQDmllbVEhjmIDgtmSOd4uiRF+TQuTQpNoLCw\nkDfeeIObb775uF533nnn8cYbbxAfH++jyJRS/lBWVcvO/DISIkNJjAolPMRRt6+ksoYnFm7lg9V7\nyCutrtveKSGC5Ogwtuwvobza2eh5/zploCaFlqCwsJBnnnnmsKTgdDpxOBxHeBXMnTvX16EppXyo\notrJ+r1FFJXX0C4mjPAQB++tyuLNZbspqaytO25I53guGd6J+MgQ/vLxJvaXVDJxYAcmDerI0PR4\nvsvIY976fZRV1zJtRGf6doihS1IUnRMjiI0IobzKSVl1LUlRoT6/Jk0KTeCee+5h27ZtDBkyhJCQ\nEKKjo0lNTWXNmjVs3LiRiy66iMzMTCorK7ntttuYMWMGcGjKjtLSUiZOnMgZZ5zBokWLSEtL48MP\nPyQiIsLPV6ZU25VZUA5Ax/gIHEGH2vNyS6p4f1UWc9buZfO+Epyu+jNNO4KECQM7MGFAB8qqasku\nquSz9fu4b/Z6APp2iGHmL4YzpPOhGoJpIzozbUTnI8YSG9583dBbXVL480cb2Li3uEnP2b9jLH+6\nYMAR9z/88MOsX7+eNWvW8NVXXzFp0iTWr19f13X0xRdfJDExkYqKCk4++WSmTp1KUlJSvXNs3bqV\nN998k+eff55p06bx3nvvMX369Ca9DqXU4XbnlzN/4z5qnIa4iBCKKmr4ZN1e1u+x95FQRxAd48MJ\nC3YgAhk5pdS6DMO7JHDLmB6c1Dme5OgwckuqOFBezWk9k0mLr/+F7vZzerFhbzE788s4d0AHQhyB\n28en1SWFQDBy5Mh6YwmeeOIJPvjgAwAyMzPZunXrYUmhW7duDBkyBIDhw4ezc+fOZotXqbamvLqW\n91ft4Z0VmazNKjps/0md47lvUj9iw0PYnldG1oFyap2GWpeLs3q349IRnemZEu31+4kIA9PiGJgW\n15SX4ROtLikc7Rt9c4mKOtQQ9NVXX7FgwQIWL15MZGQkY8aMaXSsQVhYWN1jh8NBRUVFs8SqVGtn\njOH9VXtYuiOfyNBgal0uPlqbTVFFDf1TY7l3Yl8mDU4lMSqU4opaggRSYsP9HbbftLqk4A8xMTGU\nlDS+qmFRUREJCQlERkayefNmlixZ0szRKdV6VNY4+WjtXpZsLyBIINghDO4Uz6TBqUSFBvPuykwe\nmb+FsOAg7p7Ql3F9U7hv9no+WL2HxKhQapwuqmpdjOuTwnWjuzGiS0K98T+RoXpL1N9AE0hKSuL0\n009n4MCBRERE0L59+7p9EyZMYObMmQwePJg+ffowatQoP0aqVMvjchnWZhXy2fp9vL0ikwPlNSRH\nhxHqECpqnLy5LJMH52wgNS6cnfnlDEuPp7zaya/fXE1kqIOKGie/Hd+bW8f2JChIB4AeS4tbo3nE\niBGm4SI7mzZtol+/fn6KqHm1pWtVbdu+okr+t2gHH6zaQ05JFY4g4Zx+KVx1WldO7W6ntTHG8ENW\nEe+uzGJjdjFXn9aV8wen4jLw/qos3l2Zxa/H9eKMXsn+vhy/E5GVxpgRxzpOSwpKKb9bl1XEzG+2\nsTOvjJSYMIIdQXz1Yw5Ol+Gcfu2ZOKgDY/ukEB9Zv5++iHBS53hO6lx/AKhD4NIRnbn0KN08VeM0\nKSilmk1eaRXlVU4qa51kHShnU3YJi7bl8X1GPjHhwQxLTyC3tIrC8hquPKUL153Rjc6Jkf4Ou03R\npKCU8rmCsmrun72eT9ZlH7avS1Ikd0/oy/RR6cQ04yAt1ThNCkopn6lxupi/YT9/mrOBoopqbh7T\ngx7togkLCaJ9bDh9OsQ062hddWyaFJRSTWbZjgJ25JVSWuVk6/4SPtuwj8LyGvp2iOGVa0fSv2Os\nv0NUx6BJQSl13L7dmsvz3+7gprO6c1oP27Nn1rLd3PP+urpjIkMdjO/fngsGd+SsPu0CemoHdYgm\nhSZwolNnAzz++OPMmDGDyEhtTFOBL6e4kr98sok5a/fiCBKWbM/nycuHEhocxB9nr+es3u3428WD\niA4NJjo8uN5Ecqpl0NTdBA5OnX0iHn/8ccrLy5s4IqWaVnl1LU8s3MqYR77isw37uOOc3iy+Zxz9\nU2P51Wsrufm1VfRpH8PTVw4jLT6CuMgQTQg/hcvlt7fWkkIT8Jw6e/z48aSkpPD2229TVVXFlClT\n+POf/0xZWRnTpk0jKysLp9PJ/fffz/79+9m7dy9jx44lOTmZL7/80t+XolQ9xhhmr9nDw59uZn9x\nFRMGdOCeiX3pmmzn93r9+lO4+fVV7Mgr43/XnEx0mN5SjsjlgvwMiO0IYUeYTK+mEr7+ByydCRc9\nAwOmNG+MtMak8Ok9sG/dsY87Hh0GwcSHj7jbc+rs+fPn8+6777Js2TKMMVx44YV888035Obm0rFj\nRz755BPAzokUFxfHo48+ypdffklyso64VIHD5TKs2HWAhz/dxKrdhZzUKY6nrxjGiK6J9Y6LCgvm\npWtOxmVoXSUDY2Du76HjMBh65U8/n8sF710HG963z+M6Q7s+kNwHEruBIxRcNbBkJuRvhej28OGt\nkDIA2vX+6e9/HFpfUvCz+fPnM3/+fIYOHQpAaWkpW7duZfTo0fz+97/n7rvv5vzzz2f06NF+jlSp\n+qprXSzalse8DftYsCmH3JIqkqND+b9LBjN1WKcjzhskIjj8nQ9qq6G6FCITj32sN9a8ActfAHHY\nm3aX0+z2mkqorYSIYyyhu/N7yNkIgy6BiARY8IBNCKNugcgEyP0RcjfDzu/s+Q6KS4fp70NKP5g5\nGt7+BdzwBYT6dglOT60vKRzlG31zMMZw7733cuONNx62b+XKlcydO5d7772Xn/3sZzzwwAN+iFCp\n+nbnl/PMVxnMXZdNcWUtUaEOxvRJYXz/9pzTv33gVglVlcLn98OOb6BgB4jAjd9C+/7Hd57C3bbK\npvdE6Hc+lBfY86aNgIoD8M41cOM3sOt7mHsnlOfZb/Ip/WD41dBvMgS5m2cri2D+/bDqZft84UPQ\nYyxs/BBOvgHO/auN8yCXE0pzwLjbEKJTwOEetzH1BXh1CrxwDsR0sNtG3gh9Jpzwr8wbAfpptyye\nU2efe+653H///Vx55ZVER0ezZ88eQkJCqK2tJTExkenTpxMdHc1LL71U77VafaSaU3FlDVv2lTB7\nzR5mLcvEESRMGpzKeQNTOaNXcr2F5puNMfVvmEdzYCfMutJ+G+9zHvSfbKtevnsMpj7v3TlcLljx\nX1jwoC1lrH4NzrobSrKhohB++W8bz/Nnw7OnQnk+pA6BU2+xbQO7F8M7V9sqoB5j7bY9q6CyEE77\nNfSfAouesAmhzySY+I/Dry/IAbGpjcfXYyxc8LiNq8o9Nb+rxrtr+wk0KTQBz6mzJ06cyBVXXMGp\np54KQHR0NK+99hoZGRnceeedBAUFERISwrPPPgvAjBkzmDhxIqmpqdrQrJpUVa2T7bllpCdGEhUW\nTFWtk3dXZvHidzvYllsGQHCQcNnIzvx6XC/a+3thmdk3Q+ZSOP8x6H7WkY/bswpemwrGCVe+Az3P\nsdtrKm0D7bg/QkLXo7+XywUfzIB170CPcTDx/+C7R22JAeDUW6HDQPv4gn/Dp3fC+Ids9Y/Dfdt0\nOWHjbPj2UVj5MiT3gl7j4ZQbIW24PWbay1CcDVHtbAI4XsOvtj/NyKdTZ4vIBODfgAN4wRjzcIP9\nXYAXgXZAATDdGJN1tHPq1Nlt51rV8Ssoq+a9lVks3Lyf1bsLqap14QgS+qXGkFdSzb7iSoZ0jmd8\n//b0S41hUFo87WLCjn1iXysvgEfcDaquGhh2FUx4GEIbjN+pKoWZp9ub+i9nQ1KPQ/uK9sC/T4Lh\nV8Gkf0FNhb3p799g6/BjO8LZf4KY9vD5n+D7x2HsH+HMO+03eGNg2X9g6+dw6Uv1ewgdqxRzPKUc\nP/H71Nki4gCeBsYDWcByEZljjNnocdgjwCvGmJdFZBzwd+AXvopJqdakutbFgx9tICOnlJSYMFzG\nsGBjDtVOF/1TY7nylC4M6hTLjtwyVuw6QGJUGI9cehKn90yqt9pYQNjwvk0G130Omz+G75+w2y98\nov5xnz8AB3bB1Z/UTwgAcWlw0mWw6lXodiYs+DMUbIOQKEjuCbsWweZPoP+FsOoVGHHdoYQA9t9T\nbrQ/DR3r9xVov8+fwJfVRyOBDGPMdgARmQVMBjyTQn/gDvfjL4HZPoxHqVbD6TLc8dYaPlmXzbD0\neDbsLaa8upYrTknnilPS6d0+xt8hHp+1b0FKf+h0MnQeCYj9Jt/3fOj9M3tMxkLbBnDqrdD19MbP\nc/rttg7+7V9CfBf4xQfQbYxtCM7dAnNutQmh9wSY+M9WdTNvKr5MCmlApsfzLOCUBsesBaZiq5im\nADEikmSMyfc8SERmADMA0tPTG30zY0zgfftpYi1tlTzVdIwx5JZUUVnjwuEQ/r1gC5+sy+a+Sf24\nfnR3f4f30xRsh6xlcM6Dh27SY/8AW+fDnF/bnj9b59mePMl9YNz9Rz5Xck8Yd59tOD7zzvpdOdv1\nhms+gx1fQ/qoQ20Dqh5f/lYau0M3vKv9HnhKRK4GvgH2ALWHvciY/wD/Adum0HB/eHg4+fn5JCUF\nYLG4iRhjyM/PJzzcz42Bqlnkl1axYtcBVu46wA9ZhWzeV0Jhef2eJ7ed3avlJwSAH94BBAZdemhb\ncBhMmQnPj4PHB4Kz2vb8uehZCDnG/4Ezf3/kfUFBtlePOiJfJoUswHMtvE7AXs8DjDF7gYsBRCQa\nmGqMKTreN+rUqRNZWVnk5ub+hHADX3h4OJ06dfJ3GMrHPlidxd3vraO61kWoI4j+HWOZODCVvh1i\niA4LxukyJEWHMq5vir9DPTEHxxdUHIB+F8APs6DrGRDX4G879STb2Lz5Y1tl1PMcre5pBr5MCsuB\nXiLSDVsCuAy4wvMAEUkGCowxLuBebE+k4xYSEkK3bt1+YrhK+UdljZPgICFIhMcWbOHJLzIY1T2R\nO8/tw8C0OMKC/TBmwFcKdsCsK+xo3ohE2PCB3T76d40fP/IG+6Oajc+SgjGmVkRuBeZhu6S+aIzZ\nICIPASuMMXOAMcDfRcRgq49u8VU8Svmby2XYU1hBRk4pW3NKWL+nmLVZhezKt7PkhjiEGqdh2ohO\n/OWiQYQGt7JJjLNWwutTbffN6e9Bt7Ng9xLIXlO/6kj5lU/HKfhCY+MUlAo0lTVOvtuax/a8Unbk\nlbFlfyk/7iuhtOpQk1mH2HCGdI6vW42srLqWXikxTB2W1vraxgq22+kaQqNtj6CG3UmVz/l9nIJS\nbcWq3QdYsbOAS4d3JiEqlG25pdzy+io277NTEyRGhdIzJZqpw9Lo0yGW3u2j6ZkSTXxkqJ8j9wFj\nbK+hb/4PcjbDiGtgyBV2SgrjspO9aUIIaJoUlDqGLftLmLd+H6d0T+Lkrgn1vsW/vTyTP85eR43T\n8PiCrZw3KJW567IJD3Hw7JXDOK1HMnGRbWRh+opCeP0SyFpuZ/vsMQYWP2Xn/3GEwVVzbJdRFdA0\nKag2q6rWSXZhJV2SIhutrlmbWchTX2bw+cb9ddu6t4tibJ8UYsKD2VtYwdsrshjdK5nbz+nN60t2\n8cHqPQxLj+eJy4eSGhfRnJfT/CqLISzm0BQRH94Ce1fbuYKGXGln+8zLgGXPQa+f2bEBKuBpm4Jq\n1fYUVrAuq4hN2cW4jGHS4FT6dojl6y25/OnD9ezMLyctPoKJAzswqnsSPVOiqXW5+Nf8LXy6fh9x\nESFcfVpXpp3cmUUZeby1PJP1e4uorLFTHV99Wlfum9SPYPei9IXl1cSGhxxx7YFWY+f38MqF9mY/\n6V+w7l3bzfTcv9lZRFXA8bZNQZOCahGcLsPHP+ylT4cY+naIrdteWeNkY3Yxm7NLyC6q4LxBqfRL\njaWq1smjn2/hP99sr5urLEgEp8uQFh/BnsIKuidHcfnIdBZvz+e7rXlUOw+tixsV6uCGM7tz/eju\nja4n4HQZapwu/0wx7W+11TDzDKgosGMOHCFQXQZ9J8G0V3QsQYDShmYVMKpqnQBH7G9fXes6rPul\ny2UwQJDAsh0FPPjRRjZlFxMWHMTDUwcxZWgn5m3Yx/2z15NTUlX3uie/yGBc3xT2FVWyMbuYy07u\nzOUj7VxAFTVOPvlhLws25XDFKelcP7obYcH25l9aVcuW/SVk7C+lsKKaKUM7HXX2UEeQ4DiRqZBb\nImMgb6udGlrEthPk/QiXv2W3fXyHXWtg8tOaEFoBLSmoE7ZlfwnpiZFH/LZc43Tx6uJdPLZgC44g\n4fKR6Vw8NI0f95fw7ZY8NmYXs7ugnKKKGsb1TeGOc3qTlhDBi9/t4OXFOympPNR9s2NcOHeM7827\nK7NYuqOAQWlxrNtTRL/UWG47uycDOsYRHRbMq0t28b/vdxAkwsNTBzO+f/tm+m20UtVldp2DjbPt\nWuUnX2/XQe95Nlz2+qHjWsDU0W2dVh8pn3G6DP8370dmfr2NfqmxPDd9OOlJkWQWlPPkF1vJKaki\nLDiIjJxStuWWMbpXMlGhwczfuA+X+88tJjyYIZ3j6ZIUSUSIg7dXZFFUUUNYcBDVThcTBnSgX2os\ntS5DQmQIl52cTkSogxqni7/N3cQbS3fz63E9ufGsHoQ46pcyqmtdBAl19fzqOBhjVzVzOaGqGOb8\nBvavh5Ovg21fHpqK+tZlh09LoQKaJgXlEwfKqvnNrNV8uzWP8wZ14PuMfIwxTBrckfdWZuEIEnq1\nj6aqxkV4qINbxvRgfP/2iAiZBeV89WMO/TvGcVKnuHo37ZLKGl5etJPsokquPq0rvY4x9XOt06U3\nfV9Y9jzM9ZhQLiwOLvmvXVHM5YRNc+xC9N3H+CtCdYI0KagT4nIZ9pdU0iE2vF43zVqnizeXZ/Lo\n/B8pq3Ly0OQBXDYynd355dz02ko2Zhdz8bA07jq3Lx3idCbXFqm2Gp4YYhelH3Wz3ZY+CuI7H/11\nqkXQhmZ1XDJySnlnZSZz1uwlu6iS03okcfeEvqQnRjJn7V5eXbKLjJxSRnVP5IHzB9RNzZCeFMkH\nt5xGTnEVnRMjj/EuKqCtexuK98AFT0Cvc/wdjfITTQpt3K78Mh5fsJXZa/bgEOHM3u24dERnXluy\ni8lPf183SVv/1FhmTh/OuQPaHzbQKyzYoQmhKRkDu76HtBHHXjtg72oIj4PEn7iugssJ3z0OHQbb\nRmTVZmlSaKOMMbzw7Q7+8dlmgh3CjDO7c8Po7iRH226YN4zuxsuLbA+gyUPS6koGqhmsewfev8FW\n4Uz4+5GPO7ATXpwIQQ6Y8hz0O//E33Pzx5C/FS75n/YiauO0TaENqnXaBd9fW7KbCQM68NDkAaTE\najtAQCjeC8+MgqoSCA6HOzZAZKLdt38jxHaEiHhbmnjzMtjxrR0rkL0GxvzBLkEZdJQG+JpK2LMC\nknpCTAe7LXstvHcDuGrh1uU2yahWR9sUVKMqa5zc/Poqvticw01n9eCuc/u0/ikZWgpj7JrEzhr4\n+esw63JY+hyMvddOK/HyBbYb6OVv2sVqtnwGP/sLnHwDfHQbfPU32PYFTH7K3vT3b7DPa92D+3I3\nw5Z5UF0CiG1EDo2GjM9tL6Opz2tCUJoU2pIap4tb37AJ4S8XDWT6qC7+Dkl5WvEiZCyA8x6BvudB\nn0mwdCYMngbvXgPx6VBTAS+MtwvStx8Ip9xkp5mYMtN2E513Lzx7OsSl2TUMPEUmwcApdr6i/Rtt\n99L8bTD2Pjhlhm2bUG2eVh+1EU6X4TezVttpHnq8Tc/kSOg/GbqPPXZjpi/t+MY2cJ79AHQc4t1r\nNn0EP7wFk5+B8FbS1pG91t7su54OV75nq4CyVsALZ9ubdW013LDQLmH51nTYsxKunQfpp9Q/T2kO\nfP4nKMuBvudDn/NsMgBbCtD2gjZLq49UPQ99tIFPfsjmn2dF0HPph5AdDGvftH3SZ3xl66o9GQPv\nXQ8l+2DqCxCbauu55/zafsuc9C/oNvrIb/jjZ/Dx7XaBlVG3QFRS/f2lOfDlX2HlS4eez/gKHEf5\nk3S54Ot/wNcP2+c9x8Pwq47r9wDYSdxCo7y/QTpr7DTRntdQWw3Ln4f+F9lv5Y2prYI1r8OSmRDV\nDkb/FnqMO/x9Kwrh7V9CVDJc/PyhNoFOI6DraNj5rW1Ibj/Abr/mUyjOarzHUXQKTHnWu+tSqhFa\nUmgD3luZxe/eWcv1Z3TjvsSFMP8++M1qyN0Cb/8CBk2Di56u/6KVL8NHvwFx2BvaxH/YG3LuZohJ\ntf3Zh18DP/t/dk59Ty6nbSwt2W+nSgiJhE7DAbE3yvwMKM8DCbLTLHcYbHvbHJx2uTQXFj4IJ11h\nvzmDTQjvXWsXej/pCshaBtEd4JpPDr/g4r22vj33RyjKsomp7yS7b82btv79rLvgTI+RuzmbbazJ\nvW0D7/YvbYkka4Wd2sFVC2PuhbPutse/P8P26089Ca6db0tbVaW2+ubALntM3hYoyYaOQ23SK94D\nKf3t7xPsyOB2fe217PjG3uw7j6x/LYWZtmF4wJTj/diVqkdLCgqA9XuK+MMH6xjVPZF7JvaFV++y\nN6bE7vbnlBth0VP239TB9kUHdsG8P9hvqRMehllXwDtXQXi8XU6x8yn2W/6SZ2D3Ytvw6fmtdd07\n9oY47RVI7mNX3srfZvcFBUOfifZm2H0MdBhoSyXr3oEv/waJPeCT39lvwpvnwk3f2sbVbx+xCeHs\nB+CM38I3j8CXf4HC3bau3dOHt8K2hXaOnrAY291ywMX2W/TSmXYVsCXP2AQUEgFlebaaprrUHWMI\nuGps42vX021CObADvvq7LTVgbELod6Gtl//sHhj/ELx+qV11rNMIQCB1CFz0jK2ic1bbktm6d90N\nv8ZWGW380D6e8I/DEwLY0cQ6olg1Iy0ptGKVNU7GP/Y1tU7DR78+g+TgSvhndzj1Vhj/Z3tQRSE8\nMdTOgPnLD+3N67WpsHcN3Lyhwo5hAAAeGElEQVTI3nDLC+zN9KTL6t/8t39tk4UxcMmLdtCTswae\nOhnComHGN0fvHumpYIctXdRWQmyaLYHMuQ3a9bHf6t/4uW1wnfKcrX45sBP+fZJNEqN/d+g8zhp4\nOB0G/xwmPQrGPSjrm3/aaxt5I/SZAK9OgQufhGG/tCWnxU/b52V5UJZrb+TdzoRg9zrKLhd8cseh\n6q5hV9kVxhY8CN8/DvFdbElg6n9hwEXef0g1FVBZdKh7qFI+oiUFxfcZeWQWVPD8L0fYQWkbPrXV\nIL3PPXRQRDyMuQc+vct+081caqtRLnzq0DfwyEQY+4fD36D7WXDDl7Yk8drF0PMcWwo5sAMun+V9\nQgBI7AYT/2kXfT//MfutXhw26bzxc1uyOP+xQ/XxCV0h/VRY+5YtORzcvnc11JRDj7Hu9w+Cs+60\nN+oDu+z0DcZA+0Gw5FnbE2fZCzDoUhg6/cjxBQXB+Y/bRtuS/TbhiMC4+20VU+ZSWzI6WE3lrZAI\n+6NUgNCk0IrN27CPmLBgzuydbDdsnW+rgDo1qKYYca39Brxnha0SGXix91MdJHaD6xfY/vSLn7Jd\nKtOGQ+8Jxx/w8KvqNxwPuAj23garXoGfv2obhz0NnmYXeMleY+vtAXZ+Z/9NP63+scm97A/Ym/mo\nX8GHN8OsK20J4mBbwdGI2JKJJ0cwTH/XthkkaBdf1fJp9VFrtOEDXMtfZMXOAmIiI+g38SZbp/6v\n3rZK5JIXD39NbbVt+D1a759jqS6zdeZdTjt0A24KzhrbF7+higPwSG8YcR1MdPdIem2qbZy9ddnR\nz1lTCY8PtFVFQ6fbVcOUasW8rT7SCelbm9wf4f0bqc7bAa4a0oNy4L3r4MVz7Q2w17mNvy449Kcl\nBLDf5Idf1bQJARpPCGB77/SdBGvesD1/nLWwe+mhHktHExJuG9eDI+DMu5o2XqVaME0KrYmzFj64\nEUKjeKrbM0x3/RluWQ4/+yvsW2dLAj1b2ZTIo26GqiLbs2ffD3YKhy5eJAWAM34Hd6zXah+lPGib\nQmvy3aOwdzXm0pd5f04NZ/ZKJioiDE671X6jLt5z+CCylq7TybYNY8mzh9ojup7h3WuDguyAMaVU\nHS0ptBYHdtnBZQMvYV3cGPYWVXLuAI9ujondvL9ZtiQitrRQsA0WPWnHOWj3TqVOmCaF1mLLZ7a7\n6bg/8tn6fTiChHP6tfd3VM2j/2SI6WjbTLxpT1BKHZEmhdYiYwEk9qAqtgtvr8jizF7JJESF+juq\n5uEIgZE32MddWmFpSKlmpG0KrUFNpe2fP3Q6H6/NJq+0imvP6ObvqJrXyBm26+pPWX1MKaVJoVXY\nvRhqyjE9xvHivB30SonmjJ5trAE1LBrGeDEATSl1VFp91BpsWwiOUFbKQDbsLeaa07shOm++UuoE\n+DQpiMgEEflRRDJE5J5G9qeLyJcislpEfhCR83wZT6uVsRDSR/HfZTnER4YwZegR5vdXSqlj8FlS\nEBEH8DQwEegPXC4i/Rscdh/wtjFmKHAZ8Iyv4mm1ivdCzkaKO53FvA37uHxkOhGhus6uUurE+LKk\nMBLIMMZsN8ZUA7OAyQ2OMcDB9RTjgL0+jKd1ylgIwBIZgsvARUO0lKCUOnG+TAppQKbH8yz3Nk8P\nAtNFJAuYC/y6sROJyAwRWSEiK3Jzc30Ra8u15TOI7sC83GQSo0Lp3T7a3xEppVowXyaFxlo6G07J\nejnwkjGmE3Ae8KqIHBaTMeY/xpgRxpgR7dq180GoLdSuRXZVsSFXsGRHAaO6J2oDs1LqJ/FlUsgC\nPNcR7MTh1UPXAW8DGGMWA+FAG+tLeYKcNfDxbyEunaxBt7CnsIJR3VvZvEZKqWbny6SwHOglIt1E\nJBTbkDynwTG7gbMBRKQfNilo/ZA3Fj8NuZvgvH+yOLMCgFO6aVJQSv00PksKxpha4FZgHrAJ28to\ng4g8JCIXug/7HXCDiKwF3gSuNi1t1R9/KN5rJ7/rMwn6TGTJ9gISo0LplaLtCUqpn8anI5qNMXOx\nDcie2x7weLwR0BnMjte2L+w6xOPuA2DJ9nxO6ZZIUJC2Jyilfhod0dwS5WdAUAgk9yazoFzbE5RS\nTUaTQkuUn2HXR3AEs2R7PoAmBaVUk9Ck0BLlZUBSTwBtT1BKNSlNCi2NywkF2yGpB8YYvs/IY1R3\nbU9QSjUNTQotTVEWOKsgqRfbckvZV1zJ6F46oE8p1TS8Sgoi8p6ITGpstLFqZvkZ9t+knnyzJQ+g\n7a2doJTyGW9v8s8CVwBbReRhEenrw5jU0XgkhW+35tI9OYrOiZH+jUkp1Wp4lRSMMQuMMVcCw4Cd\nwOciskhErhGREF8GqBrIz4CwWKrCk1iyvYDRvbSUoJRqOl5XB4lIEnA1cD2wGvg3Nkl87pPIVOPy\nMyCpByt3F1JR49T2BKVUk/JqRLOIvA/0BV4FLjDGZLt3vSUiK3wVnGpEXgakn8K3W/MIDhJG9dDx\nCUqppuPtNBdPGWO+aGyHMWZEE8ajjqamAooyIWk6367LZViXBKLDfDpTiVKqjfG2+qifiMQffCIi\nCSJys49iUkdSsAMwlER3Yf2eYkZrryOlVBPzNincYIwpPPjEGHMAuME3Iakjyt8KwOoyW2V0hjYy\nK6WamLdJIUg8lvQSEQcQ6puQ1BG5u6N+mx9HZKiDQWlxfg5IKdXaeFshPQ94W0RmYpfUvAn4zGdR\nqcblb4OYVL7LrGJYegLBDh1LqJRqWt7eVe4GvgB+BdwCLATu8lVQ6gjytlKb0IPN+4oZ0TXB39Eo\npVohr0oKxhgXdlTzs74NRx2Rswb2b2Bf14sxBkZ2TfR3REqpVsjbcQq9gL8D/bHrKANgjOnuo7hU\nQ9lroaaMlfQjOEgYkh5/7NcopdRx8rb66H/YUkItMBZ4BTuQTTWXnd8B8HFhVwakxREZquMTlFJN\nz9ukEGGMWQiIMWaXMeZBYJzvwlKH2fU9rqRefJ0dxEhtT1BK+Yi3SaHSPW32VhG5VUSmACk+jEt5\ncjlh9xLykkZQXetihLYnKKV8xNukcDsQCfwGGA5MB67yVVCqgX0/QFUxax0DADhZk4JSykeOWTHt\nHqg2zRhzJ1AKXOPzqFR9uxYBMLe4Bz1TokiM0nGDSinfOGZJwRjjBIZ7jmhWzWzn95DQjW/3hzKk\ns/Y6Ukr5jrddWFYDH4rIO0DZwY3GmPd9EpU6xOWC3Yuo7X0eeUur6JYc5e+IlFKtmLdJIRHIp36P\nIwNoUvC1nI1QcYDcxJMBdOlNpZRPeTuiWdsR/CVzKQAZEYOA/aRrUlBK+ZC3I5r/hy0Z1GOMubbJ\nI1L1Fe4CRyhbqhLQpKCU8jVvq48+9ngcDkwB9jZ9OOowRVkQm0bmgUqiw4JJiAzxd0RKqVbM2+qj\n9zyfi8ibwAKfRKTqK8yE+M7sLiinc2Ik2glMKeVLJzohfy8gvSkDUUdQlAVxNimkJ0b4OxqlVCvn\nbZtCCfXbFPZh11hQvuSsgZJsTGwamQXljO3Tzt8RKaVaOW+rj2JO5OQiMgH4N+AAXjDGPNxg/2PY\nWVfBTqORYozR0VkHFe8FDMVhHaiqdZGepGMUlFK+5VX1kYhMEZE4j+fxInLRMV7jAJ4GJmLXYbhc\nRPp7HmOMucMYM8QYMwR4Eh33UF9RFgDZJANozyOllM9526bwJ2NM0cEnxphC4E/HeM1IIMMYs90Y\nUw3MAiYf5fjLgTe9jKdtKMoEYGdtEqBJQSnle94mhcaOO1bVUxqQ6fE8y73tMCLSBeiGXQe6sf0z\nRGSFiKzIzc31ItxWwp0UtlTEIgJp8drQrJTyLW+TwgoReVREeohId3dbwMpjvKaxvpOHDYBzuwx4\n1z353uEvMuY/xpgRxpgR7dq1ocbWoiyITGZHkYuOcRGEBp9oZzGllPKOt3eZXwPVwFvA20AFcMsx\nXpMFdPZ43okjD3i7DK06OlxRFsR1co9R0FKCUsr3vO19VAbcc5znXg70EpFuwB7sjf+KhgeJSB8g\nAVh8nOdv/QozoV1vdmdod1SlVPPwtvfR5yIS7/E8QUTmHe01xpha4FZgHrAJeNsYs0FEHhKRCz0O\nvRyYZYw5UtVS22QMFGVRG51GbkkVXbQ7qlKqGXg791Gyu8cRAMaYAyJyzDWajTFzgbkNtj3Q4PmD\nXsbQtlQcgJoyCkLaAzpltlKqeXjbpuASkbppLUSkK0duNFZNQccoKKX8wNuSwh+B70Tka/fzM4EZ\nvglJAR5jFBIB6JygDc1KKd/zqqRgjPkMGAH8iO2B9DtsDyTlK+6SwtbKeCJDHSRGhfo5IKVUW+Dt\nhHjXA7dhu5WuAUZhewuNO9rr1E9QlAmOMLaUhNEpwalTZiulmoW3bQq3AScDu4wxY4GhQBsaWuwH\n7jEKWYWVdErQ9gSlVPPwNilUGmMqAUQkzBizGejju7BUXVI4UE4nbU9QSjUTbxuas9zjFGYDn4vI\nAXQ5Tt8qzKS629kUV9ZqUlBKNRtvRzRPcT98UES+BOKAz3wWVVvnrIXS/RSF2FHMWn2klGou3pYU\n6hhjvj72UeonKc8DDLnYJSy0pKCUai467WYgKt0PwN7aWEBLCkqp5qNJIRCV2o5du6qiiQp1kBAZ\n4ueAlFJthSaFQOQuKWwrj6RTQqSOUVBKNRtNCoGoLAeATSXh2p6glGpWmhQCUWkOhESRUWg0KSil\nmpUmhUBUmoMzKoWSylptZFZKNStNCoGoLIeqMDs7qpYUlFLNSZNCICrNoSQ4CdDuqEqp5qVJIRCV\n5lAgOnBNKdX8NCkEGmcNVBSw3xlLVKiDeB2joJRqRpoUAk2ZHbiWWROjYxSUUs1Ok0KgcQ9c21ER\npVVHSqlmp0kh0LinuPixLII0TQpKqWamSSHQuEsKu6qi6aw9j5RSzUyTQqBxT3GRZ+K0pKCUanaa\nFAJNaQ61wVFUEqZtCkqpZqdJIdCU5lAWageupcVrUlBKNS9NCoGmNIeioAQiQhwkRoX6OxqlVBuj\nSSHQlOWQ625P0DEKSqnmpkkh0JTmkF0bo+0JSim/CPZ3AMpDbRVUFrJLorU9QSnlF1pSCCQHp7io\njtHZUZVSfqFJIZC4B67lmTitPlJK+YUmhUDinuIiVweuKaX8xKdJQUQmiMiPIpIhIvcc4ZhpIrJR\nRDaIyBu+jCfgaUlBKeVnPmtoFhEH8DQwHsgClovIHGPMRo9jegH3AqcbYw6ISIqv4mkR3FNcFAUn\nkBwV5udglFJtkS9LCiOBDGPMdmNMNTALmNzgmBuAp40xBwCMMTk+jCfwleVRERRFSnwcQUE6RkEp\n1fx8mRTSgEyP51nubZ56A71F5HsRWSIiExo7kYjMEJEVIrIiNzfXR+EGgLI8ConV9gSllN/4Mik0\n9lXXNHgeDPQCxgCXAy+ISPxhLzLmP8aYEcaYEe3atWvyQANGeT55rmhtT1BK+Y0vk0IW0NnjeSdg\nbyPHfGiMqTHG7AB+xCaJNslVlsd+pw5cU0r5jy+TwnKgl4h0E5FQ4DJgToNjZgNjAUQkGVudtN2H\nMQU0Z2keB4wOXFNK+Y/PkoIxpha4FZgHbALeNsZsEJGHRORC92HzgHwR2Qh8CdxpjMn3VUwBzRiC\nKgsoIEbbFJRSfuPTuY+MMXOBuQ22PeDx2AC/df+0bdVlOJxVFJhYbVNQSvmNjmgOFOW2gFQsMaTE\nhPs5GKVUW6VJIVCU5wEQHJOCQ8coKKX8RJNCoCgvACA2qb2fA1FKtWWaFAJEdYkdlJeU0nB8n1JK\nNR9NCgGiIMcO4UhN1aSglPIfTQoBorhgHzXGQfdOHf0dilKqDdOkECAqC3M4QAxd20X5OxSlVBum\nSSFAuMryKHXEERbs8HcoSqk2TJNCgHBUHqA6NMHfYSil2jhNCgGgutZFZG0hEpXk71CUUm2cJoUA\nsCOvjASKCY1t2wvPKaX8T5NCANi6v5B4yohK0KSglPIvTQoBIGvPXoLEkJCc6u9QlFJtnCaFAJCz\n3w5cC4lpxavKKaVaBE0KAaAoz70gXaQ2NCul/EuTgp/VOF1UFtl5j4hK9m8wSqk2T5OCn/24r4Q4\nU2yfaElBKeVnmhT8bMn2fBIosU80KSil/EyTgp8t3VFAl4gKCI2B4DB/h6OUauM0KfiRy2VYvrOA\nnlGVEJno73CUUkqTgj9tySmhsLyGjqHl2sislAoImhT8aNkOuwRnIsXanqCUCgiaFPxo6fYCOsaF\nE1J9ACK1pKCU8j9NCn5ijGHpjgJO6Z6ElBdom4JSKiBoUvCT7Xll5JVWcWp6BNSUa/WRUiogaFLw\nk6XbbXvCabF5dkN8uh+jUUopS5OCn3y/LY92MWGk5S0CBLqP8XNESimlScEvSqtqWbhpPxMGdEC2\nLYTUk7RLqlIqIGhS8IPP1u+jssbFxQNiIHMZ9Dzb3yEppRSgScEvZq/eQ3piJENq1oBxQs9z/B2S\nUkoBmhTqlFbVUlZV65Nz/5BVyLqsIgD2F1fy/bY8LhrSEdn2hZ3zqNPJPnlfpZQ6XsH+DiAQbN5X\nzNUvLqeoooZJg1O5fGQ6w7skNMm5MwvKueL5pVTXunjqiqHsyi/HGJg8pCO8vhC6nwWOkCZ5L6WU\n+ql8WlIQkQki8qOIZIjIPY3sv1pEckVkjfvnel/G05gl2/O5dOZiDIYLT+rIp+uymfrsIpZszz+h\n82UdKKeoogawE97d+e5ajDH0TY3hV6+v4j/fbuekTnH0kGwoytT2BKVUQPFZSUFEHMDTwHggC1gu\nInOMMRsbHPqWMeZWX8XRqD2rICaVT3bCHW+tIT0pkpevHUlafAR/PK8Xv334CeauTmVU92MMKNu1\nGNr1gchEqmqdPP3lNr7+aj4VoUlcM/F0yqudLNlewD+mDmLS4I5c+9Jylu0o4OYxPSDjU3uOHpoU\nlFKBw5fVRyOBDGPMdgARmQVMBhomhea1dhbM/hXlIUk8WnoXgzoP5L9XjSA+MhScNcR+fBMvBM3m\nm/XzMRd+goSEN36e75+Az++HpJ7smDSLGz/cS9/cebwf+iwlEselH9zLVtOJcX1TmDaiMyLCy9eM\n5KMf9nJh7wj477OQ0h8SujTv9Sul1FH4svooDcj0eJ7l3tbQVBH5QUTeFZHOPowHVr2K+eAmdkYO\noqyqmg8i/sobF8bYhFBbDe9cDRtnk5U6njPNCkpengY1FYef55tHbELocTamZB+hr07ivOJ3+HfY\nszg6jyQuMoyPY/7O1d1LefjiQYgIABGhDqYNSyP8o19BSTZMfsqnl6uUUsfLlyUFaWSbafD8I+BN\nY0yViNwEvAyMO+xEIjOAGQDp6Sc2HcQPHz3F4JV/5BvnYG4quIObh4Rya+ZvkZd+Zmcora2AslyY\n8A8iB13LPX/7A3/PegEeG4AzOJL80iqCgoRwhxBdtQ8z6FJqLniGPz/3Onfn/YHb5VU7KvmyN5GS\nbMJeOp8HC+6CDfkw/GoIjbSBfPcobJ0P5z0CacNP6FqUUspXxJiG9+kmOrHIqcCDxphz3c/vBTDG\n/P0IxzuAAmNM3NHOO2LECLNixYrjjuebr+YhS55h5+n/5NwhXUiJCYf8bbDoSaitsgf1OgcGTgVg\n2nOLGVD0DX/qtZOlO/LZU1hBRKiDsion210deD/iEjokRLMms5D/TQhjrFkGZ9wBIRH2XAU74MNb\nYdd3ENUOOgyC3C1QnAWDLoWLnwdpLG8qpVTTE5GVxpgRxzzOh0khGNgCnA3sAZYDVxhjNngck2qM\nyXY/ngLcbYwZdbTznmhSOF7Pf7Odv87dxF8uGsh9s9dz+zm9uP2c3uwrquS7jDy+3JzD99vyuOrU\nrtwxvveRT7RrEXz3GJTuh3Z9of1AOPn6QyUHpZRqBn5PCu4gzgMeBxzAi8aYv4rIQ8AKY8wcEfk7\ncCFQCxQAvzLGbD7aOZsrKezIK2PsI18hAt2To5h722jCgh0+f1+llPIFb5OCTwevGWPmAnMbbHvA\n4/G9wL2+jOFEdUuOomdKNBk5pfz94sGaEJRSbYKOaD6Keyf2ZW9hBSO76apoSqm2QZPCUZzdr72/\nQ1BKqWalE+IppZSqo0lBKaVUHU0KSiml6mhSUEopVUeTglJKqTqaFJRSStXRpKCUUqqOJgWllFJ1\nfDr3kS+ISC6w6wRfngzkNWE4/tSargVa1/XotQSmtn4tXYwx7Y51UItLCj+FiKzwZkKolqA1XQu0\nruvRawlMei3e0eojpZRSdTQpKKWUqtPWksJ//B1AE2pN1wKt63r0WgKTXosX2lSbglJKqaNrayUF\npZRSR6FJQSmlVJ02kxREZIKI/CgiGSJyj7/jOR4i0llEvhSRTSKyQURuc29PFJHPRWSr+98Ef8fq\nLRFxiMhqEfnY/bybiCx1X8tbIhLq7xi9ISLxIvKuiGx2fz6nttTPRUTucP99rReRN0UkvCV9LiLy\noojkiMh6j22NfhZiPeG+H/wgIsP8F/nhjnAt/+f+O/tBRD4QkXiPffe6r+VHETn3p7x3m0gKIuIA\nngYmAv2By0Wkv3+jOi61wO+MMf2AUcAt7vjvARYaY3oBC93PW4rbgE0ez/8BPOa+lgPAdX6J6vj9\nG/jMGNMXOAl7TS3ucxGRNOA3wAhjzEDAAVxGy/pcXgImNNh2pM9iItDL/TMDeLaZYvTWSxx+LZ8D\nA40xg4EtuNe3d98LLgMGuF/zjPued0LaRFIARgIZxpjtxphqYBYw2c8xec0Yk22MWeV+XIK98aRh\nr+Fl92EvAxf5J8LjIyKdgEnAC+7nAowD3nUf0iKuRURigTOB/wIYY6qNMYW00M8FuzxvhIgEA5FA\nNi3oczHGfAMUNNh8pM9iMvCKsZYA8SKS2jyRHltj12KMmW+MqXU/XQJ0cj+eDMwyxlQZY3YAGdh7\n3glpK0khDcj0eJ7l3tbiiEhXYCiwFGhvjMkGmziAFP9FdlweB+4CXO7nSUChxx98S/l8ugO5wP/c\nVWEviEgULfBzMcbsAR4BdmOTQRGwkpb5uXg60mfR0u8J1wKfuh836bW0laQgjWxrcX1xRSQaeA+4\n3RhT7O94ToSInA/kGGNWem5u5NCW8PkEA8OAZ40xQ4EyWkBVUWPcde2TgW5ARyAKW8XSUEv4XLzR\nUv/mEJE/YquUXz+4qZHDTvha2kpSyAI6ezzvBOz1UywnRERCsAnhdWPM++7N+w8Wed3/5vgrvuNw\nOnChiOzEVuONw5Yc4t3VFtByPp8sIMsYs9T9/F1skmiJn8s5wA5jTK4xpgZ4HziNlvm5eDrSZ9Ei\n7wkichVwPnClOTTIrEmvpa0kheVAL3dPilBso8wcP8fkNXed+3+BTcaYRz12zQGucj++CviwuWM7\nXsaYe40xnYwxXbGfwxfGmCuBL4FL3Ie1lGvZB2SKSB/3prOBjbTAzwVbbTRKRCLdf28Hr6XFfS4N\nHOmzmAP80t0LaRRQdLCaKVCJyATgbuBCY0y5x645wGUiEiYi3bCN58tO+I2MMW3iBzgP22K/Dfij\nv+M5ztjPwBYHfwDWuH/Ow9bFLwS2uv9N9Hesx3ldY4CP3Y+7u/+QM4B3gDB/x+flNQwBVrg/m9lA\nQkv9XIA/A5uB9cCrQFhL+lyAN7HtITXYb8/XHemzwFa5PO2+H6zD9rry+zUc41oysG0HB+8BMz2O\n/6P7Wn4EJv6U99ZpLpRSStVpK9VHSimlvKBJQSmlVB1NCkoppepoUlBKKVVHk4JSSqk6mhSUakYi\nMubgzLBKBSJNCkoppepoUlCqESIyXUSWicgaEXnOvf5DqYj8S0RWichCEWnnPnaIiCzxmOf+4Jz9\nPUVkgYisdb+mh/v00R5rMLzuHkGsVEDQpKBUAyLSD/g5cLoxZgjgBK7EThK3yhgzDPga+JP7Ja8A\ndxs7z/06j+2vA08bY07CziN0cBqFocDt2LU9umPng1IqIAQf+xCl2pyzgeHAcveX+AjsRGou4C33\nMa8B74tIHBBvjPnavf1l4B0RiQHSjDEfABhjKgHc51tmjMlyP18DdAW+8/1lKXVsmhSUOpwALxtj\n7q23UeT+BscdbY6Yo1UJVXk8dqL/D1UA0eojpQ63ELhERFKgbp3fLtj/LwdnDL0C+M4YUwQcEJHR\n7u2/AL42dr2LLBG5yH2OMBGJbNarUOoE6DcUpRowxmwUkfuA+SIShJ2p8hbsIjoDRGQldmWyn7tf\nchUw033T3w5c497+C+A5EXnIfY5Lm/EylDohOkuqUl4SkVJjTLS/41DKl7T6SCmlVB0tKSillKqj\nJQWllFJ1NCkopZSqo0lBKaVUHU0KSiml6mhSUEopVef/AwswEZcCHcfCAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "%matplotlib inline\n",
    "print(history.history.keys())\n",
    "# summarize history for accuracy\n",
    "plt.plot(history.history['acc'])\n",
    "plt.plot(history.history['val_acc'])\n",
    "plt.title('model accuracy')\n",
    "plt.ylabel('accuracy')\n",
    "plt.xlabel('epoch')\n",
    "plt.legend(['train', 'test'], loc='upper left')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Evaluating on Given Test Set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model.load_weights(filename)\n",
    "pred_results = model.predict(([inputs_test, queries_test]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Mary',\n",
       " 'got',\n",
       " 'the',\n",
       " 'milk',\n",
       " 'there',\n",
       " '.',\n",
       " 'John',\n",
       " 'moved',\n",
       " 'to',\n",
       " 'the',\n",
       " 'bedroom',\n",
       " '.']"
      ]
     },
     "execution_count": 74,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_data[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Mary got the milk there . John moved to the bedroom .\n"
     ]
    }
   ],
   "source": [
    "story =' '.join(word for word in test_data[0][0])\n",
    "print(story)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Is John in the kitchen ?\n"
     ]
    }
   ],
   "source": [
    "query = ' '.join(word for word in test_data[0][1])\n",
    "print(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "True Test Answer from Data is: no\n"
     ]
    }
   ],
   "source": [
    "print(\"True Test Answer from Data is:\",test_data[0][2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted answer is:  no\n",
      "Probability of certainty was:  0.9999999\n"
     ]
    }
   ],
   "source": [
    "#Generate prediction from model\n",
    "val_max = np.argmax(pred_results[0])\n",
    "\n",
    "for key, val in tokenizer.word_index.items():\n",
    "    if val == val_max:\n",
    "        k = key\n",
    "\n",
    "print(\"Predicted answer is: \", k)\n",
    "print(\"Probability of certainty was: \", pred_results[0][val_max])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Writing Your Own Stories and Questions\n",
    "\n",
    "Remember you can only use words from the existing vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'.',\n",
       " '?',\n",
       " 'Daniel',\n",
       " 'Is',\n",
       " 'John',\n",
       " 'Mary',\n",
       " 'Sandra',\n",
       " 'apple',\n",
       " 'back',\n",
       " 'bathroom',\n",
       " 'bedroom',\n",
       " 'discarded',\n",
       " 'down',\n",
       " 'dropped',\n",
       " 'football',\n",
       " 'garden',\n",
       " 'got',\n",
       " 'grabbed',\n",
       " 'hallway',\n",
       " 'in',\n",
       " 'journeyed',\n",
       " 'kitchen',\n",
       " 'left',\n",
       " 'milk',\n",
       " 'moved',\n",
       " 'no',\n",
       " 'office',\n",
       " 'picked',\n",
       " 'put',\n",
       " 'the',\n",
       " 'there',\n",
       " 'to',\n",
       " 'took',\n",
       " 'travelled',\n",
       " 'up',\n",
       " 'went',\n",
       " 'yes'}"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['John',\n",
       " 'left',\n",
       " 'the',\n",
       " 'kitchen',\n",
       " '.',\n",
       " 'Sandra',\n",
       " 'dropped',\n",
       " 'the',\n",
       " 'football',\n",
       " 'in',\n",
       " 'the',\n",
       " 'garden',\n",
       " '.']"
      ]
     },
     "execution_count": 80,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Note the whitespace of the periods\n",
    "my_story = \"John left the kitchen . Sandra dropped the football in the garden .\"\n",
    "my_story.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "my_question = \"Is the football in the garden ?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Is', 'the', 'football', 'in', 'the', 'garden', '?']"
      ]
     },
     "execution_count": 82,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "my_question.split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "mydata = [(my_story.split(),my_question.split(),'yes')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "my_story,my_ques,my_ans = vectorize_stories(mydata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pred_results = model.predict(([ my_story, my_ques]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted answer is:  yes\n",
      "Probability of certainty was:  0.97079676\n"
     ]
    }
   ],
   "source": [
    "#Generate prediction from model\n",
    "val_max = np.argmax(pred_results[0])\n",
    "\n",
    "for key, val in tokenizer.word_index.items():\n",
    "    if val == val_max:\n",
    "        k = key\n",
    "\n",
    "print(\"Predicted answer is: \", k)\n",
    "print(\"Probability of certainty was: \", pred_results[0][val_max])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Great Job!"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
