{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "___\n", "\n", " \n", "___\n", "# Question and Answer Chat Bots" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "----\n", "\n", "------" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Loading the Data\n", "\n", "We will be working with the Babi Data Set from Facebook Research.\n", "\n", "Full Details: https://research.fb.com/downloads/babi/\n", "\n", "- Jason Weston, Antoine Bordes, Sumit Chopra, Tomas Mikolov, Alexander M. Rush,\n", " \"Towards AI-Complete Question Answering: A Set of Prerequisite Toy Tasks\",\n", " http://arxiv.org/abs/1502.05698\n" ] }, { "cell_type": "code", "execution_count": 1, "metadata": { "collapsed": true }, "outputs": [], "source": [ "import pickle\n", "import numpy as np" ] }, { "cell_type": "code", "execution_count": 2, "metadata": { "collapsed": true }, "outputs": [], "source": [ "with open(\"train_qa.txt\", \"rb\") as fp: # Unpickling\n", " train_data = pickle.load(fp)" ] }, { "cell_type": "code", "execution_count": 3, "metadata": { "collapsed": true }, "outputs": [], "source": [ "with open(\"test_qa.txt\", \"rb\") as fp: # Unpickling\n", " test_data = pickle.load(fp)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "----" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Exploring the Format of the Data" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "list" ] }, "execution_count": 4, "metadata": {}, "output_type": "execute_result" } ], "source": [ "type(test_data)" ] }, { "cell_type": "code", "execution_count": 5, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "list" ] }, "execution_count": 5, "metadata": {}, "output_type": "execute_result" } ], "source": [ "type(train_data)" ] }, { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "1000" ] }, "execution_count": 6, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(test_data)" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "10000" ] }, "execution_count": 7, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(train_data)" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "(['Mary',\n", " 'moved',\n", " 'to',\n", " 'the',\n", " 'bathroom',\n", " '.',\n", " 'Sandra',\n", " 'journeyed',\n", " 'to',\n", " 'the',\n", " 'bedroom',\n", " '.'],\n", " ['Is', 'Sandra', 'in', 'the', 'hallway', '?'],\n", " 'no')" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "train_data[0]" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'Mary moved to the bathroom . Sandra journeyed to the bedroom .'" ] }, "execution_count": 9, "metadata": {}, "output_type": "execute_result" } ], "source": [ "' '.join(train_data[0][0])" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'Is Sandra in the hallway ?'" ] }, "execution_count": 10, "metadata": {}, "output_type": "execute_result" } ], "source": [ "' '.join(train_data[0][1])" ] }, { "cell_type": "code", "execution_count": 11, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "'no'" ] }, "execution_count": 11, "metadata": {}, "output_type": "execute_result" } ], "source": [ "train_data[0][2]" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "-----\n", "\n", "## Setting up Vocabulary of All Words" ] }, { "cell_type": "code", "execution_count": 12, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Create a set that holds the vocab words\n", "vocab = set()" ] }, { "cell_type": "code", "execution_count": 13, "metadata": { "collapsed": true }, "outputs": [], "source": [ "all_data = test_data + train_data" ] }, { "cell_type": "code", "execution_count": 14, "metadata": { "collapsed": true }, "outputs": [], "source": [ "for story, question , answer in all_data:\n", " # In case you don't know what a union of sets is:\n", " # https://www.programiz.com/python-programming/methods/set/union\n", " vocab = vocab.union(set(story))\n", " vocab = vocab.union(set(question))" ] }, { "cell_type": "code", "execution_count": 15, "metadata": { "collapsed": true }, "outputs": [], "source": [ "vocab.add('no')\n", "vocab.add('yes')" ] }, { "cell_type": "code", "execution_count": 16, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'.',\n", " '?',\n", " 'Daniel',\n", " 'Is',\n", " 'John',\n", " 'Mary',\n", " 'Sandra',\n", " 'apple',\n", " 'back',\n", " 'bathroom',\n", " 'bedroom',\n", " 'discarded',\n", " 'down',\n", " 'dropped',\n", " 'football',\n", " 'garden',\n", " 'got',\n", " 'grabbed',\n", " 'hallway',\n", " 'in',\n", " 'journeyed',\n", " 'kitchen',\n", " 'left',\n", " 'milk',\n", " 'moved',\n", " 'no',\n", " 'office',\n", " 'picked',\n", " 'put',\n", " 'the',\n", " 'there',\n", " 'to',\n", " 'took',\n", " 'travelled',\n", " 'up',\n", " 'went',\n", " 'yes'}" ] }, "execution_count": 16, "metadata": {}, "output_type": "execute_result" } ], "source": [ "vocab" ] }, { "cell_type": "code", "execution_count": 17, "metadata": { "collapsed": true }, "outputs": [], "source": [ "vocab_len = len(vocab) + 1 #we add an extra space to hold a 0 for Keras's pad_sequences" ] }, { "cell_type": "code", "execution_count": 18, "metadata": { "collapsed": true }, "outputs": [], "source": [ "max_story_len = max([len(data[0]) for data in all_data])" ] }, { "cell_type": "code", "execution_count": 19, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "156" ] }, "execution_count": 19, "metadata": {}, "output_type": "execute_result" } ], "source": [ "max_story_len" ] }, { "cell_type": "code", "execution_count": 20, "metadata": { "collapsed": true }, "outputs": [], "source": [ "max_question_len = max([len(data[1]) for data in all_data])" ] }, { "cell_type": "code", "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "6" ] }, "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], "source": [ "max_question_len" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Vectorizing the Data" ] }, { "cell_type": "code", "execution_count": 22, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'.',\n", " '?',\n", " 'Daniel',\n", " 'Is',\n", " 'John',\n", " 'Mary',\n", " 'Sandra',\n", " 'apple',\n", " 'back',\n", " 'bathroom',\n", " 'bedroom',\n", " 'discarded',\n", " 'down',\n", " 'dropped',\n", " 'football',\n", " 'garden',\n", " 'got',\n", " 'grabbed',\n", " 'hallway',\n", " 'in',\n", " 'journeyed',\n", " 'kitchen',\n", " 'left',\n", " 'milk',\n", " 'moved',\n", " 'no',\n", " 'office',\n", " 'picked',\n", " 'put',\n", " 'the',\n", " 'there',\n", " 'to',\n", " 'took',\n", " 'travelled',\n", " 'up',\n", " 'went',\n", " 'yes'}" ] }, "execution_count": 22, "metadata": {}, "output_type": "execute_result" } ], "source": [ "vocab" ] }, { "cell_type": "code", "execution_count": 23, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Reserve 0 for pad_sequences\n", "vocab_size = len(vocab) + 1" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "-----------" ] }, { "cell_type": "code", "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Using TensorFlow backend.\n" ] } ], "source": [ "from keras.preprocessing.sequence import pad_sequences\n", "from keras.preprocessing.text import Tokenizer" ] }, { "cell_type": "code", "execution_count": 25, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# integer encode sequences of words\n", "tokenizer = Tokenizer(filters=[])\n", "tokenizer.fit_on_texts(vocab)" ] }, { "cell_type": "code", "execution_count": 26, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'.': 13,\n", " '?': 12,\n", " 'apple': 3,\n", " 'back': 18,\n", " 'bathroom': 23,\n", " 'bedroom': 29,\n", " 'daniel': 21,\n", " 'discarded': 5,\n", " 'down': 22,\n", " 'dropped': 4,\n", " 'football': 37,\n", " 'garden': 36,\n", " 'got': 34,\n", " 'grabbed': 15,\n", " 'hallway': 27,\n", " 'in': 8,\n", " 'is': 30,\n", " 'john': 10,\n", " 'journeyed': 35,\n", " 'kitchen': 11,\n", " 'left': 31,\n", " 'mary': 25,\n", " 'milk': 7,\n", " 'moved': 28,\n", " 'no': 6,\n", " 'office': 24,\n", " 'picked': 1,\n", " 'put': 33,\n", " 'sandra': 2,\n", " 'the': 32,\n", " 'there': 16,\n", " 'to': 14,\n", " 'took': 9,\n", " 'travelled': 20,\n", " 'up': 17,\n", " 'went': 26,\n", " 'yes': 19}" ] }, "execution_count": 26, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tokenizer.word_index" ] }, { "cell_type": "code", "execution_count": 27, "metadata": { "collapsed": true }, "outputs": [], "source": [ "train_story_text = []\n", "train_question_text = []\n", "train_answers = []\n", "\n", "for story,question,answer in train_data:\n", " train_story_text.append(story)\n", " train_question_text.append(question)" ] }, { "cell_type": "code", "execution_count": 28, "metadata": { "collapsed": true }, "outputs": [], "source": [ "train_story_seq = tokenizer.texts_to_sequences(train_story_text)" ] }, { "cell_type": "code", "execution_count": 29, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "10000" ] }, "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(train_story_text)" ] }, { "cell_type": "code", "execution_count": 30, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "10000" ] }, "execution_count": 30, "metadata": {}, "output_type": "execute_result" } ], "source": [ "len(train_story_seq)" ] }, { "cell_type": "code", "execution_count": 31, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# word_index = tokenizer.word_index" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Functionalize Vectorization" ] }, { "cell_type": "code", "execution_count": 32, "metadata": { "collapsed": true }, "outputs": [], "source": [ "def vectorize_stories(data, word_index=tokenizer.word_index, max_story_len=max_story_len,max_question_len=max_question_len):\n", " '''\n", " INPUT: \n", " \n", " data: consisting of Stories,Queries,and Answers\n", " word_index: word index dictionary from tokenizer\n", " max_story_len: the length of the longest story (used for pad_sequences function)\n", " max_question_len: length of the longest question (used for pad_sequences function)\n", "\n", "\n", " OUTPUT:\n", " \n", " Vectorizes the stories,questions, and answers into padded sequences. We first loop for every story, query , and\n", " answer in the data. Then we convert the raw words to an word index value. Then we append each set to their appropriate\n", " output list. Then once we have converted the words to numbers, we pad the sequences so they are all of equal length.\n", " \n", " Returns this in the form of a tuple (X,Xq,Y) (padded based on max lengths)\n", " '''\n", " \n", " \n", " # X = STORIES\n", " X = []\n", " # Xq = QUERY/QUESTION\n", " Xq = []\n", " # Y = CORRECT ANSWER\n", " Y = []\n", " \n", " \n", " for story, query, answer in data:\n", " \n", " # Grab the word index for every word in story\n", " x = [word_index[word.lower()] for word in story]\n", " # Grab the word index for every word in query\n", " xq = [word_index[word.lower()] for word in query]\n", " \n", " # Grab the Answers (either Yes/No so we don't need to use list comprehension here)\n", " # Index 0 is reserved so we're going to use + 1\n", " y = np.zeros(len(word_index) + 1)\n", " \n", " # Now that y is all zeros and we know its just Yes/No , we can use numpy logic to create this assignment\n", " #\n", " y[word_index[answer]] = 1\n", " \n", " # Append each set of story,query, and answer to their respective holding lists\n", " X.append(x)\n", " Xq.append(xq)\n", " Y.append(y)\n", " \n", " # Finally, pad the sequences based on their max length so the RNN can be trained on uniformly long sequences.\n", " \n", " # RETURN TUPLE FOR UNPACKING\n", " return (pad_sequences(X, maxlen=max_story_len),pad_sequences(Xq, maxlen=max_question_len), np.array(Y))" ] }, { "cell_type": "code", "execution_count": 33, "metadata": { "collapsed": true }, "outputs": [], "source": [ "inputs_train, queries_train, answers_train = vectorize_stories(train_data)" ] }, { "cell_type": "code", "execution_count": 34, "metadata": { "collapsed": true }, "outputs": [], "source": [ "inputs_test, queries_test, answers_test = vectorize_stories(test_data)" ] }, { "cell_type": "code", "execution_count": 35, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[ 0, 0, 0, ..., 32, 29, 13],\n", " [ 0, 0, 0, ..., 32, 36, 13],\n", " [ 0, 0, 0, ..., 32, 36, 13],\n", " ...,\n", " [ 0, 0, 0, ..., 32, 3, 13],\n", " [ 0, 0, 0, ..., 32, 36, 13],\n", " [ 0, 0, 0, ..., 3, 16, 13]])" ] }, "execution_count": 35, "metadata": {}, "output_type": "execute_result" } ], "source": [ "inputs_test" ] }, { "cell_type": "code", "execution_count": 36, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[30, 10, 8, 32, 11, 12],\n", " [30, 10, 8, 32, 11, 12],\n", " [30, 10, 8, 32, 36, 12],\n", " ...,\n", " [30, 25, 8, 32, 29, 12],\n", " [30, 2, 8, 32, 36, 12],\n", " [30, 25, 8, 32, 36, 12]])" ] }, "execution_count": 36, "metadata": {}, "output_type": "execute_result" } ], "source": [ "queries_test" ] }, { "cell_type": "code", "execution_count": 37, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([[0., 0., 0., ..., 0., 0., 0.],\n", " [0., 0., 0., ..., 0., 0., 0.],\n", " [0., 0., 0., ..., 0., 0., 0.],\n", " ...,\n", " [0., 0., 0., ..., 0., 0., 0.],\n", " [0., 0., 0., ..., 0., 0., 0.],\n", " [0., 0., 0., ..., 0., 0., 0.]])" ] }, "execution_count": 37, "metadata": {}, "output_type": "execute_result" } ], "source": [ "answers_test" ] }, { "cell_type": "code", "execution_count": 38, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "array([ 0., 0., 0., 0., 0., 0., 503., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 497., 0., 0.,\n", " 0., 0., 0., 0., 0., 0., 0., 0., 0., 0., 0.,\n", " 0., 0., 0., 0., 0.])" ] }, "execution_count": 38, "metadata": {}, "output_type": "execute_result" } ], "source": [ "sum(answers_test)" ] }, { "cell_type": "code", "execution_count": 39, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "19" ] }, "execution_count": 39, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tokenizer.word_index['yes']" ] }, { "cell_type": "code", "execution_count": 40, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "6" ] }, "execution_count": 40, "metadata": {}, "output_type": "execute_result" } ], "source": [ "tokenizer.word_index['no']" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Creating the Model" ] }, { "cell_type": "code", "execution_count": 41, "metadata": { "collapsed": true }, "outputs": [], "source": [ "from keras.models import Sequential, Model\n", "from keras.layers.embeddings import Embedding\n", "from keras.layers import Input, Activation, Dense, Permute, Dropout\n", "from keras.layers import add, dot, concatenate\n", "from keras.layers import LSTM" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Placeholders for Inputs\n", "\n", "Recall we technically have two inputs, stories and questions. So we need to use placeholders. `Input()` is used to instantiate a Keras tensor.\n" ] }, { "cell_type": "code", "execution_count": 42, "metadata": { "collapsed": true }, "outputs": [], "source": [ "input_sequence = Input((max_story_len,))\n", "question = Input((max_question_len,))" ] }, { "cell_type": "markdown", "metadata": { "collapsed": true }, "source": [ "### Building the Networks\n", "\n", "To understand why we chose this setup, make sure to read the paper we are using:\n", "\n", "* Sainbayar Sukhbaatar, Arthur Szlam, Jason Weston, Rob Fergus,\n", " \"End-To-End Memory Networks\",\n", " http://arxiv.org/abs/1503.08895" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Encoders\n", "\n", "### Input Encoder m" ] }, { "cell_type": "code", "execution_count": 43, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Input gets embedded to a sequence of vectors\n", "input_encoder_m = Sequential()\n", "input_encoder_m.add(Embedding(input_dim=vocab_size,output_dim=64))\n", "input_encoder_m.add(Dropout(0.3))\n", "\n", "# This encoder will output:\n", "# (samples, story_maxlen, embedding_dim)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Input Encoder c" ] }, { "cell_type": "code", "execution_count": 44, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# embed the input into a sequence of vectors of size query_maxlen\n", "input_encoder_c = Sequential()\n", "input_encoder_c.add(Embedding(input_dim=vocab_size,output_dim=max_question_len))\n", "input_encoder_c.add(Dropout(0.3))\n", "# output: (samples, story_maxlen, query_maxlen)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Question Encoder" ] }, { "cell_type": "code", "execution_count": 45, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# embed the question into a sequence of vectors\n", "question_encoder = Sequential()\n", "question_encoder.add(Embedding(input_dim=vocab_size,\n", " output_dim=64,\n", " input_length=max_question_len))\n", "question_encoder.add(Dropout(0.3))\n", "# output: (samples, query_maxlen, embedding_dim)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Encode the Sequences" ] }, { "cell_type": "code", "execution_count": 46, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# encode input sequence and questions (which are indices)\n", "# to sequences of dense vectors\n", "input_encoded_m = input_encoder_m(input_sequence)\n", "input_encoded_c = input_encoder_c(input_sequence)\n", "question_encoded = question_encoder(question)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "##### Use dot product to compute the match between first input vector seq and the query" ] }, { "cell_type": "code", "execution_count": 47, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# shape: `(samples, story_maxlen, query_maxlen)`\n", "match = dot([input_encoded_m, question_encoded], axes=(2, 2))\n", "match = Activation('softmax')(match)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Add this match matrix with the second input vector sequence" ] }, { "cell_type": "code", "execution_count": 48, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# add the match matrix with the second input vector sequence\n", "response = add([match, input_encoded_c]) # (samples, story_maxlen, query_maxlen)\n", "response = Permute((2, 1))(response) # (samples, query_maxlen, story_maxlen)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "#### Concatenate" ] }, { "cell_type": "code", "execution_count": 49, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# concatenate the match matrix with the question vector sequence\n", "answer = concatenate([response, question_encoded])" ] }, { "cell_type": "code", "execution_count": 50, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "" ] }, "execution_count": 50, "metadata": {}, "output_type": "execute_result" } ], "source": [ "answer" ] }, { "cell_type": "code", "execution_count": 51, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Reduce with RNN (LSTM)\n", "answer = LSTM(32)(answer) # (samples, 32)" ] }, { "cell_type": "code", "execution_count": 52, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# Regularization with Dropout\n", "answer = Dropout(0.5)(answer)\n", "answer = Dense(vocab_size)(answer) # (samples, vocab_size)" ] }, { "cell_type": "code", "execution_count": 53, "metadata": { "collapsed": true }, "outputs": [], "source": [ "# we output a probability distribution over the vocabulary\n", "answer = Activation('softmax')(answer)\n", "\n", "# build the final model\n", "model = Model([input_sequence, question], answer)\n", "model.compile(optimizer='rmsprop', loss='categorical_crossentropy',\n", " metrics=['accuracy'])" ] }, { "cell_type": "code", "execution_count": 54, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "__________________________________________________________________________________________________\n", "Layer (type) Output Shape Param # Connected to \n", "==================================================================================================\n", "input_1 (InputLayer) (None, 156) 0 \n", "__________________________________________________________________________________________________\n", "input_2 (InputLayer) (None, 6) 0 \n", "__________________________________________________________________________________________________\n", "sequential_1 (Sequential) multiple 2432 input_1[0][0] \n", "__________________________________________________________________________________________________\n", "sequential_3 (Sequential) (None, 6, 64) 2432 input_2[0][0] \n", "__________________________________________________________________________________________________\n", "dot_1 (Dot) (None, 156, 6) 0 sequential_1[1][0] \n", " sequential_3[1][0] \n", "__________________________________________________________________________________________________\n", "activation_1 (Activation) (None, 156, 6) 0 dot_1[0][0] \n", "__________________________________________________________________________________________________\n", "sequential_2 (Sequential) multiple 228 input_1[0][0] \n", "__________________________________________________________________________________________________\n", "add_1 (Add) (None, 156, 6) 0 activation_1[0][0] \n", " sequential_2[1][0] \n", "__________________________________________________________________________________________________\n", "permute_1 (Permute) (None, 6, 156) 0 add_1[0][0] \n", "__________________________________________________________________________________________________\n", "concatenate_1 (Concatenate) (None, 6, 220) 0 permute_1[0][0] \n", " sequential_3[1][0] \n", "__________________________________________________________________________________________________\n", "lstm_1 (LSTM) (None, 32) 32384 concatenate_1[0][0] \n", "__________________________________________________________________________________________________\n", "dropout_4 (Dropout) (None, 32) 0 lstm_1[0][0] \n", "__________________________________________________________________________________________________\n", "dense_1 (Dense) (None, 38) 1254 dropout_4[0][0] \n", "__________________________________________________________________________________________________\n", "activation_2 (Activation) (None, 38) 0 dense_1[0][0] \n", "==================================================================================================\n", "Total params: 38,730\n", "Trainable params: 38,730\n", "Non-trainable params: 0\n", "__________________________________________________________________________________________________\n" ] } ], "source": [ "model.summary()" ] }, { "cell_type": "code", "execution_count": 55, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Train on 10000 samples, validate on 1000 samples\n", "Epoch 1/120\n", "10000/10000 [==============================] - 7s 701us/step - loss: 0.8846 - acc: 0.4966 - val_loss: 0.6938 - val_acc: 0.4970\n", "Epoch 2/120\n", "10000/10000 [==============================] - 4s 364us/step - loss: 0.7022 - acc: 0.4987 - val_loss: 0.6935 - val_acc: 0.5030\n", "Epoch 3/120\n", "10000/10000 [==============================] - 3s 347us/step - loss: 0.6958 - acc: 0.5042 - val_loss: 0.6937 - val_acc: 0.4970\n", "Epoch 4/120\n", "10000/10000 [==============================] - 3s 348us/step - loss: 0.6946 - acc: 0.5097 - val_loss: 0.6977 - val_acc: 0.4970\n", "Epoch 5/120\n", "10000/10000 [==============================] - 3s 345us/step - loss: 0.6943 - acc: 0.5073 - val_loss: 0.6932 - val_acc: 0.5030\n", "Epoch 6/120\n", "10000/10000 [==============================] - 3s 347us/step - loss: 0.6954 - acc: 0.4873 - val_loss: 0.6938 - val_acc: 0.4970\n", "Epoch 7/120\n", "10000/10000 [==============================] - 3s 345us/step - loss: 0.6946 - acc: 0.4970 - val_loss: 0.6953 - val_acc: 0.4970\n", "Epoch 8/120\n", "10000/10000 [==============================] - 3s 347us/step - loss: 0.6948 - acc: 0.4955 - val_loss: 0.6939 - val_acc: 0.4970\n", "Epoch 9/120\n", "10000/10000 [==============================] - 3s 346us/step - loss: 0.6944 - acc: 0.4937 - val_loss: 0.6933 - val_acc: 0.5030\n", "Epoch 10/120\n", "10000/10000 [==============================] - 4s 360us/step - loss: 0.6939 - acc: 0.5011 - val_loss: 0.6937 - val_acc: 0.4970\n", "Epoch 11/120\n", "10000/10000 [==============================] - 4s 365us/step - loss: 0.6941 - acc: 0.5051 - val_loss: 0.6934 - val_acc: 0.5030\n", "Epoch 12/120\n", "10000/10000 [==============================] - 4s 352us/step - loss: 0.6941 - acc: 0.5014 - val_loss: 0.6955 - val_acc: 0.4970\n", "Epoch 13/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.6936 - acc: 0.5104 - val_loss: 0.6943 - val_acc: 0.4940\n", "Epoch 14/120\n", "10000/10000 [==============================] - 3s 350us/step - loss: 0.6938 - acc: 0.5045 - val_loss: 0.6938 - val_acc: 0.4950\n", "Epoch 15/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.6914 - acc: 0.5216 - val_loss: 0.6944 - val_acc: 0.5030\n", "Epoch 16/120\n", "10000/10000 [==============================] - 4s 351us/step - loss: 0.6830 - acc: 0.5467 - val_loss: 0.6825 - val_acc: 0.5250\n", "Epoch 17/120\n", "10000/10000 [==============================] - 4s 356us/step - loss: 0.6663 - acc: 0.5840 - val_loss: 0.6656 - val_acc: 0.6020\n", "Epoch 18/120\n", "10000/10000 [==============================] - 4s 368us/step - loss: 0.6404 - acc: 0.6339 - val_loss: 0.6247 - val_acc: 0.6690\n", "Epoch 19/120\n", "10000/10000 [==============================] - 4s 364us/step - loss: 0.6049 - acc: 0.6829 - val_loss: 0.5708 - val_acc: 0.7210\n", "Epoch 20/120\n", "10000/10000 [==============================] - 4s 356us/step - loss: 0.5569 - acc: 0.7290 - val_loss: 0.5159 - val_acc: 0.7460\n", "Epoch 21/120\n", "10000/10000 [==============================] - 4s 358us/step - loss: 0.5180 - acc: 0.7549 - val_loss: 0.4775 - val_acc: 0.7870\n", "Epoch 22/120\n", "10000/10000 [==============================] - 4s 354us/step - loss: 0.4891 - acc: 0.7774 - val_loss: 0.4449 - val_acc: 0.7970\n", "Epoch 23/120\n", "10000/10000 [==============================] - 4s 354us/step - loss: 0.4528 - acc: 0.8020 - val_loss: 0.4142 - val_acc: 0.8190\n", "Epoch 24/120\n", "10000/10000 [==============================] - 4s 354us/step - loss: 0.4253 - acc: 0.8161 - val_loss: 0.4205 - val_acc: 0.8280\n", "Epoch 25/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.4009 - acc: 0.8354 - val_loss: 0.4094 - val_acc: 0.8280\n", "Epoch 26/120\n", "10000/10000 [==============================] - 3s 348us/step - loss: 0.3815 - acc: 0.8432 - val_loss: 0.3919 - val_acc: 0.8240\n", "Epoch 27/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.3653 - acc: 0.8496 - val_loss: 0.3926 - val_acc: 0.8450\n", "Epoch 28/120\n", "10000/10000 [==============================] - 4s 351us/step - loss: 0.3535 - acc: 0.8549 - val_loss: 0.3939 - val_acc: 0.8430\n", "Epoch 29/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.3435 - acc: 0.8581 - val_loss: 0.3716 - val_acc: 0.8320\n", "Epoch 30/120\n", "10000/10000 [==============================] - 4s 350us/step - loss: 0.3403 - acc: 0.8603 - val_loss: 0.3677 - val_acc: 0.8340\n", "Epoch 31/120\n", "10000/10000 [==============================] - 4s 351us/step - loss: 0.3302 - acc: 0.8570 - val_loss: 0.3681 - val_acc: 0.8430\n", "Epoch 32/120\n", "10000/10000 [==============================] - 3s 348us/step - loss: 0.3295 - acc: 0.8593 - val_loss: 0.3476 - val_acc: 0.8380\n", "Epoch 33/120\n", "10000/10000 [==============================] - 3s 347us/step - loss: 0.3239 - acc: 0.8628 - val_loss: 0.3521 - val_acc: 0.8430\n", "Epoch 34/120\n", "10000/10000 [==============================] - 4s 350us/step - loss: 0.3171 - acc: 0.8677 - val_loss: 0.3443 - val_acc: 0.8390\n", "Epoch 35/120\n", "10000/10000 [==============================] - 3s 347us/step - loss: 0.3168 - acc: 0.8629 - val_loss: 0.3507 - val_acc: 0.8340\n", "Epoch 36/120\n", "10000/10000 [==============================] - 4s 351us/step - loss: 0.3121 - acc: 0.8664 - val_loss: 0.3558 - val_acc: 0.8310\n", "Epoch 37/120\n", "10000/10000 [==============================] - 4s 351us/step - loss: 0.3107 - acc: 0.8662 - val_loss: 0.3411 - val_acc: 0.8430\n", "Epoch 38/120\n", "10000/10000 [==============================] - 4s 355us/step - loss: 0.3061 - acc: 0.8698 - val_loss: 0.3460 - val_acc: 0.8400\n", "Epoch 39/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.3065 - acc: 0.8671 - val_loss: 0.3493 - val_acc: 0.8400\n", "Epoch 40/120\n", "10000/10000 [==============================] - 4s 352us/step - loss: 0.3060 - acc: 0.8688 - val_loss: 0.3446 - val_acc: 0.8410\n", "Epoch 41/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.3000 - acc: 0.8696 - val_loss: 0.3542 - val_acc: 0.8450\n", "Epoch 42/120\n", "10000/10000 [==============================] - 4s 354us/step - loss: 0.3039 - acc: 0.8665 - val_loss: 0.3692 - val_acc: 0.8350\n", "Epoch 43/120\n", "10000/10000 [==============================] - 3s 348us/step - loss: 0.3015 - acc: 0.8695 - val_loss: 0.3513 - val_acc: 0.8400\n", "Epoch 44/120\n", "10000/10000 [==============================] - 3s 347us/step - loss: 0.2986 - acc: 0.8694 - val_loss: 0.3577 - val_acc: 0.8320\n", "Epoch 45/120\n", "10000/10000 [==============================] - 3s 346us/step - loss: 0.2952 - acc: 0.8730 - val_loss: 0.3496 - val_acc: 0.8400\n", "Epoch 46/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.2969 - acc: 0.8681 - val_loss: 0.3424 - val_acc: 0.8450\n", "Epoch 47/120\n", "10000/10000 [==============================] - 3s 347us/step - loss: 0.2923 - acc: 0.8721 - val_loss: 0.3549 - val_acc: 0.8280\n", "Epoch 48/120\n", "10000/10000 [==============================] - 3s 346us/step - loss: 0.2911 - acc: 0.8732 - val_loss: 0.4681 - val_acc: 0.8140\n", "Epoch 49/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.2917 - acc: 0.8703 - val_loss: 0.3502 - val_acc: 0.8390\n", "Epoch 50/120\n", "10000/10000 [==============================] - 3s 344us/step - loss: 0.2900 - acc: 0.8746 - val_loss: 0.3515 - val_acc: 0.8400\n", "Epoch 51/120\n", "10000/10000 [==============================] - 4s 352us/step - loss: 0.2855 - acc: 0.8757 - val_loss: 0.3499 - val_acc: 0.8360\n", "Epoch 52/120\n", "10000/10000 [==============================] - 4s 354us/step - loss: 0.2864 - acc: 0.8735 - val_loss: 0.3531 - val_acc: 0.8410\n", "Epoch 53/120\n", "10000/10000 [==============================] - 4s 356us/step - loss: 0.2864 - acc: 0.8772 - val_loss: 0.3905 - val_acc: 0.8270\n", "Epoch 54/120\n", "10000/10000 [==============================] - 3s 346us/step - loss: 0.2857 - acc: 0.8752 - val_loss: 0.3618 - val_acc: 0.8390\n", "Epoch 55/120\n", "10000/10000 [==============================] - 3s 346us/step - loss: 0.2819 - acc: 0.8742 - val_loss: 0.3501 - val_acc: 0.8380\n", "Epoch 56/120\n", "10000/10000 [==============================] - 3s 348us/step - loss: 0.2853 - acc: 0.8775 - val_loss: 0.3484 - val_acc: 0.8400\n", "Epoch 57/120\n", "10000/10000 [==============================] - 4s 355us/step - loss: 0.2767 - acc: 0.8804 - val_loss: 0.3463 - val_acc: 0.8410\n", "Epoch 58/120\n", "10000/10000 [==============================] - 4s 355us/step - loss: 0.2802 - acc: 0.8780 - val_loss: 0.3763 - val_acc: 0.8350\n", "Epoch 59/120\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "10000/10000 [==============================] - 3s 350us/step - loss: 0.2844 - acc: 0.8777 - val_loss: 0.3483 - val_acc: 0.8420\n", "Epoch 60/120\n", "10000/10000 [==============================] - 4s 350us/step - loss: 0.2759 - acc: 0.8828 - val_loss: 0.3819 - val_acc: 0.8340\n", "Epoch 61/120\n", "10000/10000 [==============================] - 4s 351us/step - loss: 0.2722 - acc: 0.8799 - val_loss: 0.3596 - val_acc: 0.8400\n", "Epoch 62/120\n", "10000/10000 [==============================] - 4s 366us/step - loss: 0.2704 - acc: 0.8845 - val_loss: 0.3751 - val_acc: 0.8400\n", "Epoch 63/120\n", "10000/10000 [==============================] - 4s 372us/step - loss: 0.2691 - acc: 0.8854 - val_loss: 0.3745 - val_acc: 0.8430\n", "Epoch 64/120\n", "10000/10000 [==============================] - 4s 356us/step - loss: 0.2698 - acc: 0.8865 - val_loss: 0.3562 - val_acc: 0.8400\n", "Epoch 65/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.2674 - acc: 0.8875 - val_loss: 0.3534 - val_acc: 0.8400\n", "Epoch 66/120\n", "10000/10000 [==============================] - 4s 356us/step - loss: 0.2622 - acc: 0.8888 - val_loss: 0.3763 - val_acc: 0.8390\n", "Epoch 67/120\n", "10000/10000 [==============================] - 3s 347us/step - loss: 0.2593 - acc: 0.8885 - val_loss: 0.3670 - val_acc: 0.8470\n", "Epoch 68/120\n", "10000/10000 [==============================] - 3s 347us/step - loss: 0.2567 - acc: 0.8937 - val_loss: 0.3699 - val_acc: 0.8560\n", "Epoch 69/120\n", "10000/10000 [==============================] - 4s 361us/step - loss: 0.2573 - acc: 0.8951 - val_loss: 0.3676 - val_acc: 0.8430\n", "Epoch 70/120\n", "10000/10000 [==============================] - 4s 364us/step - loss: 0.2489 - acc: 0.8962 - val_loss: 0.3564 - val_acc: 0.8510\n", "Epoch 71/120\n", "10000/10000 [==============================] - 4s 363us/step - loss: 0.2479 - acc: 0.8961 - val_loss: 0.3605 - val_acc: 0.8460\n", "Epoch 72/120\n", "10000/10000 [==============================] - 4s 353us/step - loss: 0.2406 - acc: 0.9026 - val_loss: 0.3605 - val_acc: 0.8560\n", "Epoch 73/120\n", "10000/10000 [==============================] - 4s 354us/step - loss: 0.2404 - acc: 0.9020 - val_loss: 0.3490 - val_acc: 0.8510\n", "Epoch 74/120\n", "10000/10000 [==============================] - 4s 358us/step - loss: 0.2374 - acc: 0.9045 - val_loss: 0.3400 - val_acc: 0.8470\n", "Epoch 75/120\n", "10000/10000 [==============================] - 4s 381us/step - loss: 0.2299 - acc: 0.9060 - val_loss: 0.3453 - val_acc: 0.8490\n", "Epoch 76/120\n", "10000/10000 [==============================] - 4s 352us/step - loss: 0.2301 - acc: 0.9046 - val_loss: 0.3372 - val_acc: 0.8490\n", "Epoch 77/120\n", "10000/10000 [==============================] - 4s 353us/step - loss: 0.2250 - acc: 0.9076 - val_loss: 0.3354 - val_acc: 0.8510\n", "Epoch 78/120\n", "10000/10000 [==============================] - 3s 346us/step - loss: 0.2147 - acc: 0.9087 - val_loss: 0.3416 - val_acc: 0.8490\n", "Epoch 79/120\n", "10000/10000 [==============================] - 4s 351us/step - loss: 0.2111 - acc: 0.9119 - val_loss: 0.3774 - val_acc: 0.8520\n", "Epoch 80/120\n", "10000/10000 [==============================] - 4s 351us/step - loss: 0.2148 - acc: 0.9139 - val_loss: 0.3209 - val_acc: 0.8650\n", "Epoch 81/120\n", "10000/10000 [==============================] - 3s 347us/step - loss: 0.2045 - acc: 0.9158 - val_loss: 0.3157 - val_acc: 0.8650\n", "Epoch 82/120\n", "10000/10000 [==============================] - 3s 345us/step - loss: 0.1916 - acc: 0.9194 - val_loss: 0.3012 - val_acc: 0.8700\n", "Epoch 83/120\n", "10000/10000 [==============================] - 3s 350us/step - loss: 0.1881 - acc: 0.9228 - val_loss: 0.2922 - val_acc: 0.8670\n", "Epoch 84/120\n", "10000/10000 [==============================] - 3s 348us/step - loss: 0.1783 - acc: 0.9266 - val_loss: 0.2849 - val_acc: 0.8770\n", "Epoch 85/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.1933 - acc: 0.9209 - val_loss: 0.3006 - val_acc: 0.8730\n", "Epoch 86/120\n", "10000/10000 [==============================] - 4s 352us/step - loss: 0.1824 - acc: 0.9279 - val_loss: 0.2729 - val_acc: 0.8810\n", "Epoch 87/120\n", "10000/10000 [==============================] - 3s 350us/step - loss: 0.1779 - acc: 0.9282 - val_loss: 0.2774 - val_acc: 0.8840\n", "Epoch 88/120\n", "10000/10000 [==============================] - 3s 345us/step - loss: 0.1710 - acc: 0.9303 - val_loss: 0.2758 - val_acc: 0.8810\n", "Epoch 89/120\n", "10000/10000 [==============================] - 3s 348us/step - loss: 0.1658 - acc: 0.9345 - val_loss: 0.2854 - val_acc: 0.8880\n", "Epoch 90/120\n", "10000/10000 [==============================] - 4s 358us/step - loss: 0.1637 - acc: 0.9347 - val_loss: 0.2634 - val_acc: 0.8930\n", "Epoch 91/120\n", "10000/10000 [==============================] - 4s 351us/step - loss: 0.1577 - acc: 0.9358 - val_loss: 0.2546 - val_acc: 0.8910\n", "Epoch 92/120\n", "10000/10000 [==============================] - 4s 354us/step - loss: 0.1611 - acc: 0.9387 - val_loss: 0.2445 - val_acc: 0.9080\n", "Epoch 93/120\n", "10000/10000 [==============================] - 4s 368us/step - loss: 0.1560 - acc: 0.9381 - val_loss: 0.2369 - val_acc: 0.9040\n", "Epoch 94/120\n", "10000/10000 [==============================] - 4s 352us/step - loss: 0.1470 - acc: 0.9409 - val_loss: 0.2764 - val_acc: 0.8950\n", "Epoch 95/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.1475 - acc: 0.9428 - val_loss: 0.2634 - val_acc: 0.8980\n", "Epoch 96/120\n", "10000/10000 [==============================] - 3s 350us/step - loss: 0.1418 - acc: 0.9454 - val_loss: 0.2367 - val_acc: 0.9070\n", "Epoch 97/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.1436 - acc: 0.9453 - val_loss: 0.2460 - val_acc: 0.9120\n", "Epoch 98/120\n", "10000/10000 [==============================] - 4s 350us/step - loss: 0.1434 - acc: 0.9430 - val_loss: 0.2593 - val_acc: 0.9130\n", "Epoch 99/120\n", "10000/10000 [==============================] - 3s 348us/step - loss: 0.1348 - acc: 0.9465 - val_loss: 0.2851 - val_acc: 0.9000\n", "Epoch 100/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.1406 - acc: 0.9431 - val_loss: 0.2609 - val_acc: 0.9040\n", "Epoch 101/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.1342 - acc: 0.9478 - val_loss: 0.2705 - val_acc: 0.9050\n", "Epoch 102/120\n", "10000/10000 [==============================] - 3s 346us/step - loss: 0.1352 - acc: 0.9475 - val_loss: 0.2505 - val_acc: 0.9010\n", "Epoch 103/120\n", "10000/10000 [==============================] - 3s 345us/step - loss: 0.1291 - acc: 0.9502 - val_loss: 0.2708 - val_acc: 0.9080\n", "Epoch 104/120\n", "10000/10000 [==============================] - 4s 351us/step - loss: 0.1250 - acc: 0.9523 - val_loss: 0.2634 - val_acc: 0.9120\n", "Epoch 105/120\n", "10000/10000 [==============================] - 3s 347us/step - loss: 0.1203 - acc: 0.9519 - val_loss: 0.2725 - val_acc: 0.9070\n", "Epoch 106/120\n", "10000/10000 [==============================] - 4s 350us/step - loss: 0.1187 - acc: 0.9540 - val_loss: 0.2557 - val_acc: 0.9170\n", "Epoch 107/120\n", "10000/10000 [==============================] - 3s 347us/step - loss: 0.1182 - acc: 0.9531 - val_loss: 0.2664 - val_acc: 0.9090\n", "Epoch 108/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.1181 - acc: 0.9530 - val_loss: 0.2334 - val_acc: 0.9130\n", "Epoch 109/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.1158 - acc: 0.9554 - val_loss: 0.2899 - val_acc: 0.9120\n", "Epoch 110/120\n", "10000/10000 [==============================] - 3s 350us/step - loss: 0.1167 - acc: 0.9567 - val_loss: 0.2754 - val_acc: 0.9090\n", "Epoch 111/120\n", "10000/10000 [==============================] - 4s 350us/step - loss: 0.1120 - acc: 0.9561 - val_loss: 0.2898 - val_acc: 0.9100\n", "Epoch 112/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.1118 - acc: 0.9588 - val_loss: 0.2541 - val_acc: 0.9140\n", "Epoch 113/120\n", "10000/10000 [==============================] - 3s 346us/step - loss: 0.1083 - acc: 0.9583 - val_loss: 0.2511 - val_acc: 0.9110\n", "Epoch 114/120\n", "10000/10000 [==============================] - 4s 359us/step - loss: 0.1131 - acc: 0.9560 - val_loss: 0.2496 - val_acc: 0.9180\n", "Epoch 115/120\n", "10000/10000 [==============================] - 4s 364us/step - loss: 0.1050 - acc: 0.9599 - val_loss: 0.3021 - val_acc: 0.9170\n", "Epoch 116/120\n", "10000/10000 [==============================] - 3s 342us/step - loss: 0.1038 - acc: 0.9619 - val_loss: 0.2673 - val_acc: 0.9160\n", "Epoch 117/120\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ "10000/10000 [==============================] - 4s 350us/step - loss: 0.1100 - acc: 0.9599 - val_loss: 0.2855 - val_acc: 0.9160\n", "Epoch 118/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.1012 - acc: 0.9610 - val_loss: 0.2887 - val_acc: 0.9200\n", "Epoch 119/120\n", "10000/10000 [==============================] - 4s 352us/step - loss: 0.1040 - acc: 0.9618 - val_loss: 0.2774 - val_acc: 0.9120\n", "Epoch 120/120\n", "10000/10000 [==============================] - 3s 349us/step - loss: 0.1010 - acc: 0.9615 - val_loss: 0.3139 - val_acc: 0.9120\n" ] } ], "source": [ "# train\n", "history = model.fit([inputs_train, queries_train], answers_train,batch_size=32,epochs=120,validation_data=([inputs_test, queries_test], answers_test))" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Saving the Model" ] }, { "cell_type": "code", "execution_count": 72, "metadata": { "collapsed": true }, "outputs": [], "source": [ "filename = 'chatbot_120_epochs.h5'\n", "model.save(filename)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Evaluating the Model\n", "\n", "### Plotting Out Training History" ] }, { "cell_type": "code", "execution_count": 57, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "dict_keys(['val_loss', 'val_acc', 'loss', 'acc'])\n" ] }, { "data": { "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEWCAYAAACJ0YulAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xd4VGX68PHvnUnvjUAIhN6LNBEL\nCigriIqIshZ27eiqu+ru2nbVdX23uPtz1bXi6rp2sSMqCoJdepUqhJpAII30PvO8fzxDmIQAA2Yy\nk+T+XFcuZs45c+Y+mXDueboYY1BKKaUAgvwdgFJKqcChSUEppVQdTQpKKaXqaFJQSilVR5OCUkqp\nOpoUlFJK1dGkoNoUEXlJRP7i5bE7ReQcX8ekVCDRpKCUUqqOJgWlWiARCfZ3DKp10qSgAo672uZO\nEflBRMpE5L8i0l5EPhWREhFZICIJHsdfKCIbRKRQRL4SkX4e+4aKyCr3694Cwhu81/kissb92kUi\nMtjLGCeJyGoRKRaRTBF5sMH+M9znK3Tvv9q9PUJE/iUiu0SkSES+c28bIyJZjfweznE/flBE3hWR\n10SkGLhaREaKyGL3e2SLyFMiEurx+gEi8rmIFIjIfhH5g4h0EJFyEUnyOG64iOSKSIg3165aN00K\nKlBNBcYDvYELgE+BPwDJ2L/b3wCISG/gTeB2oB0wF/hIRELdN8jZwKtAIvCO+7y4XzsMeBG4EUgC\nngPmiEiYF/GVAb8E4oFJwK9E5CL3edPd8T7pjmkIsMb9ukeA4cBp7pjuAlxe/k4mA++63/N1wAnc\n4f6dnAqcDdzsjiEGWAB8BnQEegILjTH7gK+AaR7nnQ7MMsbUeBmHasU0KahA9aQxZr8xZg/wLbDU\nGLPaGFMFfAAMdR/3c+ATY8zn7pvaI0AE9qY7CggBHjfG1Bhj3gWWe7zHDcBzxpilxhinMeZloMr9\nuqMyxnxljFlnjHEZY37AJqaz3LuvBBYYY950v2++MWaNiAQB1wK3GWP2uN9zkfuavLHYGDPb/Z4V\nxpiVxpglxphaY8xObFI7GMP5wD5jzL+MMZXGmBJjzFL3vpexiQARcQCXYxOnUpoUVMDa7/G4opHn\n0e7HHYFdB3cYY1xAJpDm3rfH1J/1cZfH4y7A79zVL4UiUgh0dr/uqETkFBH50l3tUgTchP3Gjvsc\n2xp5WTK2+qqxfd7IbBBDbxH5WET2uauU/uZFDAAfAv1FpDu2NFZkjFl2gjGpVkaTgmrp9mJv7gCI\niGBviHuAbCDNve2gdI/HmcBfjTHxHj+Rxpg3vXjfN4A5QGdjTBwwEzj4PplAj0ZekwdUHmFfGRDp\ncR0ObNWTp4ZTGj8LbAZ6GWNisdVrx4oBY0wl8Da2RPMLtJSgPGhSUC3d28AkETnb3VD6O2wV0CJg\nMVAL/EZEgkXkYmCkx2ufB25yf+sXEYlyNyDHePG+MUCBMaZSREYCV3jsex04R0Smud83SUSGuEsx\nLwKPikhHEXGIyKnuNowtQLj7/UOA+4BjtW3EAMVAqYj0BX7lse9joIOI3C4iYSISIyKneOx/Bbga\nuBB4zYvrVW2EJgXVohljfsTWjz+J/SZ+AXCBMabaGFMNXIy9+R3Atj+87/HaFdh2hafc+zPcx3rj\nZuAhESkBHsAmp4Pn3Q2ch01QBdhG5pPcu38PrMO2bRQA/wCCjDFF7nO+gC3llAH1eiM14vfYZFSC\nTXBvecRQgq0augDYB2wFxnrs/x7bwL3K3R6hFACii+wo1TaJyBfAG8aYF/wdiwocmhSUaoNE5GTg\nc2ybSIm/41GBQ6uPlGpjRORl7BiG2zUhqIa0pKCUUqqOlhSUUkrVaXGTaiUnJ5uuXbv6OwyllGpR\nVq5cmWeMaTj25TAtLil07dqVFStW+DsMpZRqUURk17GP0uojpZRSHjQpKKWUqqNJQSmlVJ0W16bQ\nmJqaGrKysqisrPR3KD4VHh5Op06dCAnRtVCUUr7RKpJCVlYWMTExdO3alfoTYrYexhjy8/PJysqi\nW7du/g5HKdVKtYrqo8rKSpKSklptQgAQEZKSklp9aUgp5V+tIikArTohHNQWrlEp5V+tovpIKaVa\nGqfLsHxnAev3FHFKtyQGpsUiIrhchj2FFezKLyfzQDmllbVEhjmIDgtmSOd4uiRF+TQuTQpNoLCw\nkDfeeIObb775uF533nnn8cYbbxAfH++jyJRS/lBWVcvO/DISIkNJjAolPMRRt6+ksoYnFm7lg9V7\nyCutrtveKSGC5Ogwtuwvobza2eh5/zploCaFlqCwsJBnnnnmsKTgdDpxOBxHeBXMnTvX16EppXyo\notrJ+r1FFJXX0C4mjPAQB++tyuLNZbspqaytO25I53guGd6J+MgQ/vLxJvaXVDJxYAcmDerI0PR4\nvsvIY976fZRV1zJtRGf6doihS1IUnRMjiI0IobzKSVl1LUlRoT6/Jk0KTeCee+5h27ZtDBkyhJCQ\nEKKjo0lNTWXNmjVs3LiRiy66iMzMTCorK7ntttuYMWMGcGjKjtLSUiZOnMgZZ5zBokWLSEtL48MP\nPyQiIsLPV6ZU25VZUA5Ax/gIHEGH2vNyS6p4f1UWc9buZfO+Epyu+jNNO4KECQM7MGFAB8qqasku\nquSz9fu4b/Z6APp2iGHmL4YzpPOhGoJpIzozbUTnI8YSG9583dBbXVL480cb2Li3uEnP2b9jLH+6\nYMAR9z/88MOsX7+eNWvW8NVXXzFp0iTWr19f13X0xRdfJDExkYqKCk4++WSmTp1KUlJSvXNs3bqV\nN998k+eff55p06bx3nvvMX369Ca9DqXU4XbnlzN/4z5qnIa4iBCKKmr4ZN1e1u+x95FQRxAd48MJ\nC3YgAhk5pdS6DMO7JHDLmB6c1Dme5OgwckuqOFBezWk9k0mLr/+F7vZzerFhbzE788s4d0AHQhyB\n28en1SWFQDBy5Mh6YwmeeOIJPvjgAwAyMzPZunXrYUmhW7duDBkyBIDhw4ezc+fOZotXqbamvLqW\n91ft4Z0VmazNKjps/0md47lvUj9iw0PYnldG1oFyap2GWpeLs3q349IRnemZEu31+4kIA9PiGJgW\n15SX4ROtLikc7Rt9c4mKOtQQ9NVXX7FgwQIWL15MZGQkY8aMaXSsQVhYWN1jh8NBRUVFs8SqVGtn\njOH9VXtYuiOfyNBgal0uPlqbTVFFDf1TY7l3Yl8mDU4lMSqU4opaggRSYsP9HbbftLqk4A8xMTGU\nlDS+qmFRUREJCQlERkayefNmlixZ0szRKdV6VNY4+WjtXpZsLyBIINghDO4Uz6TBqUSFBvPuykwe\nmb+FsOAg7p7Ql3F9U7hv9no+WL2HxKhQapwuqmpdjOuTwnWjuzGiS0K98T+RoXpL1N9AE0hKSuL0\n009n4MCBRERE0L59+7p9EyZMYObMmQwePJg+ffowatQoP0aqVMvjchnWZhXy2fp9vL0ikwPlNSRH\nhxHqECpqnLy5LJMH52wgNS6cnfnlDEuPp7zaya/fXE1kqIOKGie/Hd+bW8f2JChIB4AeS4tbo3nE\niBGm4SI7mzZtol+/fn6KqHm1pWtVbdu+okr+t2gHH6zaQ05JFY4g4Zx+KVx1WldO7W6ntTHG8ENW\nEe+uzGJjdjFXn9aV8wen4jLw/qos3l2Zxa/H9eKMXsn+vhy/E5GVxpgRxzpOSwpKKb9bl1XEzG+2\nsTOvjJSYMIIdQXz1Yw5Ol+Gcfu2ZOKgDY/ukEB9Zv5++iHBS53hO6lx/AKhD4NIRnbn0KN08VeM0\nKSilmk1eaRXlVU4qa51kHShnU3YJi7bl8X1GPjHhwQxLTyC3tIrC8hquPKUL153Rjc6Jkf4Ou03R\npKCU8rmCsmrun72eT9ZlH7avS1Ikd0/oy/RR6cQ04yAt1ThNCkopn6lxupi/YT9/mrOBoopqbh7T\ngx7togkLCaJ9bDh9OsQ062hddWyaFJRSTWbZjgJ25JVSWuVk6/4SPtuwj8LyGvp2iOGVa0fSv2Os\nv0NUx6BJQSl13L7dmsvz3+7gprO6c1oP27Nn1rLd3PP+urpjIkMdjO/fngsGd+SsPu0CemoHdYgm\nhSZwolNnAzz++OPMmDGDyEhtTFOBL6e4kr98sok5a/fiCBKWbM/nycuHEhocxB9nr+es3u3428WD\niA4NJjo8uN5Ecqpl0NTdBA5OnX0iHn/8ccrLy5s4IqWaVnl1LU8s3MqYR77isw37uOOc3iy+Zxz9\nU2P51Wsrufm1VfRpH8PTVw4jLT6CuMgQTQg/hcvlt7fWkkIT8Jw6e/z48aSkpPD2229TVVXFlClT\n+POf/0xZWRnTpk0jKysLp9PJ/fffz/79+9m7dy9jx44lOTmZL7/80t+XolQ9xhhmr9nDw59uZn9x\nFRMGdOCeiX3pmmzn93r9+lO4+fVV7Mgr43/XnEx0mN5SjsjlgvwMiO0IYUeYTK+mEr7+ByydCRc9\nAwOmNG+MtMak8Ok9sG/dsY87Hh0GwcSHj7jbc+rs+fPn8+6777Js2TKMMVx44YV888035Obm0rFj\nRz755BPAzokUFxfHo48+ypdffklyso64VIHD5TKs2HWAhz/dxKrdhZzUKY6nrxjGiK6J9Y6LCgvm\npWtOxmVoXSUDY2Du76HjMBh65U8/n8sF710HG963z+M6Q7s+kNwHEruBIxRcNbBkJuRvhej28OGt\nkDIA2vX+6e9/HFpfUvCz+fPnM3/+fIYOHQpAaWkpW7duZfTo0fz+97/n7rvv5vzzz2f06NF+jlSp\n+qprXSzalse8DftYsCmH3JIqkqND+b9LBjN1WKcjzhskIjj8nQ9qq6G6FCITj32sN9a8ActfAHHY\nm3aX0+z2mkqorYSIYyyhu/N7yNkIgy6BiARY8IBNCKNugcgEyP0RcjfDzu/s+Q6KS4fp70NKP5g5\nGt7+BdzwBYT6dglOT60vKRzlG31zMMZw7733cuONNx62b+XKlcydO5d7772Xn/3sZzzwwAN+iFCp\n+nbnl/PMVxnMXZdNcWUtUaEOxvRJYXz/9pzTv33gVglVlcLn98OOb6BgB4jAjd9C+/7Hd57C3bbK\npvdE6Hc+lBfY86aNgIoD8M41cOM3sOt7mHsnlOfZb/Ip/WD41dBvMgS5m2cri2D+/bDqZft84UPQ\nYyxs/BBOvgHO/auN8yCXE0pzwLjbEKJTwOEetzH1BXh1CrxwDsR0sNtG3gh9Jpzwr8wbAfpptyye\nU2efe+653H///Vx55ZVER0ezZ88eQkJCqK2tJTExkenTpxMdHc1LL71U77VafaSaU3FlDVv2lTB7\nzR5mLcvEESRMGpzKeQNTOaNXcr2F5puNMfVvmEdzYCfMutJ+G+9zHvSfbKtevnsMpj7v3TlcLljx\nX1jwoC1lrH4NzrobSrKhohB++W8bz/Nnw7OnQnk+pA6BU2+xbQO7F8M7V9sqoB5j7bY9q6CyEE77\nNfSfAouesAmhzySY+I/Dry/IAbGpjcfXYyxc8LiNq8o9Nb+rxrtr+wk0KTQBz6mzJ06cyBVXXMGp\np54KQHR0NK+99hoZGRnceeedBAUFERISwrPPPgvAjBkzmDhxIqmpqdrQrJpUVa2T7bllpCdGEhUW\nTFWtk3dXZvHidzvYllsGQHCQcNnIzvx6XC/a+3thmdk3Q+ZSOP8x6H7WkY/bswpemwrGCVe+Az3P\nsdtrKm0D7bg/QkLXo7+XywUfzIB170CPcTDx/+C7R22JAeDUW6HDQPv4gn/Dp3fC+Ids9Y/Dfdt0\nOWHjbPj2UVj5MiT3gl7j4ZQbIW24PWbay1CcDVHtbAI4XsOvtj/NyKdTZ4vIBODfgAN4wRjzcIP9\nXYAXgXZAATDdGJN1tHPq1Nlt51rV8Ssoq+a9lVks3Lyf1bsLqap14QgS+qXGkFdSzb7iSoZ0jmd8\n//b0S41hUFo87WLCjn1iXysvgEfcDaquGhh2FUx4GEIbjN+pKoWZp9ub+i9nQ1KPQ/uK9sC/T4Lh\nV8Gkf0FNhb3p799g6/BjO8LZf4KY9vD5n+D7x2HsH+HMO+03eGNg2X9g6+dw6Uv1ewgdqxRzPKUc\nP/H71Nki4gCeBsYDWcByEZljjNnocdgjwCvGmJdFZBzwd+AXvopJqdakutbFgx9tICOnlJSYMFzG\nsGBjDtVOF/1TY7nylC4M6hTLjtwyVuw6QGJUGI9cehKn90yqt9pYQNjwvk0G130Omz+G75+w2y98\nov5xnz8AB3bB1Z/UTwgAcWlw0mWw6lXodiYs+DMUbIOQKEjuCbsWweZPoP+FsOoVGHHdoYQA9t9T\nbrQ/DR3r9xVov8+fwJfVRyOBDGPMdgARmQVMBjyTQn/gDvfjL4HZPoxHqVbD6TLc8dYaPlmXzbD0\neDbsLaa8upYrTknnilPS6d0+xt8hHp+1b0FKf+h0MnQeCYj9Jt/3fOj9M3tMxkLbBnDqrdD19MbP\nc/rttg7+7V9CfBf4xQfQbYxtCM7dAnNutQmh9wSY+M9WdTNvKr5MCmlApsfzLOCUBsesBaZiq5im\nADEikmSMyfc8SERmADMA0tPTG30zY0zgfftpYi1tlTzVdIwx5JZUUVnjwuEQ/r1gC5+sy+a+Sf24\nfnR3f4f30xRsh6xlcM6Dh27SY/8AW+fDnF/bnj9b59mePMl9YNz9Rz5Xck8Yd59tOD7zzvpdOdv1\nhms+gx1fQ/qoQ20Dqh5f/lYau0M3vKv9HnhKRK4GvgH2ALWHvciY/wD/Adum0HB/eHg4+fn5JCUF\nYLG4iRhjyM/PJzzcz42Bqlnkl1axYtcBVu46wA9ZhWzeV0Jhef2eJ7ed3avlJwSAH94BBAZdemhb\ncBhMmQnPj4PHB4Kz2vb8uehZCDnG/4Ezf3/kfUFBtlePOiJfJoUswHMtvE7AXs8DjDF7gYsBRCQa\nmGqMKTreN+rUqRNZWVnk5ub+hHADX3h4OJ06dfJ3GMrHPlidxd3vraO61kWoI4j+HWOZODCVvh1i\niA4LxukyJEWHMq5vir9DPTEHxxdUHIB+F8APs6DrGRDX4G879STb2Lz5Y1tl1PMcre5pBr5MCsuB\nXiLSDVsCuAy4wvMAEUkGCowxLuBebE+k4xYSEkK3bt1+YrhK+UdljZPgICFIhMcWbOHJLzIY1T2R\nO8/tw8C0OMKC/TBmwFcKdsCsK+xo3ohE2PCB3T76d40fP/IG+6Oajc+SgjGmVkRuBeZhu6S+aIzZ\nICIPASuMMXOAMcDfRcRgq49u8VU8Svmby2XYU1hBRk4pW3NKWL+nmLVZhezKt7PkhjiEGqdh2ohO\n/OWiQYQGt7JJjLNWwutTbffN6e9Bt7Ng9xLIXlO/6kj5lU/HKfhCY+MUlAo0lTVOvtuax/a8Unbk\nlbFlfyk/7iuhtOpQk1mH2HCGdI6vW42srLqWXikxTB2W1vraxgq22+kaQqNtj6CG3UmVz/l9nIJS\nbcWq3QdYsbOAS4d3JiEqlG25pdzy+io277NTEyRGhdIzJZqpw9Lo0yGW3u2j6ZkSTXxkqJ8j9wFj\nbK+hb/4PcjbDiGtgyBV2SgrjspO9aUIIaJoUlDqGLftLmLd+H6d0T+Lkrgn1vsW/vTyTP85eR43T\n8PiCrZw3KJW567IJD3Hw7JXDOK1HMnGRbWRh+opCeP0SyFpuZ/vsMQYWP2Xn/3GEwVVzbJdRFdA0\nKag2q6rWSXZhJV2SIhutrlmbWchTX2bw+cb9ddu6t4tibJ8UYsKD2VtYwdsrshjdK5nbz+nN60t2\n8cHqPQxLj+eJy4eSGhfRnJfT/CqLISzm0BQRH94Ce1fbuYKGXGln+8zLgGXPQa+f2bEBKuBpm4Jq\n1fYUVrAuq4hN2cW4jGHS4FT6dojl6y25/OnD9ezMLyctPoKJAzswqnsSPVOiqXW5+Nf8LXy6fh9x\nESFcfVpXpp3cmUUZeby1PJP1e4uorLFTHV99Wlfum9SPYPei9IXl1cSGhxxx7YFWY+f38MqF9mY/\n6V+w7l3bzfTcv9lZRFXA8bZNQZOCahGcLsPHP+ylT4cY+naIrdteWeNkY3Yxm7NLyC6q4LxBqfRL\njaWq1smjn2/hP99sr5urLEgEp8uQFh/BnsIKuidHcfnIdBZvz+e7rXlUOw+tixsV6uCGM7tz/eju\nja4n4HQZapwu/0wx7W+11TDzDKgosGMOHCFQXQZ9J8G0V3QsQYDShmYVMKpqnQBH7G9fXes6rPul\ny2UwQJDAsh0FPPjRRjZlFxMWHMTDUwcxZWgn5m3Yx/2z15NTUlX3uie/yGBc3xT2FVWyMbuYy07u\nzOUj7VxAFTVOPvlhLws25XDFKelcP7obYcH25l9aVcuW/SVk7C+lsKKaKUM7HXX2UEeQ4DiRqZBb\nImMgb6udGlrEthPk/QiXv2W3fXyHXWtg8tOaEFoBLSmoE7ZlfwnpiZFH/LZc43Tx6uJdPLZgC44g\n4fKR6Vw8NI0f95fw7ZY8NmYXs7ugnKKKGsb1TeGOc3qTlhDBi9/t4OXFOympPNR9s2NcOHeM7827\nK7NYuqOAQWlxrNtTRL/UWG47uycDOsYRHRbMq0t28b/vdxAkwsNTBzO+f/tm+m20UtVldp2DjbPt\nWuUnX2/XQe95Nlz2+qHjWsDU0W2dVh8pn3G6DP8370dmfr2NfqmxPDd9OOlJkWQWlPPkF1vJKaki\nLDiIjJxStuWWMbpXMlGhwczfuA+X+88tJjyYIZ3j6ZIUSUSIg7dXZFFUUUNYcBDVThcTBnSgX2os\ntS5DQmQIl52cTkSogxqni7/N3cQbS3fz63E9ufGsHoQ46pcyqmtdBAl19fzqOBhjVzVzOaGqGOb8\nBvavh5Ovg21fHpqK+tZlh09LoQKaJgXlEwfKqvnNrNV8uzWP8wZ14PuMfIwxTBrckfdWZuEIEnq1\nj6aqxkV4qINbxvRgfP/2iAiZBeV89WMO/TvGcVKnuHo37ZLKGl5etJPsokquPq0rvY4x9XOt06U3\nfV9Y9jzM9ZhQLiwOLvmvXVHM5YRNc+xC9N3H+CtCdYI0KagT4nIZ9pdU0iE2vF43zVqnizeXZ/Lo\n/B8pq3Ly0OQBXDYynd355dz02ko2Zhdz8bA07jq3Lx3idCbXFqm2Gp4YYhelH3Wz3ZY+CuI7H/11\nqkXQhmZ1XDJySnlnZSZz1uwlu6iS03okcfeEvqQnRjJn7V5eXbKLjJxSRnVP5IHzB9RNzZCeFMkH\nt5xGTnEVnRMjj/EuKqCtexuK98AFT0Cvc/wdjfITTQpt3K78Mh5fsJXZa/bgEOHM3u24dERnXluy\ni8lPf183SVv/1FhmTh/OuQPaHzbQKyzYoQmhKRkDu76HtBHHXjtg72oIj4PEn7iugssJ3z0OHQbb\nRmTVZmlSaKOMMbzw7Q7+8dlmgh3CjDO7c8Po7iRH226YN4zuxsuLbA+gyUPS6koGqhmsewfev8FW\n4Uz4+5GPO7ATXpwIQQ6Y8hz0O//E33Pzx5C/FS75n/YiauO0TaENqnXaBd9fW7KbCQM68NDkAaTE\najtAQCjeC8+MgqoSCA6HOzZAZKLdt38jxHaEiHhbmnjzMtjxrR0rkL0GxvzBLkEZdJQG+JpK2LMC\nknpCTAe7LXstvHcDuGrh1uU2yahWR9sUVKMqa5zc/Poqvticw01n9eCuc/u0/ikZWgpj7JrEzhr4\n+esw63JY+hyMvddOK/HyBbYb6OVv2sVqtnwGP/sLnHwDfHQbfPU32PYFTH7K3vT3b7DPa92D+3I3\nw5Z5UF0CiG1EDo2GjM9tL6Opz2tCUJoU2pIap4tb37AJ4S8XDWT6qC7+Dkl5WvEiZCyA8x6BvudB\nn0mwdCYMngbvXgPx6VBTAS+MtwvStx8Ip9xkp5mYMtN2E513Lzx7OsSl2TUMPEUmwcApdr6i/Rtt\n99L8bTD2Pjhlhm2bUG2eVh+1EU6X4TezVttpHnq8Tc/kSOg/GbqPPXZjpi/t+MY2cJ79AHQc4t1r\nNn0EP7wFk5+B8FbS1pG91t7su54OV75nq4CyVsALZ9ubdW013LDQLmH51nTYsxKunQfpp9Q/T2kO\nfP4nKMuBvudDn/NsMgBbCtD2gjZLq49UPQ99tIFPfsjmn2dF0HPph5AdDGvftH3SZ3xl66o9GQPv\nXQ8l+2DqCxCbauu55/zafsuc9C/oNvrIb/jjZ/Dx7XaBlVG3QFRS/f2lOfDlX2HlS4eez/gKHEf5\nk3S54Ot/wNcP2+c9x8Pwq47r9wDYSdxCo7y/QTpr7DTRntdQWw3Ln4f+F9lv5Y2prYI1r8OSmRDV\nDkb/FnqMO/x9Kwrh7V9CVDJc/PyhNoFOI6DraNj5rW1Ibj/Abr/mUyjOarzHUXQKTHnWu+tSqhFa\nUmgD3luZxe/eWcv1Z3TjvsSFMP8++M1qyN0Cb/8CBk2Di56u/6KVL8NHvwFx2BvaxH/YG3LuZohJ\ntf3Zh18DP/t/dk59Ty6nbSwt2W+nSgiJhE7DAbE3yvwMKM8DCbLTLHcYbHvbHJx2uTQXFj4IJ11h\nvzmDTQjvXWsXej/pCshaBtEd4JpPDr/g4r22vj33RyjKsomp7yS7b82btv79rLvgTI+RuzmbbazJ\nvW0D7/YvbYkka4Wd2sFVC2PuhbPutse/P8P26089Ca6db0tbVaW2+ubALntM3hYoyYaOQ23SK94D\nKf3t7xPsyOB2fe217PjG3uw7j6x/LYWZtmF4wJTj/diVqkdLCgqA9XuK+MMH6xjVPZF7JvaFV++y\nN6bE7vbnlBth0VP239TB9kUHdsG8P9hvqRMehllXwDtXQXi8XU6x8yn2W/6SZ2D3Ytvw6fmtdd07\n9oY47RVI7mNX3srfZvcFBUOfifZm2H0MdBhoSyXr3oEv/waJPeCT39lvwpvnwk3f2sbVbx+xCeHs\nB+CM38I3j8CXf4HC3bau3dOHt8K2hXaOnrAY291ywMX2W/TSmXYVsCXP2AQUEgFlebaaprrUHWMI\nuGps42vX021CObADvvq7LTVgbELod6Gtl//sHhj/ELx+qV11rNMIQCB1CFz0jK2ic1bbktm6d90N\nv8ZWGW380D6e8I/DEwLY0cQ6olg1Iy0ptGKVNU7GP/Y1tU7DR78+g+TgSvhndzj1Vhj/Z3tQRSE8\nMdTOgPnLD+3N67WpsHcN3Lyhwo5hAAAeGElEQVTI3nDLC+zN9KTL6t/8t39tk4UxcMmLdtCTswae\nOhnComHGN0fvHumpYIctXdRWQmyaLYHMuQ3a9bHf6t/4uW1wnfKcrX45sBP+fZJNEqN/d+g8zhp4\nOB0G/xwmPQrGPSjrm3/aaxt5I/SZAK9OgQufhGG/tCWnxU/b52V5UJZrb+TdzoRg9zrKLhd8cseh\n6q5hV9kVxhY8CN8/DvFdbElg6n9hwEXef0g1FVBZdKh7qFI+oiUFxfcZeWQWVPD8L0fYQWkbPrXV\nIL3PPXRQRDyMuQc+vct+081caqtRLnzq0DfwyEQY+4fD36D7WXDDl7Yk8drF0PMcWwo5sAMun+V9\nQgBI7AYT/2kXfT//MfutXhw26bzxc1uyOP+xQ/XxCV0h/VRY+5YtORzcvnc11JRDj7Hu9w+Cs+60\nN+oDu+z0DcZA+0Gw5FnbE2fZCzDoUhg6/cjxBQXB+Y/bRtuS/TbhiMC4+20VU+ZSWzI6WE3lrZAI\n+6NUgNCk0IrN27CPmLBgzuydbDdsnW+rgDo1qKYYca39Brxnha0SGXix91MdJHaD6xfY/vSLn7Jd\nKtOGQ+8Jxx/w8KvqNxwPuAj23garXoGfv2obhz0NnmYXeMleY+vtAXZ+Z/9NP63+scm97A/Ym/mo\nX8GHN8OsK20J4mBbwdGI2JKJJ0cwTH/XthkkaBdf1fJp9VFrtOEDXMtfZMXOAmIiI+g38SZbp/6v\n3rZK5JIXD39NbbVt+D1a759jqS6zdeZdTjt0A24KzhrbF7+higPwSG8YcR1MdPdIem2qbZy9ddnR\nz1lTCY8PtFVFQ6fbVcOUasW8rT7SCelbm9wf4f0bqc7bAa4a0oNy4L3r4MVz7Q2w17mNvy449Kcl\nBLDf5Idf1bQJARpPCGB77/SdBGvesD1/nLWwe+mhHktHExJuG9eDI+DMu5o2XqVaME0KrYmzFj64\nEUKjeKrbM0x3/RluWQ4/+yvsW2dLAj1b2ZTIo26GqiLbs2ffD3YKhy5eJAWAM34Hd6zXah+lPGib\nQmvy3aOwdzXm0pd5f04NZ/ZKJioiDE671X6jLt5z+CCylq7TybYNY8mzh9ojup7h3WuDguyAMaVU\nHS0ptBYHdtnBZQMvYV3cGPYWVXLuAI9ujondvL9ZtiQitrRQsA0WPWnHOWj3TqVOmCaF1mLLZ7a7\n6bg/8tn6fTiChHP6tfd3VM2j/2SI6WjbTLxpT1BKHZEmhdYiYwEk9qAqtgtvr8jizF7JJESF+juq\n5uEIgZE32MddWmFpSKlmpG0KrUFNpe2fP3Q6H6/NJq+0imvP6ObvqJrXyBm26+pPWX1MKaVJoVXY\nvRhqyjE9xvHivB30SonmjJ5trAE1LBrGeDEATSl1VFp91BpsWwiOUFbKQDbsLeaa07shOm++UuoE\n+DQpiMgEEflRRDJE5J5G9qeLyJcislpEfhCR83wZT6uVsRDSR/HfZTnER4YwZegR5vdXSqlj8FlS\nEBEH8DQwEegPXC4i/Rscdh/wtjFmKHAZ8Iyv4mm1ivdCzkaKO53FvA37uHxkOhGhus6uUurE+LKk\nMBLIMMZsN8ZUA7OAyQ2OMcDB9RTjgL0+jKd1ylgIwBIZgsvARUO0lKCUOnG+TAppQKbH8yz3Nk8P\nAtNFJAuYC/y6sROJyAwRWSEiK3Jzc30Ra8u15TOI7sC83GQSo0Lp3T7a3xEppVowXyaFxlo6G07J\nejnwkjGmE3Ae8KqIHBaTMeY/xpgRxpgR7dq180GoLdSuRXZVsSFXsGRHAaO6J2oDs1LqJ/FlUsgC\nPNcR7MTh1UPXAW8DGGMWA+FAG+tLeYKcNfDxbyEunaxBt7CnsIJR3VvZvEZKqWbny6SwHOglIt1E\nJBTbkDynwTG7gbMBRKQfNilo/ZA3Fj8NuZvgvH+yOLMCgFO6aVJQSv00PksKxpha4FZgHrAJ28to\ng4g8JCIXug/7HXCDiKwF3gSuNi1t1R9/KN5rJ7/rMwn6TGTJ9gISo0LplaLtCUqpn8anI5qNMXOx\nDcie2x7weLwR0BnMjte2L+w6xOPuA2DJ9nxO6ZZIUJC2Jyilfhod0dwS5WdAUAgk9yazoFzbE5RS\nTUaTQkuUn2HXR3AEs2R7PoAmBaVUk9Ck0BLlZUBSTwBtT1BKNSlNCi2NywkF2yGpB8YYvs/IY1R3\nbU9QSjUNTQotTVEWOKsgqRfbckvZV1zJ6F46oE8p1TS8Sgoi8p6ITGpstLFqZvkZ9t+knnyzJQ+g\n7a2doJTyGW9v8s8CVwBbReRhEenrw5jU0XgkhW+35tI9OYrOiZH+jUkp1Wp4lRSMMQuMMVcCw4Cd\nwOciskhErhGREF8GqBrIz4CwWKrCk1iyvYDRvbSUoJRqOl5XB4lIEnA1cD2wGvg3Nkl87pPIVOPy\nMyCpByt3F1JR49T2BKVUk/JqRLOIvA/0BV4FLjDGZLt3vSUiK3wVnGpEXgakn8K3W/MIDhJG9dDx\nCUqppuPtNBdPGWO+aGyHMWZEE8ajjqamAooyIWk6367LZViXBKLDfDpTiVKqjfG2+qifiMQffCIi\nCSJys49iUkdSsAMwlER3Yf2eYkZrryOlVBPzNincYIwpPPjEGHMAuME3Iakjyt8KwOoyW2V0hjYy\nK6WamLdJIUg8lvQSEQcQ6puQ1BG5u6N+mx9HZKiDQWlxfg5IKdXaeFshPQ94W0RmYpfUvAn4zGdR\nqcblb4OYVL7LrGJYegLBDh1LqJRqWt7eVe4GvgB+BdwCLATu8lVQ6gjytlKb0IPN+4oZ0TXB39Eo\npVohr0oKxhgXdlTzs74NRx2Rswb2b2Bf14sxBkZ2TfR3REqpVsjbcQq9gL8D/bHrKANgjOnuo7hU\nQ9lroaaMlfQjOEgYkh5/7NcopdRx8rb66H/YUkItMBZ4BTuQTTWXnd8B8HFhVwakxREZquMTlFJN\nz9ukEGGMWQiIMWaXMeZBYJzvwlKH2fU9rqRefJ0dxEhtT1BK+Yi3SaHSPW32VhG5VUSmACk+jEt5\ncjlh9xLykkZQXetihLYnKKV8xNukcDsQCfwGGA5MB67yVVCqgX0/QFUxax0DADhZk4JSykeOWTHt\nHqg2zRhzJ1AKXOPzqFR9uxYBMLe4Bz1TokiM0nGDSinfOGZJwRjjBIZ7jmhWzWzn95DQjW/3hzKk\ns/Y6Ukr5jrddWFYDH4rIO0DZwY3GmPd9EpU6xOWC3Yuo7X0eeUur6JYc5e+IlFKtmLdJIRHIp36P\nIwNoUvC1nI1QcYDcxJMBdOlNpZRPeTuiWdsR/CVzKQAZEYOA/aRrUlBK+ZC3I5r/hy0Z1GOMubbJ\nI1L1Fe4CRyhbqhLQpKCU8jVvq48+9ngcDkwB9jZ9OOowRVkQm0bmgUqiw4JJiAzxd0RKqVbM2+qj\n9zyfi8ibwAKfRKTqK8yE+M7sLiinc2Ik2glMKeVLJzohfy8gvSkDUUdQlAVxNimkJ0b4OxqlVCvn\nbZtCCfXbFPZh11hQvuSsgZJsTGwamQXljO3Tzt8RKaVaOW+rj2JO5OQiMgH4N+AAXjDGPNxg/2PY\nWVfBTqORYozR0VkHFe8FDMVhHaiqdZGepGMUlFK+5VX1kYhMEZE4j+fxInLRMV7jAJ4GJmLXYbhc\nRPp7HmOMucMYM8QYMwR4Eh33UF9RFgDZJANozyOllM9526bwJ2NM0cEnxphC4E/HeM1IIMMYs90Y\nUw3MAiYf5fjLgTe9jKdtKMoEYGdtEqBJQSnle94mhcaOO1bVUxqQ6fE8y73tMCLSBeiGXQe6sf0z\nRGSFiKzIzc31ItxWwp0UtlTEIgJp8drQrJTyLW+TwgoReVREeohId3dbwMpjvKaxvpOHDYBzuwx4\n1z353uEvMuY/xpgRxpgR7dq1ocbWoiyITGZHkYuOcRGEBp9oZzGllPKOt3eZXwPVwFvA20AFcMsx\nXpMFdPZ43okjD3i7DK06OlxRFsR1co9R0FKCUsr3vO19VAbcc5znXg70EpFuwB7sjf+KhgeJSB8g\nAVh8nOdv/QozoV1vdmdod1SlVPPwtvfR5yIS7/E8QUTmHe01xpha4FZgHrAJeNsYs0FEHhKRCz0O\nvRyYZYw5UtVS22QMFGVRG51GbkkVXbQ7qlKqGXg791Gyu8cRAMaYAyJyzDWajTFzgbkNtj3Q4PmD\nXsbQtlQcgJoyCkLaAzpltlKqeXjbpuASkbppLUSkK0duNFZNQccoKKX8wNuSwh+B70Tka/fzM4EZ\nvglJAR5jFBIB6JygDc1KKd/zqqRgjPkMGAH8iO2B9DtsDyTlK+6SwtbKeCJDHSRGhfo5IKVUW+Dt\nhHjXA7dhu5WuAUZhewuNO9rr1E9QlAmOMLaUhNEpwalTZiulmoW3bQq3AScDu4wxY4GhQBsaWuwH\n7jEKWYWVdErQ9gSlVPPwNilUGmMqAUQkzBizGejju7BUXVI4UE4nbU9QSjUTbxuas9zjFGYDn4vI\nAXQ5Tt8qzKS629kUV9ZqUlBKNRtvRzRPcT98UES+BOKAz3wWVVvnrIXS/RSF2FHMWn2klGou3pYU\n6hhjvj72UeonKc8DDLnYJSy0pKCUai467WYgKt0PwN7aWEBLCkqp5qNJIRCV2o5du6qiiQp1kBAZ\n4ueAlFJthSaFQOQuKWwrj6RTQqSOUVBKNRtNCoGoLAeATSXh2p6glGpWmhQCUWkOhESRUWg0KSil\nmpUmhUBUmoMzKoWSylptZFZKNStNCoGoLIeqMDs7qpYUlFLNSZNCICrNoSQ4CdDuqEqp5qVJIRCV\n5lAgOnBNKdX8NCkEGmcNVBSw3xlLVKiDeB2joJRqRpoUAk2ZHbiWWROjYxSUUs1Ok0KgcQ9c21ER\npVVHSqlmp0kh0LinuPixLII0TQpKqWamSSHQuEsKu6qi6aw9j5RSzUyTQqBxT3GRZ+K0pKCUanaa\nFAJNaQ61wVFUEqZtCkqpZqdJIdCU5lAWageupcVrUlBKNS9NCoGmNIeioAQiQhwkRoX6OxqlVBuj\nSSHQlOWQ625P0DEKSqnmpkkh0JTmkF0bo+0JSim/CPZ3AMpDbRVUFrJLorU9QSnlF1pSCCQHp7io\njtHZUZVSfqFJIZC4B67lmTitPlJK+YUmhUDinuIiVweuKaX8xKdJQUQmiMiPIpIhIvcc4ZhpIrJR\nRDaIyBu+jCfgaUlBKeVnPmtoFhEH8DQwHsgClovIHGPMRo9jegH3AqcbYw6ISIqv4mkR3FNcFAUn\nkBwV5udglFJtkS9LCiOBDGPMdmNMNTALmNzgmBuAp40xBwCMMTk+jCfwleVRERRFSnwcQUE6RkEp\n1fx8mRTSgEyP51nubZ56A71F5HsRWSIiExo7kYjMEJEVIrIiNzfXR+EGgLI8ConV9gSllN/4Mik0\n9lXXNHgeDPQCxgCXAy+ISPxhLzLmP8aYEcaYEe3atWvyQANGeT55rmhtT1BK+Y0vk0IW0NnjeSdg\nbyPHfGiMqTHG7AB+xCaJNslVlsd+pw5cU0r5jy+TwnKgl4h0E5FQ4DJgToNjZgNjAUQkGVudtN2H\nMQU0Z2keB4wOXFNK+Y/PkoIxpha4FZgHbALeNsZsEJGHRORC92HzgHwR2Qh8CdxpjMn3VUwBzRiC\nKgsoIEbbFJRSfuPTuY+MMXOBuQ22PeDx2AC/df+0bdVlOJxVFJhYbVNQSvmNjmgOFOW2gFQsMaTE\nhPs5GKVUW6VJIVCU5wEQHJOCQ8coKKX8RJNCoCgvACA2qb2fA1FKtWWaFAJEdYkdlJeU0nB8n1JK\nNR9NCgGiIMcO4UhN1aSglPIfTQoBorhgHzXGQfdOHf0dilKqDdOkECAqC3M4QAxd20X5OxSlVBum\nSSFAuMryKHXEERbs8HcoSqk2TJNCgHBUHqA6NMHfYSil2jhNCgGgutZFZG0hEpXk71CUUm2cJoUA\nsCOvjASKCY1t2wvPKaX8T5NCANi6v5B4yohK0KSglPIvTQoBIGvPXoLEkJCc6u9QlFJtnCaFAJCz\n3w5cC4lpxavKKaVaBE0KAaAoz70gXaQ2NCul/EuTgp/VOF1UFtl5j4hK9m8wSqk2T5OCn/24r4Q4\nU2yfaElBKeVnmhT8bMn2fBIosU80KSil/EyTgp8t3VFAl4gKCI2B4DB/h6OUauM0KfiRy2VYvrOA\nnlGVEJno73CUUkqTgj9tySmhsLyGjqHl2sislAoImhT8aNkOuwRnIsXanqCUCgiaFPxo6fYCOsaF\nE1J9ACK1pKCU8j9NCn5ijGHpjgJO6Z6ElBdom4JSKiBoUvCT7Xll5JVWcWp6BNSUa/WRUiogaFLw\nk6XbbXvCabF5dkN8uh+jUUopS5OCn3y/LY92MWGk5S0CBLqP8XNESimlScEvSqtqWbhpPxMGdEC2\nLYTUk7RLqlIqIGhS8IPP1u+jssbFxQNiIHMZ9Dzb3yEppRSgScEvZq/eQ3piJENq1oBxQs9z/B2S\nUkoBmhTqlFbVUlZV65Nz/5BVyLqsIgD2F1fy/bY8LhrSEdn2hZ3zqNPJPnlfpZQ6XsH+DiAQbN5X\nzNUvLqeoooZJg1O5fGQ6w7skNMm5MwvKueL5pVTXunjqiqHsyi/HGJg8pCO8vhC6nwWOkCZ5L6WU\n+ql8WlIQkQki8qOIZIjIPY3sv1pEckVkjfvnel/G05gl2/O5dOZiDIYLT+rIp+uymfrsIpZszz+h\n82UdKKeoogawE97d+e5ajDH0TY3hV6+v4j/fbuekTnH0kGwoytT2BKVUQPFZSUFEHMDTwHggC1gu\nInOMMRsbHPqWMeZWX8XRqD2rICaVT3bCHW+tIT0pkpevHUlafAR/PK8Xv334CeauTmVU92MMKNu1\nGNr1gchEqmqdPP3lNr7+aj4VoUlcM/F0yqudLNlewD+mDmLS4I5c+9Jylu0o4OYxPSDjU3uOHpoU\nlFKBw5fVRyOBDGPMdgARmQVMBhomhea1dhbM/hXlIUk8WnoXgzoP5L9XjSA+MhScNcR+fBMvBM3m\nm/XzMRd+goSEN36e75+Az++HpJ7smDSLGz/cS9/cebwf+iwlEselH9zLVtOJcX1TmDaiMyLCy9eM\n5KMf9nJh7wj477OQ0h8SujTv9Sul1FH4svooDcj0eJ7l3tbQVBH5QUTeFZHOPowHVr2K+eAmdkYO\noqyqmg8i/sobF8bYhFBbDe9cDRtnk5U6njPNCkpengY1FYef55tHbELocTamZB+hr07ivOJ3+HfY\nszg6jyQuMoyPY/7O1d1LefjiQYgIABGhDqYNSyP8o19BSTZMfsqnl6uUUsfLlyUFaWSbafD8I+BN\nY0yViNwEvAyMO+xEIjOAGQDp6Sc2HcQPHz3F4JV/5BvnYG4quIObh4Rya+ZvkZd+Zmcora2AslyY\n8A8iB13LPX/7A3/PegEeG4AzOJL80iqCgoRwhxBdtQ8z6FJqLniGPz/3Onfn/YHb5VU7KvmyN5GS\nbMJeOp8HC+6CDfkw/GoIjbSBfPcobJ0P5z0CacNP6FqUUspXxJiG9+kmOrHIqcCDxphz3c/vBTDG\n/P0IxzuAAmNM3NHOO2LECLNixYrjjuebr+YhS55h5+n/5NwhXUiJCYf8bbDoSaitsgf1OgcGTgVg\n2nOLGVD0DX/qtZOlO/LZU1hBRKiDsion210deD/iEjokRLMms5D/TQhjrFkGZ9wBIRH2XAU74MNb\nYdd3ENUOOgyC3C1QnAWDLoWLnwdpLG8qpVTTE5GVxpgRxzzOh0khGNgCnA3sAZYDVxhjNngck2qM\nyXY/ngLcbYwZdbTznmhSOF7Pf7Odv87dxF8uGsh9s9dz+zm9uP2c3uwrquS7jDy+3JzD99vyuOrU\nrtwxvveRT7RrEXz3GJTuh3Z9of1AOPn6QyUHpZRqBn5PCu4gzgMeBxzAi8aYv4rIQ8AKY8wcEfk7\ncCFQCxQAvzLGbD7aOZsrKezIK2PsI18hAt2To5h722jCgh0+f1+llPIFb5OCTwevGWPmAnMbbHvA\n4/G9wL2+jOFEdUuOomdKNBk5pfz94sGaEJRSbYKOaD6Keyf2ZW9hBSO76apoSqm2QZPCUZzdr72/\nQ1BKqWalE+IppZSqo0lBKaVUHU0KSiml6mhSUEopVUeTglJKqTqaFJRSStXRpKCUUqqOJgWllFJ1\nfDr3kS+ISC6w6wRfngzkNWE4/tSargVa1/XotQSmtn4tXYwx7Y51UItLCj+FiKzwZkKolqA1XQu0\nruvRawlMei3e0eojpZRSdTQpKKWUqtPWksJ//B1AE2pN1wKt63r0WgKTXosX2lSbglJKqaNrayUF\npZRSR6FJQSmlVJ02kxREZIKI/CgiGSJyj7/jOR4i0llEvhSRTSKyQURuc29PFJHPRWSr+98Ef8fq\nLRFxiMhqEfnY/bybiCx1X8tbIhLq7xi9ISLxIvKuiGx2fz6nttTPRUTucP99rReRN0UkvCV9LiLy\noojkiMh6j22NfhZiPeG+H/wgIsP8F/nhjnAt/+f+O/tBRD4QkXiPffe6r+VHETn3p7x3m0gKIuIA\nngYmAv2By0Wkv3+jOi61wO+MMf2AUcAt7vjvARYaY3oBC93PW4rbgE0ez/8BPOa+lgPAdX6J6vj9\nG/jMGNMXOAl7TS3ucxGRNOA3wAhjzEDAAVxGy/pcXgImNNh2pM9iItDL/TMDeLaZYvTWSxx+LZ8D\nA40xg4EtuNe3d98LLgMGuF/zjPued0LaRFIARgIZxpjtxphqYBYw2c8xec0Yk22MWeV+XIK98aRh\nr+Fl92EvAxf5J8LjIyKdgEnAC+7nAowD3nUf0iKuRURigTOB/wIYY6qNMYW00M8FuzxvhIgEA5FA\nNi3oczHGfAMUNNh8pM9iMvCKsZYA8SKS2jyRHltj12KMmW+MqXU/XQJ0cj+eDMwyxlQZY3YAGdh7\n3glpK0khDcj0eJ7l3tbiiEhXYCiwFGhvjMkGmziAFP9FdlweB+4CXO7nSUChxx98S/l8ugO5wP/c\nVWEviEgULfBzMcbsAR4BdmOTQRGwkpb5uXg60mfR0u8J1wKfuh836bW0laQgjWxrcX1xRSQaeA+4\n3RhT7O94ToSInA/kGGNWem5u5NCW8PkEA8OAZ40xQ4EyWkBVUWPcde2TgW5ARyAKW8XSUEv4XLzR\nUv/mEJE/YquUXz+4qZHDTvha2kpSyAI6ezzvBOz1UywnRERCsAnhdWPM++7N+w8Wed3/5vgrvuNw\nOnChiOzEVuONw5Yc4t3VFtByPp8sIMsYs9T9/F1skmiJn8s5wA5jTK4xpgZ4HziNlvm5eDrSZ9Ei\n7wkichVwPnClOTTIrEmvpa0kheVAL3dPilBso8wcP8fkNXed+3+BTcaYRz12zQGucj++CviwuWM7\nXsaYe40xnYwxXbGfwxfGmCuBL4FL3Ie1lGvZB2SKSB/3prOBjbTAzwVbbTRKRCLdf28Hr6XFfS4N\nHOmzmAP80t0LaRRQdLCaKVCJyATgbuBCY0y5x645wGUiEiYi3bCN58tO+I2MMW3iBzgP22K/Dfij\nv+M5ztjPwBYHfwDWuH/Ow9bFLwS2uv9N9Hesx3ldY4CP3Y+7u/+QM4B3gDB/x+flNQwBVrg/m9lA\nQkv9XIA/A5uB9cCrQFhL+lyAN7HtITXYb8/XHemzwFa5PO2+H6zD9rry+zUc41oysG0HB+8BMz2O\n/6P7Wn4EJv6U99ZpLpRSStVpK9VHSimlvKBJQSmlVB1NCkoppepoUlBKKVVHk4JSSqk6mhSUakYi\nMubgzLBKBSJNCkoppepoUlCqESIyXUSWicgaEXnOvf5DqYj8S0RWichCEWnnPnaIiCzxmOf+4Jz9\nPUVkgYisdb+mh/v00R5rMLzuHkGsVEDQpKBUAyLSD/g5cLoxZgjgBK7EThK3yhgzDPga+JP7Ja8A\ndxs7z/06j+2vA08bY07CziN0cBqFocDt2LU9umPng1IqIAQf+xCl2pyzgeHAcveX+AjsRGou4C33\nMa8B74tIHBBvjPnavf1l4B0RiQHSjDEfABhjKgHc51tmjMlyP18DdAW+8/1lKXVsmhSUOpwALxtj\n7q23UeT+BscdbY6Yo1UJVXk8dqL/D1UA0eojpQ63ELhERFKgbp3fLtj/LwdnDL0C+M4YUwQcEJHR\n7u2/AL42dr2LLBG5yH2OMBGJbNarUOoE6DcUpRowxmwUkfuA+SIShJ2p8hbsIjoDRGQldmWyn7tf\nchUw033T3w5c497+C+A5EXnIfY5Lm/EylDohOkuqUl4SkVJjTLS/41DKl7T6SCmlVB0tKSillKqj\nJQWllFJ1NCkopZSqo0lBKaVUHU0KSiml6mhSUEopVef/AwswEZcCHcfCAAAAAElFTkSuQmCC\n", "text/plain": [ "
" ] }, "metadata": {}, "output_type": "display_data" } ], "source": [ "import matplotlib.pyplot as plt\n", "%matplotlib inline\n", "print(history.history.keys())\n", "# summarize history for accuracy\n", "plt.plot(history.history['acc'])\n", "plt.plot(history.history['val_acc'])\n", "plt.title('model accuracy')\n", "plt.ylabel('accuracy')\n", "plt.xlabel('epoch')\n", "plt.legend(['train', 'test'], loc='upper left')\n", "plt.show()" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "### Evaluating on Given Test Set" ] }, { "cell_type": "code", "execution_count": 73, "metadata": { "collapsed": true }, "outputs": [], "source": [ "model.load_weights(filename)\n", "pred_results = model.predict(([inputs_test, queries_test]))" ] }, { "cell_type": "code", "execution_count": 74, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['Mary',\n", " 'got',\n", " 'the',\n", " 'milk',\n", " 'there',\n", " '.',\n", " 'John',\n", " 'moved',\n", " 'to',\n", " 'the',\n", " 'bedroom',\n", " '.']" ] }, "execution_count": 74, "metadata": {}, "output_type": "execute_result" } ], "source": [ "test_data[0][0]" ] }, { "cell_type": "code", "execution_count": 75, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Mary got the milk there . John moved to the bedroom .\n" ] } ], "source": [ "story =' '.join(word for word in test_data[0][0])\n", "print(story)" ] }, { "cell_type": "code", "execution_count": 76, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Is John in the kitchen ?\n" ] } ], "source": [ "query = ' '.join(word for word in test_data[0][1])\n", "print(query)" ] }, { "cell_type": "code", "execution_count": 77, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "True Test Answer from Data is: no\n" ] } ], "source": [ "print(\"True Test Answer from Data is:\",test_data[0][2])" ] }, { "cell_type": "code", "execution_count": 78, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Predicted answer is: no\n", "Probability of certainty was: 0.9999999\n" ] } ], "source": [ "#Generate prediction from model\n", "val_max = np.argmax(pred_results[0])\n", "\n", "for key, val in tokenizer.word_index.items():\n", " if val == val_max:\n", " k = key\n", "\n", "print(\"Predicted answer is: \", k)\n", "print(\"Probability of certainty was: \", pred_results[0][val_max])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "## Writing Your Own Stories and Questions\n", "\n", "Remember you can only use words from the existing vocab" ] }, { "cell_type": "code", "execution_count": 79, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "{'.',\n", " '?',\n", " 'Daniel',\n", " 'Is',\n", " 'John',\n", " 'Mary',\n", " 'Sandra',\n", " 'apple',\n", " 'back',\n", " 'bathroom',\n", " 'bedroom',\n", " 'discarded',\n", " 'down',\n", " 'dropped',\n", " 'football',\n", " 'garden',\n", " 'got',\n", " 'grabbed',\n", " 'hallway',\n", " 'in',\n", " 'journeyed',\n", " 'kitchen',\n", " 'left',\n", " 'milk',\n", " 'moved',\n", " 'no',\n", " 'office',\n", " 'picked',\n", " 'put',\n", " 'the',\n", " 'there',\n", " 'to',\n", " 'took',\n", " 'travelled',\n", " 'up',\n", " 'went',\n", " 'yes'}" ] }, "execution_count": 79, "metadata": {}, "output_type": "execute_result" } ], "source": [ "vocab" ] }, { "cell_type": "code", "execution_count": 80, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['John',\n", " 'left',\n", " 'the',\n", " 'kitchen',\n", " '.',\n", " 'Sandra',\n", " 'dropped',\n", " 'the',\n", " 'football',\n", " 'in',\n", " 'the',\n", " 'garden',\n", " '.']" ] }, "execution_count": 80, "metadata": {}, "output_type": "execute_result" } ], "source": [ "# Note the whitespace of the periods\n", "my_story = \"John left the kitchen . Sandra dropped the football in the garden .\"\n", "my_story.split()" ] }, { "cell_type": "code", "execution_count": 81, "metadata": { "collapsed": true }, "outputs": [], "source": [ "my_question = \"Is the football in the garden ?\"" ] }, { "cell_type": "code", "execution_count": 82, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "['Is', 'the', 'football', 'in', 'the', 'garden', '?']" ] }, "execution_count": 82, "metadata": {}, "output_type": "execute_result" } ], "source": [ "my_question.split()" ] }, { "cell_type": "code", "execution_count": 83, "metadata": { "collapsed": true }, "outputs": [], "source": [ "mydata = [(my_story.split(),my_question.split(),'yes')]" ] }, { "cell_type": "code", "execution_count": 84, "metadata": { "collapsed": true }, "outputs": [], "source": [ "my_story,my_ques,my_ans = vectorize_stories(mydata)" ] }, { "cell_type": "code", "execution_count": 85, "metadata": { "collapsed": true }, "outputs": [], "source": [ "pred_results = model.predict(([ my_story, my_ques]))" ] }, { "cell_type": "code", "execution_count": 86, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "Predicted answer is: yes\n", "Probability of certainty was: 0.97079676\n" ] } ], "source": [ "#Generate prediction from model\n", "val_max = np.argmax(pred_results[0])\n", "\n", "for key, val in tokenizer.word_index.items():\n", " if val == val_max:\n", " k = key\n", "\n", "print(\"Predicted answer is: \", k)\n", "print(\"Probability of certainty was: \", pred_results[0][val_max])" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ "# Great Job!" ] } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.6" } }, "nbformat": 4, "nbformat_minor": 2 }