mirror of
https://github.com/saymrwulf/nvidia-AI.summerschool17-notebooks.git
synced 2026-11-21 09:54:43 +00:00
1261 lines
759 KiB
Text
1261 lines
759 KiB
Text
|
|
{
|
||
|
|
"metadata": {
|
||
|
|
"name": "20-intro-mlp"
|
||
|
|
},
|
||
|
|
"nbformat": 3,
|
||
|
|
"nbformat_minor": 0,
|
||
|
|
"worksheets": [
|
||
|
|
{
|
||
|
|
"cells": [
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Multilayer Perceptrons and Backpropagation\n",
|
||
|
|
"==========================================="
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"\n",
|
||
|
|
"import scipy,pylab,numpy\n",
|
||
|
|
"from numpy import *\n",
|
||
|
|
"from scipy import *\n",
|
||
|
|
"from pylab import *\n",
|
||
|
|
"from urllib import urlopen\n",
|
||
|
|
"from gzip import GzipFile\n",
|
||
|
|
"from scipy.spatial import distance\n",
|
||
|
|
"from scipy.spatial.distance import cdist"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 13
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "heading",
|
||
|
|
"level": 1,
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Nonlinear Classification Problem"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Let's start by generating a simple linear non-classification problem. Actually, this is a pretty simple problem as far as non-linear classification problems go: it's low dimensional, and the decision boundary is the combination of two axis aligned linear boundaries. Still, it's a good way to get started."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"data = random_sample((1000,2))\n",
|
||
|
|
"# labels = (data[:,0]**4 * 0.7+data[:,1]*0.4>0.3)\n",
|
||
|
|
"labels = ((data[:,0]-0.3) * (0.7-data[:,1]) > 0.0)\n",
|
||
|
|
"sum(labels)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 14,
|
||
|
|
"text": [
|
||
|
|
"607"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 14
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"def plotlabels():\n",
|
||
|
|
" clf()\n",
|
||
|
|
" d0 = data[labels==False]\n",
|
||
|
|
" d1 = data[labels]\n",
|
||
|
|
" plot(d0[:,0],d0[:,1],\"bo\")\n",
|
||
|
|
" plot(d1[:,0],d1[:,1],\"ro\")\n",
|
||
|
|
" savefig(\"tmp.png\")\n",
|
||
|
|
" print len(d0),len(d1)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 15
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"plotlabels()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"393 607\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD9CAYAAABHnDf0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXt0VNd9NvwIBpgxEmIsYQ9CYIxIbZBlqvmCI1Zbifhd\n1thV7hcJfJFsJJAsGRD5EqfWxaOAcbzwer0+BHJwY39pGvw2JO1q0kouyKvEI7dFttrwFXNJl63g\ngJBliCIbQUYgwf7+mDkz57L3Pvvsc85I2HrWYtk6s8++nXN+e+/f5fmlEUIIpjGNaUxjGp8ozJjs\nDkxjGtOYxjScx7Rwn8Y0pjGNTyCmhfs0pjGNaXwCMS3cpzGNaUzjE4hp4T6NaUxjGp9ATAv3aUxj\nGtP4BMJUuG/YsAG33norCgoKmGW2bNmCz3zmM1i1ahWOHj3qaAenMY1pTGMa1mEq3B977DEcPHiQ\n+ftrr72G9957D++++y7++q//Go8//rijHZzGNKYxjWlYh6lw/4u/+Av4/X7m7//0T/+EqqoqAMDn\nPvc5fPTRR/jwww+d6+E0pjGNaUzDMmzr3M+dO4fFixcn/s7NzcXAwIDdaqcxjWlMYxo24HGiEj2D\nQVpamqEM7do0pjGNaUzDHDIsMbZ37osWLcLZs2cTfw8MDGDRokXUskT3L1xSAkKIo//CJSWGdgiA\nsMvtN5eWUtttCYWMfQyHU9amlX7R/nV2RlBa2oySkjBKS5vR2Rmx1efS0mbqkwmFWhx7Fk73Wf2v\npIT2JsWu2353bbwXbvZLP7d5eU2aNvLymhydY9ZcGN8dYvndEa2DVy4/fxOAJt1vTQA2mc4H6zkl\nJVSEUrccbO/cv/SlL2Hv3r1Yt24dent7MX/+fNx6663Usj0AuuONTgD4cHTUbvMGXLh4kXr9Gu2a\n1yvdTk9XF7rb2+G5cgUTc+bgwrlz1HIzx8Ycqb90yxYUl5VpypRu2YLm/n7s7O9PXGvKy8P9mzcn\n/vZcuSLdr66uHmzdegj9/TsT1/r7mwEAZWXFmnLt7d24csWDOXMmsGVLqeZ3NbZsKUV/f7Omzry8\nJmzefL9pf0Qg2mdZzJkzQb3u9dLeMDqszFeq+iXap/b2bs3cAkB//05UVVXgxz92Zo5ZcOLdEa2D\nV66qqgPAS7qadwJYByA2H3v2tFLngvWckhKqO16XfZgK9/Xr1yMSieD3v/89Fi9ejO9973sYHx8H\nANTW1uIv//Iv8dprr2H58uWYO3cufvSjHzHrOqTr9rcGB9HT1WUQWrLo6erC2AcfoFnXToPfj7E5\nc4ChocQ1vRC02s6hrVs1QrXO50MPAP3jPDc6ipZQSCOkZepvjv9/cVkZXmxrQ2TvXvgmJvDR9etY\nt2wZ7ly8GNe8Xty/ebNmPifmzKG2IbKwsT5k9YtrVZgq1/bsacXY2EwMDPwKu3fvkBYKeqF04cII\n+vs7uH22A7sChjdfk9Wvrq4e1NT8AkNDLySuHTv2Lbz8svEZXrlCFxnDwyuwdeshAO4JeP274/Ve\nw+bN91tqT7QOXrmcnH/E8DCt9gkALwKox9jYTGr7tOcUCGwDcDEunhzRlMdAUgQAhFD+tYRCjrXR\nXFpKCEAiAGkBSDj+35pgkEQ6O0lLKETCJSWkJRQikc5OQgghkc5O0lxaSsIlJaS5tDRx3aydCECa\n4200x9ss9/k0Y3ssECDbAgHNtaa8PPL/fP/73PqrCwsNdStz1REOk1qPR1NnrcdDOsJhal2Rzk7S\nlJenKf9UXp7QOEtKwrRHRkpKkm2VljZTy4RCLab1E0LIr371K6FyNHR2RkheXpOmXa+30rTPdtHZ\nGSGhUAspKQmTUKiFdHZGhO/lzZedubDTr8LCemqfgsF64f4DLQSIkKysclJSEialpc1C7Xd2Rkhp\nabPhHrtzYVa/HfDnoJYAHdz3n/aclGt+fwWlXjkxPenCPVxSIiVgaQiXlDDboIEm+JoEBN+m/HzS\npGujCSAP3XabZgGpLyy0vKBFOjtJnddrqDsSH0d5Vha1zorsbG6dtIXNDCKCW2QBsAvWB0rvn73F\nxs3+EpKa+bIKv5++IPr9lYaytAUVeIoAHQTQXs/La+IKU1pdZvdYgVv1s+cgQgBCZsz4gnQb9Lpv\nEOGu3/F+fdkyg4Ct9fmYO1EelJ27qDC1Wl6BqIC1uthw+xTvV1VmJvX3qsxMq9PFRWdnhBQWVhOv\nt073cTyleXHt7txF+sH6QJOCMhIX6mECVJNZs2q4fXYTZgLF7fmSAX23SIjfv45avrMztkOPzXeL\nav6tjcvtuXCz/s7OCJk588u6OYjVP3fuQ7bqDoc7SFZWOcnMrIrPs5yYdlDBY47qQACBoSGNPnzT\nwABCV69qyu2LRlGxaxfuWr3akj5exMCohqyxMScnBzSl28KFCzV/y+i7WX064/WievNmdPznf1J/\nH5s1y3BNxChLQ1Iv/DJiZvBWeL1nsHJlOrZvr9DoJ63oemUMiTy9/5w5JN4/rTUnLa0cwWADMjIW\nSOll7cDMTuG2QVkGS5emY2SkGsBCJN0dBnH77enU8mVlxfjxj6GzHRymlmXpngG2/p53jxVYqd/q\nu1lWVoz58zswPNxm+M3nYxlNzdHV1YP9+89hePiA6urPpOpKqXD3LlyInSqjJgD89dWraIXRELki\nGsXre/ZYEu5K2dY9ezBzbIxqYFRD1tg4d+FC4J13DNfTc3I0f5stNjThy+pT+sqVKC4rw/EnnkDd\nzp3YN5F8gWo9HhQ3NGjKmxlledAKqGIAxRgbAxYsMBolRQ1Usl4s2g806W/19tvvYsuWYrz5Zgei\n0QOae65e/RkWLGjFwYNt3HFagejHbyZQnDAKOo0vfSmIY8eO4tq1ZxLXZs6swRe/WMi8Rz+O48dP\nUY2MPG8dJzyPeBCtX/bdfOKJEuzcWYeJiX2Jax5PLRoaks4GTmxmpGHr/GABUNQRNDUFQw3BU184\ngUhnJ9kQCGjURI8FAqY6aStGSp4hl6bv7wiHTevuCIdJRXY2qcrMJBXZ2VQVlqzKiRB39MKyR+Tk\nfRFC0+nedttDjvdVDyu626modjGDE32mzxFfHSZ7j6iBVLR+O+MPhztIdnYFycysItnZFSQc7uC0\nba7vp397N4BahrUrPTZjBnD9euLvJgD3A3jdhh+6KDIBPKP6e9Mf/oDjfX0oLitjqjWsnBDU5dXo\nbm/X7KoBYGd/P1p7exHavZtbd31bG+rb2rjjsuPf7saOSvYInlRjpEHv/9vfvxNZWRXU+5za/QFi\nLqHq/vb1PYGRkb2Ja35/AzZvpvdzMqHsLN96i04XYkU9InMisXqPHbfbwcFLGBwchNc7H+3t3Zrf\n7aiH2trq0dZWb7hu5Z1Rg+0Hbx0pFe4sNcVn7rkHFb/4BVZEo7iGmGA/aMMPXRTd7e14gaImqti1\nCwBwbv9+plqDJbRFwRO+dusG7Pm3u6EX5i0YvOOr8t9HHnkFIyPG+3NycjB/vrs67A8+uEy9Pjh4\nyXCtr+84Pv74PIBWADMBXMPHH19AX9/xSVW96KEVlC3UMlYXyLKyYqo6jqeaoN3DgozAVK5v3XoI\nw8MHMDwMnDihXRSm0mZmzZocHD6sVfXIIqXCnbfj7enqwuvx66+b6MqdAkvArohGEdm7Fwd0SsSd\n/f1otWgHYMGO8BUBayHNLSoyBFXpx+OGXpi1YBQV5ZruxsrKirF6dTe6u4315uSkY/Pm+1zVYQ8O\nDlKvf/DBB4Zre/dGcP261gB2/TrQ0bGOusOThd0oV62gLAV0oX9OLJBORwvLCszJMHLLLhhHjgxi\nYuJBqDcHskipcAfYagrebpWlHpH1BlHAFLAAfBP0hyNLJ6CHFc8e/Thz1qzB4JEj3HHTFtLcoiLu\naUQPQojmv3bAWjBEd2O8D1Bk92dHGAYC8zE8rI97bsKtt2Yayk5M+Kh1jI87p2J0QmhqBaVyTysy\nM8+iqGixIwukrGqCBVmBORlG
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 16
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Forward Step\n",
|
||
|
|
"==============\n",
|
||
|
|
"\n",
|
||
|
|
"The function we use to predict the output from the neural network is two \"layers\" of linear discriminant functions with intervening sigmoid non-linearities.\n",
|
||
|
|
"\n",
|
||
|
|
"\n",
|
||
|
|
"$ y = \\sigma(A x + a) $\n",
|
||
|
|
"\n",
|
||
|
|
"$ z = \\sigma(B y + b) $\n",
|
||
|
|
"\n",
|
||
|
|
"The sigmoid function (applied element-wise) is defined as:\n",
|
||
|
|
"\n",
|
||
|
|
"$\\sigma(x) = \\frac{1}{1+e^{-x}}$"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"def sigmoid(x):\n",
|
||
|
|
" return 1/(1+exp(-x))\n",
|
||
|
|
" \n",
|
||
|
|
"def forward(x):\n",
|
||
|
|
" global A,a,B,b\n",
|
||
|
|
" y = sigmoid(dot(A,x)+a)\n",
|
||
|
|
" z = sigmoid(dot(B,y)+b)\n",
|
||
|
|
" return z"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 17
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Let's apply this, using randomly generated initial weights.\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"A = random(size=(5,2))-0.5\n",
|
||
|
|
"a = random(size=5)-0.5\n",
|
||
|
|
"B = random(size=(2,5))-0.5\n",
|
||
|
|
"b = random(size=2)-0.5"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 18
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"As output, we take an indicator vector as the target output. For example, in a three class problem with output classes {0,1,2}, when the output class is 1, we require the MLP to output the vector [0,1,0]. This is also called a unary code. Therefore, our network returns two values.\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"x = array([0.3,0.7])\n",
|
||
|
|
"forward(x)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 19,
|
||
|
|
"text": [
|
||
|
|
"array([ 0.43903366, 0.45311104])"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 19
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"The vector of outputs can be viewed as the outputs from a set of discriminant functions, so we identify the class as the maximum.\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"argmax(forward(x))"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 20,
|
||
|
|
"text": [
|
||
|
|
"1"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 20
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"We can also treat the outputs as an approximation to a probability vector; in that case, we normalize the output.\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"z = forward(x)\n",
|
||
|
|
"z/sum(z)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 21,
|
||
|
|
"text": [
|
||
|
|
"array([ 0.49211037, 0.50788963])"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 21
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"We can plot either the posterior probabilities or the decision regions.\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"def plotgrid(f,dr=0):\n",
|
||
|
|
" result = zeros((100,100))\n",
|
||
|
|
" for i in range(100):\n",
|
||
|
|
" for j in range(100):\n",
|
||
|
|
" x = array([i/100.0,j/100.0])\n",
|
||
|
|
" z = f(x)\n",
|
||
|
|
" if dr==0:\n",
|
||
|
|
" z /= sum(z) \n",
|
||
|
|
" result[i,j] = z[1]\n",
|
||
|
|
" else:\n",
|
||
|
|
" result[i,j] = argmax(z)\n",
|
||
|
|
" imshow(result.T[::-1])\n",
|
||
|
|
"subplot(121); plotgrid(forward)\n",
|
||
|
|
"subplot(122); plotgrid(forward,dr=1)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAWwAAAC1CAYAAAB72cXqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztvWGsZUd1LvhV7XM9DxETBoa+PbhtNSFut7sN2I4d9HhR\npqHftSUe7TjYsRIi00JOMnpR9DBCI/FjJIxGstviBzYBaUYIRT0gYayZkeOHHE9oWY2SMTbj1xAQ\nMdM8cIvG6e73RKcDBuN7zt41P/auXatWrVV733v7nOvm1Ic295yzV1Xte7z72+t+9a0q45xzKCgo\nKCh41cNu9wUUFBQUFIxDIeyCgoKCSwSFsAsKCgouERTCLigoKLhEUAi7oKCg4BJBIeyCgoKCSwRb\nIuwnn3wSe/fuxdVXX40HH3zwYl1TQcG2otzXBa9WmM36sOu6xjXXXINjx47hiiuuwM0334wvfelL\nuPbaay/2NRYULAzlvi54NWPTGfY3vvEN/OZv/iZ2796NlZUV/OEf/iH++q//+mJeW0HBwlHu64JX\nMyabbfjiiy/iyiuv7N/v2rULzz77bBRjjNn8lRUUjMDFLtQt93XBqwXSvb1pwh5703783wP3/Qe0\nufwEwEp7NBOgXgFmKwazlQrTlQkaVKhRYYYJZqgwwwpmmGCKSffZCmpUaGAx7c7Rw8fVmOD/uO//\nw233vY18Hvry49Soon6mXduaXIc0zgwTNLDi+C/edxT/3X3/Y3/e9znFCmauQtNUqKcVptMVzKYT\n1N2B6QSYWaA2wBThWCevawAzxOfp54/fB/zb+0IbLX7WnePjbCSejv3yfQDua1/3cF3DdfJLrHef\n0c995zTuFdL5lMXOSNz/POoe3AjG3tf/A4ADF330cTi+TWMv27gXc+zvYj+O4d/i53gtplgZ0eIT\n4qeblkSuuOIKnD59un9/+vRp7Nq1Kw10ABr2ugZMA1QNYBsH2zSomgbGORg4WDTRUZHXBq678Lo7\n0jiDBkDoq2IxpjvHx6rYGNo4tvuF2va12Hc4T9uFfo1xsFUNUzUw7RcBWBf+q/ijIofpDv/ex4Cc\nsywe5DxvA9ZGiq+UeH9k+c0/pScDv4R0MdWIC7n4GH1fFxRsALtxCrfjMVyBF7fUz6YJ+6abbsL3\nv/99nDp1Cuvr6/jyl7+M2267LQ3sCLrj0P4wriVtWwO2drB1A+somcaEx0lbiglx4Vw4H5MuJW3e\nR3oufYjQuPgzfs7/rGFNA2saGOtgjIMxgLENrPVPLxdIkBIwPzivVaxNrl3F3o8Zh3/GSZ4ePQz5\nyS+YPo2kzqXP+WfzcaSOvq8LCjaA1+LnuAIv4l/hl1vqZ9OSyGQywWc+8xnceuutqOsa99xzjziT\nfuBGtGRNCRvte1MD1nbJZdMSV5sd++zK9QQYSNfCokENCwiE6eP2HXiDQKihfTqOHFfDKmP4PuLz\nrzvwjr59011j+6r7xIajri2scXC2zbJd3bRfiDEpz/kHH+VBzl9XH4jP8Xa8jT/P430bbRzD+lg5\nELfpwVN238AH0vfa06gWPvMXevEx9r7ePZfRx2G7xl62cbd7bAmbtvWN6twYuGfQ69bR0f2l7Cqg\nngDTTsueVRNMrdeNq04vljVlr017zTno37Fu3WrVQZ8OMbE+3VK1rl1rujXXvbW+e12+mWA2m2A2\n7Y7ZBE1t4aYTuOkEWLfA1AT92B9D2vI6O6e1o1r1+oh4aRwtvkZ4KPcaGNWnfUevKB3yX2Id6S/o\nP/+fLvqk4xgYY/DxhY9acKljhgn+T9yB72HviOhPiPf2/Csd/b9Z6SDySKtlO1jnM9y6lyI0aYNq\nzuGQJRQLSf6I9emchJIbh0soXNs2fUz7hRgTsmzjD+N1IqdLHTkVgUu9Q1KHpHfnJBI+Tk5S6SFd\nuCaPSDIJH4xffEHBpQMDh7fiB/gN/BB0nmwjWBxh1+wgf0Ib57XsppVGBKIEqF5NybROyNb0JJ2b\nLExllNw4hvwJTsfhbbX+Qfo2xrWEXXWHJX/eG9dp2U7mudykIj/v/+tWGCZzes73NzQO7QtCbNKR\nNOnIB9E0bT5jWgi74NJChRo34Tn8Fv7Tq5iwG/KT6qMdaZs6xBjiGKlccFZw4m4vXHeQAJ40dWI2\ncKii9hsfR9e2/fnQNmlvAGsbVF7T7t0ixCnCeUpzcVBylrJwHy+ROUg7St4ScW9knATSnwDaE0Z7\nKlkE10kh7ILlw+IybJppM2nENEENaI9UVpCcF3LGrckf4xwimxlHejDEDpHwsAgulpa0jXWoukzb\nyyOwjW6uyEkdUlJakfiNOE84p25mnB70hPSLaHoP75hfVEHBcmH+dz3JoBM9u8u0vSc7TiBToqWZ\nbhvD9eiUdMGIX/JaA9jUOMETnurhlSK/0AeRgYOxnTxi65BlV/4phpTHcspCTu6gJCqpDLzdVsaJ\nID0ttOyZD+o/myC9mELYBZcuaL3GRrAYScRLIPT6GHmbGpEnWyuiiaWNpvslYskhJlZJvkiLYdqY\nzYwT28u4DVHSvi2a1pPtJxv9eeNCll11Oja38UkSiaQsaPY/LhdLRLzRcST5hn0r6ZND0rA14Zxe\nbNGwCy5t7MKP8X78X7gKP9pw28Vl2NyLzTNtX+zX6di2cQJp80lETbqIqx6HJBAqYcgPB32cVGYJ\nxCy7WMg4vJDGtgd8lm29RoT4kHhOItmcVDzGDSJl6LxPyV3CHxqA0JgPIGXhki6jPhUKCi4JvA4/\nxT78I16PCxtuu5gMm0820mybnDM1UPksu2lgnT65RzNdDy3OW+/CL+2ymbLZ4jhahWaa7dd9pl1V\nbbZt4GCt67LsRpYtuEQiqQ2SqWIjcgdYO8ktoikbPDPPNuD6jO8kN9Mpai8FBb/ymD9hO8RWPsWL\nDS/b+nJ1n2lnLXNct051Y4MGXOqQtG0vk7TxYOMNj5M/H48TZdrek121Orax3ZdiXMiyh7iLk+PQ\nZKXk7vDtNG2bPyQ0OYRn4lHnIMET5H8hCJ/zX6Kg4NLFjTiBf42v4zKsj26zmAxbmWyMCNsX0Xhp\npMuyKxcXnqQErK0RwjPctEBGzoQl+UR3rKS6d530wceJZBpDJh57t0gnjVBPtiZ3SPKwJGloagRv\nkyNh6bPcOCJpayn9kP4i/RIFBZcursKP8Bv4Iap0TQcVi8mwgZSomX7t0SeWvvIx8mTHHmjub441\nYxASlSSQVG7RZBLD2uvjDPnCKWnHs8Te3ud92XQyUiRRis1OBvJ5P23S0bfTyFwbR+XWjdhRLNKn\nRUHBcmJxGjYQyyMaefu6EZ9ld6v4UdcHJ8M0w03JcqjIxcdpJeljxgl9pPp2eg01k266CsiqCcuu\nek+2d4twjpMyYGkCMadt5yYetXEgxIN8rsrMnMH9e18Ik9OwpV+koGC5MH/C1uQPrmVT8u6XX/WO\nEU/asmyhySSyLCGvXx3kjVyxDS17Hz+OBZNBeP+mm3AkThHbreDXVz9qGrQmW3BlQZp01D6T2mlS\nSE6KGcyux7hC6CD8YgsKLm28Bi/jSpzG5fjZqPjFZdg00wbSbJt5sj1pUy1bIlIqW3gS5tIGJcu0\nffBkm+7CcuOEsVKZhY4jySvBzy2cT2SQNtsGz7I3kjFr8glXG4CUCyX5RJOZc5Oioo4tDeQ/m7AG\nuT8ZCgoubaziHA7hP+I38MNR8YvLsMf6sf0aI2RZDYDKFalkQTVpnmVLmrNm26PZu0ba6aRkvoqS\ne7Lp+WQcsla2Nd210upHusEBIBOmlukOyR0YaAekPKslxJJMo1r8tMxa07bFtL2g4JLEBDP8Gl4a\n7RRZfIbNJxoVecR0Jet+fRH0RTR5jThXIBPkDAcqbXDSphsaULlF2vVGXkckLo3XCmmSjRO8J9s2\nvUvEtGvPtke1gR1pgJTQh+QOCO00YuZErxG2KI0M2Uu4XiPJI4W0C5YP8ydsaXlVnml7EDKPPNl1\nQyoftcy3zb49ZMtd8GDTOFlz
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 22
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "heading",
|
||
|
|
"level": 1,
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Backward Step"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"We start off with the total squared error.\n",
|
||
|
|
"\n",
|
||
|
|
"$$ e = \\sum_i (x_i - t_i)^2$$\n",
|
||
|
|
"\n",
|
||
|
|
"We want to minimize that by changing the parameters $\\theta = (A,a,B,b)$, so we\n",
|
||
|
|
"look for places where the partial derivatives vanish."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"(Here, I use a shorthand of $\\nabla_\\theta z$ to stand for\n",
|
||
|
|
"the collection of partial derivatives with respect to the components of $\\theta$.\n",
|
||
|
|
"This can be ambiguous at times, so if in doubt, work it out in components.)"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"$$ \\nabla_\\theta e = \\sum_i 2(z_i-t_i) \\frac{\\partial z_i}{\\partial \\theta}$$\n",
|
||
|
|
"\n",
|
||
|
|
"Now let's look at these components one-by-one.\n",
|
||
|
|
"\n",
|
||
|
|
"$$ \\nabla_\\theta z_i = (\\nabla_A z_i,\\nabla_a z_i,\\nabla_B z_i,\\nabla_b z_i)$$"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"(second layer)\n",
|
||
|
|
"\n",
|
||
|
|
"$$ \\nabla_B e = \\sum 2(z-t) \\nabla_B z$$\n",
|
||
|
|
"$$ ~~~~~ = \\sum 2(z-t) \\sigma'(By+b) y^T$$\n",
|
||
|
|
"$$ ~~~~~ = \\sum 2(z-t) z(1-z) y^T$$\n",
|
||
|
|
"\n",
|
||
|
|
"$$ \\nabla_b e = \\sum 2(z-t) \\nabla_b z$$\n",
|
||
|
|
"$$ ~~~~~ = \\sum 2(z-t) \\sigma'(By+b)$$\n",
|
||
|
|
"$$ ~~~~~ = \\sum 2(z-t) z(1-z)$$"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"(first layer)\n",
|
||
|
|
"\n",
|
||
|
|
"$$\\nabla_A e = \\sum 2(z-t)\\nabla_A z$$\n",
|
||
|
|
"$$ ~~~~~ = \\sum 2(z-t)\\sigma'(By+b) B \\nabla_A y$$\n",
|
||
|
|
"$$ ~~~~~ = \\sum 2(z-t) z(1-z) B \\sigma'(Ax+a) x^T$$\n",
|
||
|
|
"$$ ~~~~~ = \\sum 2(z-t) z(1-z) B y(1-y) x^T$$\n",
|
||
|
|
"\n",
|
||
|
|
"$$\\nabla_a e = \\sum 2(z-t)\\nabla_a z$$\n",
|
||
|
|
"$$ ~~~~~ = \\sum 2(z-t)\\sigma'(By+b) B \\nabla_a y$$\n",
|
||
|
|
"$$ ~~~~~ = \\sum 2(z-t) z(1-z) B \\sigma'(Ax+a)$$\n",
|
||
|
|
"$$ ~~~~~ = \\sum 2(z-t) z(1-z) B y(1-y)$$"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"(deltas)\n",
|
||
|
|
"\n",
|
||
|
|
"Now, let's write $\\delta_z = 2(z-t) z(1-z)$\n",
|
||
|
|
"\n",
|
||
|
|
"and\n",
|
||
|
|
"\n",
|
||
|
|
"$\\delta_y = 2(z-t) z(1-z) B y(y-1)$.\n",
|
||
|
|
"\n",
|
||
|
|
"Then $\\delta_y = \\delta_z B y(y-y)$."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"(backpropagation of deltas)\n",
|
||
|
|
"\n",
|
||
|
|
"We obtain a general update rule:\n",
|
||
|
|
"\n",
|
||
|
|
"- compute the $\\delta$ for a layer by taking the $\\delta$ from the\n",
|
||
|
|
"next layer and propagating it back by right multiplying with the\n",
|
||
|
|
"weight matrix connecting the two layers -- multiply it with the\n",
|
||
|
|
"activations for the previous layer to obtain the gradient"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"(gradient descent)\n",
|
||
|
|
"\n",
|
||
|
|
"For gradient descent, we now use the standard update rule\n",
|
||
|
|
"\n",
|
||
|
|
"$$\\theta(t+1) = \\theta(t) - \\eta \\nabla_\\theta e$$\n",
|
||
|
|
"\n",
|
||
|
|
"For _batch gradient descent_, we do this after summing up the total error.\n",
|
||
|
|
"\n",
|
||
|
|
"For _stochastic gradient descent_, we compute the error gradient separately for each sample (or for a minibatch) and then update the parameters."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"(derivative of the sigmoid function)\n",
|
||
|
|
"\n",
|
||
|
|
"The derivative of the sigmoid function is:\n",
|
||
|
|
"\n",
|
||
|
|
"$\\sigma'(x) = -\\frac{1}{(1+e^{-x})^2} e^{-x} (-1)$\n",
|
||
|
|
"\n",
|
||
|
|
"$ ~~~~~ = \\frac{e^{-x}}{(1+e^{-x})^2} = \\frac{1}{1+e^{-x}} \\frac{e^{-x}}{1+e^{-x}}$\n",
|
||
|
|
"\n",
|
||
|
|
"We can relate this back to the value of the sigmoid function itself (this saves computation).\n",
|
||
|
|
"\n",
|
||
|
|
"$ ~~~~~ = \\sigma(x) (1-\\sigma(x)) $"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"# Python implementation\n",
|
||
|
|
"def backward(x,target,eta):\n",
|
||
|
|
" global A,a,B,b\n",
|
||
|
|
" y = sigmoid(dot(A,x)+a)\n",
|
||
|
|
" z = sigmoid(dot(B,y)+b)\n",
|
||
|
|
" delta_z = 2*(z-target)*z*(1-z)\n",
|
||
|
|
" delta_y = dot(delta_z,B)*y*(1-y)\n",
|
||
|
|
" B -= eta * outer(delta_z,y)\n",
|
||
|
|
" b -= eta * delta_z\n",
|
||
|
|
" A -= eta * outer(delta_y,x)\n",
|
||
|
|
" a -= eta * delta_y"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 23
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"# target array\n",
|
||
|
|
"targets = array([1-labels,labels]).T\n",
|
||
|
|
"targets[30:40]"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 24,
|
||
|
|
"text": [
|
||
|
|
"array([[0, 1],\n",
|
||
|
|
" [1, 0],\n",
|
||
|
|
" [0, 1],\n",
|
||
|
|
" [0, 1],\n",
|
||
|
|
" [1, 0],\n",
|
||
|
|
" [0, 1],\n",
|
||
|
|
" [1, 0],\n",
|
||
|
|
" [0, 1],\n",
|
||
|
|
" [1, 0],\n",
|
||
|
|
" [0, 1]])"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 24
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"In order to train on the input data, we need to perform multiple updates.\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"def update(steps,eta):\n",
|
||
|
|
" for i in range(steps):\n",
|
||
|
|
" index = randint(0,len(data))\n",
|
||
|
|
" x = data[index]\n",
|
||
|
|
" target = targets[index]\n",
|
||
|
|
" backward(x,target,eta)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 25
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Before and after the updates.\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"subplot(121); plotgrid(forward)\n",
|
||
|
|
"update(10000,1.0)\n",
|
||
|
|
"subplot(122); plotgrid(forward)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAWwAAAC1CAYAAAB72cXqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvVuMJdlZLvitFfuel8qsvFZfoD0Y424zMAw26GgQKuTT\nRuIIyzq2DBKyWuCH0bwMZqQxSIzmNJoHyoIBDObRQj0vCGseoI0sHvpIboEQBskMSBzPMRps07Sr\n65aV99yXiLXmYa1/xx9//Cv2zsyqzOru/EtROzJiXWLHjvjii2/9/7+M997jyq7syq7syp54s5d9\nAFd2ZVd2ZVc2n10B9pVd2ZVd2dvErgD7yq7syq7sbWJXgH1lV3ZlV/Y2sSvAvrIru7Ire5vYFWBf\n2ZVd2ZW9TexcgP0Xf/EXeP/734/v//7vx+c+97lHdUxXdmWXalfX9ZU9qWbO6oddFAV+4Ad+AK+9\n9hqefvppfOhDH8If//Ef4/nnn3/Ux3hlV3ZhdnVdX9mTbGdm2H/7t3+L9773vXjuuefQbrfx8z//\n8/izP/uzR3lsV3ZlF25X1/WVPcnWOmvFN998E88+++z072eeeQZf+9rXKmWMMWc/siu7sjnsUQfq\nXl3XV/akmHZtnxmw571o/9P/BLz8PyNw+RaAdlhcCyjaQN42yNsZJu0WHDIUyJCjhRwZcrSRo4UJ\nWnFbGwUyOFhM4j6+ULkCLfzfL/9XfPTl/5ZtL9uifgpklXYmsW7BjkPrJ0cLDlbt/82XX8H6y//j\ndD+1OUEbuc/gXIZikmEyaSOftFDEBZMWkFugMMAE5TJm6wWAHNX9fPurLwP//uWyTqp8HvfJfk5T\nnvd98jKAl8P61HysOGZfYhy38e3UOC83Yo1PRNmclfvf5roGT2PzXtf/Pi6PpE+E24Ned+V6i312\nAPwZgF8A0AOwEJclANe6wLUFoL0G2A0A2wCeBfA9AJ4D3HsN9p5exJtrm/gX/Df4Ft6Db+E9+Dae\nwxv+WbxZPIO922sYvrEIfBfAHQB3AewAeAjg714GnnoZOARwHJcThJ+LfpIc4aefmo9LwRYnttO6\nE5XpGvodAP/LKc6og7gYz2GfB/DLj6gtMiPWM6XMs8q2c0giTz/9NN54443p32+88QaeeeaZekH6\nHfh6ARgHZA6wzsM6h8w5GO9h4GHhKkvG1k38QS2KuNTLmfjDU1uZKGPiPtlXJvpI9WPjFwr1C7Xt\ncj+vV7ZrjIfNCpjMwYQTAVhf/iq0ZGwxKH/fDNU7m/ZZUR5sv6wDUUcrnyXK09KIbwQzrRlfQjuY\njG1LlX30Nu91/ah6p2+stce/dYbyTFoE3tOLywDAggUGXaC9CNhlACsAVgGsAdgA/CZwstnB7vIS\n7mED97CB+1jHA6zhIVbxcLKK/Z0VjO93A0DvAtgDcADgCAGcJwgAPYwLf9461PGW3/A1sKbtKbB+\np1omltNB8JkB+4Mf/CD++Z//Gd/+9rcxHo/xJ3/yJ/joRz9aL0i/C3+oesD4ANq2AGzhYQsH6zmY\nVgFPgrZWpixX7iv3V0GXg7Zso76v/hDh5arb5D76LGCNgzUOxnoY42EMYKyDtfT08uXdywFYLhLX\nMlGnqV4m/p6nH7lNgjxfpmbYpzxg/jTSGte2y22PxyN13uuavtF5jIO1Edu05zWBtkVg2h0EwO5b\nYKEDdAaAXQJwDSVYrwPYBPyWwcHiAu63r08B+x428ABr2Cmu42C4jPHDPtzDVgDrfZRgfYQA1PzF\niDNqAm2VWWsLBwNeXtZ3eHtb0w2k3jQz7cySSKvVwhe+8AX89E//NIqiwKc//Wl1JP3mf4/q05d+\nFweYArA2kksXgCuwY2JXfgqAJehaWDgUsIACmFTuhZvXFUAt69f70csVsIk+qI3q/uWbPzyt7+Ix\nhrW4xZZLUVhY4+FtYNm+cOGEGFPHOXrwpe5sC+D7b1b3yXoaGjilPH+jTF1zvI32zcRbqKTsVIEK\n8r9TT6NC2UYH+uht3uv6++IRSdg5jaVedjSgJrDuAPiR+NkF0DPAIAMGfaBN2ggx7Miu3aZBvpFh\nt3sN97CBu9jAPayXDHuyisOj5QDWD02VWRO7HgJYuJmW6GqADVRZM//UpA+tLv3G/67xPD5e+/Fz\n1E3JHedo8XGmVzXGwP8Nprp1ZYnvdz4DihYwiVp2nrUwsaQbZ1Ev1jVl0qZJcy7176puHbTqUp8u\ny1T16QDVae06pVtL3TvV9lSXdy3keQv5JC55C66w8JMW/KQFjC0wMVUGM4+2PBb7UvX4jSaZ0rz9\npMpXbly6OTklo4ZGiQbllxij/gVp+//6yAcd5zFjDP7PuE7f8DSPjyZmLR9NbNhnCtIDBN16EcBK\nG1juAYMVwK4B2AJwA0G3/l4A7wWOv7eLnaeX8K3sPfgX8x78C96D7+A5fAffi3/Fs3hwbxsHb63A\nvZkBt03QrR/E5SEC2ybg5pLICKV+rYI1f72mC4OvUzntsUf1zmKPUsOe11Li1lnFs2cf7aDj3Mav\naLkweSRo2T6wz8hriZX6Cost2W8GF/9ZJmG4yj4b/7fwcCqT5v0YtR9aaH+9n6qEUmrbJZs3MPGn\n8zCmZNmGFmfgTTwZkulKts0JqWP7ZzFqWSfFwrV6Wj9aeRpHAhIH7pSOJe2nbQZ1ls3bvDwzCF+V\nH8k8bFsOMPJtqZ9LAncXUbs2QL8NdAeAWUSQQlYAXAewBvgNoNg0OFzp415rHfcQlgdx2SmuY3e8\nipP9gS6FEDhLGUQ+yJNgLW92TafWpJHzvLc8bkvJGBdzTT7+HrRxB/7QRQTsAkHHdrpWDHC9muvR\nxbQM7Tfx7+bBwrqM0tSPYU973o+sm2ofrG1j4oMpi4tlTML4qGX7qsyrvS9rioOUioHqe7XEPCkn\nc8o3qx9JE2XZWkPaoKPsJKVpyxHTywdsvq5+dcWawFoTfjRJpIMghSxYoN8NUoiRA43rgN80GG61\nsL+0MJVC7kfd+gGu42F+Hfv7y5jsdQJYE2AfQgdsCdY1jxCgqlNLNPfKNq3+rDKXadq1e/rBw7Pa\n42fYJBXzhy0jVaYAfLxSDfMY8cbBGab9Rh5LxgcZOZMmSw8WFlOmnMHBT+ufvp+Utl3uL+uW/YSt\nMIC1Dpl1cBG8vXPwzpbnR5JQ/obIma48rykWntKpgTp4U19n7af2NssvdF5Q6tWFso0fZAtPgkcB\nHT3nghKM521jlnadQTBrAIsWWCTdehFBt17FdJARW4DbtNjtruCupUHGzciw40DjyTLcTg/+YVbq\n1gTW5LbHPUK4IsU9QxqtiVk/rt/RIFwn5wH+R68/Pwq7OEnEoA4+cTEuEMtyiSxUyBIkfVh4+Iq8\nwcul5A8OqGkPkdP04+M/7cFg2HHwh0UGE+HbAwYw1iPLHJxzcEWQR7x1gMt0kG2SOjhoSmBOMWte\nT5NIztIP7ZveKxyauMShKbf8CUBwlbNtmqhw8UZgzT1WT6NWSmatgbYmg/QBDGyQQjp9IFtAOch4\nHQGwN4B8I8PR9R52OqvCjS8A9t5wFSf7i/A7bKBRgjXp05JhE7tW3fg0DVTKHPPKI2c1/rbG+5Rt\nN11Dp/fguAh7/Fc9HzuQv2UkVOSTXX3NrANt6banSRu62x6ENKL5WgM4Uz+lTzhtr/ZTr1uVagw8\njI3yiC1gySc7o6cYqnKFlEPk21mT3CFHuKTKkEKNs/RTsaaXfl5Y65QruPJgLh+w6fO0t7U8Iyn1\nqiKBIIB1H8GNr9sB7AAxYgYV7RobwHC9jZ1rS7jfWsM9bEY5ZC3KIWs4OFzBaGcAv2PqA4vErGlc\nOOUZUmHXWoBMIbZThRSzPu3w7SxLXWvAbAHqcq+vlD3+o+L6tZSzOMsuUPHJTgXRVDVnF79E1U+7\nCqxaEEs9GCaUOUs/1QtMuiFq2reFCz7Z1sGYajANDUIiizo2B1kJ3lL2lYDK62mgKq9XSXbn7Udu\nq11VvFCThp0SzvnBcqi8PEsd
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 26
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"More updates.\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"subplot(121); plotgrid(forward)\n",
|
||
|
|
"update(10000,1.0)\n",
|
||
|
|
"subplot(122); plotgrid(forward)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAWwAAAC1CAYAAAB72cXqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvWmMLNd15/m7EZGRkVl71auq98jHfXkkRXERqc2WJfbM\ndBuDmTHcMGAbGMiCIaA/tu0ZQDLQs3TPl6amuz0Q0EADAzQaxgxg2B8atmQYPWjZENuiTG0URUoU\nKVIUH6nHt9deucQ6H27cjJM3b2RlLa/ee3Id4CIjY8vIiHP/ce7/LFcVRVFwIidyIidyIre8eDf7\nAk7kRE7kRE5kMjkB7BM5kRM5kdtETgD7RE7kRE7kNpETwD6REzmRE7lN5ASwT+RETuREbhM5AewT\nOZETOZHbRA4F2P/pP/0nHnnkER566CG+9KUvHdU1nciJ3FQ50esTuVVFHTQOO8syzp07x9e+9jXu\nvPNOPvrRj/Inf/InPProo0d9jSdyIscmJ3p9IreyHNjC/va3v82DDz7IvffeS6PR4Ld/+7f5i7/4\ni6O8thM5kWOXE70+kVtZgoMeeOHCBe66667B97Nnz/Ktb31raB+l1MGv7EROZAI56kTdE70+kVtF\nXLp9YMCeVGn/m7IdhSj0kMAMC8yyWe+XrQF8FfhtoAlEwBQwDcwoWIhgahbCJWAVOAPcBdwDxf1Q\nPKS4tLzE+zN38A738zPu42fcx3nu4f38Li717mDnvQXSn4dwEbgMXAPWgBf/Odz1z2Eb6JRtF+iX\nLSlbZv+7olyZA2n5mZfbcnGAXG/EHPdHwP804d00v3cUgPdl4PeO4DxSlLXsO/a5y7HukL86oV7/\nr/8z/B+fRz/zy2W7BHxQtf4l2LoCl3K4Wu66g1aJuGxJeT75hPeSrwPP1V0/ulP75ecMsAzc4cHD\nK7D4JPCP4P3/dpWXHnmWPy1+m++++UnOv/wAvAD8EHgHWMsh2QbW0QrcAf4d8D+KKzd6V4hPxPpJ\nZBId/HPg1yc831HLzfrt33WuPTBg33nnnbz//vuD7++//z5nz54d2e+obBHTZe3z2WBtmof+cw00\nYLeAtoIpD6IWNGaAOWAeWAROASuQrXjsLoest+e4yjJXWeYap1hjkTUW2OzMs7M2S3Y90Lq8AWyh\nAXoXrccdoFc2A9IGg536acDTNAnK8oDcdfAvoLie9PHIpHrtSYySloSxGELwQggbECYQ5hCidVLu\n6nr9HlTMOY3uN9GAvYQG7Ogs8BTwK/CTlXO8lH+S7/We4er5ZfgxcB794tnMIeujFbmLVuI6BbbB\n+iBXbc5xInvJgTnsZ599lrfeeot3332XOI750z/9U37t135tZD+jy4cRcw7ZhT2GFV820ynKfjOw\nstseTDeh0QZlAHsBrdElYCenAjZac1zzl4YA+3qxxHq6wO7uDOn1JsW6gk00UBuzqYPW69hqWbne\n6PyQbkuFt1smlrGW7WNvV5FoJ8dMdjsemVSvB0grcUYqYwBeAI0GhJ7WxYbYfCP+kblTBqyn0PbI\nUhsWz0D4Idh5os3P7r+LV6Y/zGu7T3D5p3fQe6utreqLwGYBcQq5BGtjcUjDQuqd/NyvPpqrtnv4\nibjkwBZ2EAT823/7b/nVX/1Vsizj85//vNOT/gDVO/Qw72AJ+rZVbYN2ADyJBmsD2C0F7Qa026Cm\n0abHLBVgr0C+ougtNLnmLXGFZa5yimtlWysW2egv0Ntuw7qnLWsD2LtUo8bp56oRo9FzCdgj4946\nG8u2Olx3z3QgI5907HMc8vFDHFtHd9wcmVSvn3uWYeyCEcX0gtLCjiuwNnTFYV5F947ZZgyVFlrF\nT3mwNA/TD0P+EcX1xxZ4eenDvJw9xVuXHqb/2jT5mz68h+ZtdjPIY6phoqE+MuBxhkeCLpmU2JFi\n7kQd2D9ygHMeldzM3x6VA4f1TXRypfg35bKBn/0MfGRXNoot7S8J2A3RmmVro3nraWAhhJkpaM2j\nSb1V4A7gHuA+4EHYvGuKy6eWeMe/j3e4j59xP+9yL+9xNxfis1y/skp8oUV+wdd8pSEmr6PBe4vK\n0u5RGSg9KkvbaV1n1qdcL/ezH9V+mE9bjpLD3o/UEVsHkbuO3Ok4iSilKH6IHlltlM3owZXhVlyB\nq9twpafdHEZNjFtDwqFtt058PQyPKCO0HXLag4dn4NSjEH5KsfmP27zy4Q/zlzP/HX9z8R/xxusf\novdCm+L7Ct5Ac++9DuTbVJaIUV5jhdiWR2Etp/u8einyn9/uI8fDyu8erdNxUlEM03wwmbXtGiRJ\nq8Szlo1hE1Bx15IKaYWaCsFBheTLimxVsTkzzVX/VEmDLHOdJa6zxFqywFZnlmS9Sb7hj1rWXUb1\n2tZtJxUiX2O5tc6mPuw7dpgxy3FIHRF2vBTHDROXf0IqemlNqAB8pfUxEM0YG+bJy/6xH6iSP2eo\nkGlgScHpAObOQPNRyD7q8d7dZ3l15nG+WzzLhQ/uovuTKU2FfACsF9A31rX8c/YrZS+9PIzYpOeN\n/K3bU44NsM2yz2TvYBcN4lvLFmU4FCVi6JCWgmkfogiCaYYdjUvAsqZCOqsBG43ZYd66BOyN3gK7\nG9Pkm35FhWxRcddSv23QNvruBGt7TG22TdJlj9JdddTy94CT3AuwRfPU8GjQs5ZtmILJx012dFSE\nVu9VD860oHk/FE8p4o82eGPhEb6bf5TXkifYeW8e3gTeR48GtnPIk/JPuRTYRYO49FQ51h1E7Bf7\nL4K/5vByLIBtc9j7ZSylUtc5G41nfMBZU4byBTDVBn+KircWUSGsQrwYcs1f5IpaHnU0Zgt0d6fJ\n10LYUG7e2ljYxso2wD02MkSKdDTacqOU1Lz2DgP844D5FxisoX5k5QDvwNOOxzCvLG2js3Cwp1Bn\nWc+j1XpxHqYegOBpuP6hBd5avZfvNZ/mzbVH2XlrkfSNhuCtgTSHQvJ45o9gXd04ZTY6dSOotnHh\nC39/gPyGA7YdtGP0eFKxaRAXWBsKRNIgLTQVEoUiKmSWkTC+eDlge67NNe/UiHW9li+ytTtPvNGC\ndV+H8bnA2hgldXSIM4zPbpISce3jOvawYncC13nHURhHEQN0m8o2lX/CWNvyJS0A2/MgUKOUiNFl\nQ4nsRR/KJ2GgUYauzgBLHqy2YP5OCB5X9J9pcOHB03yv9TSvdp/ivUv3Ef8ogp+ho0KuA11DhUhl\nluGl+6Em5D85ahmnh+PkdgDzyVDxWAA7o1LM/Qb82DSIXJYKK8P3WmgqpOVD2ARlc9cGsJdhdzHi\n+nQVc31VUCHr6SK7m/Oka6F2Km1QAba0rKVVLa0sA9pDYtMhkhKRlkxdbKodGXJYkXfW5cS0yakT\nAbQexGh8M7pgANu8pMtb6/ngexDk0CiGw/uMPeqJZXA/EVcfMNZ1C63eywGsLkH7AcieUmw80+an\nd9/Nd/KP8fraE1x672xFhVxF63SalH/A/AkT1uTirW9FGaefR91fbpRMZvzccMCWiicVbq9QeRcN\ngjiHzV/bwN1WOoSvLkEmX4XsNGy0ZgRYr1SJMv1FtrdmyddKy9qOuZaWlc1hGyt75A/e6jycK8Tu\nF5zaOKhsUj1ng3VSDyzXhE1hhNQzZvK1XRcpZc5jwvcWgDtDOLsA4UPAMxB/POTHc4/xXT7KD/In\n2Dw/p6NBDG+9BWQFFObNYyduHZQqM733VgDK/YSN3si+eTSj0WMDbCPydoyDApsGsUP5JFCbDiCp\nkFYDwjb4drx1Cdjpss/OfJO1cKGMuTZUiM5q3OrN0dtok697w05Gmwqx6ZBUNCcVIsP2XMkHZrku\n7vpGgv3tEsFxC1znJvoZGz9dHTWSl3qsRnW3Ue7m444BAneUlAzhm0LbI6c9WJ6F2bOgPgy7T7S4\n+OAqr7U/zGvdJzi/fj+dn00NUyG7BeRpCdgy3VyGNe1Ff+DYR75a7H91M4yVSXXF8AA36hoOr7PH\nxmEbcQ3v6qSOBpFgPZQcQ42j
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 27
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"\n",
|
||
|
|
"def plotclass():\n",
|
||
|
|
" pred = zeros(len(data))\n",
|
||
|
|
" for i in range(len(data)):\n",
|
||
|
|
" z = forward(data[i])\n",
|
||
|
|
" pred[i] = z[1]>z[0]\n",
|
||
|
|
" cla()\n",
|
||
|
|
" d0 = data[pred==0]\n",
|
||
|
|
" d1 = data[pred==1]\n",
|
||
|
|
" plot(d0[:,0],d0[:,1],\"bo\")\n",
|
||
|
|
" plot(d1[:,0],d1[:,1],\"ro\")\n",
|
||
|
|
" print \"err=\",sum(pred*1.0!=labels*1.0)/float(len(labels))"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 28
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"# plotting the classifications\n",
|
||
|
|
"figure(figsize=(6,6)); plotclass()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"err= 0.076\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAFtCAYAAAD8jSoDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXtwVMedNvwQCzNjI4lBIhkEOAY5iUHBZKaMF39bKxHX\nZ8m7yjpvsrsSThzJRjICyQhwVbJrJDEKGNtlqryFsBKy61TWl7wJSbYqte/IAbGLPfJWjMO7oZZr\n8hkZJ4AsQ1gZC0cCRPr74+hozqX7nD59+lxG6KmibM2c6e7Tp8+vf/38btMIIQRTmMIUpjCFSYVP\nBD2AKUxhClOYgnxMCfcpTGEKU5iEmBLuU5jCFKYwCTEl3KcwhSlMYRJiSrhPYQpTmMIkxJRwn8IU\npjCFSQhb4b569Wp86lOfwtKlS5nXtLa24jOf+QyWLVuGw4cPSx3gFKYwhSlMwTlshfujjz6KvXv3\nMr9/7bXXcOrUKbzzzjv4p3/6J6xbt07qAKcwhSlMYQrOYSvc/+Iv/gKxWIz5/b/927+hvr4eAPBn\nf/Zn+PDDD/HBBx/IG+EUpjCFKUzBMVxz7ufOncOCBQsm/p4/fz7Onj3rttkpTGEKU5iCC+TJaMSY\nwWDatGmma2ifTWEKU5jCFOwhkiXGteY+b948nDlzZuLvs2fPYt68edRriebf5tJSZNJpEEKk/suk\n09hcWqrrqykaRcbQPwHQXlUlve+2ykqkKirQVlnJvL9UKiWlv7bKStM9Ge+LNh9ezT3Pv8rKNsNo\nUwAIqqraAxmP+/ETaeN3sy68HFcQ/2hzIeMeKypS1DYqKlLcfbkZB+u3QDuzPVG4Fu4PPvggXn75\nZQDAwYMHMWvWLHzqU5+y/d32/n7s37XLbfcm9HZ1YXt/v+6z3SMj2E+59qbRUWn99vX0YN+GDXiq\ntxedmQye6u3Fvg0b0NfTI60PIypbW9FWWqr7bHNpKe5fv37ib9p8eDX3PGhtrURpaZvus9LSzVi/\n/v5AxuMUV67QD7ujozf5PBI9wjoumZCxdmbMGKN+Holc5+5Lmes+KAK5c/y/fTh48AyqqtrR09PH\n7J/1nADtc1LaVtsThS0t89BDDyGTyeAPf/gDFixYgG9/+9u4du0aAKCpqQl/9Vd/hddeew133HEH\nbr31VvzgBz/g7lymcFWRd+UKvS/KZ9cjEWn9soRox65dKK+ultaPFmq7Hbt24abRUVyPRPDA+vW6\n/pjzwTn3PT196OrqxZUreZgxYwytrZWori4XHrP62127OjA6ehPOnn0dO3duc9Wmn+AVDn7Dz3HJ\nXhO8MK6dSOQ61q9/wFHfra2V6O9vQ3//9onPFKH9AHdfHR0vA9gHYLvmF224dGk6enufQn9/m64N\nLVjPCVCfU99E25cuAb29MPTjAMQnACDE8K+9qkp6P22VlaZ+CEBqolHd30+WlpJMOi2t31RFBbXf\nVEWF6drXX39dWr92YM0Hz9yn0xlSWrpZ99PS0s0knc5IG58Xc5FOZ0hlZRupqEiRyso2qeOlz8mT\nUvpwMxdejsu+H7lrghBv35F0OkOqqtpJRUWKVFW1Ox57ItFMe6UIkP28qqqd2bdx/uLxjSQeXz3+\ndxulXTExLcWgyos+AL1Qjgsno1FUrFghvY/K1la09ffrtOjNpaWoePhhdBw8yNRw3WJsxgzq57TT\nwcqVK23b6+vpQW9XF/KuXMHYjBmobG0VGi9rPh7QUDcsdHX16jQcAOjv345duzp0WokbTY5nLpyg\np6cPGzbs043bSpNyChnaI2u+3MyFjHHxgLUm6utr8dJLcuYYkL8utKiuLnc1zoKCOYxvsp+z6DD6\nc/rKxGcHD57FpUvCQ9NDaEsQAADSZNCeN0vWngkhJJNOk4ZEgqyKxUhdLEaak0npfaj9tFVWklRF\nBWmrrCTdqRTZXFpqOh10p1K663jGkkmnTW1p58rYt12bmXSatFdVkVRFBWmvquKej4qKFFVDqahI\nTVzjlybHi8pKmubD1qT8RtjmS0Uq1U2KimpIYWE9KSqqIalUN/U61poAUqG4Dz/AWmNADQEyrtYb\nvW0xMR0oLSObmrETil73051K6YQoTeDzjMeKSsmk02RTPK77fFM87skGxiMogxamRgpm6dKNthtS\nkAh6vmhIpbpJXl6Tbjx5eU1UAc8WbO2B34dfoG3QQBMBGgiwmsTjjwpvcvS2c1S40zhpUbjhl532\nkwFIG0BS4//NUPoRHY8Vf9+cSFC/a04mpd4jIXw8Lo927xVo44tGa0InPLUIcr5YKCqiz1lxca3p\nWrrweXJcY82QWKzWka3DS/uIl+2n0xmSn/+/CFA3vrFlxudiM1m0yDxvTtvW2gREhbuvnDsNMj1W\nRDxDRLjtC+fOUWzlwB8MkbminipnP/oI7VBsE2MAKgGUQ5mry0eOUH9z+fRpyzZFwMPjBuk9QuN/\nR0ZaEI2uxcjI7onPaN4QQSGM3jZjY1Hq59eumd9N9dnX19fi4sXFULw81Lndh6GhHyOTUf6ys3V4\nbR/xsv3q6nLk5X0HwEuGb7ZjaOgh121rxzdt2lNiDbnaYhwA45SEdstXPVaccsgsONWURWmcmqIi\naj+1xcWuxqOOyUi7bAbIo+PUS20sRm1zVSzmfMIkwA8vDZb2xdKCy8rWuPKG8Gq86nd+eLU4gRPN\nXYX5PpzTTV5TVF63H4vVUduPxeqktK9CVEz7qrlX7dyJx7ZswfDp05gBYGZBAY4dOoRzr76q8+ZY\n++abOPatb6G5s9NR+049Q0R900tKSoCLF02fz50719V41DE9PzioHxOAlpISlFdX4+Xbb0fb0JDu\n1LAZwMyFC5ltegmvvTSstK+sFqz1wxrDjBl/wt6926T07xR22qJfXi1O8PjjFdi+fS3GxrKnnby8\nJrS0sMdkvI8jR85gaMh8nVUQldeBV163f/vtM6n3vHDhTFftGr2phCF1i7EAAKqm3BSNkgxl+6uJ\nRoU0eCeeIU5807VwopE79VSxG1MmnSar43HSDoXvb9do9bS+RU9EXnOhvLDSvtLpzLh/sNFveFMo\nxxtmPPTQt8j06V8iN91US6ZP/xJ56KFvOfq9yH3nuuaurL9NhrW30dXay1mDKlMo0oSZDX0hA6IG\nT9omJSsoimdMPBuGG88hr9z1RDYMOwMkK6AkKGEaRoOpHWQ8bxG6yWuKykn7osqM24AoI2S6QvpK\nyzhKDQBv0hNoIRrgwxP2zwOaMZdnTOXV1bZ9uUmHwBu85ASixi39sTRLvxw7dhI9PX3MgBLZOVV4\nA7XCaDC1g4znLUI3eU1R8bbvxvBqFRAlEtzHzj0jAFfbjAPAQnP/yic+odeCQXct9AKiAT4y+mVp\n1jLGJEo5EeKN9il6RM5qXxkT/VJaupkkEg2ea+5ONNswGkztkIunDZnwgr4RPQ3lrObO0ko/c889\nqP35z7F4ZGTCsWovZ4i8DBBCdP/1A1aa9ba9e12nRnCSDsEIL7RPUeNW1vWuGxcv7tF919+/Hclk\nC0pL7RNBuYETzTaMBlM75NJpw4ukZV4YXkVPQ7TEZqLwVbhb0Rl9PT3YP/75fg9yv9CgpunVCtm2\n8f9XxyQjvwsNbrM12sFNThnezHlO4EaAVFeX4/OfPzDhP61Ffv4cbN16n6fC1OnLf+jQMfzf//v/\nYWwsiry8ERw6NDeUwl0VlOfOXQh1bIAKN/SJ1aYQNmXm0KFjeOGF2on1Q3HM44OQvi8AH7viBosm\nUvPReJnKwI9oWjf0jmxDkRVdwWPMCtILhWW0TSabTdc6CeUPEubnkSHRaA1ZunRjILEBPHBP7dEp\nEi+oNJljFZWdN7RwZ/HSdZEIaWCE+csSvl563IQVtA2Dl5sMkstWeH1zyH0y2Wi6ViQgKAj4tVnK\ndKkVtQ3w3KufyozzseYA5x42sHjp20ZHceq996jfyaJNZHncOEVQhRYAumdBVVU7FzcZJJddUDAf\nwH0AOqD4dimWofz8A6ZrnYTy
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 29
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"# more updates\n",
|
||
|
|
"subplot(121); plotgrid(forward)\n",
|
||
|
|
"update(100000,1.0)\n",
|
||
|
|
"subplot(122); plotgrid(forward)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAWwAAAC1CAYAAAB72cXqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvVmMJNl97vc7seVaWfvSXb1ULzO9ztrDGVLiMtos0NeX\nIECY1IMpwiAgwC+CBMOUAF3DkAFLQwsSQECAgWsLwlgGJPHhgqJ0eSl7KHG4DDnDWTgzHM7a03tX\nV3fXnpVLbMcPJ07myVOR1dVLVXdT9QcOMjIyMjIi8sR3vvP9lxBSSsmO7diO7diO3fPm3O0D2LEd\n27Ed27HN2Q5g79iO7diO3Se2A9g7tmM7tmP3ie0A9o7t2I7t2H1iO4C9Yzu2Yzt2n9gOYO/Yju3Y\njt0ndluA/e1vf5ujR4/ywAMP8NWvfvVOHdOO7dhdtZ1+vWP3qolbjcNOkoQjR47w3HPPMT09zUc+\n8hH+9m//lmPHjt3pY9yxHds22+nXO3Yv2y0z7JdeeonDhw8zMzOD7/v81m/9Fv/wD/9wJ49tx3Zs\n222nX+/YvWzerX7x0qVL7N27t/N+z549vPjiiz3bCCFu/ch2bMc2YXc6UXenX+/YvWJ5ffuWAXuz\nnfZ//h/hf/0yMA9cBa4As1m7DFyC1hVYvgZXJFwDFoFVoAmEWYuz/SVAuonf/S7w9AafO4CbtSIw\nAEwB+4vw4CQEn4TmbxY4/++m+M+Dn+ab9c/y2ltPsfLCELwAvA1clrCUAAvAMrAG/CXw32VHHWUt\nyX5V5ixvBnDM721k3wA+u4nt7rTdrd/97+/4Hjfbrz/Fxv1rK+27t/jbLr1TakHvfeACAd37YdqF\nvaMw8l87vPrvnuR/+n6FufmvceY7KVx5EXVTN2/9RDZt3+X+u9q3a3+cu/aWAXt6epoLFy503l+4\ncIE9e/as285J6SKs7iG6l3hAAE4AgQdBAkGqOo1Hb0cyd3O7pjuuC/ioDloDJoCxMrgPAE9A/bEq\nr5RO8WL0FD9fPk7zvTK8D1xEjSyNGGgBjew1NI40pReM7fc3Y4LuVdgp/bKVttl+fT+a7j1u9l7S\ne2vq+8IHKsDIIAwfd1n9QpWfeE/w8zcdwuursHABRVLC7T2BHbt1DfuJJ57g/fff5+zZs4RhyN//\n/d/zmc98Zv2Gulfo3mIO655qjg+BD4HogrWXbSaMdifNBOsBYBQYrUFtPziPwsLxId6bPsjL3ine\nnz/K4vvjxO97cA4F1mspRCGKVbdQTDqmy4b1XMAEWGm83iz4btWV2DHTNt2v70OT9N6OqbFO90QX\nKAFjwOB+aH9skG/s+Rzfnn2alQ9cWhevQngFaHPnKNSObdZumWF7nsdf/uVf8pu/+ZskScKXv/zl\nXE/600+gcMzUMhx6mLbjZYAdgZ/0MmvTbgauZvqs1/twUINDGRgUMC5geBJKx0F+xOHCod28WnuE\nV5PHuXRpL/EbRfgAJeMsSAgTkG0UYLfpSh8P0QXsfgLOrc4XHDaWR47ewj7vhN2t373zttl+PbP9\nh3ZHftvsPXn3kwuUBUx4UDzmc/7j0/yfS/8Dr7w2DRcuorTMFTYn0d0pm9nG37qXfnu93TJgA3z6\n05/m05/+9IbbPP0oUKcrRNtzME8BNj74jgJR3fzsALXSGxv7vRE/nclZp3/SN/ZfQ3XOfQNQPgzR\nYx7LHynz9tSD/KT9JB/MHmfpg7GuFHIdqEtImigpJKQXnB9CgfdWmT4L6J3Uwg5g3xnbTL+e2Z5D\n2dLfNsFbT7V9oFyE2gycfuwI/9/0r7Pw58PwvUXUXPQ6vXfidtjMNv/evfLb6+22AHtT1qKXgOZJ\nIy4IF1zRlUPyNGzH+KpmBpsVFTTM2br1uIDxClT3g/sQLD1c4r2pw7xefJi3lk6y8OEY4QcFJYVc\nAVZSiCPjxLRzcSNWfae1Z5MX2aqWOcHdsR3b2GxJpABUBqDwUXh/7Bj/fPbXWXx1Dc5dRoF1m53+\ndfds61PTm3RxTUsjJmAboO2IXpA2nYO2d9tl8/KICdYasCvACDDlwtggOMchPeWwdHyYV/xTvJKc\n4oPVB2i/F3SlkHmgmWQnZIK1frW1H7g1vXqzZl5E3XY07h27dSsClUFB8is+78jjfPfbH2dp8SJq\nitlge6WQHbNt6wG7kTWNcTrSzZJHhAOe0+t41H5JzahvBqS12cy6hJrYDQOTdHVr8QRcfGCK10ZP\n8rL7BOcvHaDxeg35vguXUE7xFpDoyBAdFWKG6ZluHPMEbdPD0FaYPTzptgPkO9bfdK8ZCaA8XON7\nD36Sd+b3w7dXYWGJ7Qnf27Eb2dZLIqsooG7SK49oPDPYtpuBtpcocPXpsmuTS9qqCvRO60zJxIog\npAQMAmMeTJWgOgPiYYfGR3xO75nhNf9R3qw/yty53cTvBHAemAOWJETa0agjxE2dx2TRN5IlhPF6\np5l3P2DWV66f3Q/T3Htn0NmKf+5ume6tDlCuQTw6xHPyv+Ktq3vg7AVgCXXj7tjdtu0B7JAuaGus\nM+URWAfYZmif5ogJXZDWmra2lF7vt9vdbYddF1CAPQxMFGFqCsQRaD7mcf1UjbeLD/JK4wnemztB\n40ylK4VcB1YlpC26I4+pW2/Epu8VM69Knm02iedumjlE311z+MURBzTVcAB3HFYmR/jnD/89b1+J\ngZfZ/qiQHetnWw/Yy3RlEB1YodUEjXkWTtisWHNYe1OT05pwZLLrLACFMsrJOALsLcLEBIjjwFOw\nfHKIV7zHeYVTvN94gOgdX4H1JRRYN4A0zdi1GRWy2bzLPDOTYe4FsN8sENqRKXfS7h1AvpHdO1z/\nzpkLiH3ANPAc8Hobxbh2wPpese0B7BgFzm268q+JfYbGYToeNdjGWXNZzwHNKEGMZTOEr0jmZBSw\ny4GxYageAB6D5RMDnNm7j1fcU7y9eoLZy7tJPvC6Usgi0EpAaueiOT0wUxA2sn4xLWZUOMY2d4Pp\n3gwEbRVc3T+JQVqmu9fnJJs1gbrf0hmX8LCL/E4C57Tj6V4gFDsG2wXYCb2lNdpG02pCmoF1TqSI\n2fIgzzZTRtG6dQ0YE7DPh/Ie4CTwUTh3YJpXBx7iFXGK87MHqb89DGdENypkBQi1o1EPHXZQ+Uag\nrYeOfiwlLyzvXpYnbiSt/Nswj/xZ3/1qOiU9POiz+lBAshzCimZUvwhn+IthWw/Yi3RlXv2q9Wwz\n3C8FmYKU+UCt5RHbbMC25RQthewCJgYUWLuPQOvxgKWTA7w1fIzX4sf5+epJFs+OwgdCSSHXULPB\nVAOoZtd5wKtB184KMkUbHUW+GTaeB+I7LOdesvtjHrB5c1Az0atiN2edw7Rog2ywU7/m3rLtAWxN\nQLXsq7HPDGXOABu5PhQvojfjUVvelDQPrEcFTJZgeAr845CegpWHq7y36yBvOif5+epJZi/tITpX\nVAkyWgpZk0q77jDrPOdi3jRegzP0xrSQsy05Z2EDdtrne3nf3bHtsF+kQEkth1SA60xxmgO00fkG\nO/1r+61/z9oeSUST1NR41cqCmQVp9A3diXQauSbnJvTZaq8GehOsx4BJB0anoHQUeAyipwRzR0Z4\nyXmSV3mcM2sHSd4P4KzogvUq0JKZdm2WSL2RaWad0AXevPgVbZthMDqg0TYzNmbHttN0nsAvgmCg\n6+oMAGcZ4SLTRJ1++otylveT3U3AXspe7TJhGoFDkCGkMSRJRmhZX6dXg7cOp9KvZpUxMzmmigLr\n0QqMjEPhOIjHQX5UcGbXAV4tPM5rPMa5q4dY+nAUedZRdW2uk+nWKH2mJyLkRrHVZvK8Bu6Nan+Y\n3zNts/LHjfTknensVpnOEdBX/34XrAIBYy4Ip8QqgyRUUSKJx9bWxtmxrmks6A/LWw7YckllMao3\ndABbZqAt4wysY4iTTDKmF7A1WJvwqZ0+Go40Ey8CVQGDLowVYHBSFXXiUWg/FrB6osLPB4/xKo/z\nZvgQV2Z30zpdgQtkCTKoAnyd
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"figure(figsize=(6,6)); plotclass()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"err= 0.078\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAFtCAYAAAD8jSoDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvX9wVFWeN/xBg3QrJLQJYxPAEaKjECPT/aqLz9YmjPWY\ndiez7uPubIKzmiiJCSQSwHrHWklCZ0DUkir3JZgZZtepWX/MM8PObNXUPh2FsIt23BpQdofa8Gvm\nkYgrECMMGzE4CRDmvH903869t8+599xzz/3RIZ8qC9N9+5xzzz33e77n8/01jRBCMIUpTGEKU5hU\nuMbrAUxhClOYwhTkY0q4T2EKU5jCJMSUcJ/CFKYwhUmIKeE+hSlMYQqTEFPCfQpTmMIUJiGmhPsU\npjCFKUxCmAr3lStX4qabbkJZWRnzmtbWVtx2221YunQpDh48KHWAU5jCFKYwBeswFe5PPPEEdu3a\nxfz+rbfewvHjx/Hhhx/i7/7u77B69WqpA5zCFKYwhSlYh6lw/5M/+ROEQiHm9//8z/+Muro6AMAf\n/dEf4fPPP8dnn30mb4RTmMIUpjAFy7DNuZ8+fRoLFizI/D1//nycOnXKbrNTmMIUpjAFG8iT0Yg+\ng8G0adOyrqF9NoUpTGEKUzCHSJYY25r7vHnzcPLkyczfp06dwrx586jXEtV/G0pKkEwkQAiR+l8y\nkcCGkhJNX03BIJK6/gmA9lhMet9tlZWIV1SgrbKSeX/xeFxKf22VlVn3pL8v2nw4NfciY4479Cyc\n+q+yso0y4wSxWLvttu2sCyfH5cV/tLmQcY8VFXFqGxUVce6+7IyD9VugndmeKGwL94ceegivv/46\nAGD//v2YPXs2brrpJtPfbRkYwJ7t2+12n4Xeri5sGRjQfLZjdBR7KNdeOzYmrd++nh7sXrsWz/X2\nojOZxHO9vdi9di36enqk9aFHZWsr2kpKNJ9tKCnBA2vWZP6mzYdTc88DnjH7GRcv0g+7Y2PXujwS\nLfw6Lploba1ESUmb5rOSkg1Ys+YB7jZmzBinfh4IXOHuKzXXfUgJ5M70v33Yv/8kYrF29PT0Mftn\nPSdA/ZxSbSvticKUlnnkkUeQTCbxu9/9DgsWLMD3vvc9XL58GQDQ1NSEb37zm3jrrbdw66234oYb\nbsCPf/xj7s5lClcFeRcv0vuifHYlEJDWL0uIdmzfjvKqKmn9qKG027F9O64dG8OVQAAPrlmj6Y85\nH5xz39fTg96uLuRdvIjxGTNQ2dpq6370Y37n1Cls3rbNsTmSDV7h4DbcHFdPTx+6unpx8WIeZswY\nR2trJaqqyqX3o4fSx/btHRgbuxaBwBWsWfOgpb5bWysxMNCGgYEtmc9SQvtB7r46Ol4HsBvAFtUv\n2nD+/HT09j6HgYE2TRtqsJ4ToDynvkzb588Dvb3Q9WMBxCUAIET3X3ssJr2ftsrKrH4IQKqDQc3f\nz5aUkGQiIa3feEUFtd94RUXWte+88460fs3Amg+euU8mEmRDSYnmdxskz5sTc5FIJEllZRupqIiT\nyso2kkgkpbZdUrJBM50lJc9K6cPOXDg5LvN+Nkjvx8l3JJFIklisnVRUxEks1m557JFIM+2VIsDE\n57FYO7Nv/fyFw+tIOLwy/XcbpV0xMS3FoMqLPgC9SB0XjgWDqFi2THofla2taBsY0GjRG0pKUPHo\no+jYv5+p4drF+IwZ1M9pp4Ply5ebtidLY2bNx4McNAjvacTOWHnmwgp6evqwdu1ujWZmpElZhQzt\nkaX52pkLGePiQVdXr2ZuAWBgYAvq6mrw2mty5hiQvy7UqKoqtzXO/Pw5jG8mPmfRYfTn9HDms/37\nT+H8eeGhaSG0JQgAAGnSac+ytUBCUtpmfSRCVoRCpDYUIs3RqPQ+lH7aKitJvKKCtFVWku54PEvL\nfbakhHTH45rreMZipjHr+zZrM5lIkPZYjMQrKkh7LMY9HzynETe0eyuorKRpPmxNym24pflaRTze\nTQoLq0lBQR0pLKwm8Xg39bqKijhDa4374j7cAGuNAdUESNpab/S2xcS0p7SMbGrGLUHD6qc7HtcI\nUZrA5xmPEZWSTCTI+nBY8/n6cNgRYcpD6dihfWRAT8GUla2jvngVFXFXxmMGP24+8Xg3yctr0own\nL6+JKuDZgq3d8/twC7QNGmgiQD0BVpJw+AnhTY7edo4KdxonLQq3BE1bZSVJAqQNIPH0v0lKP6Lj\nMdKYmyMR6nfN0ajUeySEvonpbRVWbA2yQXsRgsFq3wlPNViar5ebT2Ehfc6KimqyrqULn2fTGmuS\nhEI1lmwdTtpHnGw/kUiSWbP+FwFq0xtbMj0XG8iiRdnzZrVttU1AVLi7yrnTINNjRcQzRIQvPnv6\nNMVWDvxOF5kr6qly6osv0I6UbWIcQCWAcqTm6kJ/P/U3F06cMGxTBDzeOFZsDbJB439HR1sQDK7C\n6OiOzGc0bwiv4Edvm/HxIPXzy5ezn6HCGdfV1eDcucVIeXkoc7sbw8M/QzKZ+svM1uG0fcTJ9quq\nypGX930Ar+m+2YLh4Udst60e37Rpz4k1ZGuLsQCkKQmaFmiVQ2bBqqYsSuNUFxZS+6kpKrI1HmVM\netplA0CeSFMvNaEQtc0VoZD1CZMAHu3eLljaF0sLLi1ttOUN4dR4le/c8GqxAiuau4Ls+7BONzlN\nUTndfihUS20/FKqV0r4CUTHtquYe27YNT27ciJETJzADwMz8fBw+cACn33xT45Wx6r33cPiZZ9Dc\n2WmpfaueIaK+6cXFxcC5c1mfz50719Z4lDG9PDSkHROAluJilFdV4fVbbkHb8LDm1LABwMyFC5lt\nOgke7d4OjLSvCS1Y7Yc1jhkz/oBduzZL6d8qzLRFt7xarOCppyqwZcsqjI9PnHby8prQ0sIek/4+\n+vtPYng4+zqjICqnA6+cbv+WW2ZS73nhwpm22tV7UwlD6hZjAABULa8pGCRJyvZXHQwKaX9WPENE\n+WIrGrlVTxWzMSUTCbIyHCbtSPH97Sqtnta36IlI1mnKLoy0r0QimfYP1vsNr/dME/ajwZQHjzzy\nDJk+/Vvk2mtryPTp3yKPPPKMpd+L3Heua+6p9bdet/bW2Vp7OWtQZQpFmjAzoS9kQNTg6SQVwTMm\nng3DjueQU15HIsYtMwMkK6DEK2HqR4OpGWS4Z4rQTU5TVFbaFzW82g2I0kOmK6SrtIyl1ABwJj2B\nGqIBPrKoCJoxl2dM5VVVpn3ZSYfgRCoFUeOW9lg6Qb8cPnwMPT19zIAS2TlVeEPu/WgwNQMrMGn7\n9g5uukiEbnKaouJt347h1SggSiRNAzv3jABsbTMWAAPN/eFrrtFqwaC7FjoB0QAfGf2ytGMZY7Lj\nouiEe6PoEXlC+0pm0S8lJRtIJFLvuOZuRbP1o8HUDLl42pAJJ+gb0dNQzmruLK30tnvvRc0vf4nF\no6MZx6pdnCHyMkAI0fzrBoy04827dtk2SNpxUXTCvVHUuDXheteNc+d2ar4bGNiCaLQFJSXmiaDs\nwIpm60eDqRly6bThRNIyJwyvoqchWmIzUbgq3I3ojL6eHuxJf77HgdwvNChpetVCti39/8qYZGZE\nVMNutkYz2MkpY+e3LNgRIFVV5bjzzr0Z/2k1Zs2ag02b7ndUmFp9+Q8cOIx///f/i/HxIPLyRnHg\nwFxfCndFUJ4+fdbXsQEK7NAnRpuCE5ubHWXmwIHDeOWVmsz6oTjm8UFI3xeAi11xg0UTKflonExl\n4EY0rR16RzZdZURX8BizvPRCYRlto9HmrGuthPJ7ieznkSTBYDUpK1vnSWwAD+xTe3SKxAkqTeZY\nRWXnVS3cWdxybSBA6hlh/rKErxvBP34DzbOAl5v0kstO8frZIffRaEPWtSIBQV7Arc1SZvi/qG2A\n515le72Irtec5dz9Bha3fPPYGI5//DH1O1m0idPBPyw4STWZgeZZEIu1c3GTXnLZ+fnzAdwPoAMp\n366UZWjWrL1Z11oJ5fcSblRu
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 31
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Different Learning Rates\n",
|
||
|
|
"=========================\n",
|
||
|
|
"\n",
|
||
|
|
"In the above examples, we just picked a learning rate of $\\eta=1$. That is often not the right learning rate. In fact, with that learning rate and stochastic gradient decent, we have a limit on how well the algorithm can perform. Effectively, a high learning rate means that the algorithm \"forgets\" earlier training examples quickly. Eventually, to obtain better results from larger training sets, we need to decrease the learning rate. That leads to better performance even in this case."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"subplot(121); plotgrid(forward)\n",
|
||
|
|
"update(100000,0.1)\n",
|
||
|
|
"subplot(122); plotgrid(forward)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAWwAAAC1CAYAAAB72cXqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvVmMZNl55/e7e+xb7pWVVVnV1exVzSbZJGckQa0hhxA4\nYzdo0yAFGBQNUNCjIPlB0sDCWHqw1LQeBBoC/GDLg7bgkUTNWFKTohZTNimRVDeXJimuvdaS+x77\ndpfjh3NPxIlbkdXVVZVbV3yN2xEZmVFx497v/u//+3/LMYQQgolNbGITm9ipN/Okd2BiE5vYxCZ2\nezYB7IlNbGITOyM2AeyJTWxiEzsjNgHsiU1sYhM7IzYB7IlNbGITOyM2AeyJTWxiEzsjdleA/Td/\n8zc8/PDDPPjgg3z605++V/s0sYmdqE38emKn1Yw7rcMOw5CHHnqIL37xiywuLvLe976XP/7jP+aR\nRx651/s4sYkdm038emKn2e6YYX/961/nypUrLC8v4zgOP//zP89f/uVf3st9m9jEjt0mfj2x02z2\nnb5xbW2NpaWlwc/nz5/nxRdfHPkbwzDufM8mNrHbsHvdqDvx64mdFhvn23cM2LfrtE8DP3unH3IX\n9qU7/FwDsOJHZWa8WcgDZgEukAVKBly0Yf4Jg+y/SfHfbjyA+MD/wF/+1n8JG1+Dxo+BfeA4JgB8\nibN1tO/Wfvue/4u369f/fgn++xp8rw3XA9gBuoAPhPd8r+J9Q/re/wt8MH6uNjPxs4P00TTST/PA\nLDBVhsoyRP/W4JUPXOE/v+8Z/vzP/hu+9WcPwpe/D603gI34myR99kvcX/51kp893rfvWBJZXFxk\nZWVl8PPKygrnz5+/03/u1JhAXnCR9loUb+p1deF4QN6EmTlwf8pl/RNTvH5wha/+7eOw8iK0rwNN\njgesJ3Yv7Hb9uleH1RbshdAC+kDAqN/ca1O+qR5DJKz2tS35s75flgl2FliE9ns9Xn7HFf7j6n/H\nK/+0DC/sQvc6sBe/Y+Kzp9HuGLCfeuopXn31Va5du0a/3+dP//RPeeaZZ+7lvp2YiXhTQC0SG0im\nnQdmXMi9E1594lH+t/wvsbYxze7XfehchbCKvIQmdlbsdv26343BWkCP4c38qGFO90NFIgLtMQnW\nPkPW7zrglYEr8IPFR3ih9X6u/qfLNL7Vhv2rEO4h44SjvO1M7G7sjiUR27b5gz/4A37u536OMAz5\n1Kc+NTaTvnw3e3cXdrefq7usyahEYiBDzoIFszmD6Cc9Xrj4k/zPP/530PiP8KMbyCDZ53idf/kY\nP+s0fO69t9v163cFMnbqMWSwx8VJlxmybBj6pvI0K/69wRC0IwNcD9xZEI/DC5l/wT+++tME/4eA\njRvAq8hvdCtBZ/nefpHbtpP63JP+7JvtjgEb4MMf/jAf/vCHb/k3y3fzAXdh9/Jz9YvDRl4IHpCr\ngPu4x1ee+im+ufkk/C8CVheBl5GXynGHlcvH/Hkn/blHY7fj1z8RwTWOXgYZZ8uJn8eBt3puqd8Z\nYKRAXIDgafjx+mP88JuXCfZfht4aUth5s2+S/OTjspP63JP+7JvtrgD7fjGhPQok484AmSWIftLj\nq8HP8NLLD8I3XgfWgRqTsPLtbT1x/Mz6ViYSjyABW8klAhBZOJgu8frSBa7+7SUO/iEFresQ7XMy\nBGNib9UmgH0HZgI5A9zLJs2nc/zTP/8033vpHPA1YAsZKE/s7Ww6WJ+2Ij9FFRRgB0BoADlYL5zj\n886/Ze3FefhyG8JNpBQyIRhnwSazRN6imYBnwLwH+/PL/M2lD7DzrQx8qwYcMEky3h+ml3meNsCG\noUwyqBoxICrDSnaJ/7v5X7PStyC6BrQ5ukLEid1rO3LAPo3OfKc2qBCxITcNa7kH+Vzn37C9ImBr\nE6kDTpz/fjAF2KpG3+B0+XqyiiQwIJqCuijy8jcepb7bB7HJsMZlYmfBjhyw3y4UXk/sWDbYF+A1\n82Ge/8FH2G62gBscf1XIxE7KVIMKSKA+rX6uQDs0gWkQLYvwz9KIG01kQ9e4BpmJnVabMOy3aAZg\necDjQAf4TwI2W/EPE7tfLNlleOrNAOaQO/wisKsqtSd2luxYGPbbCbQtwEoZ+E/YBB7wgg/VHhPt\n+v4y25D6tRpVoEsjp8VGmr1MiOZAZAS8EULTR4olEztLduRVIjbDsOztEHiZSMBuPenR8U1Y74CY\nSCH3m7kGuELO61CgrRpWTpMiPBizYEAwC0E7hEYPhCr4m9hZsokk8hbNAUxcXjffwZoxDaKKTNxM\nAPt+spQlfUEH69Ps68IwqDlF6k6G4fQTfdjCxM6CHTlgn3ZHfitmoBiVwzUus8EUk8qQk7KT9SrP\nlt2uLuomfvokEWVG/P+6ladhZhkmxydgfdbsyAFbMZDT6MhvxVSbbwpI4bDFOfaoMJzfd9a/4Vmz\nkz3erim7XdNI4D7Nfi6rWAw6ZGmT5YylSyem2bEB9tvFPbImTFkmXSNPmwLyslWX7MSO3tSt0znR\nvbBtefPWWbY+L/00AbcaBRxi45MGCsi9P61tPxM7zI4cQ1VS5rSGi2/FTCDjQDFjEFoe/mA0fJph\nYHzWv+VpNlXxrJaQODlzHEibsuvVZdTPT5MXqMmSWcDAIrJc8NJgppF7fZr2dmJvZkcO2CmGUHaW\nTTl+KgOpKTBtkN+ugrwcUsgLYOL8R2c6WKdPdE9sD9IepA3JXlUC8rT5uYn0zAJxDOgiXdbzkNGh\nAu2JnQU78jOlnPms38eV47t5MGcFrutj5yM4Z8dT4cuMyiOnLTA+q6ZAWl/0Koc83idnlgeeCylT\n7pHiq6cJtA3k/uRMmLIhbUSYuQiuAPkiMIX02bcDpbo/7FgYtp5JP4sQNnB8wMuBMSdIuV3cUh8e\nsCAzg2wjKyHZtgLts/qNT4spCcRmCNZ5JEWcP8H9AtMDx5UMWwG2IicqX3PSZ34gh1hQTkPKCrGK\nITwCFCpIny0wlPQUyTjpPb/f7fDjf+SZMj0p43A2m2hU0qYIZLJgzkTknCapmS48ZsH1JTgwkeVS\nNnIedjt+96RB4c5MJRcVu04hb5klJDNcOvytx2EuGC54McPWiclpaaJRC22k0pCeAdcLsUohvBN4\naS7+ixZDgOgyXC1yUvZ3cnaCgK2c2WO4vtxZcwXFrueBQg7M2Yis2yI124F3G/CGC90Z2Aa5gMEO\ncrBOB3kR9Bm9TZ21I3Dcpg8v1Zn1FDANzEKpAtWT20M8MDxwXUgHkA5HxbCTFhhMhsnGdFrKeLYX\nYpV9zHf1iF5wEWsl2LwCkaJVNaS/6guf6auaTuxoTUU31qF/cWyArbfwniW+qVhKwYB5GwoFCKYj\nsk6T3EwT98kewRsOUTMHnQz0XAjSEKWAOnI4fBt5EajJaIrFwORCUKazCsWqVWKsAEyDOQvWDHjT\nsGSfPGCnwEtB2odUOPTz0yAsqJ6BIpDNAHNgpUK8YpfswzU67yzir2SgdRFaNgQukmjUkKy7i/RX\nFSGqTt6Jvx6dKfH1BAE7Y0BKjOp7Klw87c3cSgPMAWUbirOQmoN2OSJnt5jOb7PwwA2233uOjp8F\nx4Q3ZmAnB905YBPYRS5soCb6Kfaibl2KxdzvZmqbi/SYLFICmQEWZXK3koVlS+qw3zuxnZWSiAdO\nGrwupLuqqUru/UmPVrKRR28OKGXAmAfTCynaNS7n3+D6Yw9ysJ2CrgWvzcNOFgnvyl/rSKLRZrju\n+lkUNM+KqVyNIirj7egB25Iho86wT0NC5s1M3evSyBTXrAPpeTDnwSgIMnabGXubpcI1ek94hKZJ\nP52GKReuW7DlQc2BThGCOvIiqAINhjKJWtNaXQT3o1SiuKgK4h0kq84jwXoenFnIlWAhBRdteBTM\nxwOi//3k9lqKw5JlO45MPmbEELD7jHLT4zqr6kimgZIJsxkoTgFzYKYickaTi9Z1qu+YotnN4fse\n5D14w4R9E1oF6DWAPaTPqnyM
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 32
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"If the learning rate is too large, the stochastic gradient descent algorithm may diverge altogether."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"subplot(121); plotgrid(forward)\n",
|
||
|
|
"update(100000,10.0)\n",
|
||
|
|
"subplot(122); plotgrid(forward)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAWwAAAC1CAYAAAB72cXqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztvWuQXFeZrvnsvXPn/VIX1UWXkuWbLMs2RsZu3Jw5tMEj\nM3Cw243dRDcdxkPQ3RPR0RHAMDHNj5k53fMDiyBiwDTE+QFMhwJiMETMOeAg3JzG4zEBY0xDy8aA\nbWSwZJUuVVJdsvJ+23vNj7VX5sqtzFJJVVlZUq0nYkWlSlm5d2WtfPe33/V937KEEAKDwWAwbHns\nYZ+AwWAwGNaGEWyDwWC4SjCCbTAYDFcJRrANBoPhKsEItsFgMFwlGME2GAyGq4R1CfYPfvADDhw4\nwM0338znPve5jTong2GomHlt2KpYV5qH7Xket9xyC88++yy7d+/mnnvu4Vvf+ha33nrrRp+jwbBp\nmHlt2MpccYT9r//6r9x0003s27cP13X5sz/7M773ve9t5LkZDJuOmdeGrUzkSn/wzJkzzMzMtP+9\nZ88efvazn3U9x7KsKz8zg2ENbHShrpnXhq1Cr7l9xYK91kn7v83Ap1bglTKc8mABqAItwLvSg18C\nG3geeG/w2AmG/liNKBADEkAaGAEmgbHdkHob+B+0+X//3b/nP488zH/++p8y/0wN/u1V4CSwEvwm\nOs8D9w3oN7sUwzr2sI77Dxv+imsW46n/Eesjn8f5sxoHDrzKu7L/H3dxjDvErzjg/Zb07+pET3hw\nCjgHnAcWkVOmEIwyiBq06tBsQKMJzRY0fGggPx9qAPjB+E/A/0Dvz48InjOIfhNfA/5yAK+7VY87\nzGPf2+f7VyzYu3fvZnZ2tv3v2dlZ9uzZc9HzagU4XYYlH8p0JqJ/pQdeA2rCenQmrg9YXCzYaN9v\nAcIC1wJnBLwbbUrvjvNi5l6+9/qHWH4uCsffAmaBEoO75BiGxVrnNYkkzk0eY5MLTKXmmOACO1hg\ntLlCtljHWfGhCFSAGlDXhv4h8MD3wAtGS/T+bOgirES533NMc6Brlyv2sO+++27eeOMNTp48SaPR\n4Nvf/jYPPfTQRc9r1mDJg7LonqeDnlRq4gafCVraaCLPJTxaABbEYxCZguL1GX46/U5e/v0hzn5v\ngsbvzkJxFhkmNTfhtzBsNmud12QdnH0e45klJp15JrjAKEukG2UiBQ+rJGSEUkHeUtbonmxNwAOh\nxNqXQ4jesyosxP1mnpmR1zZXHGFHIhG+/OUv8773vQ/P8/j4xz/ecyX97S0ZaNSRgrgZYg2wL/iq\nIm2QVyeLTrStohQ9wrYcSCQgMgNLN4zxjP8fePWFW+DbRSgcB04Hv82ljjwMhnXsYR1341nrvObQ\nHxHZ22QsvsQU59nBAiPkSTSrbbuDChcLdjMYLRAtED74aojuKLof79jA3/dyuGubHXfYx+7FFQs2\nwPvf/37e//73r/qcO3x4i45Ybxb7enxPv1jotxZKsD3kAysF/gFYvinLz1/6Q069loDi6+AtsLpY\n9zvyZjGsYw/ruINhLfPa/W/vJbV7gR0xKdajLDPSWiFRDQS7SEe067RFun2L6Uux9kIfCo8ut6Tn\nZ+buPt8fNEawh8/AKx0bYnMj60sRtkp0u8QDhANk4czUTn4TvZUzP5yh/KsmNE+BKHLxIqNhO5LY\nVWUsc4EdzgXGWWBE5EnXK8TKLSnYJTr+dY/oGk+LrIPoWn1G9K9qEPq/MMa73h6sK8JeC5sdWa+V\nsFXSXpF3gRy8mjrIjwv/DZXvx+CNAnABGSptxd/GsNmkdpSZisyzgwXGWGJE5EmUm0QKoiPWVboX\nGpVYB4ItPGgFwxPdi+Q6umBvdmaIYWsx8Ahbz8zYio1LfOTnp70WFAUxCb+K3MFzlT+k1JoDcQ4Z\nJpmPhEGSGim3vetRlsmKIpFysGAT9qzVSrd67Ie862CAEV7D6gxcQ3WxVot+W6nsQEUn7cAnCmIa\nzi9Pc+o3u2nUzoNYDP7XfJQMkuzIchBdL5PzCzI7pOx1WyEqstaF25NiLdRX0ckMCVshBkOYgVsi\nSrCVUG/FKFv3tP0YsAvEmxG8120oLiLDJpNzbegwll1gnEVGyJNtlUhVqkTKdBYaa3T716EFx3BW\niBJp5UWb2WboxaZE2Hp0veWJA9chU61/LaCuEhINhg5t75oVko0qTgks3bfWLRF9dVvlXvud/Ot+\n9256pN0v6jb+9fZi4IIdAdxgbGXxbi/sxMGfAb8p4ExL1gybeMcQQkbXy2RZIdGsYZeQ0XV4oVFP\nQQoeh4tl9EhbR8/86JcFYrJDthcDF+yo3RFsJdpbbQFS3YL6SMFu7gE/1oJyA/ytmudiGCYjLJOj\nQIYisWb9Yu9ajaY2lG8dWCKe11lsNBjWwsA97JgDUV8eSAn1VouudZq2y0IsSyXiID+B/ZKtDNsZ\nmRlSINMoE6s0pGCrCDucdx0qlvGDxUaFFxpGww39GHigG4vIbnhROhH2VssUUVhAy3JZjoxStaOY\nzBBDP0bJk/ULJKs13LLXXYKuR9Z6Exstha9fZki4gZPA2B6GDptiiSSAJFK09ayRrYbsL+JQJEuD\nBFfHPYFhGIyQJyuKRKsedjgzpEdHvq5iGbXg2KdYRv2IEmoTNhgUAxdsNyIFO6oNZY9sNeGWPrtN\ngygtksgO2TE6jVgNBkmWAim/jFP1ulP5dCukR7GMCFRY0G2LXC4mO2R7MvgsERcSNsStji2i+9lb\nBQuZ0RfHwsPBd2LgJsGKI896K52tYdhkRJGkV8Gu+hcXyoStkMCcDhfLKK7U9jCCvf0YvGDHIB6V\ngh2nY4tspchalc8ngRTBm5IEcjY4SeQ9gsvWOmvDMMm0iiRrVRlhK+9a969Dq4jtzBCVyqctPOqN\nyIwAG1ZjUwQ7FpNRdoKOaKuYdStIoI08n6wNI47AxcPeIWBPBNxxIIe0RiJsjTM2DJtks45bFrJY\nRhdqfZeMUJ9UgbboSLdXbTCshYELth0DN4iwE8ihpG+rFNHYyHPKRCEbE0QtD3vKhxui4E4DE0AG\neblRZ75Vc122C8N97+PNBk4lVN0YbqGqUvj1niGiE133ql5cy64yhu3LwPOwiYIdhViPCDtC9/6L\nw8IOziuRgfiIIOK0sHf5cHMMfnwd8pNXDZ5t0WmzauKj4TFcwXabjc7mBH2KZPQpIkRQKCM6871f\nOXq4v4jBoBi8YMfAikHMhXgLEp6MZreKJeIQRNdANAvOmMC1Wzi7G9h3tvD3R8GbkrsIE0VufV2g\n+xOqb4tgGCzqzma4mTtus9F3669elTC9tgK71KV+tX0bzUzbnmyaYEfjkGh2C3YEOceHKdoR5Pri\nKBDPgjUuI+z4ZIXEwRLVQzn88ihUY1C3wY8jfwPV+Fg1hwpny5rIezBYdPKMhke00bq0YGvVjbpg\nh9P5LjdDxBTSbF82xRKx4hBNQKwO8XrHGnGRc32Ygh0FssAuIJWTgu3YHhOxC+yePMNbf5CkXo1D\nKwFvXg+VLHIRchHZ0q9IJ6/LpnMjaxpGbTw2nXZisaGeSbTqXZwZ0uTiDn2hDQp0RPfTDIZLsikR\nNjEp2q4LCQsSouNl662CNzNyUDfVKWAsAtMpSO6A1pjAdjzG7CV25c5w4dAUrbqDJ1xIJeDcOCxH\noTIGXh4p3Hk6wh0OtYwTuT5UF3UVWUeRl/zUME8KuyK6Gz3pE1n/s2uLjuGS9HAZusFwKTZNsIlJ\nwU46kPQgLuS3wx0oN2vyOsiLRg7YEYPxCZkQUh4D2/YZIc/OxFlO3TJD03epuFn8MRteT8DvEnAe\nqBSglQVvCcQyMuIOt2xT4m3skstH32DOoSPWGWBkiOeFtEP03iFqIus3WOFFR82/7jXXzQwxXIpL\npvXNzs7ynve8h9tuu43bb7+dL33pSwAsLS1x+PBh9u/fzwMPPEA+n+/9AlFUCSFuQgapCaf9ra6W\nq5tljaiFxiwwDUzEwd4d/GNE
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 33
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Different Number of Hidden Units\n",
|
||
|
|
"===================================\n",
|
||
|
|
"\n",
|
||
|
|
"The minimum number of hidden units capable of solving this classification problem perfectly is 2. However, that doesn't mean that 2 is the best number of units to use. In fact, finding a good solution for the minimum number of units may be a computationally hard problem. On the other hand, using \"too many\" hidden units may mean that the algorithm learns irrelevant detail from the training set (overtraining, see below)."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"def init(nhidden):\n",
|
||
|
|
" global A,a,B,b\n",
|
||
|
|
" A = random(size=(nhidden,2))-0.5\n",
|
||
|
|
" a = random(size=nhidden)-0.5\n",
|
||
|
|
" B = random(size=(2,nhidden))-0.5\n",
|
||
|
|
" b = random(size=2)-0.5"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 34
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"init(2)\n",
|
||
|
|
"update(100000,1.0)\n",
|
||
|
|
"subplot(121); plotgrid(forward)\n",
|
||
|
|
"subplot(122); plotclass()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"err= 0.247\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAD9CAYAAABOd5eOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvX18VOWd9/+ehyQz5AFCQCcBFAyuRSlWKhbv3Q2se5vo\nL3ddH1qDCmIFBY2G6qvVLSGSFV1d/f3si6cWqtx721pb2t27WzVdCC4L8bW3iD/LrwhaKyMgEAIY\nAnlgJpmH8/vjnOvMdc5cZzITntSeD68hkzPXnHOdM5nP9T2f75NH0zQNFy5cuHDxhYb3fE/AhQsX\nLlycPlwyd+HChYsvAVwyd+HChYsvAVwyd+HChYsvAVwyd+HChYsvAVwyd+HChYsvAVwyd/Fng3vv\nvZcLL7yQr371q45jGhoauPTSS7nyyivZsWPHOZydCxenB5fMXfzZ4Dvf+Q4bNmxwfP13v/sde/bs\n4eOPP+YnP/kJDzzwwDmcnQsXpweXzF382eCv//qvKS0tdXz9tddeY+7cuQB84xvf4MSJExw5cuRc\nTc+Fi9OCS+YuXBg4dOgQ48aNM38fO3YsBw8ePI8zcuEie/jP9wRcuPg8wV7dwuPxpI1RbXPh4kwj\n10orrmXuwoWBMWPGcODAAfP3gwcPMmbMGOVYTXosnTEDTdPSHktnzLCMM8cP8t7G6mrl+5bU1LB0\n6VLlsRzfo9iHpmlsfeMNGqurWTpjBo3V1Wx9442M+1yq2If9kc0+M14Xp+uoOOdc97H1jTdYXFlp\nGbu4stJxjuK4mT4L1WuN0vOtwGLb64uB+4G7A4GMfxtDgWuZu3Bh4KabbmLVqlXMmjWLbdu2MWLE\nCC688ELl2CXoX544cKSnRzkmXlCg3J6QnwcCltfaWlrY8+67NBv7rgaqjNd80Wja2NYVK/D399PR\n3c2joRAvdHSYry8GbrAd2xeN0tbSwsZFi3g6HDa3P7pzJz8tL2dsSQnxggKqGxqobmigMRy2jFtc\nWckNDz+cdk6qfTYaz6tqay1jHa+LcS3k84oXFHBo1Ki0sbnuo+PYMV6S5gbwdDhM/RNPpM1PxmDX\nwP5aeyjEo8ALHR20Ak/b9vc0MAuYaPsszfk7ziQLaC5c/Jlg1qxZWnl5uZaXl6eNHTtWW7dunbZm\nzRptzZo15pj6+nqtsrJSmzJlivbee+8p9wNomvR4JBTStr7xRtq41UuXagv8fsvY+0FbbTz/QWWl\n5X1b33hDW1xZad03aPNAWwra7WVl2j133qkcuxq0Gzwe7Vs+n3aT369VB4PaVts8NdCW1NRojdXV\nads10JZIzxcbc9v6xhvakpoabcbFF2tLamqU57n1jTe028vK1PusqVGOt5/nD6Tj2V/769LStOMO\nto9HQiHrdc/PV16PuwMB5TlpmqYtXbrUPNaSmhpt6YwZaddA9ZrYdldhofKazAFtK2gLAwHr/I3t\nmmGo5wqXzF24yBF2Mncircbqam2rQZJLjZ9bQbulsFC7vaxM++5Xv6qPMchBjG80xjca42WSvaui\nQtv6xhuWsfeBdrNEBBpo830+bfaIEUqyWzpjhpJklmY4p//8z/9UXgtBqvb3mvucMcPxfSqCVC00\n/5lhUVDt48Grrsq4WMnX+HbQ5k+datlnY3W1tnTGDG321Vc7Er08Tv4M7Z+/ah63GIvv/KlTzfnf\nWlxs+fxcMnfh4hxAJnNBDHOHD0/7UjuR5t02i0xYwfdfcYW22DZ2Mbo1bydZp7EyIdxcXJw1Ydot\n88GIWBDZ7WVl5jXI1jLPtL+60lKlBe00FxXuLi1VzmWWcX3s122hYZ2rrP3FtrsnMd9sx93r9Sqt\n7//h9Wa8K3PJ3IWLcwAM8vquYdnJ5CNLLtmSpiA9J6miTkFsjrKG9Hzu8OHK+a9eulRbEAwqSWYw\nIlYR2b3GHBfa3m+XkVRQEmOWc3FCnQOZ/19er3Z7hkXH8fOyHTvbcZqmaX83bFjanZkGugSj+Fxu\nLyvT5g4fPiQydx2gLlwMAU9JzxuNn1Xoji/hVFM5zxYGAtypcH75olEqKiqgszPttXLb74lAwHGs\nT3p+MplkSU2N6QSsbmgA4NArr3BnJEKTMX5Xfj7JYcP4xxMnzPcurqxk7PTpae9vXbHCcj5tQAjd\nsdcGNAGfAt3FxTyyfHlG5yKQtj+MfTWRcvw6OV2dUDR+PPO6uign5aRuB4rHj6egvR0crr8T7K/5\n+/uV43rb29OuV0EwyLJTp9LGxoNBy+9tLS0ceuUV1huf6csZz1ANl8xduDhN2Mmnd+9eIBXF0bRy\nJb5olEQgQPzoUaoUNV8SgQCF5eXw/vtprxVJzx8Jheg5epReh2QmEQ0x3+ejyOfjqdZW87XGcJgT\nJSWsFlEm4oWBAeonT6Zp9GhznmOnT+fQK6+kRad8JqJFgFbgADDO+L1K2mf9pZcqidweZdJ3+LDy\nPD4tLaV5yhQSgQA3PPzwoIuCjKk33cSOnTt5KpGKDZnv81E1Zw67X3sNHK6/pmnK/dkjjpwiadr3\n7GG99Pk1hsOMr65m4a9/zZp4HNCv0w+B0YWFLKmpobqhgaraWuWilitcMnfh4gxAtogHpOdVtbUW\nImpraaFx0SJqwmFa0b+AHwaDzJg+ncnTpqVZ8o+EQvRWVNBcXMyxnh6i7e2s27GDNvQ7Ajn07V6v\nl+OBALOGDYOSEn75ySeWOT4dDjPXoZzB6OJimqW6NUtqatIt5nCYurIy2oCNtmPLdydif3aowhfr\nbBaqwEXXXGOZTy5of/ttXkxYg/xeSiRo2raNumXLaLTNIVOooeoOpeLaa9PGLQgGqY9ELMd8Ohym\naeJEpjQ2Mmv1aqKnTlEWjfKbZBL274f9+83wTSdrPxe4ZO7CxRmAoI7FQNGECY7jqmpr2fXuu/zs\nmWd4ccCg/UiE+555hj9ceimJkhLqp05ldHExiUCAWySrdElNDU/9/vf6foz9NaFbsRddcw33SGOb\nZ84EG5kDOFGG3fp0IpeKigpWnzrFejtxYb07se8P1JJKfSTCffn5qWsBzM/PZ+r06Q4zzYy2lhYO\nvPuu8jVfNKq8W7Jb/vJrTncoY2bPpv711+ndu5cBoHtgwH4485gPNjfzYHOz/vlJd0pgEP7KlWgO\n1n4ucMnchYvTxALjZxPQEQpxz5NPZhy/+7XXLOQF8OLAAPW7d7MWaKys5Lonn0yTFuwEK2SN5ilT\n0qxYJymgeMIEGkeOHDQRyP5+IascP3iQhM9nyioyxN2Jk8btJKl0xeOmfp8AtIEB3lu7lrZp03KS\nV4TlP66rS/m6WGDkuyUh+2x+/nlT516WxR1K/euvM+LkSVZLx7LfncjHBOcF0heNcsHMmSzcvNmU\nY4YCl8xduBgCmmpq8EWjHOvpIQmMMSzpe7LQd3v37VNvN34Kay3XzEkZsvPV1LcDAYo0jTGzZ9O0\nbZujZap6vymrGOSlIq6PSktpuuYaR427vb09bVsr8C/JZNr2po4ONimuAaTr7nbdWSVBqRaYbLJW\nHZ2de/daiBwGd9xm+vza336bO6VFbShwydyFiyFgmULPla28g93dFACjpfR4QRBOUodsq6uiK1TR\nMcIhumDyZE50dFBRUUFheTnVDQ3ULF/OfU88gf+DD/hxNKpHcezYwbzDhwmUlzO6pMTR6SfLER9v\n384vsyCuB4zolbaWlrSojqraWkaEQjR2dlpI9gBq+EAZdZKJgAXxyhKUD32ReUARWaOMpLEtpE4E\nrBZVMjtuM5UG2Pz88xYH8j847D8TXDJ34eIMQCYZpYNQsviKxo+nsavLajmiR62YVvTOnZZoB/Fe\nSGm6wiE6d8cONgJrQQ9XfP99GsNhapYv58JRo3hKIsU2INTRwdNSDRenGipCjmieORO2bk07ZxVx\nZSLb0WPGUL17t0VSKUrbq44EQJa6uyDg493dqbmTIsY7jPfZzzGT7CHgRMCFJSXmXYqMTI7bTHq9\nmN9pIefIdBcu/syh+trIiSSD
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 35
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"init(20)\n",
|
||
|
|
"update(100000,1.0)\n",
|
||
|
|
"subplot(121); plotgrid(forward)\n",
|
||
|
|
"subplot(122); plotclass()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"err= 0.037\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAD9CAYAAABOd5eOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsnXt4E+ed7z+yZFuyLd9t5AtXQRMCgcYthGxbm83ZWOl6\nm03TJiYJCUkhgYRgmjy7yS7GwRvKJk2eJ3245bINZzctTUt7TttNcJdAloKzXSichC2YkJu42sIX\nbPmKZFnSnD9GI82MZmzZ3Jv5Po8wnnln5h3J+r6/+f5uJkEQBAwYMGDAwDWNpCs9AQMGDBgwcOEw\nyNyAAQMG/gxgkLkBAwYM/BnAIHMDBgwY+DOAQeYGDBgw8GcAg8wNGDBg4M8ABpkb+MLge9/7HuPG\njePGG2/UHVNTU8O0adOYPXs2hw4duoyzM2DgwmCQuYEvDB5++GF27Nihu/93v/sdn3/+OZ999hn/\n8i//wmOPPXYZZ2fAwIXBIHMDXxh84xvfICcnR3f/22+/zaJFiwC4+eab6e7upq2t7XJNz4CBC4JB\n5gYMRNDS0sL48eOjv5eWltLc3HwFZ2TAQOKwXOkJGDBwNUFd3cJkMsWN0dpmwMDFxmgrrRiWuQED\nEZSUlHDmzJno783NzZSUlGiOFWSvNRUVCIIQ91pTUaEYFx0/wrG1lZWax612uVizZo3mtXSP0TiH\nIAjs3b6d2spK1lRUUFtZyd7t2+POWVlZqzlrl2u15hy2b99LZWUtFRVrqKysZfv2vZrjKirWaMxU\n3K75Pmrc82jPsX37XpzOVYqxTucq3TlK11W+B4LiPdDeJ9+2F1il2r8KeBSr9UHN88be59HDsMwN\nGIjgjjvuYNOmTSxYsID9+/eTnZ3NuHHjNMeuRvzyBIG2vj7NMcHUVM3tIfn/rVbFvsaGBj4/eJD6\nyLkrgfLIPrPfHzd254YNWAYHae3t5SmHg5dbW6P7VwG3q65t9vtpbGjg3ZUrWed2R7d//3ATdUVf\nx5Q5ndTUIDU1ldTUVOJ21+J2r4uOczpXsWKF+qzQ0NDIypXvKsa63bUAVFWVK8ampgY13hWwWkPR\nc23YsJPBQQupqUHy81vixo72HB0drbjdbyjGut3rePbZ5XHzk2Ok90C9z+HwAE/R2voysBNYpzwh\n64AF+P1Tda4Y0tmeAAQDBr4gWLBggVBUVCQkJycLpaWlwpYtW4TXXntNeO2116Jjli9fLjidTmHW\nrFnCBx98oHkeQBBkrycdDmHv9u1x4zavWSMstVgUYx8FYXPk///odCqO27t9u7DK6VSeG4TFIKwB\n4Z68POGh++7THLsZhNtNJuG7ZrNwh8UiVNpswl7VPAUQVrtcQm1lZdx2AYSv4or+6nSuErZv3yts\n375XcLlWCxMnVggu12ph+/a9cfe5ffteIS/vHq1TCi7Xas3xTucqxTin8x+j11Pvy8n5Rtx1RzqH\nw/GkYl9KyqMC7I2bn9X6oOY9CYIgrFmzJnotl2u1UFGxJu490NonbUtPv1/zPYEHBNgrWK3LVNv/\nUTbH0VOzQeYGDIwSajKXSFKN2spKYS8IqyNkvBqEvSB8Oz1duCcvT/j+jTeKYyKELo2vjYyvjYxf\nLbvO/cXFwt7t2xVjHwHhzshYadwSs1lYmJ2tmKO0eKypqNAk8woqdIn497//veZ7ESPVNZrEVVGx\nRvc4LYKsrKzVOM/vdRcFrXPcdNPjOiS6OvJzrwC1kTnfI5SVLVGcs7KyVqioWCN89asLdYlePq6y\nslZznPa9CAJ8W4DNQlnZkuj87fa7VIuNQeYGDFxyyMlcItRFWVkKYhYEQZc0H7RaFb+vipDsozNm\nCKtUY1chWvPqhUNvrJzQ77TbhdUul7CmokJY7XIpFo2RLPORiFgiMtEil8gxMct8uPPl5FRrWtB6\nc9FCTs6DOiS6IHJupUVvtS7TfSqQnlDU8010XFLS9zSt76Skv4mz8JXnNMjcgIFLDiIE/n0Q7lER\nqFxy0SPN1VrbXC7hnrw8zfHVqt/XVFTojpWfe1FWlub8N69ZIyy12RTH3Y1TyGD7iESsRWTwPQGq\nBVDKBpLsMRy0z7cqjtATWRQkiAtC/NuTlPTXAujLQXqWtPraiY4TBEFIS/tbQXwiWBP5Kd5Xevr9\ncWPXrNks5OXdI2RlLRoTmRsOUAMGxoAfyP5fG/lZDrzc2sryZ5+lvKqKypoaat1uhaNxmdXKfSpH\nJoiOyeLiYujsjNtXpPo9ZLXqjjXL/t8TDrPa5cIyOEgwNZXKmhoAWrZu5T6fj7rI+KaUVI6kOejv\nrooe63SuYt68Ulyu1VEnYk1NJRs27FQ4/KARcCA69hqBOuA0dnsv69c/OaxzEdA4H5Fz1SG5fvWc\nrnqYNCkDr3cx4jsnuak9TJpkx+NJRePtx+83x2/U2Tc4qE2bHk9/3Ptls6Vy/vzauLE2m9KB29DQ\nyNatLXR2botseVP/BnVgkLkBAxcIJfVA/4kTAJRXieRYt3EjZr+fkNVKsL2dco2aLyGrlfSiIjhy\nJG5fhuz/Tzoc9LW306+TzCTFQiwxm8kwm/nBzp3RfbVuN92ZmWyOLC5Rmg0MsmzmEJML6vD7zVit\nIebNK2Xr1pa46BSr9Vzkt0bEaI0zwPjI7+XRs06bph0loo4yOXt2QPM+cnJOM2tWPVZriBUrbh9x\nUZDjjjvKOHz4EKFQbMk1m5fwwAPlvP32UbRK7litIQRB0DyfFCEjQS+S5vPPPRw5si36u9tdS2Xl\nJH71q2UEg69FtjYCPyI9vQCXazU1NZVUVZXrLGqjg0HmBgxcBMhtt4Ds/+VVVVFSBzGcsHblSlxu\nNzsRv4DHbDYq5s1j5pw5cZb8kw4H/cXF1NvtdPT14fd42HLoEI2ITwTyr//3kpLoslpZkJYGmZn8\n4vhxxRzXud0s0iln4LDbeG1HzIJ0uVbHkYvbvY68vGpEQnpXdXX58wnY7QVx19AKX7TZqjXnM3fu\nBHbsqNfcNxL27fMQCv1YsS0UeoP9++tYu7aalSsTDzXUekK55ZbiuHE221J8vuWKa7rd65g6tY7a\n2lls3ryA8+f9+P15hMO/4dQpOHUqFr6pZ+2PBgaZGzBwESDZbquAjMmTdceVV1XRdPAgP33+eX4c\niNC+z8cjzz/Pn6ZNI5SZyfKyMgrsdkJWK99esSK6GKx2ufjBhx+K54mcrw44nZPDhLlzeUg2tn7+\nfFCROcCg3vxV8e565FJcXMz585vx+bap9iifT9TWLGhLKj7fclJSHiEQiJFvSsoS5s0r05np8Gho\naOTgwTOa+/x+c9TC37gx9hSitvzl+/SeUBYuLOGdd5Zz4kQ/ECAQ6NW9Zn3949TXP47LtZqdO3+g\n2O92r2PjxjpSU7WfCkYDg8wNGLhALI38rANaHQ4eeu65YccfffvtGJFH8ONAgOVHj/I6UOt0cutz\nzyksegDLoJKKJVGjftYs6lXVIPUSluyTJ1Obm6uw/lc5ndy+YoViXLyUIMoqzc1dmM0hYrKKHOLz\niZ7GrSepBINeiCr4IQIBgddf/4A5cxpHJa9Ilr/XO15zv7TAVFWVR88ryT4vvbQ7qnPvSOAJ5Z13\nltPTk43Xu1m2R/l0Ir8m6C+Qfr+Z+fML2b1bLseMHgaZGzAwBtS5XJj9fjr6+ggDJRFLWm4d66H/\n5Ent7ZGf69xu6jZujDuPbkapyqoGFM7XqLpttZIhCJQsXEjd/v1RHf92jTkrMx9jsorXK42IJ66c\nnE+YO7dOV+P2eDwas99JOPx/4ra2ttaxceOuhHT3eN05XoTSWmASyVrVI+ATJ/pVRA4jOW6Hy1rd\nt89DMHgfsUVt9DDI3ICBMWCtRl10Kb1+90sv0dzbSypQkJkZjSSRCFNP6pDb6urUfUAzOkZyiC6d\nOZPu1laKi4tJLyqisqYG1/r1PPLss1g++ohX/X7w++HQIRafPYu1qIiCzExdp59cjjhw4DO83l+o\nRsQT1/r1j1FVVU5DQ2NcVEdV
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 36
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"init(200)\n",
|
||
|
|
"update(100000,1.0)\n",
|
||
|
|
"subplot(121); plotgrid(forward)\n",
|
||
|
|
"subplot(122); plotclass()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"err= 0.607\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAD9CAYAAABOd5eOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXt0VFWaNv5UqpJUQRISwqUSggaCrQhikxYbv5lJGOcj\npV9GG7XboIKxBQwQLuLqxm5CoBqa1pZv2YubQjfMDIoX2t9MTwtxIDgMxG9axGVnmotIQ7iGUIAh\nkARSJFU5vz92nTr77PPuc04VGLS7nrWyUrXPPu/e51TVs9/z7He/26EoioIEEkgggQS+0Ui62R1I\nIIEEEkjg+pEg8wQSSCCBvwAkyDyBBBJI4C8ACTJPIIEEEvgLQILME0gggQT+ApAg8wQSSCCBvwAk\nyDyBvxo8++yzGDhwIO666y5pnTlz5uC2227D3Xffjfr6+h7sXQIJXB8SZJ7AXw1++MMfYtu2bdLj\nH3zwAY4ePYojR47g17/+NWbMmNGDvUsggetDgswT+KvB3/3d3yErK0t6/P3330d5eTkA4Lvf/S4u\nXbqEc+fO9VT3EkjgupAg8wQSiODMmTMYPHhw9H1eXh4aGxtvYo8SSMA+XDe7Awkk8HWCmN3C4XAY\n6lBlCSRwoxFrppWEZ55AAhEMGjQIp0+fjr5vbGzEoEGDJLWV6F9x8WIoimL4Ky5erKun/S02Pbek\npIo8z+dbiMWL6bZk5wALDTYURcHWrbtRUlKF4uLFKCmpwtatuy1sLjbYEP/s2DS7L7L7SF1zrDa2\nbt2NgoIFuroFBQukfVTbNfss6GN82W4AC4TjCwA8B7f7aYvvRuxIeOYJJBDBww8/jNWrV2PixInY\ns2cPMjMzMXDgQEnthWA/nxDa2mhdPTU1JDk3HH3ldod1R2pq6vDpp0cB+AGEAJQAKAIABINOQ92V\nK2tx7ZoLra0BeL0vIBB4lauxAMADunOCQSdqauowd+52NDQsi5bv2/cCcnLeQEZGHlJTQ5gzpwRz\n5pSgoaFKV6+gYAFmz9bbVPsi2mxoqAIAlJYW6erK7ot6L/jrSk0NoV+/M4a6sdq4cCGAhob1uroN\nDcuwaFGloX88rO6BeMzrbQKgfg61AJbpDWIZgIkIBodJWgxLym1ASSCBvxJMnDhRycnJUZKTk5W8\nvDxlw4YNytq1a5W1a9dG61RWVioFBQXKqFGjlM8++4y0A0ABlOif1ztP2bp1t6He4sVrFJerQlcX\neE4B1iiAohQU/FR33tatu5WCggVC/XkKMEUBFivZ2Y8rTz75jKTuGsXheEBxOr+vuFwPKx5PiQLs\nFmwpis+3UCkpqTKUs7+F0dcFBQuUrVt3K1u37lZ8voXKrbcWKz7fQvI6t27drWRnP07a9PkWkvXF\n61TvBXUsK+vvDO1a2fB65+mOpaQ8R94Pt/tp8prY57c42pbPt1ApLl5suAfUMbWsd++nJPd5sgLs\nVtzu6UL5T7k+xk7NCTJPIIEYIZK5jLQYae6OkOTiyP/dSu/ejyjZ2Y8rd931vFJSUhUlB61+VaQ+\nfz5rJzf3KWXr1t1C3WkKMEFHVk7nVCUzcxJJdsXFiyUks1h6Tf/1X/9F3guNVGmbxcWLpedRBEkP\nNP8lHRQoG6NHz7QYrPh7/LhSWDhVZ7OkpEopLl6s3HPPJCnR8/X4z9D4+VP9eEQB1iiFhVOj/U9P\nf1QYbBJknkACXzn0ZM6IoU+fcsOPWkaabvfTAskyL3jEiOcUQPTMFyjMm9eTrLyuRgjp6RNiIEy9\nZ25FxCqRMY9cJUd7nrmZvaysMoXyoGV9oZCV9bTk+iZGbOvvm9s9XfpUoH42Yn/t1ktKelboA/O+\nk5L+0eKpLEHmCSTwlYOReZUCPK8AKpmxHyEvudglTZX0ZFIFUGYgNnldzXafPuVk/xcvXqN4PKL8\nwz/iy4mYloKejfRRLxuIMhIF2t4CW32RgQ0IxnuTlPR/Ip8XPejIPi+xbbv1FEVRevX6niI+mQGK\n0rv3U+Tnkp39uNKnT3lcZJ6YAE0ggbjwc+51VeR/EQKBV6OTatTkmds9HcHgkwZrwaATubm5aG6m\n2srRvXO7wyZ1tUnS7u7L8PkWRicB58wpAQBs2nQGHR1PAqgG4ERKygH06tWNS5d+ET23oGABxo7N\nM5y/cmWt7nqAOgBesIm9uojNU0hPb8WKFfNMJxcBEPYQsVUNdeJXNukqQ35+GlpapoDdNzZJDTQh\nPz8dTU2pCAaN54iTy2bHrl2jabOpqd1wvzyeVFy9utRQ1+PRT+DW1NRh06YzaG7eHCnZKL9ACRJk\nnkAC1w09+Rw/3g5Ai+JYtaoawaATbncY58+HUF9vJDi3O4ycnN7Yv5+ynxZ95fXOw/nzbWhsbJf0\nhUVDOJ1T4XSmobZWG3QaGqqQkXEJDQ1rIiWsH52dwMiRlejfX+vn2LF52LTpjCE6xe3+MvKuDixa\n4zSAwZH3RVGbt91GR4mIUSZnz14hryIr6xRGjfLD7Q5j9uwHLAcFHg8/XIh9++oRDmvX7nROxeTJ\nRXj//YOgUu643WEoikLaEyOOZJE0R482Yf/+zdH3DQ1VKCnJx3vvTUcotDZSWgfgV+jduz98voWY\nM6cEpaVFkkEtNiTIPIEEbgh4760z+qq0tEhHRCyErwoNDT4wMnTB4zmEsWOLMWbMSCLUbR5yc9uR\nnu5HW9sFNDUFUV+/AYwUqsCHviUlPQu3+yJ69ZqIjAzg2LF3dT1saFiGrKxysvfp6f2xbZs/+t7n\nW2ggl4aGZcjOLou0vR36sDvt6US1J4IKX/R4ysj+3HvvLbr+xIKPP25COPwbXVk4vB579lRj6dKy\nyP23F2pIPaHcd1+uoZ7HU4GOjkpdmw0NyzBsWDWqqkZhzZqJuHo1iGAwG93dv8PJk8DJk1r4pszb\njwUJMk8ggRsC1XtbgCFD0qS1SkuL8OmnB/DSS2+is5MRTkcH8NJL03DbbX9CRkYYhYWVSE/vH/FK\nH4kOBj7fQvzxj6q3qQ4Q1cjKOoV7770Fs2c/E607bpwfx45RPbhG9kv0PmXkkpubi6tX16CjY7Nw\nRP90ItoDaEmlo6MSKSnTovcCAFJSpmLs2EKyfSuwOP3T5LFg0Ek+LYmeP39M9oQyadIgbNlSGXkK\n60RnZ6u0Tb9/Jvz+mfD5FuqelJitZVi1qhqpqfRTQSxIkHkCCVw3KiL/q+H1BrBkyTOmtd9//6CO\nvACgs/M3OHiwEsA6FBRUYcmS+w3SgpFgmawxapTf4MXKpIAhQ9LRt6/1QiDj+UxWaWy8CKczDE1W\n4eGU2gMglVRCoRao+j0QRmengnXrPsOYMXUxySuq59/SMpg8rg4w/NOSKvssX74zqnNv26Zp3LIn\nlC1bKnH5ciZaWtZwR/RPJ3ybgHyADAadGDduAHbu5OWY2JEg8wQSiAM+H/Pe2touAOhGevqgiJf3\njCUBnTgh07tZueqtxbpykod+8pURsdt9GoqShkmTBmHPHrlnSp/PZJWWFrWGkbiysg7j3nurpRp3\nU1MT0ftadHf/f4bSQKAaq1btsKW7G3VnowRFDTB2Vq3KCPj48XaByAGriVuzz+/jj5sQCmmT0vEg\nQeYJJBAHeO9NBe/ltbY2AkhFRkZ/HeEw0FIHr7VT0RVUdIw6ITpyZAUCgUvIzc1FTk5vzJlTghUr\nfFi0aBo+/9yFYPB1BINAfT1w9uwU5OS4kZHRXzrpx8sRe/ceQUvLu0INI3GtWDEDpaVFqKmpM0R1\nlJYWwevNRHOznmTZBCoFJxl1YkbAGvFqEhTgRFbW4WjfeFCyjziQylMydJKlZhO3ZqkBli/fCX4C\nGfiZpF05EmSeQAI3AHqSMU4Q8h4fC50TSW0BWNQK86L37Tuti3ZQzwU0TVebEC2PtLcOzc3A/v2s\nvRUrfOjXbyCCQV6nrUMg4EUgYJ1DRZUjxo3zY/du4zVTxGVGtoMG9cfBgyXgJRU+UkePMNxuY6kZ\nAbe2XuRKeWJ8AitX1hqu0Uz2
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 37
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Training Curves\n",
|
||
|
|
"===============\n",
|
||
|
|
"\n",
|
||
|
|
"For actually training neural networks with stochastic gradient descent, it's useful to plot the error rate of predictions as the network is being trained."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"def err():\n",
|
||
|
|
" pred = zeros(len(data))\n",
|
||
|
|
" for i in range(len(data)):\n",
|
||
|
|
" z = forward(data[i])\n",
|
||
|
|
" pred[i] = argmax(z)\n",
|
||
|
|
" return sum(pred*1.0!=labels*1.0)/float(len(labels))"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 38
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"def training_curve(runs,updates,eta):\n",
|
||
|
|
" errs = []\n",
|
||
|
|
" for i in range(runs):\n",
|
||
|
|
" e = err()\n",
|
||
|
|
" errs.append(e)\n",
|
||
|
|
" update(updates,eta)\n",
|
||
|
|
" pylab.clf(); plot(range(len(errs)),errs); savefig(\"temp.png\")\n",
|
||
|
|
" return errs"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 39
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"init(10)\n",
|
||
|
|
"training_curve(50,2000,1.0)[-1]"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 40,
|
||
|
|
"text": [
|
||
|
|
"0.058000000000000003"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAD9CAYAAABdoNd6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X1cVFX+B/DPKNNa2pqS+iOGRB2C0RDYQNdcdylF1JJ8\naFda13Vd1sgis6ft1/ZrQ7dVKduysA1L3UwXaddNrJTQ1UnFlDY1zIfEB9aJRPEpBVFgPL8/TgPM\nMHPnDsyj83m/Xr5kZs6593CB7z33e849VyOEECAiomtaB183gIiIPI/BnogoCDDYExEFAQZ7IqIg\nwGBPRBQEGOyJiIKA02BfVFSEmJgYREVFIScnx2G5zz//HCEhIVi9erXLdYmIyLM0SvPszWYzoqOj\nsXHjRoSHhyMpKQn5+fkwGAytyqWkpOCGG27AtGnTMHHiRNV1iYjI8xR79qWlpdDr9YiMjIRWq0V6\nejoKCwtblXvjjTdw//33o0ePHi7XJSIiz1MM9pWVlYiIiGh6rdPpUFlZ2apMYWEhZsyYAQDQaDSq\n6xIRkXeEKH1oCdxKZs2ahfnz50Oj0UAIAUtWSE1dV8oREZE1V1a7UQz24eHhMJlMTa9NJhN0Op1V\nmS+++ALp6ekAgNOnT2P9+vXQarWq6ralwdey7OxsZGdn+7oZfoHHohmPRTMei2audpQVg31iYiLK\ny8tRUVGBW265BQUFBcjPz7cqc/To0aavp02bhrFjxyItLQ2NjY1O6xIRkXcoBvuQkBDk5uYiNTUV\nZrMZGRkZMBgMyMvLAwBkZma6XJeIiLxPceqlVxrwfa6fAKPRiOTkZF83wy/wWDTjsWjGY9HM1djJ\nYE9EFIBcjZ1cLoGIKAj4RbCvq/N1C4iIrm1+Eeyrq33dAiKiaxuDPRFREPCLYH/qlK9bQER0bfOL\nYM+ePRGRZzHYExEFAb8I9kzjEBF5ll8Ee/bsiYg8i8GeiCgI+EWwZxqHiMiz/CLYs2dPRORZDPZE\nREHAL4J9fT3XxyEi8iS/CPY9e7J3T0TkSX4R7Hv0YLAnIvIkvwn2nJFDROQ5fhHsmcYhIvIsp8G+\nqKgIMTExiIqKQk5OTqvPCwsLERcXh4SEBNxxxx3YtGlT02eRkZEYOHAgEhISMGjQIIf7YBqHiMiz\nFJ9BazabER0djY0bNyI8PBxJSUnIz8+HwWBoKlNbW4vOnTsDAPbu3Yvx48fj8OHDAIA+ffrgiy++\nQPfu3R03QKPB3LkC588Dds4lRERkh1ufQVtaWgq9Xo/IyEhotVqkp6ejsLDQqowl0ANATU0Nbr75\nZqvP1TSGaRwiIs9SDPaVlZWIiIhoeq3T6VBZWdmq3Jo1a2AwGDB69Gi8/vrrTe9rNBqMGDECiYmJ\nePvttx3uh2kcIiLPClH6UKPRqNrIuHHjMG7cOGzduhVTpkzB119/DQAoKSlBWFgYqqurkZKSgpiY\nGAwbNqxV/bVrs7F7N5CdDSQnJyM5Odnlb4SI6FpmNBphNBrbXF8xZ79jxw5kZ2ejqKgIADBv3jx0\n6NABzzzzjMMN9uvXD6WlpQgNDbV6f/bs2ejSpQuefPJJ6wZoNDh8WCAlBTh6tM3fBxFRUHFrzj4x\nMRHl5eWoqKhAfX09CgoKkJaWZlXmyJEjTTvctWsXACA0NBSXLl3CxYsXAchB3OLiYsTGxtrdD9M4\nRESepZjGCQkJQW5uLlJTU2E2m5GRkQGDwYC8vDwAQGZmJlavXo3ly5dDq9WiS5cuWLVqFQCgqqoK\nEyZMAAA0NjZi8uTJGDlypN393Hhj8/o411/vzm+PiIgAJ2kcrzTg+0uRiAigpAS49VZftoaIKDC4\nNY3jTUzlEBF5jl8Fe66PQ0TkGX4T7HljFRGR5/hNsGcah4jIc/wq2DONQ0TkGX4T7JnGISLyHL8J\n9kzjEBF5jl8Fe6ZxiIg8w2+CPdM4RESe4zfBnmkcIiLP8Ztg33J9HCIici+/CfYaDVM5RESe4jfB\nHmAqh4jIU/wu2HNGDhGR+/lVsGcah4jIM/wq2DONQ0TkGX4X7JnGISJyP78K9kzjEBF5hl8Fe6Zx\niIg8w2mwLyoqQkxMDKKiopCTk9Pq88LCQsTFxSEhIQF33HEHNm3apLquLaZxiIg8Q/GB42azGdHR\n0di4cSPCw8ORlJSE/Px8GAyGpjK1tbXo3LkzAGDv3r0YP348Dh8+rKouYP3Q3CNHgJQU4OhRT3yr\nRETXDrc+cLy0tBR6vR6RkZHQarVIT09HYWGhVRlLoAeAmpoa3Hzzzarr2mIah4jIM0KUPqysrERE\nRETTa51Oh507d7Yqt2bNGjz77LM4ceIEiouLXaoLANnZ2U1fX76cjLq6ZFx/vUvfBxHRNc1oNMJo\nNLa5vmKw12g0qjYybtw4jBs3Dlu3bsWUKVNw8OBBlxrRMtgvWSJ797fe6tImiIiuacnJyUhOTm56\nPXv2bJfqK6ZxwsPDYTKZml6bTCbodDqH5YcNG4bGxkacPXsWOp3OpboWTOUQEbmfYrBPTExEeXk5\nKioqUF9fj4KCAqSlpVmVOXLkSNMgwa5duwAAoaGhqurawxk5RETup5jGCQkJQW5uLlJTU2E2m5GR\nkQGDwYC8vDwAQGZmJlavXo3ly5dDq9WiS5cuWLVqlWJdZ3hjFRGR+ylOvfRKA2ymDz3xBBAeDjz5\npA8bRUTk59w69dIXmMYhInI/vwv2TOMQEbmf3wV7zsYhInI/vwz2TOMQEbmX3wV7pnGIiNzP74I9\n0zhERO7nd8H+xhuB+nqgrs7XLSEiunb4XbDXaJjKISJyN78L9gBTOURE7ua3wZ4zcoiI3Mcvgz3T\nOERE7uWXwZ5pHCIi9/LbYM80DhGR+/hlsGcah4jIvfwy2DONQ0TkXn4b7JnGISJyH78M9kzjEBG5\nl18Ge6ZxiIjcyy+DPdfHISJyL6fBvqioCDExMYiKikJOTk6rz1euXIm4uDgMHDgQQ4cORVlZWdNn\nkZGRGDhwIBISEjBo0CDVjeL6OERE7hWi9KHZbEZWVhY2btyI8PBwJCUlIS0tDQaDoalM3759sWXL\nFnTt2hVFRUV48MEHsWPHDgDygbhGoxHdu3d3uWGWVM6tt7pclYiIbCj27EtLS6HX6xEZGQmtVov0\n9HQUFhZalRkyZAi6du0KABg8eDC++eYbq89defp5S5yRQ0TkPoo9+8rKSkRERDS91ul02Llzp8Py\nS5YswZgxY5peazQajBgxAh07dkRmZiamT59ut152dnbT18nJyUhOTkbPngz2REQWRqMRRqOxzfUV\ng71Go1G9oc2bN2Pp0qUoKSlpeq+kpARhYWGorq5GSkoKYmJiMGzYsFZ1WwZ7C+bsiYiaWTrCFrNn\nz3apvmIaJzw8HCaTqem1yWSCTqdrVa6srAzTp0/H2rVr0a1bt6b3w8LCAAA9evTA+PHjUVpaqrph\nPXsCJ0+qLk5ERAoUg31iYiLKy8tRUVGB+vp6FBQUIC0tzarM8ePHMWHCBKxYsQJ6vb7p/UuXLuHi\nxYsAgNraWhQXFyM2NlZ1w3r1YrAnInIXxTROSEgIcnNzkZqaCrPZjIyMDBgMBuTl5QEAMjMzMWfO\nHJw7dw4zZswAAGi1WpSWlqKqqgoTJkwAADQ2NmLy5MkYOXKk6oYx2BMRuY9GtHW6jLsaoNHYnbGz\naxeQkQHs3u2DRhER+TlHsdMRv7yDFmDOnojInfy2Z19fD3TuDFy5AnTw21MSEZFvXDM9++uuk2vk\nnD3r65YQEQU+vw32AAdpiYjcxe+DPe+iJSJqP78O9hykJSJyD78O9kzjEBG5B4M9EVEQ8Ptgz5w9\nEVH7+XWwZ86eiMg9/DrYM41DROQeDPZEREHAr4O95WlVvl3QgYgo8Pl1sO/cGejYEfh+WXwiImoj\nvw72AFM5RETuwGBPRBQEAiLYc649EVH7+H2w51x7IqL28/tgzzQOEVH7OQ32RUVFiImJQVRUFHJy\nclp9vnLlSsTFxWHgwIEYOnQo
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 40
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"init(10)\n",
|
||
|
|
"training_curve(50,2000,0.1)[-1]"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 41,
|
||
|
|
"text": [
|
||
|
|
"0.040000000000000001"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD9CAYAAABDaefJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt0U1W+B/BvSjMCwuUNg0nHqoltyqNUW5FBMApYRAyP\n6kyRiw4WbnUuo8xVR3Gu13ZGhYqMgnWcqqCiUjoqUkQIUCGXl7Rc3kiB8KiECixAXqVg23DuH7/p\ni6Z5lDzak+9nra42zT5nb87SX3/5nX321iiKooCIiFQhItQDICIi/2FQJyJSEQZ1IiIVYVAnIlIR\nBnUiIhVhUCciUhGPQd1qtSI2NhZGoxFZWVkN3n/jjTeQkJCAhIQE9OnTB5GRkTh79mxABktERO5p\n3M1TdzqdiImJQUFBAXQ6HZKSkpCbmwuTyeSy/dKlS/HWW2+hoKAgYAMmIqLGuc3Ui4qKYDAYEB0d\nDa1Wi9TUVOTn5zfafsGCBRg3bpzfB0lERN5xG9RLS0sRFRVV81qv16O0tNRl2/LycqxYsQIpKSn+\nHSEREXkt0t2bGo3G6xN9/fXXuOuuu9CxY8drPhcREdXyZTUXt5m6TqeDw+Goee1wOKDX6122Xbhw\nocfSi6Io/FIUvPzyyyEfQ3P54rXgteC1cP/lK7dBPTExEXa7HSUlJaioqEBeXh4sFkuDdufOncPa\ntWsxatQonwdARET+47b8EhkZiezsbCQnJ8PpdCItLQ0mkwk5OTkAgPT0dADA4sWLkZycjDZt2gR+\nxERE1Ci3Uxr92pFG06SPEmpks9lgNptDPYxmgdeiFq9FLV6LWr7GTgZ1IqJmzNfYyWUCiIhUhEGd\niEhFGNSJiFSEQZ2ISEUY1ImIVIRBnYhIRRjUiYhUhEGdiEhFGNSJiFSEQZ2ISEUY1ImIVIRBnYhI\nRRjUiYhUhEGdiEhFGNSJiFSEQZ2ISEUY1ImIVIRBnYhIRRjUiYhUhEGdiEhFGNSJiFSEQZ2ISEU8\nBnWr1YrY2FgYjUZkZWW5bGOz2ZCQkIDevXvDbDb7e4xEROQljaIoSmNvOp1OxMTEoKCgADqdDklJ\nScjNzYXJZKppc/bsWQwcOBArVqyAXq/HqVOn0LVr14YdaTRw0xUREbnga+x0m6kXFRXBYDAgOjoa\nWq0WqampyM/Pr9dmwYIFSElJgV6vBwCXAZ2IiIIj0t2bpaWliIqKqnmt1+tRWFhYr43dbkdlZSXu\nueceXLhwAU8//TQmTJjg8nwZGRk1P5vNZpZqiIiuYrPZYLPZmny826Cu0Wg8nqCyshJbt27Ft99+\ni/LycgwYMAB33nknjEZjg7Z1gzoRETV0dcKbmZnp0/Fug7pOp4PD4ah57XA4asos1aKiotC1a1e0\nadMGbdq0weDBg7Fjxw6XQZ2IiALLbU09MTERdrsdJSUlqKioQF5eHiwWS702o0aNwvr16+F0OlFe\nXo7CwkLExcUFdNBEROSa20w9MjIS2dnZSE5OhtPpRFpaGkwmE3JycgAA6enpiI2NxfDhw9G3b19E\nRERg8uTJDOpERCHidkqjXzvilEYiIp/5dUojERG1LEEN6lVVweyNiCj8BDWonz8fzN6IiMJPUIP6\nuXPB7I2IKPwENaifPRvM3oiIwg8zdSIiFWGmTkSkIszUiYhUhJk6EZGKMFMnIlIRZupERCrCTJ2I\nSEWYqRMRqQgzdSIiFWGmTkSkIszUiYhUhJk6EZGKBD1T5+ZHRESBE9Sg3qoVcOlSMHskIgovQQ3q\nHTqwrk5EFEhBDeodO7KuTkQUSMzUiYhUhJk6EZGKeAzqVqsVsbGxMBqNyMrKavC+zWZDhw4dkJCQ\ngISEBLzyyiuNnouZOhFRYEW6e9PpdGLKlCkoKCiATqdDUlISLBYLTCZTvXZ33303lixZ4rEzZupE\nRIHlNlMvKiqCwWBAdHQ0tFotUlNTkZ+f36Cd4uXkc2bqRESB5TZTLy0tRVRUVM1rvV6PwsLCem00\nGg02btyI+Ph46HQ6vPHGG4iLi3N5vq1bM1BRAZSXA2azGWaz+dr/BUREKmKz2WCz2Zp8vNugrtFo\nPJ7gtttug8PhQNu2bbF8+XKMHj0a+/fvd9l2zJgMFBcDGRlNGisRkepdnfBmZmb6dLzb8otOp4PD\n4ah57XA4oNfr67Vp37492rZtCwC4//77UVlZiZ9++snl+VhTJyIKLLdBPTExEXa7HSUlJaioqEBe\nXh4sFku9NidOnKipqRcVFUFRFHTu3Nnl+VhTJyIKLLfll8jISGRnZyM5ORlOpxNpaWkwmUzIyckB\nAKSnp+OLL77Au+++i8jISLRt2xYLFy5s9HzM1ImIAkujeDt15Vo70miwY4eC8eOBXbuC0SMRUcun\n0Wi8nmEI8IlSIiJV4dovREQqEtTyi9OpQKsFKipkbXUiInKvWZdfIiKA9u2B8+eD2SsRUfgIalAH\nWFcnIgqkoAd11tWJiAInJJk6gzoRUWCEJFNn+YWIKDCYqRMRqQgzdSIiFWGmTkSkIszUiYhUhJk6\nEZGKMFMnIlIRZupERCrCTJ2ISEWYqRMRqQgzdSIiFQnZgl7BWcWdiCi8BD2ot24NaDTA5cvB7pmI\nSP2CHtQB1tWJiAIlJEGddXUiosBgpk5EpCIeg7rVakVsbCyMRiOysrIabbd582ZERkZi0aJFHjtl\npk5EFBhug7rT6cSUKVNgtVqxZ88e5Obmori42GW7559/HsOHD/dq12tm6kREgeE2qBcVFcFgMCA6\nOhparRapqanIz89v0O7tt9/GQw89hG7dunnVKTN1IqLAiHT3ZmlpKaKiompe6/V6FBYWNmiTn5+P\n1atXY/PmzdBoNI2eLyMjAwCwZw/QqpUZgLmp4yYiUiWbzQabzdbk490GdXcButrUqVMxY8YMaDQa\nKIritvxSHdQjI4FLl3wbKBFRODCbzTCbzTWvMzMzfTrebVDX6XRwOBw1rx0OB/R6fb02W7ZsQWpq\nKgDg1KlTWL58ObRaLSwWS6Pn7dgROH7cp3ESEZEX3Ab1xMRE2O12lJSU4IYbbkBeXh5yc3PrtTl0\n6FDNzxMnTsSDDz7oNqADrKkTEQWK26AeGRmJ7OxsJCcnw+l0Ii0tDSaTCTk5OQCA9PT0JnXK2S9E\nRIGhUbyZg+iPjv5VcweAtWuBP/8ZWLcuGD0TEbVcdWOnN/hEKRGRinDtFyIiFWGmTkSkIiGpqV+5\nAmi1QEUF0KpVMHonImqZWkRNPSICaN8euHAhFL0TEalXSII6wLo6EVEghCyos65OROR/zNSJiFSE\nmToRkYowUyciUhFm6kREKsJMnYhIRZipExGpCDN1IiIVYaZORKQizNSJiFSEmToRkYowUyciUhFm\n6kREKsJMnYhIRUIW1Fu3BjQa4PLlUI2AiEh9QhbUAWbrRET+FtKgzro6EZF/eQzqVqsVsbGxMBqN\nyMrKavB+fn4+4uPjkZCQgNtvvx2rV6/2unNm6kRE/uV242mn04mYmBgUFBRAp9MhKSkJubm5MJlM\nNW0uXryI66+/HgCwa9cujBkzBgcOHGjYkYvNU++7D3j2WflOREQN+XXj6aKiIhgMBkRHR0Or1SI1\nNRX5+fn12lQHdAAoKytD165dve68qZn6mjXAZ5/5fhwRkdpFunuztLQUUVFRNa/1ej0KCwsbtFu8\neDGmTZuGY8eOYeXKlY2eLyMjo+Zns9mMjh3NTaqpv/QSsH07EBkJ/Pa3vh9PRNRc2Ww22Gy2Jh/v\nNqhrNBqvTjJ69GiMHj0a69atw4QJE7Bv3z6X7eoGdQBYutT3TH3vXuDgQWDtWuD++4HOnYFhw3w7\nBxFRc2U2m2E2m2teZ2Zm+nS82/KLTqeDw+Goee1wOKDX6xttP2jQIFRVVeH06dNedd6U2S9z5wKP\nPQbcdhvwxRfA+PHA5s2+nYOISK3cBvXExETY7XaUlJSgoqICeXl5sFgs9docPHiwpoi/detWAECX\nLl286tzXmnpFBTB/PvD44/J6
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 41
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "heading",
|
||
|
|
"level": 1,
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Test Set Error"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"The problem we have been using has been quite easy: we have lots of training data and the classes are easily separated. Also, we didn't look at the performance on new data.\n",
|
||
|
|
"\n",
|
||
|
|
"Let us now change that. Let's change the training set to contain 100 samples and then measure the performance on the remaining 900 samples (samples that the classifier hasn't seen)."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"if len(data)>100:\n",
|
||
|
|
" test_data = data[100:]\n",
|
||
|
|
" test_labels = labels[100:]\n",
|
||
|
|
" data = data[:100]\n",
|
||
|
|
" labels = labels[:100]"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 42
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Using a smaller number of training samples seems to result in better performance... we're getting an error rate of (close to) 0."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"init(10)\n",
|
||
|
|
"training_curve(50,2000,1.0)[-1]"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 43,
|
||
|
|
"text": [
|
||
|
|
"0.0"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD9CAYAAABDaefJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X14VNWdB/DvwEzlRZYXgYgzkaAJyQRDEk1AliojLwas\nBBVbo13ashEja6xUu63atYTWKmm1liXURhZdFA1pxTWKMijCrCBL4kOUgIkSkcgYgQKCEEJNmPz2\nj+OEhEzmLXdecuf7eZ55nszcM/eeHMJ3zpx77rkGEREQEZEu9Il0BYiISDsMdSIiHWGoExHpCEOd\niEhHGOpERDrCUCci0hGfoW6325GSkoKkpCQUFxd32f7EE08gMzMTmZmZSEtLg9FoxIkTJ0JSWSIi\n8s7gbZ66y+VCcnIyNm3aBLPZjOzsbJSVlcFqtXosv379evzpT3/Cpk2bQlZhIiLqnteeelVVFRIT\nE5GQkACTyYS8vDxUVFR0W/6ll17C7bffrnkliYjIP15DvbGxEfHx8e3PLRYLGhsbPZZtbm7Gxo0b\nMXfuXG1rSEREfjN622gwGPze0euvv47vfve7GDJkSI/3RURE5wSymovXnrrZbIbT6Wx/7nQ6YbFY\nPJZdu3atz6EXEeFDBIsXL454HaLlwbZgW7AtvD8C5TXUs7KyUF9fj4aGBrS0tKC8vBy5ubldyn39\n9dd49913MWfOnIArQERE2vE6/GI0GlFSUoKcnBy4XC7k5+fDarWitLQUAFBQUAAAePXVV5GTk4P+\n/fuHvsZERNQtr1MaNT2QwRDUVwk9cjgcsNlska5GVGBbnMO2OIdtcU6g2clQJyKKYoFmJ5cJICLS\nEYY6EZGOMNSJiHSEoU5EpCMMdSIiHWGoExHpCEOdiEhHGOpERDrCUCci0hGGOhGRjjDUiYh0hKFO\nRKQjDHUiIh1hqBMR6QhDnYhIRxjqREQ6wlAnItIRhjoRkY4w1ImIdIShTkSkIwx1IiIdYagTEemI\nz1C32+1ISUlBUlISiouLPZZxOBzIzMzEFVdcAZvNpnUdiYjITwYRke42ulwuJCcnY9OmTTCbzcjO\nzkZZWRmsVmt7mRMnTmDy5MnYuHEjLBYLjh49iuHDh3c9kMEAL4ciIiIPAs1Orz31qqoqJCYmIiEh\nASaTCXl5eaioqOhU5qWXXsLcuXNhsVgAwGOgExFReBi9bWxsbER8fHz7c4vFgsrKyk5l6uvr0dra\niuuuuw6nTp3Cfffdh3nz5nncX1FRUfvPNpuNQzVEROdxOBxwOBxBv99rqBsMBp87aG1tRXV1Nd55\n5x00Nzdj0qRJuPrqq5GUlNSlbMdQJyKirs7v8C5ZsiSg93sNdbPZDKfT2f7c6XS2D7O4xcfHY/jw\n4ejfvz/69++Pa6+9Frt27fIY6iKAH58TREQUJK9j6llZWaivr0dDQwNaWlpQXl6O3NzcTmXmzJmD\nbdu2weVyobm5GZWVlUhNTfW4v9Ontas4ERF15bWnbjQaUVJSgpycHLhcLuTn58NqtaK0tBQAUFBQ\ngJSUFMycORPjx49Hnz59sGDBgm5D/fhx4MILtf8liIhI8TqlUdMDGQyoqRGkpYXjaERE+qDplEat\nnTgRzqMREcUehjoRkY6ENdSPHw/n0YiIYg976kREOsJQJyLSEQ6/EBHpCHvqREQ6wlAnItIRhjoR\nkY5wTJ2ISEfYUyci0hGGOhGRjoQ11E+dAlyucB6RiCi2hDXUBw0CTp4M5xGJiGJLWEN9yBAOwRAR\nhVJYQ33oUM6AISIKJfbUiYh0hKFORKQjDHUiIh3hmDoRkY6wp05EpCMMdSIiHeHwCxGRjvgMdbvd\njpSUFCQlJaG4uLjLdofDgcGDByMzMxOZmZl49NFHu90Xe+pERKFl9LbR5XKhsLAQmzZtgtlsRnZ2\nNnJzc2G1WjuVmzJlCl577TWfB2OoExGFlteeelVVFRITE5GQkACTyYS8vDxUVFR0KScifh2MoU5E\nFFpee+qNjY2Ij49vf26xWFBZWdmpjMFgwPbt25Geng6z2YwnnngCqampHvf33HNFOHAAKCoCbDYb\nbDZbj38BIiI9cTgccDgcQb/fIF662evWrYPdbsfKlSsBAGvWrEFlZSWWL1/eXubUqVPo27cvBgwY\ngA0bNuC+++7D3r17ux7IYMCpU4K4OOD06aDrS0QUUwwGg9+jIYCP4Rez2Qyn09n+3Ol0wmKxdCoz\naNAgDBgwAAAwa9YstLa24quvvvK4v4EDgZYW9SAiIu15DfWsrCzU19ejoaEBLS0tKC8vR25ubqcy\nhw8fbv8Uqaqqgohg2LBhHvdnMHBcnYgolLyOqRuNRpSUlCAnJwculwv5+fmwWq0oLS0FABQUFODl\nl1/G008/DaPRiAEDBmDt2rVeD+gO9ZEjtfsliIhI8TqmrumBvh0Xys4GVqwAJkwIx1GJiHo3TcfU\nQ2HoUA6/EBGFSthDfcgQLhVARBQqEQl19tSJiEKDoU5EpCMRGVPn8AsRUWiwp05EpCMMdSIiHeHw\nCxGRjrCnTkSkIwx1IiIdYagTEelIxK4oDc+KM0REsSXsod6vH9C3L3DmTLiPTESkf2EPdYBDMERE\noRKRUOe0RiKi0GBPnYhIRxjqREQ6wlAnItIRjqkTEekIe+pERDrCUCci0hEOvxAR6Qh76kREOuIz\n1O12O1JSUpCUlITi4uJuy73//vswGo145ZVXfB6UoU5EFBpeQ93lcqGwsBB2ux21tbUoKytDXV2d\nx3K//OUvMXPmTIgfK3Ux1ImIQsNrqFdVVSExMREJCQkwmUzIy8tDRUVFl3LLly/HrbfeihEjRvh1\nUI6pExGFhtHbxsbGRsTHx7c/t1gsqKys7FKmoqICmzdvxvvvvw+DwdDt/oqKigCoFRqPHLEBsAVb\nbyIiXXI4HHA4HEG/32uoewtot0WLFmHp0qUwGAwQEa/DL+5QP3sWePJJoK0N6BORU7VERNHJZrPB\nZrO1P1+yZElA7/ca6mazGU6ns/250+mExWLpVGbnzp3Iy8sDABw9ehQbNmyAyWRCbm5u9wc1AgMG\nAKdOAYMHB1RfIiLywmuoZ2Vlob6+Hg0NDbjkkktQXl6OsrKyTmU+++yz9p/nz5+P2bNnew10N/fJ\nUoY6EZF2vIa60WhESUkJcnJy4HK5kJ+fD6vVitLSUgBAQUFB0Ad2h/ro0UHvgoiIzmMQf+YganGg\nb8fc3aZMAZYsAToMHRER0XnOz05fInaaknPViYi0x1AnItIRhjoRkY5ELNR5VSkRkfbYUyci0hGG\nOhGRjnD4hYhIR9hTJyLSEYY6EZGOMNSJiHSEY+pERDoSsbVf2toAkwn45hu1FC8REXXVa9Z+6dNH\nLbv79deRqgERkf5E9L5DHIIhItJWREOdJ0uJiLTFUCci0hGGOhGRjnBMnYhIR9hTJyLSEYY6EZGO\ncPiFiEhH2FMnItIRhjoRkY74DHW73Y6UlBQkJSWhuLi4y/aKigqkp6cjMzMTV111FTZv3uz3wTn8\nQkSkLa8LerlcLiQnJ2PTpk0wm83Izs5GWVkZrFZre5nTp09j4MCBAIDdu3fj5ptvxqefftr1QB4W\npamtBebOBerqtPp1iIj0RdMFvaqqqpCYmIiEhASYTCbk5eWhoqKiUxl3oANAU1MThg8f7vfBOfxC\nRKQtr4veNjY2Ij4+vv25xWJBZWVll3KvvvoqHnroIRw8eBBvvfVWt/srKipq/9lms2HCBBtDnYio\nA4fDAYfDEfT7vQ6/rFu3Dna7HStXrgQArFmzBpWVlVi+fLnH8lu3bsWdd96JTz75pOuBPHyFEAH6\n9VPL7/brF/TvQESkW5oOv5jNZjidzvbnTqcTFoul2/LXXHMNzp49i2PHjvl1cIOBQzBERFryGupZ\nWVmor69HQ0MDWlpaUF5ejtzc3E5l9u3b1/4pUl1dDQC46KKL/K7A8OHA4cOBVpuIiDzxOqZuNBpR\nUlKCnJwcuFwu5Ofnw2q1orS0
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 43
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"If we plot the classes, however, we can already see that the sparse samples simply can't contain enough information about the decision boundary: in the white, empty regions, the classifier simply doesn't know where to \"draw\" the decision boundary."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"plotclass()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"err= 0.0\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD9CAYAAABHnDf0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X9sFNe5N/CvgxG2Um4DISX4h0QY02CS4NzKFSFV7E0j\n7zg4pVIT6TpSehMKhMu9eCFRpaTetTCBRKR/vJIxf4SqSe6b0CCqtHqdemAxSbBXTQJGIQK9EEXO\nFtq1U3ghhZLbBGNvz/uH2cXr/TU7np05M/P9SCvh8bB7PJhnzjznOeeUCCEEiIjIVW6yuwFERGQ+\nBnciIhdicCciciEGdyIiF2JwJyJyIQZ3IiIXyhvcf/azn2H+/Pm45557sp4TCASwePFi1NXV4ZNP\nPjG1gUREVLi8wX316tUIh8NZv79//358/vnnGBoawq9+9Sts2LDB1AYSEVHh8gb3Bx54AHPmzMn6\n/XfeeQdPPvkkAGD58uW4fPkyzp8/b14LiYioYNPOuY+MjKC6ujr5dVVVFYaHh6f7tkRENA2lZrzJ\n1BUMSkpK0s7JdIyIiPIzskrMtHvulZWViMViya+Hh4dRWVmZ8VwhhC2vLY2NEEDaa0tjoz3t2bLF\ntmth98vvD079VwAgoKoh29tm98vLvxe8FtlfRk07uK9atQpvvPEGAODIkSO45ZZbMH/+/Om+ranG\nZ83KeDxeVmZxS/SLaBpCqopOnw8hVUVE0+xuUlaaFoGqhuDzdUJVQ9C0SNZzAwE/FCWYckxR2tHW\n1lTsZhJ5St60zOOPP46BgQFcvHgR1dXV2Lp1K8bGxgAA69evx8qVK7F//37U1NTg5ptvxuuvv170\nRhfKHwggGI3ixWg0eaxdUdDc1mZjq7KLaBoObtqU0t7g9T83tLQgomno27kTpaOjGJ81C/5AAA0t\nLba0VdMi2LTpIKLRF5PHotGJ4N3S0pB2fuJYd3cHrl6dgeHhw+jq2pbxXCKaBmERCz8qo4HeXhFS\nVbGlsVGEVFUM9PYmjwf9frGlsVEE/f7k8WI6fPhwzu8H/X4hgLRXot3tipJyvF1RLGl3Jn5/MFNT\nhaqGdP39fNfCS3gtbuC1uMFo7DRlQNUJGlpa0nq3+XrIxeLz+XJ+v3R0NOPxGVevom/nzpT2AsCL\n0Sg6urtt6b2Pjmb+Fbp6dYauv5/vWngJr8UNvBbT5+nlB7IFykPd3Ta1aEKuMYJcgd8Os2aNZzxe\nVha3uCVENJmng7tsgTLBHwggqCgpx9oVBU1tbQUNDlsxKMsBUiI5eSYtk4msVTSJ9EpHdzdmXL2K\neFkZmtvaksf1DA5blXKaOkBaVhZHW1szB0iJbFZyPWFf/A8qKZlWzWYxZAqA7YqC5q4u26pP9Iho\nGg5NCvxNkwJ/QkhVsb2vL+3vdqgqtuVYK4iI5GI0dnq6556vhyxTyeFkmQaHp5I15URE1vB0cAey\nB0q7KmnMImvKiYis4ekB1VxkraTRK9egLBG5n+d77tk4Pa2RL+VERO7G4J6FG9IaenLzROROTMtM\nkagNvzAygv8oL0/5HtMaROQU7LlPMnUQNQLg38rLUVFTg29VVDCtQUSOweA+ydRB1AYADd98g46K\nCtaGE5GjMC0zidMHUYmIEhjcJ3HDICoREcDgnoK14UTkFp5eWyYTPeu2EBFZxWjsZHAnIpKY0djJ\ntAwRkQsxuBMRuZAUde6yLq1LRORUtgd3py+tSxM0LYKdO/swOlqKWbPGEQj4uRsTkY1sD+7Zltbt\n6O5mcHcITYtg06aDiEZfTB6LRif2VWWAJ7KH7Tl3zgp1vp07+1ICOwBEoy+iu/uQTS0iItuDO2eF\nOt/oaOYHwKtXZ1jcEiJKsD24c1ao882aNZ7xeFlZ3OKWEFGC7Tl37hhUONmqiwIBP6LRYEpqRlHa\n0dbWbFubiLyOM1QdJmN1kaJA7eqyNcBrWgTd3Ydw9eoMlJXF0dbWxMFUIhNw+QGPCKkqtvf1pR3v\nUFWuOU/kQlx+wCNYXUREejC4Owyri4hIDwZ3h2F1ERHpwZy7A3HNeSLvLHnBAVUi8oxMS14oShBd\nXarrAjwHVInIM7jkRX62T2IiIiqUk5e8sCqdlDe4h8NhbN68GfF4HGvXrsVzzz2X8v2LFy/iiSee\nwLlz5zA+Po6f//zneOqpp0xvKBFRglOXvLB0BVWRw/j4uFAURZw5c0Zcu3ZN1NXVidOnT6ecs2XL\nFvH8888LIYS4cOGCmDt3rhgbG0t7rzwfRUSkW2/vgFCUdgGI5EtRfiF6ewfsblpOfn8wpc2Jl6qG\nsv4do7EzZ899cHAQNTU1WLhwIQCgtbUVPT09qK2tTZ6zYMECnDx5EgBw5coV3HrrrSgtZbaHiIon\n0cvt7u6YtORFs/SDqVamk3JG4ZGREVRXVye/rqqqwtGjR1POWbduHX74wx+ioqICX331FX77299m\nfb/Ozs7kn+eUleHC4cPSLH5FRM7S0tIgfTCfSk86qb+/H/39/dP+rJzBvaSkJO8bvPTSS7j33nvR\n39+PaDSKpqYmnDhxArNnz047NxHcubUeEXmRnhVUfT4ffD5f8uutW7ca+qycwb2yshKxWCz5dSwW\nQ1VVVco5H374IYLB4PVGKrjjjjvw2Wefob6+Puv7cms9IuezaxKRkycvWZlOyhnc6+vrMTQ0hLNn\nz6KiogL79u3D3r17U85ZsmQJ3n33XfzgBz/A+fPn8dlnn2HRokW5P5SLXxE5ml375jp9v96pN6a2\ntiLemPKNuO7fv19897vfFYqiiJdeekkIIcQrr7wiXnnlFSHERIXMI488IpYtWybuvvtu8Zvf/Cbv\niG/Q708fLgZESFUNjQoTkbWMVH04+XPNkLnCpz1vhY+OMJ1R3rKWhx9+GA8//HDKsfXr1yf/PG/e\nPPzhD38o6IbiDwQQjEZTUjPtioJmLn5FDubkdEGh7JpE5OTJS9ln1XbYM4mpGLi1HrmN09MFhbJr\nEpFTJy8B1t+YbCtIb2hpYTB3OC/1VPOxuldmN7v2zXXyfr1W35g424gM8VpPNR8npwuMsGsSkVMn\nLwHW35i45C8Zoqoh9PVtz3C8A+HwNhtaZC9eD2O89vRnZCN5o7GTPXcyxGs91XycnC6wixef/qyc\nVcvgToY4eWCrGJycLrCL18YprMbgPklE09C3cyfXu9GBPdV0TlzrxE58+isuBvfruN5NYVpaGnDs\n2P/Frl3/hvHxcpSWfoMnnmhkcCPd+PRXXNxm77ps690c6u62qUX5RTQNIVVFp8+HkKoiommWfbam\nRbBnzwi+/HIf/v73/8aXX+7Dnj0j0LSIZW0gZwsE/FCUYMqxiae/Jpta5C7suV/ntPVu7H7SYL6U\npovjFMXF4H7d+KxZGY/Hy8osbok+dq+syXwpmYHjFMXDtMx1/kAAQUVJOdauKGiSdL0bu580mC8l\nkpulPXeZq1Gctt6N3U8arJYhkpulM1TbFSU1R6woULu6pA2gMsuUc29XFDRbeD2NzLYj5/HaLFLZ\nGJ2hau3yAxmOd6gqtoXDVjTBdSKahkOTnjSaJH7SIGfKNItUUYLo6lIZ4C3i2ODe2diIThM2gyUi\n83HNHPsZDe62D6jKWo1CRKyKcjJLg7uTqlGIiFVRTmZptYza1eWYahQiYlWUk3E9d8pI5rJVshar\nouzljAFVBndHyLi0ActWiWzh2AFVko8TF1EjolQM7pTG7qUNiGj6GNwpjd1LGxDR9DG4UxqnLaJG\nROk4oEoZcWkDIjmwWoaIyIWMxk5u1mEzrrhH5D1WzCNhcC+QmcE404p70ejEnpIM8ETuZNkWmcIi\nFn5U0fT2DghFaReASL4UpV309g4Yej+/P5jyXomXqoZMbjm5yUBvrwj6/WJLY6MI+v1ioLfX7iZR\nAYJ+f/p/ekCEVDXj+UZjJ3vuBTB7U2iuuEeFsntjdJq+f/z1rxmP/88XX5j6OZ4uhdS0CFQ1BJ+v\nE6oagqZFcp5vdjDmintUKM4e
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 44
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Let's now evaluate the error rate on previously unseen data."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"def cv_err():\n",
|
||
|
|
" pred = zeros(len(test_data))\n",
|
||
|
|
" for i in range(len(test_data)):\n",
|
||
|
|
" z = forward(test_data[i])\n",
|
||
|
|
" pred[i] = argmax(z)\n",
|
||
|
|
" return sum(pred*1.0!=test_labels*1.0)/float(len(test_labels))"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 45
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"cv_err()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 46,
|
||
|
|
"text": [
|
||
|
|
"0.054444444444444441"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 46
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"That's obviously much higher than the error rate on the training data. So, performance on new data can be much worse than performance on the training data.\n",
|
||
|
|
"\n",
|
||
|
|
"Let's look at this with training curves again."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"def cv_training_curve(runs,updates,eta):\n",
|
||
|
|
" errs = []\n",
|
||
|
|
" cv_errs = []\n",
|
||
|
|
" for i in range(runs):\n",
|
||
|
|
" e = err()\n",
|
||
|
|
" cv = cv_err()\n",
|
||
|
|
" errs.append(e)\n",
|
||
|
|
" cv_errs.append(cv)\n",
|
||
|
|
" update(updates,eta)\n",
|
||
|
|
" pylab.clf()\n",
|
||
|
|
" plot(range(len(errs)),errs)\n",
|
||
|
|
" plot(range(len(cv_errs)),cv_errs)\n",
|
||
|
|
" savefig(\"temp.png\")\n",
|
||
|
|
" return errs"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 47
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"init(10)\n",
|
||
|
|
"_=cv_training_curve(50,2000,1.0)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAD9CAYAAABdoNd6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlcVFX/B/DPsCiJCggIyFAoIKDJEoiPmoWpkVbk8lSY\nWhkl+qg97cuvpxJLBZfc0FzKwjSkMsVccEknd3BHAxUVFBEUEFB2mPn+/rjNKMzCDM4wA3zfrxcv\nmbnn3nvm4nznzPmee46IiAiMMcZaNTNjV4AxxpjhcbBnjLE2gIM9Y4y1ARzsGWOsDeBgzxhjbQAH\ne8YYawMaDfbJycnw8fGBl5cXYmNj1ZY7duwYLCwssHHjRp33ZYwxZlgiTePspVIpvL29sWfPHri6\nuqJv375ISEiAr6+vUrlhw4ahQ4cOmDhxIsaMGaP1vowxxgxPY8s+NTUVnp6ecHd3h6WlJSIiIpCU\nlKRUbunSpfj3v/8NR0dHnfdljDFmeBqDfW5uLtzc3BSPxWIxcnNzlcokJSVhypQpAACRSKT1vowx\nxpqHhaaN8sCtyTvvvIOYmBiIRCIQEeS9Qtrsq0s5xhhj9eky243GYO/q6oqcnBzF45ycHIjF4npl\nTpw4gYiICABAYWEhduzYAUtLS632bUqFW7MZM2ZgxowZxq6GSeBrcQ9fi3v4Wtyja0NZY7APDg5G\nZmYmsrOz0a1bNyQmJiIhIaFemStXrih+nzhxIp5//nmEh4ejrq6u0X0ZY4w1D43B3sLCAnFxcQgL\nC4NUKkVkZCR8fX2xcuVKAEBUVJTO+zLGGGt+GodeNksF/unrZ4BEIkFoaKixq2ES+Frcw9fiHr4W\n9+gaOznYM8ZYC6Rr7OTpEhhjrA0wiWAvI5mxq8AYY62aSQT7spoyY1eBMcZaNZMI9qVVpcauAmOM\ntWomEexLqkqMXQXGGGvVTCLYl1Zzy54xxgzJJII9t+wZY8ywONgzxlgbYBLBvriSu3EYY8yQTCLY\n5xVzy54xxgzJJIJ9fgkHe8YYMySTCPYFd7gbhzHGDMkkgv3tCm7ZM8aYIZlEsC/hBC1jjBmUSQT7\nOzXcsmeMMUMyiWBfJuVgzxhjhmQSwb6KuBuHMcYMySSCfY0Zt+wZY8yQGg32ycnJ8PHxgZeXF2Jj\nY5W2JyUlwd/fH4GBgQgKCsLevXsV29zd3eHn54fAwECEhISoPQcRoaquqokvgTHGWGM0rkErlUrh\n7e2NPXv2wNXVFX379kVCQgJ8fX0VZcrLy2FtbQ0AOHv2LEaNGoVLly4BALp3744TJ06gS5cu6isg\nEsHsY0fc+N9ZOHV00tfrYoyxVk2va9CmpqbC09MT7u7usLS0REREBJKSkuqVkQd6ACgrK4ODg0O9\n7dpUhipteTI0xhgzII3BPjc3F25uborHYrEYubm5SuU2b94MX19fDB8+HEuWLFE8LxKJMHToUAQH\nB2P16tXqT1Rli5ulnKRljDFDsdC0USQSaXWQkSNHYuTIkThw4AAmTJiACxcuAAAOHToEFxcXFBQU\nYNiwYfDx8cGgQYOU9jc7XoRFs5dir5sHQkNDERoaqvsrYYyxVkwikUAikTR5f43B3tXVFTk5OYrH\nOTk5EIvFassPGjQIdXV1KCoqgr29PVxcXAAAjo6OGDVqFFJTU1UG+05BjyHk+efxyfMvNfV1MMZY\nq9awIRwdHa3T/hq7cYKDg5GZmYns7GzU1NQgMTER4eHh9cpcvnxZ0S9/8uRJAIC9vT0qKipw9+5d\nAEISd9euXejTp4/K81ib2yKvmLtxGGPMUDS27C0sLBAXF4ewsDBIpVJERkbC19cXK1euBABERUVh\n48aNWLt2LSwtLdGxY0ds2LABAJCfn4/Ro0cDAOrq6jBu3Dg8/fTTKs/TqZ0Nbt3hBC1jjBmKxqGX\nzVIBkQj/+ugrdLStxO5PZxmzKowx1mLodehlc7G3tuGlCRljzIBMItg7drJFaTV34zDGmKGYRLB3\ntrVFWS237BljzFBMIti7dLFBhYxb9owxZigmEezdHG1RLeJgzxhjhmISwf4RJ1vUmnM3DmOMGYqJ\nBHsbyNqVQCo1dk0YY6x1Molgb2fdCbAsR0EhR3vGGDMEkwj2ZiIzmNV1Qnb+HWNXhTHGWiWTCPYA\nYCm1xdWbnKRljDFDMJlgbwVb5BRwsGeMMUMwmWD/kJkN8m7ziBzGGDMEkwn2nSxskV/CLXvGGDME\nkwn2Nu1tUXiXW/aMMWYIJhPs7TrYoKiCW/aMMWYIJhPsHTraoqSKgz1jjBmCyQT7rjY2uFvD3TiM\nMWYIJhPsXexsUS7llj1jjBmCyQR7sb0tKomDPWOMGUKjwT45ORk+Pj7w8vJCbGys0vakpCT4+/sj\nMDAQQUFB2Lt3r9b73q+rjQ3QvhQVFU14FYwxxjQjDerq6sjDw4OysrKopqaG/P39KT09vV6ZsrIy\nxe9paWnk4eGh9b7/LHZORETHco+R5bTHKDtbU40YY4wR3Yud2tLYsk9NTYWnpyfc3d1haWmJiIgI\nJCUl1StjbW2t+L2srAwODg5a73s/WytbiKxKUVjY9A8uxhhjqllo2pibmws3NzfFY7FYjJSUFKVy\nmzdvxqeffoq8vDzs2rVLp30BYMaMGaiorUBdyg386ShBUFBoU14LY4y1WhKJBBKJpMn7awz2IpFI\nq4OMHDkSI0eOxIEDBzBhwgScP39ep0rMmDEDNdIazP9qIVxcntRpX8YYawtCQ0MRGhqqeBwdHa3T\n/hq7cVxdXZGTk6N4nJOTA7FYrLb8oEGDUFdXh9u3b0MsFuu0bzvzdjCHJXILOEPLGGP6pjHYBwcH\nIzMzE9nZ2aipqUFiYiLCw8Prlbl8+TKEXAFw8uRJAIC9vb1W+zb0kMgWN4p4+CVjjOmbxm4cCwsL\nxMXFISwsDFKpFJGRkfD19cXKlSsBAFFRUdi4cSPWrl0LS0tLdOzYERs2bNC4rybW5rbIKykB4Kqf\nV8cYYwwAICJ5s9xYFRCJFN8MvOf2h3PaAvy1boAxq8QYYybv/tipDZO5gxYAbB+yRVE5d+Mwxpi+\nmVSwd7C2RXEFT4bGGGP6ZlrBvpMNSmu4Zc8YY/pmUsHeycYWFbISSKXGrgljjLUuJhXs7R6yQfvO\npSguNnZNGGOsdTGpYG9rZYv2NiUoKDB2TRhjrHUxuWBv2amEJ0NjjDE9M6lgb2NlA7MOpdyyZ4wx\nPTOpYG9rZQu055Y9Y4zpm8kFe6klt+wZY0zfTCrY27S3Qa154y37kNUhKK7kITuMMaYtkwr2tla2\nqILm0TgF5QU4duMYrpVea76KMcZYC2dSwb6DZQdIUYtbRTVqy2QUZgAAbpbfbK5qMcZYi2dSwV4k\nEqGjhQ3yS9TPj5NekA4AyC/Lb65qMcZYi2dSwR4QunIKy9TPj5NekA4rCyvcLOOWPWOMacvkgr1d\nBxvcLtfcsh/gNgD55dyyZ4wxbZlcsO/ykC1k7UpQoWYp2ozCDAx2H8wte8YY04HJBXvbh2zRuWup\nyuGXJVUlKK0qRd9ufTlByxhjOjC5YG/T3gbWXVQPv8woyICvoy9cOrlwgpYxxnTQaLBPTk6Gj48P\nvLy8EBsbq7R9/fr18Pf3h5+fHwYOHIi0tDTFNnd3d/j5+SEwMBAhISFaVcjWyhZWtqpvrEovSEcv\nx15wsnbibhzGGNOBhaaNUqkU06ZNw549e+Dq6oq+ffsiPDwcvr6+ijI9evTA/v37YWNjg+TkZEya\nNAlHjx4FIAyllEgk6NKli9YVsmlvg3adVU+ZkF6Yjl4OveDQwQHFVcWok9XBwkzjS2CMMYZGWvap\nqanw9PSEu7s7LC0tERERgaSkpHpl+vfvDxsbGwBAv379cP369XrbdVn9HBBa9ubWmlv25mbmsH/I\nHgXlPIkOY4xpQ2OzODc3F25uborHYrEYKSkpast///33GDFihOKxSCTC0KFDYW5ujqioKLz11lsq\n95sxY4bi92pxNUQPqe6zlwd7
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 48
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"What you can see here is that initially, both the error rate on the training data and the test set error rate decrase together. However past the first six rounds of training, the test set error rate doesn't decrease any further, while the error rate on the training data keeps decreasing. We call this _overtraining_. \n",
|
||
|
|
"\n",
|
||
|
|
"In this case, it would be a good idea to take the weights of the network not at 100000 update steps, but at around 12000."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"subplot(121); plotgrid(forward)\n",
|
||
|
|
"subplot(122); plotclass()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"err= 0.0\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAD9CAYAAABOd5eOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXl0HOWZh/v03i2ptW/WYstuyVg2XrFZE1AWu0kcGCYL\nMJNJGAYSJjdBkGQO5GI844RkcCZzMjEm90xmwuQwkzMeuPfmDgYRYzvBNmExBtvY2GDLWmxJ1r63\n1Gt13T+qSyq1WltLvUj6nnPqqFWqrvpatn711vv93vfTybIsIxAIBIJ5jT7RAxAIBALB7BFiLhAI\nBAsAIeYCgUCwABBiLhAIBAsAIeYCgUCwABBiLhAIBAsAIeaCRcPf/M3fUFBQwNq1ayc8prq6moqK\nCtavX8+pU6fiODqBYHYIMRcsGu6//34OHDgw4c9fffVVLl26RG1tLf/2b//Gt771rTiOTiCYHULM\nBYuGT37yk2RlZU348/3793PfffcBcMMNN9DX10d7e3u8hicQzAoh5gJBiJaWFkpLS0e+Lykpobm5\nOYEjEgimjzHRAxAIkonw7hY6nW7cMZH2CQRzzUw7rYjIXCAIUVxcTFNT08j3zc3NFBcXRzxWluUZ\nbf9w223IMG77h9tum9l5/uEfZnztaLdt23ZoRzry2ul8Mm5jiPdnTobrynJ07bJEZC4QhLjzzjt5\n9tlnuffee3nnnXfIzMykoKBgTs4dsFgi7pes1qjPeaymhoPPPIPR6yVgsbCtuppbt2+P6lw1Ncd4\n5pmDeL1GLJYA1dXbqK7eRl3dDurqfjJynMPxBA8/fHvUYxbEDiHmgkXDX/zFX3D06FG6urooLS3l\nhz/8IX6/H4CHHnqIz3/+87z66quUl5eTmprKb37zmzm79rbqanbU1fGTurqRfU84HNz+8MNRne9Y\nTQ2vPfLIyPmOAb984w3+v/JyUpcsmZGw19Qc45FHXhsj2nV1O9izx8mePU727t3Jxx+/wapVO3n4\n4dvZvv3WqMYsiDGyQCCYEdH+2Rx95RX5SadT/uaaNfLdOTnyo2vXyju2bZOPvvLKtM/x+uuvy7Is\nyzu2bZNlkGWQj4L8ROi1uj3hcEz7vNu27ZDD3i6DLDudT467biJI1LUT+Zmj+T8mInOBIE6okfJr\njzzCr7q7obsbzp5lRyi6nk4kXVVVBYDR6x3ZdxD4SdhxP6mrY+fevdM6p9cbWQY8HsO46yaCRF07\nkZ85GsQEqEAQRw4+88yYVAsownto794ZnUebg58oIjN4PNM6l8USiLjfapVmNCZBYhFiLhDEEW1E\nrWW6wquyrbqaHQ4HAJGlGFoGB3nS6WRXVRVPOp0cq6mJeFx19TYcjh1j9ikTnVtnNCZBYhFpFoEg\njsyVq0VNn+zcu5fO5mb+tr6ef3W7R37+N4WFZF69yo9PnhzZN1E6R53Q3Lt3Jx6PAatVEhOd8xBd\nKNkuEAimiU6ni9oLHO5CgZCrZc+eqG2F6nkP7d2LweNBslrp6ejglxEahe10Onlqkv40guQgmv9j\nQswFghkyGzGH8cK75MYbufr223PiF1fZVVXFrqNHx++/7TZ2HTkyq3MLYk80/8dEmkUgiDO3bt8+\nItaRIvWZuFsmIhZFSoLkRkyACgQJZK7cLeFoJ0hVnnA42BplkZIg+RGRuUCQQObK3RKOdoJUTefc\n/vDDs07fCJIXIeYCQQKJZTpEm84RLHxEmkUgSBDHampo6+zkW2HCLdIhgmgQbhaBYIbM1s0CYyc+\njwGHgCtWK2mrV3PPj34kIupFjnCzCATzBO3E562hDY+HnXl5QsgFUSHSLAJBAojVxKdg8SLEXCBI\nAMIHLphrhJgLBAlA+MAFc42YABUIZshcTIDC+LL+rcIHLggherMIBHFgrsRcIJiIaP6PiTSLQCAQ\nLACEmAsEAsECQPjMBYJZcKymhoPPPDOn7WsFgmgQYi4QREms2tcmipqaYzzzzEG8XiMWS4Dq6m1i\ntaF5hBBzgSBKJmpfu3Pv3nkn5jU1x3jkkdeoq/vJyL66OmVdUCHo8wORMxcIomQhVXE+88zBMUIO\nUFf3E/buPZSgEQlmihBzgSBKFlIVp9cb+SHd4zHEeSSCaBFiLhBEyUKq4rRYAhH3W61SnEciiBaR\nMxcIoiSZVvOZraumunobdXU7xqRaHI4nePjh22MxXEEMmFUF6IEDB3j00UeRJIkHH3yQxx9/fC7H\nJhAkJclWARrRVeNw4NyzZ0aCXlNzjL17D+HxGLBaJR5+eKuY/EwQcS3nlySJa665hsOHD1NcXMyW\nLVvYt28flZWV0ZxOIJg3JJuYP+l08uODB8ft3+l08tSBAwkYkWC2xLWc/91336W8vJyysjJMJhP3\n3nsvL730UrSnEwgEUbKQXDWC6Ik6Z97S0kJpaenI9yUlJRw/fnzMMTqdLvqRCQTTIJki5ESxkFw1\nguiJWsynK9QrgOWAHPpaBkih7yfDAGivMNHVdCiPF+E/Pwx8NvSz8J8bQvvUaxgAE2ABrIAdKNBB\nQRrkOEC3VU/btgJe2PhFXjr+RY799yfgcCO0XwRqgSFAdQMcAaqm+HSxIlHXjtd1G0ObytE4XDP5\n2VZdzY66ujE58yccDm6fh64aQfRELebFxcU0NTWNfN/U1ERJScm448qBTwFBFBGfjpATOl6LKrrh\nyGHHhgt3MHRM+M1BDo3FpHmPDuUXYgUyLZCVBQYH9G1M4+K6Fbzs+nM+eG8THPZDbzPQBngijFYQ\nG8pCm4oQc0guV81CZT60OohazDdv3kxtbS2NjY0UFRXxwgsvsG/fvinfN92H4kjHhYt2pGNlzabd\nF2RUtMPPrRVyC5AG2NLAXApshNoKB68bbuPSGyvpfxdob0QR8gGUiFw86gsSy63btwvxjhHzpdVB\n1BOgRqORZ599FqfTyerVq7nnnnsiOlkcEd4bDWokLTEabUciGNrKwo5R94e/hrGpFhuQoQNLFgRX\n6PFuMXM6fwMH2m6n73/tcLINOAd0Am7GP2uURf0ZZ0+irp2o6woEsWe+tDqYVdHQ5z73OT73uc9N\nesxKlETEXKIKuxY1N66yHEWwdZr9E6VcVDE3A2kGyLFBSim41qRytmIVJzq2cPHYNbjPD0JPK9CN\nIuSRqubKZvfhZkWirp2o6woEsSdWrQ7mOnUT8wrQWPlZwiPz8Ehbm04JavZHei8oYm4FUsxgywXj\naujbmMGhvM9w+vhG+l9Lg6tXwNsOuAD/BGcSJDNTFbp1dXXxV3/1V7S1tREIBPi7v/s7/vqv/zox\ngxUkBbFodRCL1E3Me7NEcprEgvA0jHZ/kPGpFS1q9G4FbBbQl4C00UD7xgJqdNs5f3YV/MENA80o\n6RXvJGcTJCuSJPGd73yHAwcOcP78efbt28dHH3005phnn32WjRs3cvr0aY4cOcL3v/99AoHIf8yC\nxUF19TYcjh1j9imtDrZGfc5YpG5iHpkbUaLeeEqfKuwT3UjU9ItqUVRtiRkmyMwE/TVwtrSSP7pv\npf13JXiPD0GwCeQuYJjpe3IEyYS20A0YKXTTzvUsWbKEM2fOADAwMEBOTg5Go2hhtJhRI+W9e3dq\nWh3cPquUSCxSN3ER83iXDmndLDrNV23qRftzA8rEZ5oNUopBdz2cKVjH4a7P0nsoE+nCVZCvAIOI\n9Mr8ZTqFbt/4xjf49Kc/TVFREYODg7z44osRz3X/V79K3fHj6AMBSvPy+MauXcJNsoDZvv3WOXWu\njE/dHAGO0Nz8Ort2RXfOmKdZEtkNWetth9GIPVyKjUAqYMsAowPkT+k4lX0dhy87cb1ngeZeoAMl\nvSKEfL4ynUK3f/zHf2TDhg1cvXqV06dP8+1vf5vBwcFxxxUdP86xujqOXL7Mf733Hq898gjHampi\nMWzBAmR86qYKh8PHnj1PsStKNY95ZK4W5SQrqrc8FTAWQrcjh+MZm7j09gr4nRv6LwEtKJ6cACJX\nPn+ZTqHbW2+9xY4dyh+Zw+Fg
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 49
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"(cross-validation)\n",
|
||
|
|
"\n",
|
||
|
|
"When we have limited amounts of data, we often perform _cross-validation_:\n",
|
||
|
|
"\n",
|
||
|
|
"- partition the data $D$ into 10 separate subsets $T_i$ of about equal size\n",
|
||
|
|
"- train a classifier on $D - T_i$\n",
|
||
|
|
"- test the classifier on $T_i$ and compute the error rate $e_i$\n",
|
||
|
|
"\n",
|
||
|
|
"Our estimate of the final error rate is the average of the error rates $e_i$.\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"(leave-one-out cross validation)\n",
|
||
|
|
"\n",
|
||
|
|
"An extreme case of cross-validation is to train a model on each subset $D-\\\\{x_i\\\\}$ for $x_i\\in D$ and average the resulting error rates.\n",
|
||
|
|
"\n",
|
||
|
|
"This gives a good error estimate but is computationally very costly.\n",
|
||
|
|
"\n",
|
||
|
|
"For some classifiers (e.g., nearest neighbor), leave-one-out cross-validation can be computed efficiently."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"(observations)\n",
|
||
|
|
"\n",
|
||
|
|
"- classifiers are prone to overtraining\n",
|
||
|
|
"- the more parameters a classifier has, the more likely it is to overtrain\n",
|
||
|
|
"- test set error is a better reflection of real-world performance than training set error\n",
|
||
|
|
"- cross-validation gives us better estimates of real-world error than a single test set\n",
|
||
|
|
"\n",
|
||
|
|
"Keep in mind:\n",
|
||
|
|
"\n",
|
||
|
|
"- if your test set is not representative (by accident or lack of randomization), your error rate estimates may be wildly wrong\n",
|
||
|
|
"- cross-validated error rates do not always converge asymptotically to the true error rate"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "heading",
|
||
|
|
"level": 1,
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Test Set Error in the Presence of Errors"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"We introduced the notion of test set error in the case of fully separable data. The difference between training set and test set error becomes even more significant in the case of data that isn't fully separable.\n",
|
||
|
|
"\n",
|
||
|
|
"In this case, the test set error rate simply stops changing after a while. When the training data is noisier and there are a lot of hidden units, the cross-validated error rate can actually start increasing again.\n",
|
||
|
|
"\n",
|
||
|
|
"Let's introduce some errors and see what happens (there is no guarantee that anything interesting happens)."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"labels = abs(labels-(random(len(labels))>0.9))\n",
|
||
|
|
"test_labels = abs(test_labels-(random(len(test_labels))>0.9))"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 50
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"plotlabels()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"38 62\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD9CAYAAABHnDf0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X9sFOeZB/CvgxGglGsgpAT/kAhjN5gA5ipXhFSxl0be\nMTilUhPpHKm9hALhuMMLiSqReNfCBIJI/zjJLH+EXhN0CQ2iSqtz6oXFJMFeNQSMIAIJEHI20K5N\n4SCFkmsC2Nv3/gAvXnt/zM7Ozrwz8/1IK3nHw+7rwX7m3fd93uctEkIIEBGRo9xndQOIiMh4DO5E\nRA7E4E5E5EAM7kREDsTgTkTkQAzuREQOlDW4//znP8f06dMxb968tOf4fD5UVlaiuroan332maEN\nJCKi3GUN7suXL0c4HE77/X379uHzzz9HX18ffvWrX2HNmjWGNpCIiHKXNbg/+eSTmDJlStrvf/DB\nB3j++ecBAAsXLsT169dx+fJl41pIREQ5y3vMfWBgAOXl5YnnZWVl6O/vz/dliYgoD8VGvMjoCgZF\nRUVjzkl1jIiIstNTJSbvnntpaSlisVjieX9/P0pLS1OeK4Sw5LGxrg4CGPPYWFdnTXs2brTsWlj9\n8Hr9o/8XAAioasDytln9cPPvBa9F+odeeQf3ZcuW4Z133gEAHDlyBA888ACmT5+e78saamjChJTH\n4xMnmtwS7SKhEAKqijaPBwFVRSQUsrpJaeXSVp/PC0XxJx1TlBY0N9cXuplErpJ1WOa5555DT08P\nrl69ivLycmzatAmDg4MAgNWrV2Pp0qXYt28fKioqcP/992PXrl0Fb3SuvD4f/NEoXo9GE8daFAUN\nzc0Wtiq9SCiEA+vWJbXXf/fr2sZGREIhdG3fjuJbtzA0YQK8Ph9qGxulbOtojY21AIBgsBU3b45D\nf/8htLdvThwnIoMIk5j4Vin1dHaKgKqKjXV1IqCqoqezM3Hc7/WKjXV1wu/1Jo4X0qFDhzJ+3+/1\nCgGMeQy3u0VRko63KIop7c61rVpkuxZuwmtxD6/FPXpjpyETqnZQ29g4pieZa6/TKB6PJ+P3i2/d\nSnl83M2b6Nq+Pam9APB6NIrWYNCS3numtmqR7Vq4Ca/FPbwW+XN1+YF0gfJgMGhRi+7INEeQbzA1\nmh3nM4jcwNXBXbZAOczr88GvKEnHWhQF9c3NOQVTMyZlM7WViKzjmmGZVGTtdQ4Pr7QGgxh38ybi\nEyeiobk5cVzL5LBZQ07Z2kpE1ii6O2Bf+DcqKsorZ7MQUgXAFkVBQ3u71MEpEgrh4IhgWp8imAZU\nFVu6usb821ZVxeYMtYKISC56Y6ere+7Zep0ypRyOlGpyeDRZh5yIyByuDu5A+kBpVSaNUWQdciIi\nc7h6QjUTWTNptOJEJ5G7ub7nno7dhzU40UnkbgzuaThhWEPL2DwROROHZUYZzg2/MjCAf5s0Kel7\nHNYgIrtgz32E0ZOoEQD/MmkSSioq8K2SEg5rEJFtMLiPMHoStRZA7TffoLWkhLnhRGQrHJYZwe6T\nqEREwxjcR3DCJCoREcDgnoS54UTkFK6uLZOKlrotRERm0Rs7GdyJiCSmN3ZyWIaIyIEY3ImIHEiK\nPHdZS+sSEdmV5cHd7qV16Y5QKILt27tw61YxJkwYgs/nRWNjrdXNInIty4N7utK6rcEgg7tNhEIR\nrFt3ANHo64lj0agfABjgiSxi+Zg7V4Xa3/btXUmBHQCi0dcRDB60qEVEZHlw56pQ+7t1K/UHwJs3\nx5ncEiIaZnlw56pQ+5swYSjl8YkT4ya3hIiGWT7mzh2DcidbdpHP50U06k8amlGUFjQ3N1jWJiK3\n4wpVm0mZXaQoUNvbLQ3woVAEweBB3Lw5DhMnxtHcXM/JVCIDsPyASwRUFVu6usYcb1VV1pwnciCW\nH3AJZhcRkRYM7jbD7CIi0oLB3WaYXUREWnDM3YZYc57IPSUvOKFKRK6RquSFovjR3q46LsBzQpWI\nXIMlL7KzfBETEVGu7FzywqzhpKzBPRwOY/369YjH41i5ciU2bNiQ9P2rV6/ipz/9KS5duoShoSH8\n4he/wAsvvGB4Q4mIhtm15IWpFVRFBkNDQ0JRFHH+/Hlx+/ZtUV1dLc6cOZN0zsaNG8Urr7wihBDi\nypUrYurUqWJwcHDMa2V5KyIizTo7e4SitAhAJB6K8qro7OyxumkZeb3+pDYPP1Q1kPbf6I2dGXvu\nvb29qKiowMyZMwEATU1N6OjoQFVVVeKcGTNm4NSpUwCAGzdu4MEHH0RxMUd7iKhwhnu5wWDriJIX\nDdJPppo5nJQxCg8MDKC8vDzxvKysDEePHk06Z9WqVfjhD3+IkpISfPXVV/jtb3+b9vXa2toSX0+Z\nOBFXDh2SpvgVEdlLY2Ot9MF8NC3DSd3d3eju7s77vTIG96KioqwvsHXrVixYsADd3d2IRqOor6/H\nyZMnMXny5DHnDgd3bq1HRG6kpYKqx+OBx+NJPN+0aZOu98oY3EtLSxGLxRLPY7EYysrKks45fPgw\n/H7/3UYqeOSRR3Du3DnU1NSkfV1urUdkf1YtIrLz4iUzh5MyBveamhr09fXhwoULKCkpwd69e7Fn\nz56kc2bPno0PP/wQP/jBD3D58mWcO3cOs2bNyvymLH5FZGtW7Ztr9/16R9+YmpsLeGPKNuO6b98+\n8d3vflcoiiK2bt0qhBDizTffFG+++aYQ4k6GzNNPPy3mz58v5s6dK37zm99knfH1e71jp4sBEVBV\nXbPCRGQuPVkfdn5fI6TO8GnJmuGjIUynlDWtZcmSJViyZEnSsdWrVye+njZtGv7whz/kdEPx+nzw\nR6NJQzMtioIGFr8iG7PzcEGurFpEZOfFS+lX1bZas4ipELi1HjmN3YcLcmXVIiK7Ll4CzL8xWZaQ\nXtvYyGBuc27qqWZjdq/Malbtm2vn/XrNvjFxtRHp4raeajZ2Hi7Qw6pFRHZdvASYf2NiyV/SRVUD\n6OrakuJ4K8LhzRa0yFq8Hvq47dOfno3k9cZO9txJF7f1VLOx83CBVdz46c/MVbUM7qSLnSe2CsHO\nwwVWcds8hdkY3EeIhELo2r6d9W40YE91LDvWOrESP/0VFoP7Xax3k5vGxlqcPfYpPtqhYMJQHLeK\nx+Gpn77I4Eaa8dNfYXGbvbvS1bs5GAxa1KLsIqEQAqqKNo8HAVVFJBQy9b2v7f4v7P/yC/zP3/6E\n/V9+gWu7/8vUNpC9+XxeKIo/6didT3/1FrXIWdhzv8tu9W6s/qTB4m+UL85TFBaD+11DEyakPB6f\nONHklmhjdXC1282Q5MR5isLhsMxdXp8PfkVJOtaiKKiXtN6N1cHVbjdDIrcxtecuczaK3erdWB1c\nWfyNSG6mrlBtUZTkMWJFgdreLm0AlVmqMfcWRUGDidczEgrh4IibYb3EN0PSz22rSGWjd4WqueUH\nUhxvVVVsDofNaILjMLhSoaVaRaoofrS3qwzwJrFtcG+rq0ObAZvBEpHxWDPHenqDu+UTqpyAI5IX\nV5Hal6nB3U7ZKETEVaR2Zmq2jNrebptsFCJiDSE7Yz13SknmtFUyl54a5GQce0yoMrjbQsrSBkxb\nJbKEbSdUST52LKJGRMkY3GkMq0sbEFH+GNxpDKtLGxBR/hjcaQy7FVEjorE4oUopsbQBkRyYLUNE\n5EB6Yyc367AYK+4RuY8Z60gY3HNkZDBOVXEvGr2zpyQDPJEzmbZFpjCJiW9VMJ2dPUJRWgQgEg9F\naRGdnT26Xs/r9Se91vBDVQMGt5ycpKezU/i9XrGxrk74vV7R09lpdZMoB36vd+wfPSACqpryfL2x\nkz33HGzf3pXUywaAaPR1BIOtunrarLhHubJ6Y3TK39//8peUx//v4kVD38fVqZChUASqGoDH0wZV\nDSAUimQ83+hgzIp7lCuuHra/i2mC+F/SBH29XNtz1zPebXQwZsU9yhVXD9vfAw8/DP+XX2LkGEAL\ngG9Pn27o+7g2uOsZYjE6GA+/
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 51
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Depending on whether we got lucky with the choice of data points, we may or may not observe the test set error rate curve going up again.\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"init(10)\n",
|
||
|
|
"_=cv_training_curve(50,2000,1.0)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD9CAYAAABDaefJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlcVNX/P/DXIOOCAiooIkOigjAgAoVpmTWumBWKmqFl\nfgyXb8anvU/1qT6hv3LLcglTtCyXRHILK0VcGPfAnRRMXNAREQUB2WRg5v374waIDMMMzIIz7+fj\n4UNm7rnnHm72mjPnnnuuiIgIjDHGLIKNuRvAGGPMcDjUGWPMgnCoM8aYBeFQZ4wxC8KhzhhjFoRD\nnTHGLEiDoZ6QkAAfHx94eXlh/vz5dbYvXLgQQUFBCAoKgr+/P2xtbVFQUGCUxjLGGNNOpG2eukql\ngre3N/bs2QM3Nzf07dsXsbGxkEqlGsv//vvvWLx4Mfbs2WO0BjPGGKuf1p56SkoKPD094eHhAbFY\njPDwcMTHx9dbfsOGDZgwYYLBG8kYY0w3WkM9KysL7u7u1a8lEgmysrI0li0tLcWuXbswduxYw7aQ\nMcaYzmy1bRSJRDpX9Ntvv+Gpp55C+/btm1wXY4yxGvqs5qK1p+7m5gaFQlH9WqFQQCKRaCy7cePG\nBodeiIj/EOHzzz83exuayx8+F3wu+Fxo/6MvraEeHByMjIwMZGZmQqlUIi4uDqGhoXXKFRYW4sCB\nAxg1apTeDWCMMWY4WodfbG1tER0djZCQEKhUKkREREAqlSImJgYAMGPGDADAr7/+ipCQELRp08b4\nLWaMMVYvrVMaDXogkahRXyUskVwuh0wmM3czmgU+FzX4XNTgc1FD3+zkUGeMsWZM3+zkZQIYY8yC\ncKgzxpgF4VBnjDELwqHOGGMWhEOdMcYsCIc6Y4xZEA51xhizIBzqjDFmQTjUGWPMgnCoM8aYBeFQ\nZ4wxC8KhzhhjFoRDnTHGLAiHOmOMWRAOdcYYsyAc6owxZkE41BljzIJwqDPGmAXhUGeMMQvCoc4Y\nYxaEQ50xxiwIhzpjjFmQBkM9ISEBPj4+8PLywvz58zWWkcvlCAoKQu/evSGTyQzdRsYYYzoSERHV\nt1GlUsHb2xt79uyBm5sb+vbti9jYWEil0uoyBQUFGDBgAHbt2gWJRILc3Fw4OzvXPZBIBC2HYowx\npoG+2am1p56SkgJPT094eHhALBYjPDwc8fHxtcps2LABY8eOhUQiAQCNgc4YY8w0bLVtzMrKgru7\ne/VriUSC5OTkWmUyMjJQUVGBQYMGoaioCG+99RYmTZqksb6oqKjqn2UyGQ/VMMbYA+RyOeRyeaP3\n1xrqIpGowQoqKipw8uRJ7N27F6WlpXjiiSfQv39/eHl51Sl7f6gzxhir68EO76xZs/TaX2uou7m5\nQaFQVL9WKBTVwyxV3N3d4ezsjDZt2qBNmzZ4+umncebMGY2hzhhjzLi0jqkHBwcjIyMDmZmZUCqV\niIuLQ2hoaK0yo0aNwqFDh6BSqVBaWork5GT4+voatdGMMcY009pTt7W1RXR0NEJCQqBSqRAREQGp\nVIqYmBgAwIwZM+Dj44MRI0agT58+sLGxwbRp0+oN9fJyoFUrw/8SjDHGBFqnNBr0QCIRbt8m8OQY\nxhjTnUGnNBra3bumPBpjjFkfk4Z6YaEpj8YYY9aHe+qMMWZBONQZY8yCmHj4hdd+YYwxYzJpqOfw\noDpjjBmVaUP9bp4pD8cYY1bHpKF+qyTXlIdjjDGrY9JQzyvlnjpjjBmTSUM9/x6HOmOMGZNpZ79U\ncKgzxpgxmTTUi1Qc6owxZkwmDfUS4guljDFmTCYN9Xsi7qkzxpgxmTTUlS041BljzJhMGuoVYg51\nxhgzJpOGOrXOg1JpyiMyxph1MWmoo20uiopMekTGGLMqJg11EYCcO6WmPCRjjFkVk4a6bYUTFLk8\nrs4YY8Zi0lBvWemErHwOdcYYMxaThnprckJ2AYc6Y4wZi0lD3Q5OuFnEd5UyxpixNBjqCQkJ8PHx\ngZeXF+bPn19nu1wuh6OjI4KCghAUFIQvvvii3rratXBGbgn31BljzFhstW1UqVSIjIzEnj174Obm\nhr59+yI0NBRSqbRWuWeeeQbbt29v8GCOYifcKeNQZ4wxY9HaU09JSYGnpyc8PDwgFosRHh6O+Pj4\nOuWIdHugdIdWTsgv51BnjDFj0dpTz8rKgru7e/VriUSC5OTkWmVEIhGOHDmCgIAAuLm5YeHChfD1\n9dVY3/VDB3Cj7BKioqIgk8kgk8ma/hswxpgFkcvlkMvljd5fa6iLRKIGK3j00UehUChgZ2eHnTt3\nYvTo0bhw4YLGskNGT8Wmq9GIiopqVGMZY8zSPdjhnTVrll77ax1+cXNzg0KhqH6tUCggkUhqlbG3\nt4ednR0A4Nlnn0VFRQXu3LmjsT4XB2eUEA+/MMaYsWgN9eDgYGRkZCAzMxNKpRJxcXEIDQ2tVSYn\nJ6d6TD0lJQVEhI4dO2qsz9XRCeU2HOqMMWYsWodfbG1tER0djZCQEKhUKkREREAqlSImJgYAMGPG\nDGzevBnLly+Hra0t7OzssHHjxnrrc+voBKUthzpjjBmLiHSdutLUA4lEOJemgt/GllD+rwziFmJT\nHJYxxh5qIpFI5xmGgInvKO3Q3gai8g64U6Z5zJ0xxljTmDTUHRwAlDojj29AYowxozDt2i92AJU4\n4VYRhzpjjBmDaR+SIfpnTfU8DnXGGDMG0z7ODkArlROy8nmlRsYYMwaTh3obcsbNu9xTZ4wxYzB5\nqLe14TF1xhgzFpOHun0LJ+SWcqgzxpgxmDzUHcVOyL/Hoc4YY8Zg8lDv0NoJ+eV8oZQxxozB5KHu\nbOeMokruqTPGmDGYPNQ7tXVCsZpDnTHGjMHkoe7i0BH3kK/XAjWMMcZ0Y/JQ79hejBZkh8LyQlMf\nmjHGLJ7JQ93BARBXOCG3lC+WMsaYoZkl1G2VzsjjueqMMWZwZgl1UZkTL7/LGGNGYJZQp1In7qkz\nxpgRmCXUK4u4p84YY8ZgllCvKOQLpYwxZgwmD/V27YDKu87ILeGeOmOMGZrJQ10kAlqTE3J4+V3G\nGDM4k4c6ALSzccKtYg51xhgztAZDPSEhAT4+PvDy8sL8+fPrLXfs2DHY2tpi69atDR7U3pZnvzDG\nmDFoDXWVSoXIyEgkJCQgLS0NsbGxSE9P11juww8/xIgRI3Ra06V9SyfcuccXShljzNC0hnpKSgo8\nPT3h4eEBsViM8PBwxMfH1yn37bffYty4cejUqZNOB+3Y2hmFSu6pM8aYodlq25iVlQV3d/fq1xKJ\nBMnJyXXKxMfHY9++fTh27BhEIlG99UVFRQEAsq8BKhc1SitKYSe2a0LzGWPMssjlcsjl8kbvrzXU\ntQV0lbfffhvz5s2DSCQCEWkdfqkK9awsIOuR75FXmgc7Rw51xhirIpPJIJPJql/PmjVLr/21hrqb\nmxsUCkX1a4VCAYlEUqvMiRMnEB4eDgDIzc3Fzp07IRaLERoaWm+9Dg5AGwh3lbo7utdbjjHGmH60\nhnpwcDAyMjKQmZmJrl27Ii4uDrGxsbXKXL58ufrnKVOm4IUXXtAa6IAQ6q1UfFcpY4wZmtZQt7W1\nRXR0NEJCQqBSqRAREQGpVIqYmBgAwIwZMxp1UAcHQJzLy+8yxpihichEz5WrGnMHgB9+AL5Kfx1v\nvuSPmX1nmuLwjDH2ULo/O3VhljtKq9dU5546Y4wZlFlC3dERUJfw8ruMMWZoZuupV97lC6WMMWZo\nZgv1ikJn7qkzxpiBmS3Uy/J4TJ0xxgzNbKFemsdj6owxZmjmWU+9HVB2h3vqjDFmaGYJdRsboG0L\nRxQri1GhqjBHExhjzCKZJdQBoL2jDdq36og7ZXfM1QTGGLM4Zgt1BwfAQczj6owxZkha134xJgcH\nwLYFj6szxpghmTXUAe6pM8aYIZk11JXEd5UyxpghmXVMvZWKl99ljDFDMluoOzoCtkoefmGMMUMy\na0/dppxDnTHGDMmsoY5Snv3C
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 52
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"subplot(121); plotgrid(forward)\n",
|
||
|
|
"subplot(122); plotclass()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"err= 0.08\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAD9CAYAAABOd5eOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXlwHPWZ///quUej+xxdtmzJJ4cxsQOEBBQSWyRe2GyS\nBTabhCSQsPtLMGTzLZPCOOtAWMzuVjbGbNWyGyrl7ze13lBb2YVExICT2CYc5rKN8W3ZsnXfx8xo\n7unfHz2taY1mdIzmkvR5VXV51O7p/mgkvfvp5/N+no8ky7KMQCAQCOY1ukwPQCAQCARzR4i5QCAQ\nLACEmAsEAsECQIi5QCAQLACEmAsEAsECQIi5QCAQLACEmAsWDd/61reoqKjgmmuuiXvM1q1bWbFi\nBevWrePo0aNpHJ1AMDeEmAsWDd/85jfZv39/3P9/+eWXuXDhAufPn+ff//3f+du//ds0jk4gmBtC\nzAWLhk996lMUFRXF/f+XXnqJe++9F4AbbriB4eFhenp60jU8gWBOCDEXCMJ0dHRQW1s7/nVNTQ3t\n7e0ZHJFAMHMMmR6AQJBNRHe3kCRp0jGx9gkEyWa2nVZEZC4QhKmurqatrW386/b2dqqrq2MeK8vy\nrLa/v/VWZJi0/f2tt87uPH//97O+dqLb5s3btSMdf93U9FjaxpDu7zkbrivLibXLEpG5QBDmzjvv\n5Nlnn+Wee+7h7bffprCwkIqKiqScO2A2x9wftFgSPufh5mZefeYZDF4vAbOZzVu3csuWLQmdq7n5\nMM888yperwGzOcDWrZvZunUzLS3baWl5cvy4+vpHefDB2xMesyB1CDEXLBr+6q/+ikOHDtHf309t\nbS0//vGP8fv9ADzwwAN8/vOf5+WXX6ahoQGbzcYvfvGLpF1789atbG9p4cmWlvF9j9bXc/uDDyZ0\nvsPNzbzy0EPj5zsM/Ovrr/M/DQ3YKitnJezNzYd56KFXJoh2S8t2du9uYvfuJvbs2cGZM6+zevUO\nHnzwdrZsuSWhMQtSjCwQCGZFon82h377W/mxpib5O1ddJd9VUiI/fM018vbNm+VDv/3tjM/xxz/+\nUZZlWd6+ebMsgyyDfAjkR8Ov1e3R+voZn3fz5u1y1NtlkOWmpscmXTcTZOramfyeE/kdE5G5QJAm\n1Ej5lYce4rmBARgYgBMn2B6OrmcSSTc2NgJg8HrH970KPBl13JMtLezYs2dG5/R6Y8uAx6OfdN1M\nkKlrZ/J7TgQxASoQpJFXn3lmQqoFFOF9bc+eWZ1Hm4OPF5HpPZ4ZnctsDsTcb7EEZzUmQWYRYi4Q\npBFtRK1lpsKrsnnrVrbX1wMQW4qhw+HgsaYmdjY28lhTE4ebm2Met3XrZurrt0/Yp0x0bprVmASZ\nRaRZBII0kixXi5o+2bFnD33t7fzNxYv8m9s9/v/fstsp7OzkJx98ML4vXjpHndDcs2cHHo8eiyUo\nJjrnIVI42S4QCGaIJEkJe4GjXSgQdrXs3p2wrVA972t79qD3eAhaLAz29vKvMRqF7Whq4okp+tMI\nsoNEfseEmAsEs2QuYg6ThbfyxhvpfOutpPjFVXY2NrLz0KHJ+2+9lZ0HD87p3ILUk8jvmEizCARp\n5pYtW8bFOlakPht3SzxSUaQkyG7EBKhAkEGS5W6JRjtBqvJofT2bEixSEmQ/IjIXCDJIstwt0Wgn\nSNV0zu0PPjjn9I0gexFiLhBkkFSmQ7TpHMHCR6RZBIIMcbi5me6+Pv42SrhFOkSQCMLNIhDMkrm6\nWWDixOdh4DXgisVC7tq13P344yKiXuQIN4tAME/QTnzeEt7weNhRViaEXJAQIs0iEGSAVE18ChYv\nQswFggwgfOCCZCPEXCDIAMIHLkg2YgJUIJglyZgAhcll/ZuED1wQRvRmEQjSQLLEXCCIRyK/YyLN\nIhAIBAsAIeYCgUCwABA+c4FgDhxububVZ55JavtagSARhJgLBAmSqva1maK5+TDPPPMqXq8BsznA\n1q2bxWpD8wgh5gJBgsRrX7tjz555J+bNzYd56KFXaGl5cnxfS4uyLqgQ9PmByJkLBAmykKo4n3nm\n1QlCDtDS8iR79ryWoREJZosQc4EgQRZSFafXG/sh3ePRp3kkgkQRYi4QJMhCquI0mwMx91sswTSP\nRJAoImcuECRINq3mM1dXzdatm2lp2T4h1VJf/ygPPnh7KoYrSAFzqgDdv38/Dz/8MMFgkPvvv59H\nHnkkmWMTCLKSbKsAjemqqa+naffuWQl6c/Nh9ux5DY9Hj8US5MEHN4nJzwyR1nL+YDDIqlWrOHDg\nANXV1WzcuJF9+/axZs2aRE4nEMwbsk3MH2tq4ievvjpp/46mJp7Yvz8DIxLMlbSW87/zzjs0NDRQ\nV1eH0Wjknnvu4cUXX0z0dAKBIEEWkqtGkDgJ58w7Ojqora0d/7qmpoYjR45MOEaSpMRHJhDMgGyK\nkDPFQnLVCBInYTGfqVAvBerCr+s0r5ONjshjhgT8Abgt/FqvOUav+dcAmDRbPlAMlElQWg75K4Cb\nofvPSnj/E+v4l7Hvc+y/NjDwXCGcvQCOi8AVwAX4gCBwEGhM0Xc5HZm6drqu2xreVA6l4ZrZz+at\nW9ne0jIhZ/5ofT23z0NXjSBxEhbz6upq2traxr9ua2ujpqZm0nF1pOfPXEaRUpiYO1L3R+8LRb1f\nFXijBLkGMBUCtcA10FJRz+uOW7j4xmpGXy+As24Y6wUGADfgj3FGQfKpY2I4IMQcsstVs1CZD60O\nEhbzDRs2cP78eVpbW6mqquJXv/oV+/btS+bYZoX2YTsU/lpGiczV16Hw1+oW0hyjRvAmPVitYCiF\nQJ2BkdU2jhddy+GBW+h/vRz/h25wtAN9wCiRiFw87gsyxy1btgjxThHzpdVBwhOgBoOBZ599lqam\nJtauXcvdd98d08lSN5fRJYgMLGGywAeJyK4c9VoinHbRgyUXDHbwLDPRuqyG9w3X8W7bx3G+aYWW\nfqAFJSofQ4nKtVeqS+03NyWZunamrisQpJ750upgTkVDn/vc5/jc5z435TF1c7nAHKhDEXA1+aHm\nzdVUjBqJq4KuA8yA2QhSEbAMHPV5vG26kYvHVxD8nQH5yiC4temVWNVxdan4dmZIpq6dqesKBKkn\nVa0Okp26WdAVoNGRuU6zXxV69Rg9yiSo2QhSGXjqTXTWVfCn4Ce5dKoe+XAIBnoh0Ac4AS8iTz7/\nmK7Qrb+/n69+9at0d3cTCAT4P//n//CNb3wjM4MVZAWpaHWQitTNounNoo3SISLoaopFTzgyN4NU\nCSPLc7lYvYQ3HZ/g8qkl8L4XnN1AP+ABAggxn18Eg0G+973vsX//fk6dOsW+ffs4ffr0hGOeffZZ\n1q9fz7Fjxzh48CA/+MEPCARi/zELFgdbt26mvn77hH1Kq4NNCZ8zFambBR2Zx0OdCI3GbAJrIUi1\n0Jpfx/Hh6/AcyIPTLgh0gTxEJL0iJjwzy+zjEG2hGzBe6Kad66msrOTDDz8EYHR0lJKSEgyGRfln\nIgijRsp79uzQtDq4fU4pkVSkbhbVb2m0o0XretEBRguYikFaDhcLl3PMsR73mzlwcRRC3SjuFZFe\nyTw6Yt+Op2YmhW7f/va3ue2226iqqsLhcPDCCy/EPNc3//qvaTlyBF0gQG1ZGd/euVO4SRYwW7bc\nklTnyuTUzUHgIO3tf2TnzsTOuWjSLDDZwaJ1t+gAvQV0ZcBKuFS8jGOO6/Acs0LHKNCNkisXVsTM\nIjGxRGwW75xBods//MM/cN1119HZ2cmxY8f47ne/i8PhmHRc1ZEjHG5p4eDly/y/997jlYce4nBz\n86zHJFicTE7dNFJf72P37ifYmaCaL6rIXEWNztUHGvW1ZAFfsZHBJfl0D1UyeKyEYL8TvMOAA1Ec\nlClUAVdfJ9YmYiaFbm+++Sbbtyt/ZPX19SxbtoyzZ8+yYcOGCcctlOXiFhOJukdSUTCUitTNohRz\n7eTnhMKhHHAXWmgtrKH3o3Lc
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 53
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"metadata": {}
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}
|