mirror of
https://github.com/saymrwulf/nvidia-AI.summerschool17-notebooks.git
synced 2026-11-20 09:52:58 +00:00
1039 lines
742 KiB
Text
1039 lines
742 KiB
Text
|
|
{
|
||
|
|
"metadata": {
|
||
|
|
"name": "intro-mlp"
|
||
|
|
},
|
||
|
|
"nbformat": 2,
|
||
|
|
"worksheets": [
|
||
|
|
{
|
||
|
|
"cells": [
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"Multilayer Perceptrons and Backpropagation",
|
||
|
|
"==========================================="
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"import scipy,pylab,numpy",
|
||
|
|
"from numpy import *",
|
||
|
|
"from scipy import *",
|
||
|
|
"from pylab import *",
|
||
|
|
"from urllib import urlopen",
|
||
|
|
"from gzip import GzipFile",
|
||
|
|
"from scipy.spatial import distance",
|
||
|
|
"from scipy.spatial.distance import cdist"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 56
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"Nonlinear Classification Problem",
|
||
|
|
"==================================",
|
||
|
|
"",
|
||
|
|
"Let's start by generating a simple linear non-classification problem. Actually, this is a pretty simple problem as far as non-linear classification problems go: it's low dimensional, and the decision boundary is the combination of two axis aligned linear boundaries. Still, it's a good way to get started."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"data = random_sample((1000,2))",
|
||
|
|
"# labels = (data[:,0]**4 * 0.7+data[:,1]*0.4>0.3)",
|
||
|
|
"labels = ((data[:,0]-0.3) * (0.7-data[:,1]) > 0.0)",
|
||
|
|
"sum(labels)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 57,
|
||
|
|
"text": [
|
||
|
|
"563"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 57
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"def plotlabels():",
|
||
|
|
" clf()",
|
||
|
|
" d0 = data[labels==False]",
|
||
|
|
" d1 = data[labels]",
|
||
|
|
" plot(d0[:,0],d0[:,1],\"bo\")",
|
||
|
|
" plot(d1[:,0],d1[:,1],\"ro\")",
|
||
|
|
" savefig(\"tmp.png\")",
|
||
|
|
" print len(d0),len(d1)",
|
||
|
|
"plotlabels()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"437 563"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD9CAYAAABHnDf0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvX10VNd5L/wTyGLGlhBjyckgMB8SjktkQWeucWDdNxLl\nrmhw5Np1mkhgY2EjWcKSEcJ3OamRxCgm2H7Dev0GEcWkhdvGIW1pVt8m7cgBkSqMaAM212EVA3av\nLXADiDFElY0gM4Dk/f5x5sycj73P2ft8jISr31ospKNz9t5nn3Oe/ezn4/dkEUIIJjGJSUxiEp8p\nTBnvAUxiEpOYxCScx6Rwn8QkJjGJzyAmhfskJjGJSXwGMSncJzGJSUziM4hJ4T6JSUxiEp9BTAr3\nSUxiEpP4DMJQuK9btw6f//znUVZWxjynpaUF99xzDxYvXozjx487PsBJTGISk5iEOAyF+1NPPYX9\n+/cz//7GG2/ggw8+wPvvv48///M/xzPPPOP4ACcxiUlMYhLiMBTuX/7yl+Hz+Zh//8d//EesXbsW\nAPClL30JH3/8MT766CNnRziJSUxiEpMQRradiy9cuIC777479fvs2bNx/vx5fP7zn1edl5WVZaeb\nSUxiEpP4LwurJAK2HarajlmCnCj+hSsqQAjJ6L9oJILNJSWqcVR7varfWcdrPR7qeUb30VZZSb2m\nvKQk4/du9i8c7obXWw0gDKANQBQlJZsRiUSZ11RUhCl3R5JtEBQW1qTO9flqqOd6PPeZjq2yso16\nbSjU7tj9RyLS/SrbN7t/0X+s+aqoCCefQdjW+KU5Tj8/bfs8/zIx13zvI3vMrDEWFFQz3sd2V54n\n77jk/uXvQvSZ2IEtzX3WrFk4d+5c6vfz589j1qxZpteNeTzcffT39KC3qwvZ169jdNo0VLa0oLyq\nSnisvV1d2DYwoDq2MB6nnqs9fnciQT3v8sgIs7/s69epx7NGR42GmXH09PTju989gXh8n+JoGwYG\nQti58yCqqsqp102bxrqPMQDAzJkzU0fmzcvF8HAbgG2K8zbD58sxHd/16/RXNJGYanotL7q6ejEw\nsE11bGBgG3bu7GDePw09Pf3o6urF9evZmDZtFC0tlanrWfPl8YxZH3iyz40bD2B4+G8VR9uS/5cL\ntd/SUomBgTbVXJSUbMaGDSttjVEL7TwtW1aEI0cGU78XFl5gXst6H/z+GZgxo03zHDcDkMZu5XmK\ngDZ3yv7l7wKw/8x5YUu4P/zww/j+97+PVatW4ejRo5gxY4bOJKPFJr8fI5cuoXP5cpWwpglxADiw\ncaNKKLclfxYV8DRhayye0qgE8PRtt+Evbt5MHdsMIDE4iP6eHupYRqdNo7ZNssWn3KkFjoaurl7E\n47s0R7cB6DAUoGYvc1FRburoww8HcerUIdy40QFgKoAx+P0x/NEfLTIdn1tCUQknFhBZyCrnY2BA\nErJVVeWuCU7awiQ/v5KS/ULty4Jv507p2Xs8Y9iwYaWjApE2T3196zE6+hgAqR+frxw9Pf3Uflnv\nw+zZn8OGDV/Bzp0dOHr0HD755G5I72K6DScVAi3ksW7Z0ozTp68ikZij6D/9XXi9jdiw4XHXxqEC\nMcCqVavIzJkzyW233UZmz55N9uzZQ3bt2kV27dqVOqe5uZmUlJSQRYsWkbfffpvaDgASrqggTcEg\nWef3EwKk/m0uKSHd4TDZXFKiO94UCKiOyf/aQyGjYasQjURIW2UlqfX5SBtAoop2ogBp9HpVbTd4\nvapz5H8P5uaSdoCEAdKuaIc1lmgkorunF0pKyPdefpl77IQQ0h0Ok2qvl4SB1Pg3l5SQaCQi1A4L\nFRVh2hQTIExCofbUeZFIlFRWtpGKijCprGwjkUiURCJREgw2EY+nlgDtBIgSgBC/v5UEAnWkoiJM\nAoE64vdvSv6tnQBh4vVWk3C4m/zqV78yHV8kEiUlJZtVYyspeYFEIlFH7p8QQior26hzoLx/J9qI\nRKIkFGonFRXS3CrvgWcuaGA9P5+v1tE5cgqseZLeDfnnX+nmXn7/5s59mkyZ8hABGgjQRoCo7n2Q\n+ogm/x5OnSfyPO0gEokSn29Vsu8mAtQnf24nc+Z8XfcdGcFERBtfa/lKkU6SA2yrrKQK6+qCAurx\nWp+PejxcUcHVL03AblYI5heSC0t7KETCFRWkPRSiLjQvlJSQ1rIy4bFEIxFV26ICORqJ6BYfefwi\nC5wRWB+b11udevHoAnaz6u+y0AoGm4jfv05xrn3BqRWK4XC30AfC077dBYQlZCsqwrbGZgYnFiYZ\ntAXcLrRtlpW1MpUJ1rzRng+wmQBR4vU2knC4W9VnONxNsrMbVednZ+vPcxP05yKNl/UdKe9XnrNb\nRriHKyqoAnJtfj71eA1DuPMKNtZissrnMxS2NKHMassJISu3H66oIG2VlalxMfsUWODMQPtwvN4G\n1YcgIkD05zor9MwWGqtQL1D1JBBoEhJyTgpZ0XE7sbNxY17p71Y1Q7i3M+fNTNvXzvF4a+6i967d\n3amvsy6ibdncRcGyQ8cZdui8+fPRduedKpv75pISrNywgas/llPz3kWL0GmQnFVeVUW1abcNDFge\nCwv9PT1MvwJr/FNBd0pbsc3T7ayPq+ydIjZp/bl8NnMjZ6QSTjk/taiqKkdVVTnVJnz48HoUF/8E\ns2bdxRxXppyRtHED9u3kbswrrc14vBle73qVnyc7uxGjo2k7tHbeWO+f9CXo38MLFy4DOAC1A78N\n58//zsptWALtuQwOFuGdd/TnKsdP96FYQ0aFe2VLC1VAVqxZg7a9e3XHn3jxRQBAx86dmJpIYMzj\nwcoNG7idiazFhBWtYyQc5f/NxiIqYGlRPNsGBtCxcycIY/xv5eRgvsYpDVh3PsuCjQURp6b+3EpI\n0RtsoWfmjFTC7egZukDahVOnOnDq1FbmuLQf88jIBRCSg+3b+9DV1ctcFJyA2fPjwcWL16jHBwev\nWm6T/qzKUVz8E8yenRZ6S5cuxtGjB5FI9FEXJ7PIrJGRy6qjsdjHAH6oOXcbPvpoleV7sQLtcwmF\n2qnCXfkdsRcyC3BiC2IGZTc0k0c0EiF1gQBZ5fORWp+PNAWDjjgMWU5NWttU+7yg49JKGyxTVbii\ngtre13NySLPGXLW5pITUcTqfee2qyvMCAa0dnRCPp5EEAnVUe6F2O+r3P0WCwSadI1Huw+erTW2d\nxc0+zpo/jBzMvH25ZTpyEwUFdJNBYWGN5TbTz0ptIgkG64XaodvcX0i2+wLx+9ep5nbu3Md1JhmA\nkLKyVsv34gR4TGj699u6iM64cNfCCaFqBF6nphM2dSttmF2jHT8rguiRqVN10UDyIiGDV+jQBfQm\nUlxco4uMYV3Pigox6kN2ksm/0+zybkfP8ERzmPkLMmV/d9IBWlrakJx/tQAtLW2wNT5JKdC/S1Z8\nAqFQO8nNXUWAGiJFy6TfQ3luIxG901J+rzJpc6eNv7KyjZSWNpCCgmpSVtZK/TactLmPu3B301Ep\nAiMN2s02RHYXhn3IC6NGwCvnkVfosM5jaXdWPhoeIcpqV/7Qy8paSUFBNSktbXA0uoOtJfLdbyYi\nZ5zeHaSdkFK4qiw47QrEQKDJ0YXObG55or8yiUgkSgIBOVw4vYswelZK5ciOcM+ozZ0GptOQkRUK\nOJfUo2zn3ZMnqeeIZNOK2vgBflu+jMtXrtD7SP4vpa9IqRPr/H54Fbb5KxcKqNdq7dUsu9/oqJfr\neiVYjlIzJ5mRM1K2Y27ceABDQ/swNAScOsW204tAaTsfHLyKDz4YRDzeDDkZhsdJmonEK6cdoJJD\n+IDjDuHp0++iHh8cvIpQqN3Uga6F2dyy3qsFC4pc83mwkPYldSuOSu+o0bNS2uqzsr5tuf9xF+5W\nnJ5OZK1q2+kHsD47G7sU9ACi0TAshzFPG4QQ1f8sXIfWPalOcgaAc/n5eLqkBDMGB/GqgmP/gteH\nk+jBVajnSSt0WB/QtWtDADoh
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 58
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"Forward Step",
|
||
|
|
"==============",
|
||
|
|
"",
|
||
|
|
"The function we use to predict the output from the neural network is two \"layers\" of linear discriminant functions with intervening sigmoid non-linearities.",
|
||
|
|
"",
|
||
|
|
"",
|
||
|
|
"$ y = \\sigma(A x + a) $",
|
||
|
|
"",
|
||
|
|
"$ z = \\sigma(B y + b) $",
|
||
|
|
"",
|
||
|
|
"The sigmoid function is defined as:",
|
||
|
|
"",
|
||
|
|
"$\\sigma(x) = \\frac{1}{1+e^{-x}}$",
|
||
|
|
""
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"def sigmoid(x):",
|
||
|
|
" return 1/(1+exp(-x))",
|
||
|
|
" ",
|
||
|
|
"def forward(x):",
|
||
|
|
" global A,a,B,b",
|
||
|
|
" y = sigmoid(dot(A,x)+a)",
|
||
|
|
" z = sigmoid(dot(B,y)+b)",
|
||
|
|
" return z"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 59
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"Let's apply this, using randomly generated initial weights."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"A = random(size=(5,2))-0.5",
|
||
|
|
"a = random(size=5)-0.5",
|
||
|
|
"B = random(size=(2,5))-0.5",
|
||
|
|
"b = random(size=2)-0.5"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 60
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"As output, we take an indicator vector as the target output. For example, in a three class problem with output classes {0,1,2}, when the output class is 1, we require the MLP to output the vector [0,1,0]. This is also called a unary code. Therefore, our network returns two values."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"x = array([0.3,0.7])",
|
||
|
|
"forward(x)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 61,
|
||
|
|
"text": [
|
||
|
|
"array([ 0.66679894, 0.55102865])"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 61
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"The vector of outputs can be viewed as the outputs from a set of discriminant functions, so we identify the class as the maximum."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"argmax(forward(x))"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 62,
|
||
|
|
"text": [
|
||
|
|
"0"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 62
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"We can also treat the outputs as an approximation to a probability vector; in that case, we normalize the output."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"z = forward(x)",
|
||
|
|
"z/sum(z)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 63,
|
||
|
|
"text": [
|
||
|
|
"array([ 0.54753148, 0.45246852])"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 63
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"def plotgrid(f,dr=0):",
|
||
|
|
" result = zeros((100,100))",
|
||
|
|
" for i in range(100):",
|
||
|
|
" for j in range(100):",
|
||
|
|
" x = array([i/100.0,j/100.0])",
|
||
|
|
" z = f(x)",
|
||
|
|
" if dr==0:",
|
||
|
|
" z /= sum(z) ",
|
||
|
|
" result[i,j] = z[1]",
|
||
|
|
" else:",
|
||
|
|
" result[i,j] = argmax(z)",
|
||
|
|
" imshow(result.T[::-1])",
|
||
|
|
"subplot(121); plotgrid(forward)",
|
||
|
|
"subplot(122); plotgrid(forward,dr=1)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAWwAAAC1CAYAAAB72cXqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztfW2MHMd55lPdPQoCR45iw9yFRCkUFFHUUl+UKQtnBMlK\nvJVgw1QYU1ESG7RgCM6PIDjLMO4QXH5Ywh3EFQTEkmPnj6Hk9mxAsn4ljKEICaEjkZwsxQoVW1CY\no8/hxmuGXBzMMI5sR5yurvvRXd1vVb9V3T0zPcPl1AMMOV1dVW91b83bbz/1VJVQSikEBAQEBFzy\niGbdgICAgICAdggOOyAgIGCLIDjsgICAgC2C4LADAgICtgiCww4ICAjYIggOOyAgIGCLYGSH/dJL\nL2HXrl248cYb8eSTT06yTQEBM0Xo2wGXKsQoOmwpJW666SYcPXoU11xzDe666y4899xzuPnmm/to\nY0DA1BD6dsCljGSUQn/zN3+DX/iFX8COHTsAAL/xG7+BP/3TPzU6tRBiIg0MCHChjzlfTX079OuA\nacDVt0dy2GfOnMG1115bHm/fvh2vvfZaLd9//NzduO+xvYghkUAiQlZ8JGJkiCHLjz5H01zpunx1\nLkVC8v/hY+fxnx57d5E/Lf4n+ZVEnOWfSGaIMoVIKsQSiNL8g6z4yOKTku+SnEuLTwY89kfAY4es\ndGnVI610ux5fflomtWz/LfDYbSRvSurJPOnSqs9lp0hXGZBlQJrmn//2DvCfBSAzQKo865CpTn8u\nOtJ1uaF1+UNyjub/r7XeNhm069u/DGC5pxY04diMbM+b3Vnaftx5ZiSH3TbKUBCQSKAgIKwzZV2w\nnyTKSDG/u+wmZQ6dL/PR8wJARG1nVpsAkbqLs0irell73HcObYNGesmi+MTEhl1P4ki362tI15eY\nFOnREIj1NZm3kTUVe5qgy0gmj77MvhEi6IBLGSM57GuuuQYbGxvl8cbGBrZv317LRx1nivxHVxnU\nP4wEMWTNcUujFpoeQ0EwDdceKW122IV5FQkkoJ5Zle2IUETaXeBy2PriNWImj52/DXQ9roeFcNQV\nIXeuNvQtbEgvnXYMxDGQRMSO5bRtXOEwQTEs/tdVamc99Fc9EbTt2wEBs8BIDnvv3r34zne+g/X1\ndVx99dX42te+hueee66W77rln4csvYqC9jAxBBstx8RNw3LTNqTT6ym8f/ldzQ67MCEjWk9W1FC5\nhS5Oe3kv8kucZpBW2FrejvqDQF+GfQ0CVRjLOXWXVyTp2mnfMwAG+nodTtu+HV2crv7L05eDPm9v\nu769o8cWNGFWtufN7qxt8xjJYSdJgi9+8Yu4//77IaXEI488wo6ib1++HhIZiZ5FEcNmbFStjytn\n3p4eqc4luGP5SkikrGOo/dhHoUccHmP5A6jIW5dXmgQ9wtAdyz+PKjS17TV5SI4+4cDQI/f8VM5p\no8Fp2ya6NIvma1NuHLTr2zt6bEETZmV73uzO2jaPkRw2AHzoQx/Chz70IW+ejISbieM934yqc3rE\njo7zYFAgAn33ruiRCBFipMT/5XYjZFCQNdu12Lxw2vlDpIqztWtg6REXfWDXa2NS9IidTukR15PK\nVVcLGsRIt4pGABLKd7WkRwYOExQcPTJoKDMu2vTtgIBZYGSH3QYZIvJ7VYhJ5OxDhMyiPETpPoXl\n4ZRFr8RGGZpu0yoCCrJqk/Y8AoiEAKTOT6LtDBB2Nb5L6cqBjwsfi0RfIDh0oUdcTpuz30CPUNMc\ntFiFMk1hWDBgXtG7wxaFhCH/zVIaxPzZ1dUiNnQNmg5p/2Kcq1SUYSOPzHMPVaYLQAmBKNJejegV\nBKCThf2O7oOPTumJHnHmdw02NtXnUp0QUPWIsJ/LLeiRi44mx0Vx/SwJu20EzDN6ddgpEkTEQ2j3\npzXYNrg0d90D+EJY2/+lhbzQpkdkkdtOT+MYSugbROiRwoEZvLZ2akWEblSlORaarvNLq6H69SBC\nJU6m0AJlG1lRRvMHsZXelc7hIu0rUOfIHU4bChgkhYkx6RGdrhUiPmYnIOByR68OW3PMFfQgoj+c\nVKU3q6erIj2vx08Ac2cFlGPAklI2qAYjY5C2VJw2lHUVVHkB1J22K73ewCoKth2cK51GtFRWUWd1\nTDSFqxOgRxQwEXqE3r4m2j8g4HJFzw47Kp0jR4Vw0j5VDBbqIzs9JnVpQsPneeqSMoEBhjUKJi1a\nLCyPkpaep/LGCsVkEWHRI9RoF2rBl27DFx27hgia0l24hOgRWk3gsAPmFT1z2JqBzCUEMRPyufTU\n0pA9UCTFv1UEntMd+bRzm9pQEDUKxk+P1KmZkh5JddvzR42WcDtnRbre32m6zxNp5YVdD02ndYKk\nc/QF154tSo8EBMwjpkiJ2JF1M19N3bqJhPyrXWjEaru17puDpkeU0TbTctn0LvSIfQEctdCWpugS\nNdvfuVu8xemRbsPNAQGXF3ofdIwLYreaBCGMgUiQdL4Ol8fSTlvTI1mxDBTY6Dkt8teVIrLm6H30\niBJAMil6hEMXGsQ3q7JJQTkpesSGZxq7ix7xPZM4euSKhqYGBFyu6F3WV0Eav2+OCqkmwpgRuE7P\napSHTY9kUKTeOuWhp8bz0X1i2MwdhV4psLxRUQwMeqJHFPm/LT0imPw2PeJqE/fi0pUeafFAKOkR\nuOkR12SYNpNrAgLmBVMbdOTgGnTskq4vIS6cPI3eOW23rz2cj6wGTYvvhB5RyACRAWlBjxTOcyx6\nRFjnuAa20V67vrdBF3qk6yp/Y9AjAQHzjikOOmrY6gzfL74tY5kPOmaISgVIhsg5uMjRI3m6DlZ5\nbiPRk2wKeiSKJFQm8sg8zWmfWOu0x6FHuEFFX35gsvQILdMm/wTpEZcJ0dCkgIB5wJQ4bMAcdDRV\nG76V9/ImdplQQ510ZTNy2GxDj1T1xGWLyuQoT4iRAWlmvP4LPTGGVqXpCy7SbqsecdUzSXpE23Ch\nR3rEhRBlB8w7eleJ2Mh54Wpdvird9DKKfB8FuQIkK2mNCNJr06ZKOFKGnivpEVHlFlAmPSJIJMnB\nXPeqMtOGE3BNQZ80PeJDgx7blR7okYCA0TCVQUd7tiOgAzQzTKsGHTPrDb8LPaKtVBNkVLFuX2Wt\nrsm2lSvVuhUN9AhgaPsMeqTMbVx6VSaC6ZGohrqNisNFg0yLHulC2UyYHgkImEdMTSVCFSDmpBf/\n6nsJOaO5am6CDAdKj+hV/vwBrEV5kNZXGJEe4apywaY7fOoRweS30+2mT5Ie8VE8rvykOaPSIwEB\n84ipqkSoM66/uSvn79NcMpXf/ICzQUtVEbRoiJ55n1jVbZZrRY846nWCo0d86pFR6JFuLyzNje9K\nj6iqWkqPZLQe4rSD7w4I6H3QcVBOnOHMVpNecnrEteGAzmPDVoBo3XRiOXRKj+RRP0+PuKzpyTh1\nqibPOzY9woGjR3zqkS70iD7WIudxFC12nW3yazqIFI0ARFG+R6SBDFAqPx8WfQqYd0xx0NE1vZzf\nx5Fz2q4fbGJxAdyaIBxX7aJHdGmbHtG0SeXLHPSIdkaTokf0cqu08Lj0iIbextxFg4xLj+inFS3D\nLBMjwDhrXUUGXKHCGiIBAVMcdFTku2m+ekvOZyrqDQfagp8gU/1bpdmDi030iHA4dEGCxzo9ooDC\nO0+IHuH2TNRwRcFd6ZEu6EqP6C1j7DqsfBHyAUjF/ClE4LQDAvp22FVcSVfU45pgrgninvSiELG0\nCT9BBqitCYJ8oLOKiJvokcQxEFlPr1YGFPmjgNAjOtIGGKfd1RE1ramtwM/1rvEy5LtPiDMJeoRL\nYygZAWtZVgJVOO3gtwPmFVOYOGNGtP7p5Rk7qGjy1FV5btJLfYKMO689jkcHF03EDt/E0SP0rCwI\nWlHdaZfTbj83qMo/qnpEl4Hj+yToEfu53DSiTNqo7w1bJETaAXOMyHdyY2MD99xzD3bv3o1bbrkF\nX/jCFwAA58+fx8rKCnbu3In7
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 64
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"Backward Step",
|
||
|
|
"=============",
|
||
|
|
"",
|
||
|
|
"We start off with the total squared error.",
|
||
|
|
"",
|
||
|
|
"$$ e = \\sum_i (x_i - t_i)^2$$",
|
||
|
|
"",
|
||
|
|
"We want to minimize that by changing the parameters $\\theta = (A,a,B,b)$, so we",
|
||
|
|
"look for places where the partial derivatives vanish.",
|
||
|
|
"",
|
||
|
|
"(Here, I use a shorthand of $\\frac{\\partial z}{\\partial \\theta}$ to stand for",
|
||
|
|
"the collection of partial derivatives with respect to the components of $\\theta$.",
|
||
|
|
"This can be ambiguous at times, so if in doubt, work it out in components.)",
|
||
|
|
"",
|
||
|
|
"$$ \\frac{\\partial e}{\\partial\\theta} = \\sum_i 2(z_i-t_i) \\frac{\\partial z_i}{\\partial \\theta}$$",
|
||
|
|
"",
|
||
|
|
"Now let's look at these components one-by-one.",
|
||
|
|
"",
|
||
|
|
"$$ \\frac{\\partial z_i}{\\partial\\theta} = (\\frac{\\partial z_i}{\\partial A},\\frac{\\partial z_i}{\\partial a},\\frac{\\partial z_i}{\\partial B},\\frac{\\partial z_i}{\\partial b})$$",
|
||
|
|
"",
|
||
|
|
"$$ \\frac{\\partial e}{\\partial B} = \\sum 2(z-t) \\frac{\\partial z}{B}$$",
|
||
|
|
"$$ = \\sum 2(z-t) \\sigma'(By+b) y^T$$",
|
||
|
|
"$$ = \\sum 2(z-t) z(1-z) y^T$$",
|
||
|
|
"",
|
||
|
|
"$$ \\frac{\\partial e}{\\partial b} = \\sum 2(z-t) \\frac{\\partial z}{\\partial b}$$",
|
||
|
|
"$$ = \\sum 2(z-t) \\sigma'(By+b)$$",
|
||
|
|
"$$ = \\sum 2(z-t) z(1-z)$$",
|
||
|
|
"",
|
||
|
|
"$$\\frac{\\partial e}{\\partial A} = \\sum 2(z-t)\\frac{\\partial z}{\\partial A}$$",
|
||
|
|
"$$ = \\sum 2(z-t)\\sigma'(By+b) B \\frac{\\partial y}{\\partial A}$$",
|
||
|
|
"$$ = \\sum 2(z-t) z(1-z) B \\sigma'(Ax+a) x^T$$",
|
||
|
|
"$$ = \\sum 2(z-t) z(1-z) B y(1-y) x^T$$",
|
||
|
|
"",
|
||
|
|
"Now, let's write $\\delta_z = 2(z-t) z(1-z)$ and $delta_y = 2(z-t)",
|
||
|
|
"z(1-z) B y(y-1)$.",
|
||
|
|
"",
|
||
|
|
"Then $\\delta_y = \\delta_z B y(y-y)$.",
|
||
|
|
"",
|
||
|
|
"We obtain a general update rule:",
|
||
|
|
"",
|
||
|
|
"- compute the $\\delta$ for a layer by taking the $\\delta$ from the",
|
||
|
|
"next layer and propagating it back by right multiplying with the",
|
||
|
|
"weight matrix connecting the two layers -- multiply it with the",
|
||
|
|
"activations for the previous layer to obtain the gradient",
|
||
|
|
"",
|
||
|
|
"For gradient descent, we now use the standard update rule",
|
||
|
|
"",
|
||
|
|
"$$\\theta(t+1) = \\theta(t) - \\eta \\frac{\\partial e}{\\partial\\theta}$$",
|
||
|
|
"",
|
||
|
|
"The derivative of the sigmoid function is:",
|
||
|
|
"",
|
||
|
|
"$\\sigma'(x) = -\\frac{1}{(1+e^{-x})^2} e^{-x} (-1)$",
|
||
|
|
"",
|
||
|
|
"$ = \\frac{e^{-x}}{(1+e^{-x})^2} = \\frac{1}{1+e^{-x}} \\frac{e^{-x}}{1+e^{-x}}$",
|
||
|
|
"",
|
||
|
|
"We can relate this back to the value of the sigmoid function itself (this saves computation).",
|
||
|
|
"",
|
||
|
|
"$ = \\sigma(x) (1-\\sigma(x)) $",
|
||
|
|
""
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"Stochastic Gradient Descent",
|
||
|
|
"===========================",
|
||
|
|
"",
|
||
|
|
"The above rule is a *batch update rule*. That is, we compute the gradient using all",
|
||
|
|
"samples and then update the weights. However, that turns out not to be a very efficient",
|
||
|
|
"way of training the network.",
|
||
|
|
"",
|
||
|
|
"Instead, we use *stochastic gradient descent*, where we consider training samples one-by-one",
|
||
|
|
"and then update the weights just according to that one training sample.",
|
||
|
|
"",
|
||
|
|
"For small learning rates $\\eta$, this is almost equivalent.",
|
||
|
|
"",
|
||
|
|
"This gives us the backwards propagation rule. Acutally, expressed in Python, it is quite short."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"def backward(x,target,eta):",
|
||
|
|
" global A,a,B,b",
|
||
|
|
" y = sigmoid(dot(A,x)+a)",
|
||
|
|
" z = sigmoid(dot(B,y)+b)",
|
||
|
|
" delta_z = 2*(z-target)*z*(1-z)",
|
||
|
|
" delta_y = dot(delta_z,B)*y*(1-y)",
|
||
|
|
" B -= eta * outer(delta_z,y)",
|
||
|
|
" b -= eta * delta_z",
|
||
|
|
" A -= eta * outer(delta_y,x)",
|
||
|
|
" a -= eta * delta_y"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 95
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"targets = array([1-labels,labels]).T",
|
||
|
|
"targets[30:40]"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 66,
|
||
|
|
"text": [
|
||
|
|
"array([[0, 1],",
|
||
|
|
" [1, 0],",
|
||
|
|
" [0, 1],",
|
||
|
|
" [0, 1],",
|
||
|
|
" [1, 0],",
|
||
|
|
" [0, 1],",
|
||
|
|
" [1, 0],",
|
||
|
|
" [1, 0],",
|
||
|
|
" [1, 0],",
|
||
|
|
" [0, 1]])"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 66
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"In order to train on the input data, we need to perform multiple updates."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"def update(steps,eta):",
|
||
|
|
" for i in range(steps):",
|
||
|
|
" index = randint(0,len(data))",
|
||
|
|
" x = data[index]",
|
||
|
|
" target = targets[index]",
|
||
|
|
" backward(x,target,eta)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 67
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"subplot(121); plotgrid(forward)",
|
||
|
|
"update(10000,1.0)",
|
||
|
|
"subplot(122); plotgrid(forward)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAWwAAAC1CAYAAAB72cXqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvVuMLUla7/eLzFzXqtq79n139+6ZPgPDZRA3zwxYCOGN\nOQMCCYQEmkGgYYTAyJIlLpI9IPNAH/EwPQYhMNgvFmc8fhkx58EwgxAPLWvaYCzABwQyQmc4PnT3\n7u59696XuqxrZoYfImPll5ERuXJVrVVVPVOflFVrxYqMiLz988tffBGptNaaczu3czu3czvzFp12\nA87t3M7t3M6tnZ0L9rmd27md27vEzgX73M7t3M7tXWLngn1u53Zu5/YusXPBPrdzO7dze5fYuWCf\n27md27m9S+zIgv1nf/ZnfMM3fAPvf//7+fSnP73ONp3buZ2qnZ/b53ZWTR0lDjvLMr7+67+el19+\nmeeee44Pf/jDfO5zn+Mbv/EbN9HGczu3E7Pzc/vczrIlR1npr//6r/nar/1aXnjhBQB+4id+gj/+\n4z+unNRKqbU08NzOLWSbGPO17Nw+P6/P7SQsdG4fSbDffPNNnn/++cX3W7du8Vd/9Ve1fP/617+T\n73/xQ8RkJGRE5MWSEZMTky0W+5tMC6Xb9cvfUhKR/39+8RG/8OKFIn9a/Bf5dUacmyXKcqJcE2Wa\nOIMoNQt5sWTFkorPmfgtLZYcXvy38OLHnfTMKSdz0t1ymvLLdVKn7n8PL36LyJuKcvKG9MwpL1RP\nka5zyHNIU7P8xhT+OwVZDpk2Weee4uwyC6Tb9ebO5s/FbzL/f18729Zjbc7tX78BL14EvQ/zCcyn\nMJub7c+02Rd5cb1FChQQK+jEkMTQ6UC3B1EP6BfLABgCO8AFYBe4VCyXgWtmefFz8N/+Ro/HO9s8\nVNd4wHUecp0HXOcB13jINR5ynXe4wjv6Co9mlznYu8jk4QV4hFkeA0+LZR84AA6BETDxLFPgwYuw\n9WL1QMlz0GtaLPJkR3zHSdfiM8BvAb8s8rrl56KOUBuywG9N+TXwPwH/zQrrLitXLk32rcFfjiTY\nbb0MjSIjQaNQzi+LsmqN15WU6udQvckih82XN+F5BUSybnvQy59V8CQMWFqW663P99lnbZ1Gucmq\nWGJRh1tOEkh3y1uSbjcxKdKjuREjwN2N3qrihibYdTJPHruZm7ZW53YfuGDa040hiiCKIc8gs0uh\nL1FU7LMYOokR7LgDqgvYRQq3/TwQi/zegVmnw0QNGDNgzJARQ8YMmCyWvll0n+m8SzpJjOhOMXfM\nOVXR9TkJrkPhaqP83mg24yZsE2Xbi2ndT2+23FY7LWhHEuznnnuOO3fuLL7fuXOHW7du1fJJ4Uwx\nzSwrtBdGQkxWE+7ynliV6YwYjfI03CpSulywi+p1pEgq7oFetCOi8LRXsZBg2423FnvyuPnbmC0n\ndLNQgbIi/Oe53YVL0heiHUMcQxKJepacj91AFdLmxX9bpBXreXPRa7FW53YfuFh8jiAuBFunkKfF\n00YO9qnW/p5EoDqgErMsxLpTLD2Mlz0oPjvinQ8hS2AadxgzYMSQEYNCrPuLZUZ3scxnXbJpYoTa\nLnP8jzr2iWrupFmTjvKxdFIW4Ls9n7ZtMnjOXj1H24FHEuwPfehD/PM//zOvvvoqzz77LH/4h3/I\n5z73uVq+99x+L9lCVTRWYWKU11uOK48uVZl2LQuqnuaDt7eWC3ZRRRbJcvKihPIEWkW0b38Is4kn\niTmLum7fon4jsJvhboOivE58ot70dCmKiIDv7UDHbm9AtN3dscrlaY+89Hk2uXvbnNu33wdss3Ca\nVAQqBmYFUssgzkrBth64igqhtgLdLf73qXrYA2cZmiUbRPzn/2XEOBkUXvVw4VGPHeGe6h6ztEs2\n7aKncVWsJc4I8avc+dy9XT7+LKMQXpMi7T5DNxX0XatUsiaz3vCHN1Q2mKtn5Z14NMFOkoTf//3f\n5wd+4AfIsoyf/dmf9fai37r9r8jIhfesCh8293rV9nsp5u3xSPlbwrfd3iEj9e6K2sV+FDwSUIzb\n30F58oeOwzrwiAd33H4vpWvq1rfsnPDhE5958Mj39gzTZolou1Ws0iyZbxMPq9LanNu3vxnYo7ze\nIszNcgpqBvEcYqlPcZEnwgi0FW2LRKwXLYV7WF+yQcwHPzLkobKedb9Yqh72hD6TrM983kVPOzCJ\nDYv2eddStJuQSHK77GBwve6l1pqfeOy7WI1Br9O+Y0PlHh2PHEmwAX7wB3+QH/zBH2zMkwt3Mwk8\n51e9aoNHXO/YOIOKCPnsXeKRiIiYVOifqTciR5PV6q755oVom5tI6WfbnenFIyF84Jbr2rrwiJsu\n8UjoThUqqwUGqaQ7q0ZAInlXSzzSCVQhzYdHOkvWOa4tPbefwYgqmA1IikV20sltt9enFWwr2laY\n3Y5Hu9jv25Buw7QfC3GuMmubPqXHlB6zrMts0iWfRPUORMmwXRQihXsZq248vmcNc5xFkx1P7cT7\nyILdxnIi0QRNLDznJovIHeShFueOchROO3glrqwj0927tEKTlW2yF5SCSCnTc+S0VeWg3GKaNmVV\nBn5ca6JI8gHCZ6vgkZBo++pfgkdk1T6zDp0kTaceWPccfsEeUYqxi2YXO6lYrGctOxSHVFHIVrnM\nhzHTbpexsuy65Ndjx7s2OKRHOumiJ6rsbHSxiMQhufgvWXUouKERi8hno3ercNsDtsltkGeyravZ\nNi7YqnAvTFMkBqledvVoEddsCRaHtN+JJkpFV+ownrlRqEW6Aq0UUeTpEFFgk5V7ITZZE07ZEB4J\n5m/T19E2uqQhekS59+UWeGQWaHJcrG7vJWfh8tfPg+pT4pAE01DpPVt8YM3uIJvf7VQcev5vAdug\nhzDrdRh1pFgPOWSLUeFlLzxv3WecD5jNe2STDkwjf3SI62VLBALVkE+fgC91CC3wfrea9X5PqlPU\nPoY120YFOyUhEmet3XQbg+2aLy1cdocmF9bd9LQIL3TxSFbkdtPTOEYru4MEHikErMK1ragVHnql\nKMtYZLrNnzkNtY8HESVflCbDVd30iJIfxE76qjjH52l3qTPygGijTQibgmPjEZtuI0SayM5J2fR9\nimQICbrc6JiSSSfUY5RlPinYIQxS/M+HBodMkt5CpKVXLcP6xgxMKN+sSzrpwDSGmaqKs/WyJfpw\n8YiPZUuh3lSU3jFD3vxmhXcTZZ+8bVSwLWMuzXYiNt9J9ELN6um6SDflNANg368KHeiwlMiGsjMy\nRrSlZNr2Wq3klzdjV7RD6fUG+jvTaUiXHq0Mq6hTnaotO3/XgEc0rAWPyN23DPtv2h5cvsoOIy7o\nQ3OPjik9bCva1qu1mMHqhuthS8EuPGr5PxtGTAYJo0VkSDVCxHy3XLtncMisRzZJwOIQiUTc+Gvf\n4kMh7nIk0V6GFzaFH6z3epSyj7Pu+m3Dgh0txNGHQnyhfbroLLTf3PRYlGWBRtPOrIeUKTrMawgm\nLVqsHEVJF8pTqrGmGCyiHDwiK10FLTSlu9bkHYe6CJalh+wM4RFZzGkz7P+UvI/nLr3FoDOiq3Vd\nsHsYnj2lFEZNWLAtr97BCLX9vw3pMGbUK1FIGX89rHDrse1w1F3m024h2FQXHxZxPWo5UEaaD4ms\nbEde8RTNBkCcDbyzYYYtH0USgTzKsyEUT12m+57nIRE70OAOM+zcRRsaVUMwzXikjmYWeCS1bTe3\nGhvCHRwVGXp+l+lNSqSR44H86bJMRLoPX/ja8y7FI6dp/5GvYRz30UO4+exDtuIJsUUiVoiHVEXS\netmy09EybCnWYplvKyb9DodsLZaSXw85WDDsPjN6TNM+s0mffNKth/KFPOsZdbGW4Xy+iJFGzW1S\n+nernR2scoJIxPWsl9+xpKxXLRF/7bkUeWO7bdy3zywe0ZW2VWteNH0VPOJugA8ttMUUq3jN7mff\nLn6X45Gz8HD6z7yfSdQn7mbE
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 68
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"subplot(121); plotgrid(forward)",
|
||
|
|
"update(10000,1.0)",
|
||
|
|
"subplot(122); plotgrid(forward)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAWwAAAC1CAYAAAB72cXqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvVusLGl21/mLiIyMiMzc17PP2edadepUdfXFwGB12waL\nsUpjGsZ+8KBpq82LKYElNPJI2Lx0IzEaWpoHqscIaIkRDyOE/IQwD2PayOIBRu4xY8vWyBZgbLrr\nXue6z21fMzPuMQ9ffBkrv/1F7tzXs0/VXtKnjIwdO/ISK/75X/+11vc5VVVVXNiFXdiFXdi5N/dF\nv4ELu7ALu7ALm88uAPvCLuzCLuwlsQvAvrALu7ALe0nsArAv7MIu7MJeErsA7Au7sAu7sJfELgD7\nwi7swi7sJbEjA/a//bf/li984Qt87nOf49vf/vZJvqcLu7AXahe+fWHn1Zyj1GEXRcHnP/95/t2/\n+3fcuHGDH/mRH+Ff/It/wRe/+MXTeI8XdmFnZhe+fWHn2TpH+aff//3f54033uD27dsA/NW/+lf5\n1//6X085teM4J/IGL+zC2uw0er4O8u0Lv76ws7A23z4SYN+/f59bt25Nnt+8eZPf+73f23fc31uH\nby1BtQtZDGkCWQZFVY8SygpwlDbjAB1XDb8Dvg/dAJwACOvRq8cCsAQsAyvAaj0uw7f+L/hf/xeH\n3bWA59Eyj53LPGadx1zhMVd4wuXJeMYlnpWXeJ6uMnq6SPasB8+AzXpsAzvALrAHjOoRW8ajb8Hg\nW5ABOepRb+dAafs29YUp6+0SKMR+uZ237Af4FeCXW86vzzsL4MzzHWRFfc7/A/ifD/F/s6wyxiz7\nb07oNadtHt/+OvA/oNwgRV3iFPWN6DHPJzisOcBvAX8RdeN2gAhYBNaAV3y4sQSDL4H3I1D8uMuz\nH1vkv1z9Ar/r/Xn+P77CH+39aT54/DnK/+pTvefBJ8AG8BTl7zsoPx+jfDqhdrtvAd+q34n2UfPT\nlvXBYPc5/X9tpn1K2neAXzLOYb2RWuyg15xlNt+ujnG+ea3dt48E2HOzjBBYUI7W9cB1wXOhLKEo\n6lF/966rjvM8BdSeB14HHB/o1iOoRygeo3qE4rkPZeiQdALGTsSYiBHqMSZkXI9YjyIgjbuUidc4\nqbwTNegWLUP7UCWeS9w50L/aQKpq2T6sHQTW58Wcery49zuPb3s0gKkvvWccc9ifv7neG4rYePXw\naTjMErAcwtIqcA14FfJXPR4vXOaed5P73OBRdZXteIXiSahIyTYwRIFzwjTBkH49ZTY/bXPwl8Hn\nXi47EmDfuHGDu3fvTp7fvXuXmzdv7j8wRDHg2tM8F1wPqkyBdVkqwNbs33NroHYVUDs+yis1WMvt\nHgqcJYjX7LvyIYscErdbg3WPcQ3YCrQjEkJSuiR0ScsuadKlTL3GaeWQDqxBWu8v2U8MNEgflgxM\nmWQo8hfhvJiOiU7DNDQd+cs7ss3j2w7KFX3Updcgetq8S5uLunG7NAx7BYgWgOvAK1DchvhVj0fh\nFe5xk4fVNZ4kl9ndGcATYAsVNUrA1mGCdjdJPi7sXNiRqkS+8pWv8O677/LRRx+Rpin/8l/+S37m\nZ35m33Fv3UHJFEsopj0Atw9eD/wI/FBJHnr4IXQCcOvhBDTsukvDouWI9o+/8BMu4yBg5EpmHTWM\nWoy0DMjyLmXSpYrdxnEls9YjYzbL7r41rWocGm8kHTeZzEF3zZ8/7Isd0zQb/pFTPPdp/ijYbR7f\n/mEahu0ZwxXjJN+5/jbuMA3YPWDBhZUeRKtMAHt8LeLx4hoPOtd5wHU2WGdnd4X0WTQtfYxo5A8z\nmpwC67fYHzaeFJLPOt+PndBrHMXafPvs/VLbkRh2p9Phn/yTf8Jf/st/maIo+IVf+AVrFv2tP40K\nu/R10FQkBicDLwNPxo7S430xNGBL+SOot3vTo+rBj/+kz2ZXM+tpdj02gDspAvK0C7EPsTMth2iA\n1mCtgdsmh5SA/5Y6zkaI5/Lt41DyH+fsOJ60Hz2l874YeWQe3/4KCt+0NKLdwBPvVo+TuiL6tnhD\nvO4EsDuwtAThOnATqtuws7bA3e4t7nGTB9V1HhdXGG0uUjwNFLuWgK1lQBlV7tN03mKajdg+2VGv\n06xv6s8d8ZwnYTbffrGy3ZEAG+Cnfuqn+Kmf+qnZB11Dgas27Wk+ykESpj+3JFYSsG06tQTqCOhD\ntQD5AJJuRwB0aGXXCYEaWUAW+1TaaWPx3kxJRIK3lEhmadUVFueXf7yw2ebQKMRnc5Mc5NvajQMa\nF8hpQPW0fjb1N+HXrz1ASSErPniXUWD9GqSvOzxfW+JjXuUut3hYXOPpaI1ku6vY9RbTiUVJUKSv\nH+qrPpb+d2Fz2pEBey67gfKsiiaO04x5RENPpGM4xrE6sxKxH6w1kPfVqHoOWc8j9gMB2L0p8NYM\nOyEgqULSLKCI/elko8k2DmLWptysP89ckaPkYy9jyZjL6dREaJPfiX6tF2s+DVBnNNJIh/2/4fPW\nvLSZ5DASrHWicaULi4vgrQO3oHjFY2ttgYe9de5yiwdc42l8meHzJfJnfiOHmNq1Ldk4d7Lc3Jbv\n/sVfr0+TnSpgV7dqHRoaj9NArB+1hKBNe6dm2Zak4pRm3WMKsJOgy6gTMaRfjx5DevuY9riKGJch\nWdqljDuQOPuB2pZ4lNp0xTQbsQH2gf5q6tYvk2nOdxo1EW2v9+J/1HyaS2+CtdaypR3n29HfsJRB\nQhp2vdSH/mVUNPsKpK/4POpf5Z57k/tc51F1la3xKtlGXa6qk40j2qNIK8O2fYKD2MjL5s/n304V\nsJM7Dp0edKimaUKXhj1LiQEaDzUB26JXy1H0Ie05jLyQEb1JZUiMLuvrT1WLxEWoKkPiDiTudFgo\ny/mkA89T3gcHyCDHtZNO+GjT7PUirD3IuvVjUW9Ll9AuLr9FnaI6rFSibwPNrHUqpw8sObDiQLiE\nSjS+Ctkdl71bAfe61/mEWzysrvN4vM7e9gI8p2HXMtFolrFa+wUOm2i8KC05LTtVwH68usZCNWKp\nHCpuJNPb2gNlRYZWBdoA22DUcuQ9j3HYZeT0LGV84dR2QkBSdlXttWTXWg4xwdsGzLZ6a7MC78h+\nO4uan5b8cJzwVTPfz8ZNagNsLY3ISk/NUbQrzCseObRLIRF1z5gHSz0ILgOvAHdgeC1iY+Ey990b\n3OcGG+U627srJM8jBdi2phjNsrUQ31rKd1iB57z4grwKp1EZf7af81QB+4POHW6s3qfXHeFXFY7s\nONAeKIv29S+7lkQ6TFeEaL16ARUT6scBZL0Oo25vil3r7ZjphpmEgLTokiWBKuUzE46z5BBTwzbN\npmEfyk6LQZ+mSQ75Mr3vo5kJ2CnTsoWsGKF+PIx4JNOsZqJRu/9iFwZr0LkJ3AZeh63VFT52X+Uu\nN3nINZ6Ul4m3BpRPfQXYu+xn1iYhOVC7ftmu8WnJdvpH4Gwrs04VsN/jdWIvpOo5XLvxmF4nwZPe\nF9JoaboFVnu3TDpqwB7sH9UAsoFD3O0yrGWPEb19GvaIiISAjC5xFpAlIVXsg6y9bmuWsVWGmA0G\nstJpLknErPl72UDaNH1jfPqrBXRaRgO27uWSFSOmLAINy7bJI7KyVz/KHH1ILYUAl7uwugjuDeBV\nKF5zGd4OeLS8xse8qhpl0ms8210je95ttGvNsHVlSFsi/cLOrZ0qYL/L50jcEK9b0FkuuMQmAzfG\n9Ssc7eU6Wx0zrWXLxKQGdwnWC6qMrxg4jHtdhn44BdSjGqTHdS12LJpnkiwki7sQe+1t6LbKEBku\n2oDblERa8df2h7Z/OIlQTkPAaf8gyGD+Zf7xmW1BTaOLsgFr01VmXU0N3NJkm5AeErC1FLKKkkJ6\nl8G9BbwG2asdnq6ucD+8zie8wgNu8Cy+zPD5AuWzzjRYz2qSsYL1rE8i/36c632assVp29m+71MH\n7JQuvpPhdQuqVfB7jwg6FZ72
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 69
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"def plotclass():",
|
||
|
|
" pred = zeros(len(data))",
|
||
|
|
" for i in range(len(data)):",
|
||
|
|
" z = forward(data[i])",
|
||
|
|
" pred[i] = z[1]>z[0]",
|
||
|
|
" cla()",
|
||
|
|
" d0 = data[pred==0]",
|
||
|
|
" d1 = data[pred==1]",
|
||
|
|
" plot(d0[:,0],d0[:,1],\"bo\")",
|
||
|
|
" plot(d1[:,0],d1[:,1],\"ro\")",
|
||
|
|
" print \"err=\",sum(pred*1.0!=labels*1.0)/float(len(labels))",
|
||
|
|
"figure(figsize=(6,6)); plotclass()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"err= 0.115"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAFtCAYAAAD8jSoDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvX9wFMedN/whlmE3IImNRLxIwjHIiUNk7JIqOFD1nsTj\nqmj9nHLOJXeH8AWjxMgIS+aXq2JfkORVwCQpqEo9EpaPu7Pfu0vw5eXqri5JST4QOR6vfO8Zh/dC\nnQzmXEa23wBCNtYrG3BWgEi/f6xGOzPbPdM93T0zK+tTRQG7s9M93T3f/vbn+2sOIYRgFrOYxSxm\nMaPwqaA7MItZzGIWs1CPWeE+i1nMYhYzELPCfRazmMUsZiBmhfssZjGLWcxAzAr3WcxiFrOYgZgV\n7rOYxSxmMQPhKNwfeeQR3HbbbVixYgXzmq1bt+Lzn/887r33Xpw8eVJ5B2cxi1nMYhbicBTu3/nO\nd3D48GHm9y+99BLOnj2Lt956C3/913+Nxx57THkHZzGLWcxiFuJwFO5/8Ad/gFgsxvz+l7/8JZqa\nmgAAX/nKV/Dhhx/ivffeU9vDWcxiFrOYhTAKZH584cIFLFmyZPr/FRUVOH/+PG677TbLdXPmzJFp\nZhazmMUsPrHwmkRA2qBqb5glyMnUn52VlUj19YEQEoo/qb4+tNfXI1lXh/b6eqT6+nI+600msbOy\ncvoZeJ+Ddu9kMhn4M6v4U1eXNI2G+c8GACnU1WWfs6RkLfXaT3+6yrWdvr4UKit3Wn5XWbkTfX2p\nwMdA5E99fTt1DBKJDhBCpNeF2/1l5tQ8l378cRuLvr4U6uvbUVeXRH19+/T/ZZ9f5g/7fUja/hbr\nkwykNPfy8nKcO3du+v/nz59HeXm542/2DA+jc/9+1DY0yDTtCYP9/Rjo6UHBtWuYnDcPZatX48LB\ng9gzPDx9zRNDQ/gIwAujo9OfNb7yCg6l05Z7uT3HYH8/jmzbZrl3+/AwRr7yFbUPFRDmzZtkfHM7\ngCO4cuX96U/i8YUYG2sHsMd03U7Mnx9xbaenZwDDw3ssnw0P78H+/Z1oaKgV7jcL/f2D6OkZwLVr\nBZg3bxJbt9Yrvf+1a/RXbWLiltDcnzWnkchNT33Sgf7+QWzbdsSyJoaH27F+fTmGh9stn0ejLTh/\nHkgkOpTPpx3s9+Gm7W91c+4GKeH+4IMP4tlnn8W6detw/PhxLFy4MIeSoeGWiQmZZj2BJmxpQvvH\no6PotP12ue0aA07PMdDTY2kLyGwIdWLd9g2iwm3r1vqclwnYCeABALUA2qY/LS9fhNOn6wF0ArgF\nmYX+AIqK/m/XfukWigBbYABQJhB0Cs7+/kGcOnUGQBeASQD1yMyB2P1pc1pZuRNbtjwg3UdVYG32\nx493ors7gf37OzEychVnz44gnW7D6dO1OH1a/Xza4fw+GH9n4Ndm6SjcH3roIaRSKXzwwQdYsmQJ\nvv/97+PGjRsAgJaWFvzhH/4hXnrpJdx5552YP38+/vZv/5ar0ZsRd41NNWjClim0bf9n7skOz1Fw\n7Rr186ULFjB/ExS6up7D3r0ppNPLYQiH4eEjANgvg/H5ww8/hPHxu2AIbEOoFBYumr42s/CP5AiN\n5uZvuvbND23Sj9OBm+Bcs2aNp/saG9PY2CHTp+1T9z8sJJiNZ92/vxMTE7cgErmJLVse0Krx0uA0\nFk6bfUNDLRoaapFIdOD11w9Zvtdx2jPDPHYjI1fx7ru/xe9+dw03bw4jo+hkvo/Hd2DLlm9o6YMd\njsL9Zz/7mesNnn32WaEGd1ZW4oEtW7ivt1Mp9Vu3eqJ0aMLW7SBloB7A5mgUB0ybQUs0intXrWK2\nNzlvHvXzJfG4S0/9RX//IPbuHUI6bRUOw8MJ7N9/1PFlaGioxcqVAxgY6Mr5zix8ZYSGH9qkqtOB\n0+nHbQy8CnfaxgTsQWnpOnR3twoLM0NA6oR9nFavLsOrr45Yxo0Fns3ej9MeDeaxSyQ6MDDwDIBB\nAEcBHANwE2VlV33bLKVoGRF01dXhZiSCilWrMNDTg2P79lmENU2IA6Dy1gCEBTxN2NKE9o54HJcB\nwMS5H66sRPF996Hx5z/H8nQaNwF8K53GkYMHMbhyJbUv9Vu3on142NJ30Y3ND/T0DCCdPmD7dA+A\nTq6XgUf42l/mLVv4+U8/tEkVpwMeakeH4GQJsqqqL/qucfOANk7Hjm3G5OSfw9BunSgUnvUWBttB\ndl5qYTwXABQWdvnWBxAfYDST6usjOysrCQGm/+ysrCS9yST189bqastnxp+OREK4D7S2vzfVdkci\nQZJ1daQjkSCpvj6S6uvL+ay9vl64L7T7eIHRfrKujrTX13u+Dw11dUnaYxEgSRKJDq579PWlSCLR\nQerqMr/p60tZvqus3Gm5d2XlTss1QYPex+8J9bG+vp06jrxj6BVBtesVrP4CHdz9d1pvxvey8ykL\n1nPW1DST+vp2UleXJPX17a59khHRvmnuANvI2Pjsszg0NpbzeRMjgMqLQdbQrjv378ctExO4GYng\ngS1bmCcA++fH9u0T7kttQ4O0V9BzXV1I7d2L5en0tKnsiMfTCw0sLScaPYMtW7JGUTfKwaxl9fcP\nIpHowLVrBTh16oyND9bPf4pCxekgKCogH4ygZrDGyW7pso+bsf4uXvwYIyMjiMcXorx8EfUU2NBQ\nixMnTuHZZxsxORlFQUEa69fX+breaPMSjz+CkZGF+M1vnpn+TKeh11fhzjIyRifpAoZ+tZhBVhVn\nz+LQdRqHB/v7MbR3r8Wjpx1AYngYRxW5k9IWYTTagiefzL4MIt4kudd2UduVEXo63BZlKZOgqICw\nGEFZsM/V5cvjjCut42QeN9r6Gxtrx+nTdMN/f/8gDh68YFEqDh5sx8qVg76NC21e3n8/ipMnf2y5\njqbomMdMCp51fgEYzbCojbUlJdTPm2tqqFQKLy3BooG80BosWkcFRcKiXZhUEECSdXXS7RpwO+aK\nHP1zr1VLG4SV5gkDFSCLvr6UEGXAcz/7mMTjj5B4fIfls4KCTQRIMcfNjcqxr6WwUlUsCrSuLjl9\nTe6Y5QktwzIy1q1fj3ZbMNHOyko8vGsXAH4qxQ4WDeQliEqU1uEFK9gJYJ90boHaE4Ob1ipCOeRe\nW4/MeUMNbeCH26KXk0HYNWg36PD1p83V6OgLqKl5FPfemx2nVavuxfHjRzExcYw6bm5Ujn0dZq4f\nBDCADDmRITT9Ch5iged0R/d+8gZfhbuTgBxcuZIpOL0KUKZw9BhEpYJDt8NpAyIMKuhMNIo2iteN\nKgrKDhHKIffazEtaWroOVVVflBZ6urltmpB75ZXNWLbsRZSXL3IU9H64EeqCjk2TNVeFheU4fLiL\n+z5u0Z/2dXj58nkAR2CNiG63RE0HAR77iDQVY4Kvwh1gC0gdgjMInlwUThvQ/d/9bs5JpyUaRd2T\nT+aMldMJQHZcRYx29GsPu/pc82rLurltmpBLpw/g9OlOnD69W3ukY1C4ePFj6ucjI1c931PVXDlF\nf9LX4TxYBTum/t+GIMFzumNvZOLwXbj7iXzwNXfagGgnnW8xqCCVFJQdIpSDF3pChBLQ7R3iRgHw\narO6c9WoxsjICPXzixcver6nqrmyR39evHgRt91WjIqKo9S1VVS0iHYbS9R0UHA73dE3Mm8IhXDX\nRSeI8uQq+iF6D7cNiHaiobXBS0HxCh3adYcP7+YaA1F6QoQS0M1tuyeAcqeA/MhVoxqs5G633Vbs\n+Z7Gsz799KN4550rAOahqMhb+g3zmjLW5sTELejpGbC0BYQjiMkr7Os7lZK4mQIjsCucmlHp0SID\nFf3weg+RYCdWGxs5Ar54PU2crlPpUWHcq7i4acqrJsX0IvALtGcHvmfpm5vXRVi9NZyQ6XNqygMl\nOfV3SrrPqr2beO6XTPaSaHTt1HNknivfPJcMyIjowIW7l8hPHVDRDz+ehdVGK4fbKK/QYUfXtSp7\nUelCdKeQENUFwzV0xYrtU0KC
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 70
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"subplot(121); plotgrid(forward)",
|
||
|
|
"update(100000,1.0)",
|
||
|
|
"subplot(122); plotgrid(forward)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAWwAAAC1CAYAAAB72cXqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvVmsLNtZ5/mLMSPn3PN8zj7DnW3j4XoACup2SQjcrXKh\ndsvmxfjBEg/dLRpebCReXOqSuBaIkiVa9YRo9wvCLw02spCKanwLuC5fbEx5vsMZ9zznnBkZUz+s\nWDtXxo7c09n77HOu85OWMjIyMiMyYsU/vvX//t+3tCiKIkY2spGNbGRPvOlXfQAjG9nIRjay09kI\nsEc2spGN7CmxEWCPbGQjG9lTYiPAHtnIRjayp8RGgD2ykY1sZE+JjQB7ZCMb2cieEjs3YP/N3/wN\nzz//PM888wxf+tKXLvKYRjayK7VR3x7Zk2raeXTYQRDw3HPP8bd/+7csLCzw4Q9/mD//8z/nhRde\nuIxjHNnIHpuN+vbInmQzz/OlN954g9u3b7O8vAzAb/zGb/BXf/VXA51a07QLOcCRjWyYXUbO10l9\ne9SvR/Y4bFjfPhdgr62tsbS0dPh+cXGRb3/720e2+xTw74A20IubBwRKi+J2kfYa8G8AI24WkAcq\nwBSwnIHpWSi8CPwCdH/BZvejJd7Ifpj/pn+Mf+Jl3t57gY3VBcKfWHBfg1VgG9gDqkADaAEdoBv/\nMf+LwBfjo4iAEPAT/zSM34fxdqGyLL8nt08z+f2kfRn4PxK/cVo7aZ/H2f8F/G8pvxee8/dOaz93\nKb96mr79r4FXLmXvJ9s3z7BvDXGDZ4Ei8Axw471w/X+BL//i/85XNz/BG//hBfzVH0HzbaCG6K9p\n1+0se75Iu6r9XuW+//3QT87FYZ/WyzAQHUZtRrxTPV6+LJP7MAEbcBCgXQbG8lCYBBYgWobOvMNa\nZoFVfZF15tgKZ2g2S4S7NhxoUEc8dTqAiwBnjz4WJzH30KLEcupGI3uC7N3mQesIhyUHTNlQWLY5\n+LcV/rH5y7z+nZfxH6xDcwvRyYeB9cieFDuXh72wsMDKysrh+5WVFRYXF49spyE6i4XANQmij+OW\n0Oh71zaiw5aAcQ3sCrAALENwA5qzDqvGAmsssBnMstudpFN1hDddA5oIwO4yOEyQQH1e53RkT5yd\ntm8/LWYg+n9Wh+Ii7Dy/yN/dfoX7f74EX+tA7wGik4+ciafBzuVhv/zyy7z99tvcv3+fXq/HX/zF\nX/CJT3ziyHYfYNCzlk0F7osGcB24yaCHnSH2rk3hXdtTCMC+Do3pApuFaVa1RTaYY9efpnVQxt+3\n4YA+9SEBO83DPuR1XomPQnrTF00LHPd7H73A/ZzFPpyyTlPa02Wn6dvLV3No59q3dJoKBjjPwcbS\nEv/v3idZuVeElU0I9hCd+6S+etY9X5Rd1X6vet/pdi4P2zRN/uRP/oRf/dVfJQgCPve5z6VG0V9G\nsAiSGpG8tcnRrnEWxnWYyQfAbfqUiwrYpQxUxoFZYEnQIXvlcVbMJdZYYCOaY9+doLebJ9yzhOPR\n4KiHLb1sn0Q/f4VBrnooV3JOO+6m+tgF7ucs9pEh63Uun8e+eDtN316+mkM7974zQNkA/T066zML\n/O13P0609VPwHiI6t8fJ/fQ8e74Iu6r9XvW+0+1cgA3w8Y9/nI9//OPHbiMpiQx95kA6pToXA9Jp\nJoMtNv2AyzhQdIA54JqgQrxnNLZKUzzkGqssstmb5aBZIdzXhXct6RDJXScjp2d2oh9HMG5kj2qn\n6dtPg0kHxgEqhkbrpSLVfAH+H+BBDdHJpdcxsqfBzg3YpzGLvk7CY5AakaAdpbTzmOyc0rOWD4o8\nUNZgLAP5MQ4Bu7doszdWZN2cYxVBhxy0J+juFwRgSyVIEqzlE0dtJ5oK0qObY2SPzzTA0aGc0bk3\nscRKsAhvhLDXRnB9o0Dj02SXDthSMOYySI2YHPU1HyV2J4FaDTQ6xN61BuUSZKc55K7bszlWrCVW\ntUXWmWczmqFZL+NvK8FGKduTgK3y1rKN1CEje8ItY0Ixa3DHeJ63GreIHvgQdREc3yhi/jTZpQK2\nHb8GCG836ZgmA44yAnoWeJOetQrW0rsuAhUNxgywxjlUhrg3dQ5mC9zXrgsqxJ9lpzNNp5qFffre\nteStZbBRgrakRI7087NQHiMgH9njMc0CLa+zYl5jjSlgh753PbKnyR4bYNv0sU5VjEgnVQK3BO3T\n0COqFkGlQhyEjK8IlCwoFsGcBa4BN6E+XWQzP8MKS6wzz443TbNaxksqQ2RSjARrn0EPe6h3LUH7\npH/wpHg28gxeNL8ur86T8j9/Nk0zQHM09vQJ9ighAjM9RvGUp88eK2D3GATrJC0i6WAJ5Cd1JclZ\nQz/QKAG7gNBdl7JQmAGWgJsQ3dLYLk7zULsWc9ez7PfG8fZyRHuG8K5VsE5qr1WwHnqAZ800fBJM\nnsmLPO7LehCM7LQWAegQmRpNvUALhz5Yw+iB+nTZpQJ2Jn6VgC2bxDtJi6gZj1J4MYweUWkU6b9J\noJaqkAKCCpm2oTyGAOtl6C2btK5nWM/P8oBrIlGmO0utXiHa0QQdUqOf2Sg118mMxhH+jOwpMVVY\nahLE95pNXwLgXc2BjexcdrmAHbvQftgHa4dBiV8aqyCxMC2rRwK2SoVIwM4gALsEjOlQLoIzg6BC\nbkB3McNmZYpVFlhliQ3mOWhN0NnPE+1rwrtW09DTvOuhYJ1GgajrLoKzfhq9VTWB5mk67qffDnte\nBFoYkYva5PCAScSdaDC6Nk+XXSpgZzOAB15P4J7UY6tUMKQPylR6JGnJQKOqDJH1QiZ0cMZBjwON\n3ILWbJ4HXD/UXW8xQ7tWJtzO9ME6jbtWA43H8tbJD5NyvosAbHg66Zan7UHzLrIQND9iPNynQgdR\nAi1Pn6gcSfueFrtcDtuEMIIghCCIlxHPdtk91Od7MpH5uC6k8uCHmYzAGDCWhdIYmIvADcFbHywU\nWS3Ncp9loQzxZtlpTdPdcwYr8CVrhiSlfCfWDrno7MZ3i8nH7GXUZxxZmh2OQgPQ3ZD5cJ05Zwum\nTajloZtHdHQYJdA8HXapgG1Y4hkehhBEMWgzSBIcd/umdZ+kd61mM45pMGlApQTOPLAM0S3wn9HY\nmRjnobPIfZZZY4Edd5rmbolo1xos8qR61ypgq1VRj8WbESClm4SPZCnZkV2m6YAWguaFLERrzJU2\n0J7XiH5Shu44orMnqbvRdXlS7VIBW7NiUI09bMGlDVaKTj7XVXpkGIed5K0LxDSIBnNZyE9wqAoJ\nnwH3ts56bpZ73OABQnt94E7Alg07WrruOlkz5NgyqiM7vanCzRFN8jgsikALQhaiVean1uEXgd0p\n2G4jOr6asjtKpHmS7VIBmwzoepzxGAE9iHzhcaugLOPVaiBR5bhVUznrDEJvXUZ41pMOZGZAv46o\n1v4c1BeLrOTmuGfcjL3rRfZq07S3SkIZok5GIIONaUCtzrgwYO8m4JFFaR/H/1EpEtVGT8SLNB3Q\nIiCMmG3vMlfZwvq1Nv5PcoQPZ6HZgEjW0JSeyogeeVLtcgHbEaJ9QwM7RmMNDkFQjwZLrkq4kLVH\nkvgoP5eFnXJAUYNJHSZzQsJnLgG3IHpWw71tsTMzydvmbe5yg4fREuvBHPX9MXpbjkj4GjZ7TJo6\n5MwKkau08+hr02iLy7RkCda0SMbIzmMqdagDURhRqjWYMbeZ++BDdt+zQOutCXjbBU+DSEPwgjqC\nD5SByBFF8iTZpXvYsscYOmR0sHQwemB4oPuDSg+DvjBD6rVVJYnsfBaDvPW8A/lJsJZAuwU8C8Fz\nOjuLYzwoL/IOt7nPddb8BbY6MyLQuE1fd91EAHYSrJOAfWq76sQZ1VN+mjwledwje1RTJ/AwQShF\ndmDyYIdfnH2d1z/4Cq3VG7C+DA0TfLllNf52j1NG2Uf2GO1yAduiTzrroBugmeC4oPfA6kG7B1bY\npzhc+iVpkoAt64TYxHVCbCjn
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 71
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"figure(figsize=(6,6)); plotclass()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"err= 0.045"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAFtCAYAAAD8jSoDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvX1wFNedLvwQy6AJSGIiEQ+ScAwicYgsuzQVHPzHSqyr\novFdss5NbhbhDUaJkRGWzJerYt8giVHAJCmocl2E5bB37XfvJnjzsnW3NklJXhC5XI/Yd41Dbagr\ng9mUke3XgJCN9coGnBEgct4/Rq3p7jmn+5w+53T3yHqqKGCm53x19+/8zvP7mkUIIZjBDGYwgxlM\nK3wm6AHMYAYzmMEM1GNGuM9gBjOYwTTEjHCfwQxmMINpiBnhPoMZzGAG0xAzwn0GM5jBDKYhZoT7\nDGYwgxlMQzgK98ceewx33HEHampqmNds3rwZX/ziF3Hffffh1KlTygc4gxnMYAYzEIejcP/+97+P\nw4cPM79/5ZVXcO7cObz11lv47//9v+OJJ55QPsAZzGAGM5iBOByF+5/92Z8hGo0yv//Nb36DpqYm\nAMDXvvY1fPTRR3j//ffVjnAGM5jBDGYgjAKZH1+8eBGLFi2a+n9lZSUuXLiAO+64w3LdrFmzZLqZ\nwQxmMINPLbwmEZA2qNo7ZglyMvlne1UVUr29IISE4k+qtxftDQ1I1tejvaEBqd7enM96kklsr6qa\nmgPvPGhtJ5PJwOes4k99fdK0GuY/6wCkUF+fnWdp6WrqtZ/9bLVrP729KVRVbbf8rqpqO3p7U4Gv\ngcifhoZ26hokEh0ghEg/F27ty9xT873044/bWvT2ptDQ0I76+iQaGtqn/i87f5k/7PchaftbbEwy\nkNLcKyoqcP78+an/X7hwARUVFY6/2T00hM79+1G3apVM154w0NeH/u5uFFy/jok5c1D+wAO4ePAg\ndg8NTV3z1OAgPgbw0sjI1GeNx4/jUDptacttHgN9fTiyZYul7fahIQx/7WtqJxUQ5syZYHxzJ4Aj\nuHr1g6lPYrH5GB1tB7DbdN12zJ1b6NpPd3c/hoZ2Wz4bGtqN/fs7sWpVnfC4WejrG0B3dz+uXy/A\nnDkT2Ly5QWn716/TX7Xx8dtC0z7rnhYW3vI0Jh3o6xvAli1HLM/E0FA71q6twNBQu+XzSKQFFy4A\niUSH8vtpB/t9uGX7W909d4OUcH/44Yfx/PPPY82aNThx4gTmz5+fQ8nQcNv4uEy3nkATtjSh/dzI\nCDptv11mu8aA0zz6u7stfQGZDaFebNi+QVS4bd7ckPMyAdsBPASgDkDb1KcVFQtw5kwDgE4AtyHz\noD+E4uL/x3VcuoUiwBYYAJQJBJ2Cs69vAKdPnwXQBWACQAMy90Csfdo9rarajk2bHpIeoyqwNvsT\nJzqxb18C+/d3Ynj4Gs6dG0Y63YYzZ+pw5oz6+2mH8/tg/J2BX5ulo3B/5JFHkEql8OGHH2LRokX4\n0Y9+hJs3bwIAWlpa8Bd/8Rd45ZVXsHTpUsydOxd/93d/x9XprUJ3jU01aMKWKbRt/2fuyQ7zKLh+\nnfr54nnzmL8JCl1dL2DPnhTS6WUwhMPQ0BEA7JfB+PzRRx/B2NjdMAS2IVSKihZMXZt58I/kCI3m\n5m+7js0PbdKP04Gb4Fy5cqWndo2NaXT0kOnT9sn2DwsJZmOu+/d3Ynz8NhQW3sKmTQ9p1XhpcFoL\np81+1ao6rFpVh0SiA2+8ccjyvY7TnhnmtRsevoZ3330Pf/zjddy6NYSMopP5Phbbhk2bvqVlDHY4\nCvdf/vKXrg08//zzQh1ur6rCQ5s2cV9vp1IaNm/2ROnQhK3bQcpAA4CNkQgOmDaDlkgE961Ywexv\nYs4c6ueLYjGXkfqLvr4B7NkziHTaKhyGhhLYv/+o48uwalUdli/vR39/V853ZuErIzT80CZVnQ6c\nTj9ua+BVuNM2JmA3ysrWYN++VmFhZghInbCv0wMPlOO114Yt68YCz2bvx2mPBvPaJRId6O9/FsAA\ngKMAjgG4hfLya75tllK0jAi66utxq7AQlStWoL+7G8f27rUIa5oQB0DlrQEIC3iasKUJ7W2xGK4A\ngIlzP1xVhZL770fjr36FZek0bgH4bjqNIwcPYmD5cupYGjZvRvvQkGXsohubH+ju7kc6fcD26W4A\nnVwvA4/wtb/Mmzbx859+aJMqTgc81I4OwckSZNXVX/Zd4+YBbZ2OHduIiYm/hqHdOlEoPM9bGGwH\n2ftSB2NeAFBU1OXbGEB8gNFNqreXbK+qIgSY+rO9qor0JJPUz1tray2fGX86EgnhMdD6/uFk3x2J\nBEnW15OORIKkentJqrc357P2hgbhsdDa8QKj/2R9PWlvaPDcDg319UnatAiQJIlEB1cbvb0pkkh0\nkPr6zG96e1OW76qqtlvarqrabrkmaNDH+EOhMTY0tFPXkXcNvSKofr2CNV6gg3v8Ts+b8b3s/ZQF\na57xeDNpaGgn9fVJ0tDQ7jomGRHtm+YOsI2Mjc8/j0OjozmfNzECqLwYZA3tunP/ftw2Po5bhYV4\naNMm5gnA/vmxvXuFx1K3apW0V9ALXV1I7dmDZen0lKnsiMfTCw0sLScSOYtNm7JGUTfKwaxl9fUN\nIJHowPXrBTh9+qyND9bPf4pCxekgKCogH4ygZrDWyW7psq+b8fxduvQJhoeHEYvNR0XFAuopcNWq\nOpw8eRrPP9+IiYkICgrSWLu23tfnjXZfYrHHMDw8H7///bNTn+k09Poq3FlGxsgEXcDQrxYzyKri\n7Fkcuk7j8EBfHwb37LF49LQDSAwN4agid1LaQxiJtODpp7Mvg4g3Se61XdR+ZYSeDrdFWcokKCog\nLEZQFuz36sqVMcaV1nUyrxvt+RsdbceZM3TDf1/fAA4evGhRKg4ebMfy5QO+rQvtvnzwQQSnTj1n\nuY6m6JjXTAqedX4BGN2wqI3VpaXUz5vjcSqVwktLsGggL7QGi9ZRQZGwaBcmFQSQZH29dL8G3I65\nIkf/3GvV0gZhpXnCQAXIorc3JUQZ8LRnX5NY7DESi22zfFZQsIEAKea6uVE59mcprFQViwKtr09O\nXZO7ZnlCy7CMjPVr16LdFky0vaoKj+7cCYCfSrGDRQN5CaISpXV4wQp2Atgnndug9sTgprWKUA65\n1zYgc95QQxv44bbo5WQQdg3aDTp8/Wn3amTkJcTjj+O++7LrtGLFfThx4ijGx49R182NyrE/h5nr\nBwD0I0NOZAhNv4KHWOA53dG9n7zBV+HuJCAHli9nCk6vApQpHD0GUang0O1w2oAIgwo6G4mgjeJ1\no4qCskOEcsi9NvOSlpWtQXX1l6WFnm5umybkjh/fiCVLXkZFxQJHQe+HG6Eu6Ng0WfeqqKgChw93\ncbfjFv1pfw6vXLkA4AisEdHtlqjpIMBjH5GmYkzwVbgDbAGpQ3AGwZOLwmkDevAHP8g56bREIqh/\n+umctXI6Aciuq4jRjn7tYVefa15tWTe3TRNy6fQBnDnTiTNndmmPdAwKly59Qv18ePia5zZV3Sun\n6E/6czgHVsGOyf+3IUjwnO7YG5k4fBfufiIffM2dNiDaSee7DCpIJQVlhwjl4IWeEKEEdHuHuFEA\nvNqs7lw1qjE8PEz9/NKlS57bVHWv7NGfly5dwh13lKCy8ij12SouXkBrxhI1HRTcTnf0jcwbQiHc\nddEJojy5inGItuG2AdFONLQ+eCko3vHRhNPhw7u41kCUnhChBHRz2+4JoNwpID9y1agGK7nbHXeU\neG7TmOuOHY/jnXeuApiD4mJv6TfMz5TxbI6P34bu7n5LX0A4gpi8wv58p1ISjSkwArvCqRuVHi0y\nUDEOr22IBDux+ljPEfDFOz4njxSVHhVGWyUlTZNeNSmmF4FfoM0d+KFlbG5eF2H11nBCZsypSQ+U\n5OTfKekxq/Zu4mkvmewhkcjqyXlk5pVvnksGZER04MLdS+SnDqgYhx9zYfXRyuE2yjs+dnRdq7IX\nlS5EtwsJUV0wXENrarZOCgm2
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 72
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"Different Learning Rates",
|
||
|
|
"=========================",
|
||
|
|
"",
|
||
|
|
"In the above examples, we just picked a learning rate of $\\eta=1$. That is often not the right learning rate. In fact, with that learning rate and stochastic gradient decent, we have a limit on how well the algorithm can perform. Effectively, a high learning rate means that the algorithm \"forgets\" earlier training examples quickly. Eventually, to obtain better results from larger training sets, we need to decrease the learning rate. That leads to better performance even in this case."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"subplot(121); plotgrid(forward)",
|
||
|
|
"update(100000,0.1)",
|
||
|
|
"subplot(122); plotgrid(forward)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAWwAAAC1CAYAAAB72cXqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvVlsZNl55/k7d4194c4kmczM2kvWUtZmt2G3ut1ljzBj\nwRgBtl9kPcgwMC+GjQHseRqogQFcgoE2BPhh3gzNi2H3w9iCoTHagi211LK1llVSSaotN5JJBhn7\nHnedh3MP4/AmyczKTCYzK+MDDiIYjIh7497v/u///L/liDiOY2Y2s5nNbGaPvBnnvQMzm9nMZjaz\nu7MZYM9sZjOb2WNiM8Ce2cxmNrPHxGaAPbOZzWxmj4nNAHtmM5vZzB4TmwH2zGY2s5k9JnbPgP0P\n//APPP/88zzzzDN84QtfeJD7NLOZnavNfHtmj6qJe8nDDsOQ5557jq9+9ausra3x0Y9+lL/6q7/i\nhRdeOIt9nNnMHprNfHtmj7JZ9/Kh73znOzz99NNcunQJgN/5nd/h7/7u7444tRDigezgzGZ2kp1F\nzdedfHvm1zN7GHaSb98TYO/s7LCxsXH49/r6Ot/+9rdve9+/Bz5xLxu4T/vau9yugTwQeWAReAa4\n9D9B93+5wn/5pT/gn/7rL/Czv5yD+vfB3wV6QJyM+9nyg7Tz2vZ5bfc/n8m33o1v/+8r8H9sgu2C\nEBB7EAxg3IBmB271YR9oAUPAB8IHsG8G8HXgPwEm0mdtwAFcpP+WgHlg2YGlItgXwXgO4vfD5Fds\nfvb8M3x14RN8+eb/yr997cP0/+8y8VvXoP42cBXoAmMgSm39azxZ/nWe2z7Zt+8JsN9rLEMgLwAX\nKBmwaoPxgSw3Xtrk//vJb3DtRzbsvgHUgQG3O/PM3it2N77d3oOtPcgi/SYEPKCdjBbylj5O/ndW\nvR/U9wokmJtMgdyxwM0yRfBVQXcjz/WFDb4Tf4wbr1+h980SvOrBuA7sAv3kl8z8+1G1ewLstbU1\ntra2Dv/e2tpifX39ge3UwzYF2A6Qy0H5Evzw2ffz39z/QP+vCvDdPWAHeQnOWq+8l+1ufLsFvI30\nF4H0iAjpHUPkLX2CZNYBD95j1PbM1OvKjy3AMJIdzAFzEF0w2M6s86Pw/Xzb+ziNH8/DawMItpFg\nrZj1g5gLzOys7J6yRD7ykY/w1ltvcf36dTzP46//+q/51Kc+ddv7Lt3v3t2j3ct2BZAB8kVwfh5e\nL3+Af977ZQZvDKBeQ16md2If97LlB2Xnte3z2u7Z2N349hpyrlUD9pLHA6DJlFkHSE95kGAdA5va\n90apEetD6XxZoArRqsF2Zo2r7aepvbbO6IcuXBtAtA00kLea0+YDlx7gL3k3dl7bPe9tH2/3xLAt\ny+Iv/uIv+PVf/3XCMORzn/vcsVH0S/e7d/do97rdLJCvCKJfFLwRv8h3Xv8w9H4KYQ0YcWe+dK9b\nfhB2Xts+r+2ejd2Nb68hxYM02wmTcVaCQgxcZArKIdIjlSwTaq9FSidxIS5DsGyylVln6+pFJl8v\nwE+GsN9B3nI6SDLyKPr2eW33vLd9vN0TYAN88pOf5JOf/OSD3JdzMzWVLABu2aL+CyUG/5qBf/Sh\nf4DkTQ+aL83sUbU7+fZJoByn3nNW3qI80UKCs/LfMZJWTIBQITfgZyw61SzXrU1uNS7At4H6PrCF\n9O0JM9368bB7Buz3mgkgb4GZy/HG8rPUG3PwegDjHvIymDn0zKSFSH9Jhyf1vKHjcogelKnvVTcF\ngdTLx8iskSEwDsAfgxWALyy6doGd0UVqu6vwI6DdQgo5dzNznNmjYjPA1ixnQ5wp8QPxEW61lmHX\nQ14GDyoxa2bvBQuZZmUoUxLFw4I9tT21Px7T9NQe0PNh1IfcBPzQphcX2WtfoLm9CG9HEHeRUsiM\nXT9ONgNszQwXxtk8b4pnaZBBhpFmkfOZ3W4xh4rDkdcetkXafijW7wCdCNo+OH0IRwZe7BDdMmFv\njAygdzj7xMOZPWibAbZmwgbPcdgVq/RwkJNLFe+f2cym9qhAnMoW0UF7AHRiaAZQ7SYvREIqIPUJ\n8omec/2o/JqZ3clm3fp0MyEwLTqUmWAhpZCZM8/s0TYlj/hIgWOALOBpRuC3wehFWLGP0Yyh5UGs\najDTc4SZPeo2A2yO6o+CGIsA43ByaTE7TDN7HEzJND4SjrsRTFpg9XxKcRdr5MPIZ8auH1+bIVFi\nMRBHYEUhJXo4Mm8ECdomt+cEzGxmj5bp8sgY6EXQHUDQ9Sn2h1ijEAJVkzmLyzyONgNs3SJwQo/l\nuEYBgDlk/aPNDLRn9rhYhJRGujHUxjBshRR3x9hjNY+cBRofV5sBNtPmOcKHrD/iMteoFjow54BZ\nQNZA2szkkZk96qZYttK0u8CgB+J68s/M+e3bzO7fZuiTmAGIELLBkKfid6gutWHTAWcBqCA9XQH2\njGnPTJo4Zpy3qd4iATIne9QDbiIniUUBYjZbfFxtBtiaxTHkohFP8zZzl5vwIRcy68ASsu2ZxbSJ\n5czhn3TTu+Op8ahcUIplD0mKdbeQIZl51cbvUdnTmb0bm501phceMTj+mM3+DnOXG1i/HCAuzEF2\nFVhFMu0cM3lkZocy2jGvq1v6eXqHyhiZAP4Q2AexECAugGySrZrDzuzRML2j+enveqJNXXgWIGKw\nvID5RpuV+T0WPnSA+aILi0tgXASxzBS09ZS/R2UyPLOHYcpnjNRr+kIC6v/n5RV6bnY4AtGIyVW6\n5BZ9YAFJt2ek49Ew3XNOr2V84s+WAms7ec4YxDZcHl3jg8uvkf3EGJ6vgnsZxGWOMm2XWa72k2dp\n9qxzI10wE8e891zMA6MTsW7vcCHXBjaYxmUeiT18gk2fk1lIPDnZnvgzpQA7A9gGGBFQh/XhNu+v\nvMbCv9/D/dUJfCwHSxckcHMFCdxzQJFp6p9K/3viD+t72nQ2rbPoNHCn2fa5mQ/GMGIj3mItswcL\nOXBLSN+dEY7zs9sWdkOSwJPtie4loi46myRxzwQjBlpwYXyL9+V+zA/e9xKjfpZ9f5XIqMA7Dhzk\nYJKFqM50nZERsnpMNSLW81xnOa/vJdMBW2iv6cya5P/6uotn2XL1JBMAERiTiPVomwuFPcRli/id\nCkzmkH470vZs1jfn4Vh6JU59OeWT7YkFbHWB2chDlAEspfcPYd5r8Dw/4xf4V0bP5+hkS0zcAtG3\ns/AjB7bmYFhHrvXYQHZvUAtEqcsTjjbCnNl7wfSohc6iSf2tMjWUWZztijRpOzoLiFhmj6X5ffgY\n0F6ExghZpg7ST73k+Qy0z9bSMogO1qcnyj/xgO2QLA1GAthJz6dCb8RqZ58XCz+lXlhgfCnDzV+9\nTK9awZvPwE9s2J6Hug2jOQiayC5oXeRFoBrDB0wvX1XWMLPH2XSwViKYyVGAtJgCtoLBh8mu9WC6\nKUBYMXOiRXWhBb8I/KwE28sySTu2kS1XBxxdOnjWa+TBmT4nU7d1WxuZZORP/ZYnErD1Q6YOURFw\n1LELwekGzDW7PJ19m65TJChZ8FLMTnmTdnWeYMUh+nEOfpaF/TnoVcAvQNSAuI3sNzxkun62tmbT\nEfY9s3dn55+No+df65GLtCSitz1V8yz1/7M+8wqsHSQREXZMWXSoVltkPjzE+0aG8K152PEgtJJV\ne02m0p4iG2qOMPPVe7e0/KGGSlpQtDEHSVOMk+yJBGzdmbPIQ1QGHHXsYmAITs9jLdqhR54JDiYh\n5QtdruWvUFtdZ7yZg00Bb9hwtQpbBRgvQ9BALm7aRjJuBdxK41YT45lU8u4sLUCcj6l2YGqG5jDN\nMtIzaVVanQJwfTmvs+au6qaSARwTRDamYPRZyu5y+eIb7Pz8ZTrbZWivw8CGIIO8GjrIGaLyV7W3\nD1PMea+Z7rf6rV5JIGqOn+fcAVtnHOcRdDnO1IWWIQFrAyoOuFmmEtIErH5Itd9j3dhl4mQQgJsZ\nk7MHFKwBB5VluhslvItZop/Z
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 73
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"If the learning rate is too large, the stochastic gradient descent algorithm may diverge altogether."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"subplot(121); plotgrid(forward)",
|
||
|
|
"update(100000,10.0)",
|
||
|
|
"subplot(122); plotgrid(forward)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAWwAAAC1CAYAAAB72cXqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvVmQXNd55/m7W+6ZlbVmrUARGwFS3AlSImkSXEBZtsTm\nWDLDYbfMdsvhiXmyHH6w32y9WHA4JizZcs+DW+HByBGWOT3RMttWqy1KJmWK+05RJEFQWApAbVm5\nrzfvMg8nT+atRFURWy1AnV/EiczK7d6bdfJ/v/udb9F83/dRKBQKxZZH3+wdUCgUCsWFoQRboVAo\nrhKUYCsUCsVVghJshUKhuEpQgq1QKBRXCUqwFQqF4irhkgX7Bz/4Afv372fv3r38+Z//+ZXcJ4Vi\nU1FzW7FV0S4lDtt1Xa6//nqeeeYZJiYmOHjwIP/wD//AgQMH1mMfFYoNQ81txVbGvJQ3vfLKK+zZ\ns4fp6WkAfuM3foN/+qd/WjapNU27IjuoUKzGeuR8fdLcVvNasRGsNrcvSbDPnj3L1NRU5+/JyUle\nfvnl8173h6PwxzvBCoOmgW+DU4HGEmSLcK4KC0ABqAEtwLuUHerhOeARwAiMEBBujwSQBgaBsSj0\nD4C1E7gR3Dt0mveb/HTi0/zAfJR/OvVFzn13ivpTOpz6COrHgdOBPQ5+sc8Ch67AEVwKm7Xtzdru\n19blUy9kbj+A+i9f69vdzG2vNbMvSbAv1MoozMHMHEQRoukCTYRAF4A8UG4/5rJc+q4kWuBWb++L\nCVhA2AIrDvQBw9AaN1na2ceH0b28PX8r8y9PUH8pDB8UgHNAFqgDzjrusWKzUBa0YitzSYI9MTHB\nzMxM5++ZmRkmJyfPe10eOI6wbjWEvHkIuau1RwNhp66HYHt0xTqITle4NQNhdseBIagPRTmu7+Et\n51beW/oUjRcicDIHnAIWEaeYK3UtoNhqXOjcVig2g0uKErnzzjv56KOPOHnyJLZt84//+I889thj\n571uAmGPzgNz7dtFupZ1A2GnelxZsZ5uf6b8XLfn72VDR5jaMWAIakMxjhu7OTV7HUvvDOO+A8zn\ngbNAEXE9sJpYT1/Bo7hYNmvbm7Xd9eFC5vb05uzapm57u213s7e9GpdkYZumybe+9S0++9nP4rou\nX/nKV1ZcRZ8AKpx/VnDpiuh6sLP9+UZ7G3J7zgq3vtZ+YRT8AY3qYIzjxh4WPh6l9WIYjtegmEN4\n26vtd63G9Dod0YWwWdverO2uDxcyt6c3Z9c2ddvbbbubve3VuCTBBvjc5z7H5z73uTVfs5oo+z2v\nWS9PcHDbLYR7xERY9g3ABjy5kzo0EibZZJpj2j5ypwbhZ01onEZcH6y0yKi4FrmQua1QbAaXLNgX\ngosQyV4/snRH9N6/0kifuURHiHQDIb9VoNECpwmmCw0jzJLez8nKLoqn+uG4DfY8wokjXSFKsBUK\nxeaw7oItF/ck0qe8UbIntwfCmWG396mKWGusNqBZBcOGphumaPdzbnEn5dMpmMkixLqCsq4VCsVm\ns66CDULier2+myF7Xs9+yNjsggvJJkRL4NgGrXoI/4QO+RJiibSCkPmNPM0oFArF+ax78afzojLW\ne4Nr7IeLsJOlW6QC5H0oNcHPA00NvwGcAUpVIIcIQlTWtUKh2HzW3cLeSgTdI9K3ngf6bfCyYNRd\nTNtBmweqdbphfO6Kn6dQKBQbybYsryrdIzbCyq7a4CxCuF4n7lbRKh7YTbphfMq6VigUm8+2FGxp\naTuIaJGSA6US6CWbWLWB1vTBlZKufNcKhWJrsC0FW+IifNkFF87VwF1qEc/a6K4MCFTp5wqFYuuw\nbQVbWtlStPNAM4tIaIwg0tUVCoViC7Fhi44rFWHaTEeDjFhxEY6PImBnEZF8fUBU1vZTKBSKrcGG\nKJIs12EGxlaRQo/u4mMrh6j7OgIk5Z6qcpsKhWJrsO4WtrRTe2VPijicn0K+kXiIKOsG4BRBq3lo\nu2y0AQufBN3Ti/Jnbz4rzSSFYvuwroauFGu957FgIwH5/Gb9DIMJNV4ZjLpDaiJPuE8H+hEJ7MYm\n7qFC0Nt+QqHYfqyrYEtBDm4s2LZLCrW0tjddEmsQa9TYEz9Of8gDxhENxUJskT3cpsgZEuwVpFBs\nP9bdwg5a2VLugsKtr/D3pmELwd6rH2MwUYZ0HIw0oh2NhRLtzSA4O6RYRzZ1jxSKzWLDBHslcbbo\nLkL2Wt0bSWd7DsRaNfb4HzMwWITxCJhDiLARGeunRHvjCM4eOWPCiC6hCsX2Y12dgUGXR1C05XNy\nrNToYKPTVqQMR2iwg1P07SjAp8JwdhKaDUQRqI1qvaBY2Q0SQgi2srAV25N1Feygf3olK1o25pUJ\nLDYbHzGi05UDXYeQbpNhnsTOKtxqwusD0Bhti7YJlOhW8OvtFKm4PIIxRfIUL6/DpFhHES4qhWL7\nse6CLcU6xHLRDnaiCdaqDiaFS0FfT6RghwDDAMuyGWKJ1GSR8C0t7Ok4fnkEsi54wSXTYL932UpY\nHoHi4gme3oPRIFKoQwixjiEWghWK7ce6CraMrQhe0EovcLATjbSuYbl3WDbLXU9kn8coYIbADDv0\nUWQ8PcP4nlOc/fQ0diUNJQvsEHhRRC94aWlL4ZairVwll0bQaSYta7nAGGG5WCvBVmxP1lWw5YKi\nXCqSQz4mNy7r4kkxlw4GKX/riYGQgyRghcEI+0RaTTL6PBODMyzcMYGdD0MjAWczUI5Asw+Rw15A\n5EjWEXWzWyx3lUhU0s3q9K5yyNkhxTrGcrFOogRbsV1ZM0pkZmaGBx98kBtvvJFPfepT/NVf/RUA\nuVyOw4cPs2/fPh599FEKhcKK75c/u6DnMQmkESkpg5oYQxoMtR9LIX6aGxH5rLX3L4qIAwmHQYv4\nWC2PYTfLVPwMsZurGHe7cNCA69LQPwHhXWDsBG0SGEXsvQz/i7aPWB59rwNIIQgKde+pPYL4LhOI\nGZFG/IfkzElf9tYvd24rFJvBmoJtWRZ/+Zd/yXvvvcdLL73E3/zN3/D+++9z5MgRDh8+zLFjx3j4\n4Yc5cuTIiu+X9pG8iE0ifnaDwIgOoxaMh2HCgnEdMpp4rg/xc5XW+HrEHuqIk0Ksvb1hIBoDLQ6G\nB2mvxHjoDFPjH9N3cw7uQ4w7DNgdg75JMPcCB4DdwBRd8e5rH7EU7qAHX7Hc7RFqD+n2kKf2BOJ7\nlKOf5af0y+Ny57ZCsRms6RIZHR1ldHQUgEQiwYEDBzh79ixPP/00zz33HABPPvkkhw4dWnFiR1ku\n2kkdUgZEQxAOgRUB3QDPA9eGcANCbc+CrPHRCnzelXIsSL91CCENKRP6oxBOA0nQNIhTZURfYEf0\nFPUdcWw9RD2awO03YMCAk1FYCEEhCpUYNPvBqSDcJCWgTNfHbSMcPw7d/pCbWUFlM+mNxpfWtXSY\nSeGW12PB030E9BCYRnfR4xK53LmtUGwGF+zDPnnyJG+++SZ333038/PzZDIZADKZDPPz8yu+53t0\nbaj7gUct6ItCKAG6/C2aCO1qQCwPkSJ4JbB9aPjCM3ylm/hKmQjTdtGEIDkM5iCQAt+EiFZnkCyT\nzFAZSNCIh5mPT9IYiOJldBjX4LQBMzE4F4PcIFQd8JbAy4FfQBRtLSP62tiIozFYvpy6nUICexNh\ngsvR0cBIrDDeAH7S/ZgryMXO7WcD96fbQ6G4VE62x4VwQYJdqVT44he/yDe/+U2SyeSy5zRNQ9NW\n/gX9H7RdDjoMRiGWBKsfNHmVm0Sopg+0QFuC2DxMaeBUwbaFjeoGxpVAa282haikOhgGPQMMgZ8E\n1wRLs0lRZoRFKiTwLIPwWJPF6DCF4X68sTD+Dh1mEF3W5zRYMGFpAApJqI4j2iLk6Qp3BSHesne7\njC6RR3YtdrnprdfYu7AYXOEIDmlVx0ALg/4ghB/qBo2c+9oV2btLmduHrsiWFQrBNMtP+s+t8dpP\nFOxWq8UXv/hFvvzlL/P4448D
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 74
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"Different Number of Hidden Units",
|
||
|
|
"===================================",
|
||
|
|
"",
|
||
|
|
"The minimum number of hidden units capable of solving this classification problem perfectly is 2. However, that doesn't mean that 2 is the best number of units to use. In fact, finding a good solution for the minimum number of units may be a computationally hard problem. On the other hand, using \"too many\" hidden units may mean that the algorithm learns irrelevant detail from the training set (overtraining, see below)."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"def init(nhidden):",
|
||
|
|
" global A,a,B,b",
|
||
|
|
" A = random(size=(nhidden,2))-0.5",
|
||
|
|
" a = random(size=nhidden)-0.5",
|
||
|
|
" B = random(size=(2,nhidden))-0.5",
|
||
|
|
" b = random(size=2)-0.5"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 75
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"init(2)",
|
||
|
|
"update(100000,1.0)",
|
||
|
|
"subplot(121); plotgrid(forward)",
|
||
|
|
"subplot(122); plotclass()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"err= 0.205"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAD9CAYAAABOd5eOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXt4E/eZ9/2RjxI+HwDbGAKIk/GBhJaUvs+uYbNPcFq2\n2XbbDUlLQhsgkBAgybtNdzEGF+oeyLXpYiBt2mTbJLRd+uy1TVvcC0wf3tjdFgqbEoyxISBzso0M\nOAZskGzLmvePn0Zz0G8k2SQkLfpely7bo5nRjGR9557v/b3v26YoikIMMcQQQwx/1oj7sA8ghhhi\niCGGW0eMzGOIIYYY/gIQI/MYYoghhr8AxMg8hhhiiOEvADEyjyGGGGL4C0CMzGOIIYYY/gIQI/MY\n7hg8/vjjjB07ltLSUst11qxZw9SpU5k1axZHjhy5jUcXQwy3hhiZx3DH4Ctf+Qp79uyxfP43v/kN\np0+f5tSpU/zgBz/gySefvI1HF0MMt4YYmcdwx+Cv//qvycrKsnz+V7/6FUuWLAHgE5/4BFevXqWr\nq+t2HV4MMdwSEj7sA4ghho8KOjo6GD9+fPDvwsJC2tvbGTt2bHCZzWb7MA4thjsMIynMj0XmMcSg\ng/lLJCNvRffYOG8eiqJEfDTs3s06p5NK0/YKsFGyTAEey8qSrre+oiKq19Q/dmzcyEMOBxuBSqAB\nWOd08kxpacjrmo9Rfd2Hs7JYsEB2BgoVFetDXnP37gacznWG9ZzOdeze3RCy7rx58nfhrrtC39/i\n4icCR6k/G7EPRVGGdYxWj40bZfuS79fheEh6TurD6nhgPbAxeNzqY6SIkXkMMQQwbtw4Lly4EPy7\nvb2dcePGhd1myG4HoLGujvUVFVTPn8/6igoa6+oM69XX1lLjcklvhX0W+x6wWB7v9YY9JjMa6+po\n2rKFXR4P1cA3gL1AhctFZ2dnyPoLgJWmZeuA1EmT6O+X38x7vfEhy2pr63G5agzLXK4atm3bF/y7\nrq6Rior1NDWdQZBbo2H9hAQjudXVNdLWZguchf5sGrHbhwBYs2YBTmelYTuncx2rV98vPXZ1v7Nn\nryI7ewnZ2Q/z8su/oq6u0XS+8nfG41llOCcz1qxZgMMhe0fvB4bo7b1sue1wEJNZYoghgAcffJDt\n27fz8MMPc/DgQTIzMw0Sixkr7XaunjrF551OxnR28j0dyVa6XACUL1wIQEJ/PyAn7gJgOfBD3bIV\nCQkMZGVBT0/I+uoFBARR19fWktDfjy85mQVr1gRfU0V9bS3f93gMy2qAKiAzL4+VN28ant8FnAs8\nHw/8f4AzL48vb9pETe0B6XuhEqkekYi/rq6RtWv3mghfJeFynM51fOITkw3b1tbW4/F837THGhyO\nRaxevQqAhQvLAdi2rQqvNx67fYjVqx8ILjejrq6RZcvexO3eoVtazrJlr5GfbzMsg5+gvTNDwANA\nOV7vfum+1eN5/vlmNm36HIoyS7fdHsAd2NetI0bmMdwxeOSRR2hoaODKlSuMHz+er3/96wwODgKw\nYsUKPv3pT/Ob3/yGKVOmkJKSwo9+9CPpfpbY7Uzwevmi10t5WxuLgO+Z1qlxuajato3yhQtprKuj\ntbmZasRX9zngRd26DcAqjBTxJZ+PXZmZVDqd1AQuDPMR0sgDq1cD8FJ1NQ1btlDk8eBDxI17TRcR\n0C4kZsQDV91u/KNH8w/t7ZT5/VwGvMC/AGqsGZeUxMdXrKB84UJ6ScPlqjQQsIh6HwjZf3Ky/J5D\nJX5Z5A41ZGU9wr337mP16gdISfEbnrW6QEyZUmAg64ULyy3J24za2nrc7hdNSzfhdv+Ea9cuERf3\nEH5/HJAaeO5+oB5Bn/WGc7JCdfVT/Od/HuX48ctAH7AlsP0kzpy5SF1dY9THa4UYmcdwx+BnP/tZ\nxHW2b98ecZ3XTDJHkcV68V4vjXV17F27ll3d3cHlS4ElwKDNhlNRKEDEfOav8v60NO7btImqbduI\n93oZstt5YPXq4AVClU5UVCKkk32Bi4gKX3Ky9PiagGe6uynv7qYRqE1KYmBwkF8FdNvg8QwMUHXw\nIGCMetvbL+F2X2XUqAJqa+sNz4OQF8IRvxUxl5VNZ8+eaulzVheIgoLU4O91dY3U1tbT359AcrKP\nNWsWSIlSXe+Pf2yX7DEOSMDj+aVuWSVxca0oyjYU5f8ElyYkrGTu3DLpcekxbtxojh//BkJK2ou4\nPxI3X8uWLSU/fxfp6aMj7scKMTKPIYZbhJXmPWS3B7VyPV4FHs7NpXzVKi4ePMjFQ4cs5ZTyhQsN\nxKxq86cPH+Y/LKQTs6a+YM0aKl0uw3E8HhfH//b7g4RdDpQPDLDE4lz0+1SJce3avXR3v0x3Nxw7\nBi5XpeF59eeGDcs5c6YXSCY9XSPdSJG7DJEuEDLpxnxcoeutl7xSPZL7Lfz+KsT91XoEffrw+b7I\nwYPWmnnosdtQiVygEbc7D7dbXfb1iPuSQokhhhiiBqAopkcDKCtMy/7F6VQadu9WNs6bF7K+AmJ5\nAA27dyvrnE7p9nro19so2acCyjOgPJSTo2ycN0+pXLAguI+G3buV9RUVysZ585T1FRXKE8XF0u0X\nWex3xex7lQULKpV58zYqCxZUKvfc85RsNaWiYr3hmHfvblCcznWGdez2lco99yxVNm7cEfKc0/kv\nyu7dDWE/g927G5SKivXKvHkblYqK9Yb1FyyolB7X7NlPGfZhXK9BgWdN2zwm3Q88o8BK07J1SnHx\nE1H9/+ze3aBkZZn3bT7mkdFyLDKPIYZhQq9jg0hjlSGi4vN2O6kzZ7Jo0ybKFy6kvrZWuo+O3l7W\nV1QEE5fjFi+m6uDBoJxSOHcu9bW17H/hhWBiUx/l62PaRjQF9xTwfHc35Q0N4lh1Gro+wl9fUQHH\nj4ccVxpCrtHHjcuTkmnovItTf/pGcJndLo/hza4WmS7u9X6PI0equH69g8WLx/HrX8sjdyuE08Ot\npJuWlj6DLm1cT93XKhISLpOWFo/P56G3V7anTkSKWI8auroejnjc6rHPmVNPfb1+6ftDwzEyjyGG\nYaJi61aqtm2jr7OTs2fPMuj305GUROqkSSwNkLgKqcSRl0dmZyff+NOfgssqXS4qtm4N6uF71641\nbFPpcnFz1ChtvwQ0cvTqa2DdwM9yhIa+Y8kS9peUBC8KAD2XL7PEbme818uCwLrLk5LoHjWKR69e\nDSZjm5OSORzn5Lx7JkJaEGt7veORwSyRWJErxONybebXv17OtWtj6OkRXp6eHli7NlQWiRZW0o3X\nO4Ft2/YF96mtp78UZlFWNsDbb/8w4HB5zpQYXQdcRi+xqO9Hfn5+1McYKhVZCXXDQ4zMY4hhmFDJ\neu/atfxCDd9u3KAyO9tyXX0S03HpEi+amnjp3S8ynb3G5WJRTo6238DPHcjiRFgENAMdIJKvgUj9\nuaYmrgGvut3B9Z8E3gAeHRjgzexs3pg9m3Fpabh7PTR13sV59891e1cvFQuw25/E69V0ZZmrxYpc\nhWcHzpzpDRK5CuFFrwprJbRKcK5Zs4DGRuNxCRJ+wGAfXLNmAU1NS3G789BfCjs7nwtG8K+8Ao88\n8jl6e9MRrv9BYArC2258P/QJ2GiQnt5FVtYjQBJZWf3cvGm+cIwAIxJnYojhDoX6lalcsECqLa+v\nqIi4j0g6utXzTxQXh2jrj9nt8n0FdPwG2TFGWKaeg5X+DOsVUJTJkz+v5OQ8pGRkLFFych5SNm7c\nEXKuMs0c/iWgUysS/Vg85s3bKH3vZPtzOtcZdPN77lkaOMaNgZ8NUj0/Gt1/3ryNYbRt8XA4Hoqo\n8wc/+407FIfjocCxVSogzmfjxh3BPMBIaTkWmccQwwhg5d3u6+w0aOHmIh7Vcy6DWgxkZSUcU1jI\n/atXG6L81EuXQNKqtwMYi3DO1ENQSgF5iYp+2dXOLioq1ltY9sTaeXmPc/PmBLq7tWhy585K5sxp\nDPF7A2zYsIqWlj683gmohTZO
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 76
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"init(20)",
|
||
|
|
"update(100000,1.0)",
|
||
|
|
"subplot(121); plotgrid(forward)",
|
||
|
|
"subplot(122); plotclass()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"err= 0.034"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAD9CAYAAABOd5eOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXl0FNed9/2p3lsrAgGSEBiQ7RjjJSYhwfPMCL9+B8kZ\nTZz1seyMbRIDhoABx8/YmSAECpgs+IzzsDnxxH4ytkk85M2ZcRKUg4WHsZRnAoHYJDIYLzQYI0QL\nkLWr96r3j1vVXVVd1WoJ7/T3nDrdXXXr1q1u6Xt/9f0tV1IURSGHHHLIIYePNBwf9AByyCGHHHK4\neOTIPIcccsjhY4AcmeeQQw45fAyQI/Mccsghh48BcmSeQw455PAxQI7Mc8ghhxw+BsiReQ6XDO65\n5x4mT57Mtddea9tm1apVXHHFFVx//fUcPnz4fRxdDjlcHHJknsMlg2984xvs2bPH9vjvfvc7jh8/\nzptvvsm//Mu/8M1vfvN9HF0OOVwccmSewyWDv/mbv6GkpMT2+G9+8xsWLlwIwGc/+1l6e3vp6up6\nv4aXQw4XBdcHPYAccviw4MyZM0ydOjX5ubKyko6ODiZPnpzcJ0nSBzG0HC4xjCUxP2eZ55CDDuZ/\nIivyVnTb+vnzURRlxK11927WVFXRYDpfAdZb7FOAu0tKLNutra3N6pr6bf36Hfj9t6m9NACtVFWt\n4dpr77e4snmU4solJbdTU2N1Bwq1tWvTrrl7t7iGvl1V1Rp2725Nazt/vvW3cNll6d/v7Nn3qmNM\n3QuIPhRFGdUY7b8vq76s+/X7b7O8J22zGw+sBdYnx61tY0WOzHPIQcWUKVM4ffp08nNHRwdTpkzJ\neE7C5wOgrbmZtbW1NN10E2tra2lrbja0a9m6lU2BgOWjcNym76jNfmc4nHFMZjQ3t7F5czuh0C6g\nCXgYeJ5AoJbOzk6LM2qAZaZ9a5gxo4BIxPphPhx2pu3burWFQGCTYV8gsIlt2/YaxlZbu5b29pMI\ncmsztHe5jOTW3NzGiROSeg+pe4E2fL4EAKtW1VBV1WA4r6pqDStXLrAcu9bvnDkrGD9+IePH387j\nj/+G5uY20/1afy+h0ArDPZmxalUNfn/6ebAASDAwcN723NEgJ7PkkIOKW2+9le3bt3P77bdz4MAB\nxo0bZ5BYzFjm89H75pt8paqKSZ2d/FhHsg2BAADVdXUAuCIRwJq4K4AlwE91+5a6XERLSqCnJ629\nNoGAmERatm7FFYkQ93qpWbUqeU0NW7e2EAr9xNTLJqCRsrJxDA8vMx3fBZwCGgEn8F+UlVWxYcPX\n2bq1xfK70IhUj5GIv7m5jdWrnzcRvkbC1VRVreGzn52Z1b34/fWsXLkCgLq6agC2bWskHHbi8yVY\nufKW5H4zmpvbWLz4OYLBHbq91Sxe/BTl5ZJhH/yc1PeSAG4BqgmH91n2rY3noYeOsGHDl1CU63Xn\n7QGCal8XjxyZ53DJ4I477qC1tZULFy4wdepUvvvd7xKLxQBYunQpf/d3f8fvfvc7Lr/8cvLz8/nZ\nz35m2c9Cn49p4TBfC4epPnGCeuDHpjabAgEat22juq6OtuZmjh05QhPiX/cB4FFd21ZgBUaK+Id4\nnF3jxtFQVcUmdWK4CVhTVcUtK1cC8FhTE62bNzMrFCKOsBufN00iYE+q4CQY7GXiRJmOji8jy9cB\n54Ew8B1AWJsej4OlSz+dJMNAoMFAwMLqvSWtd6/X+plDI34ryx02UVJyB5/5zF5WrryF/HzZcNTu\nXi6/vMJA1nV11bbkbcbWrS0Eg4+a9m4gGPw5fX3ncDhuQ5YdQIF6bAHQgqDPFsM92aGpaTm/+tVf\nOHr0PDAIbFbPn8HJk2dpbm7Lerx2yJF5DpcMnn322RHbbN++fcQ2T5lkjlk27ZzhMG3NzTy/ejW7\nuruT+xcBC4GYJFGlKFQgbD7zv/K+wkJu3rCBxm3bcIbDJHw+blm5MjlBtG/ezK5QKNm+AagNBNir\nTiIa7EgV2unuvp/u7mqgDY9nK7FYFEX5jXpcjCgahQMHGgGj1dvRcY5gsJe8vIqkxa4npFWrajIS\nvx0xX3fdJ9izp8nymN29VFQUJN83N7exdWsLkYgLrzfOqlU1lkSptfvjHzssenQALkKhX+v2NeBw\nHENRtqEo/19yr8u1jHnzrrMclx5Tpkzk6NGHEVLS84inI/HwtXjxIsrLd1FUNHHEfuyQI/MccrhI\n2FFlwudLauV6PAncXlpK9YoVnD1wgLMHD9rKKdV1dQZi1rT544cO8W86IgdNOEnX1K1I1eG4B1n+\nW1JTSDXRaDVimkmHXhPXiHH16ufp7n6c7m545RVhseuPa6/r1i3h5MkBwEtRUYp0R7LcrTDSBGEl\n3ZjHld5urcWVWrB43kKWGxHPV2sR9BknHv8aBw7Ya+bpY5fQiFygjWCwjGBQ2/fdEfuyhJJDDjlk\nDUBRTFsrKEtN+75TVaW07t6trJ8/P629AmK/itbdu5U1VVWW5+uhb7feok8FlPtBuW3CBGX9/PlK\nQ01Nso/du1uV2tq1yvz565Xa2rXK7Nn3Wp2uQL3l/jlzlis1NQ3K/PnrlZqaBuWGG5ZbtqutXWsY\n8+7drUpV1RpDG59vmXLDDYuU9et3pB2rqvqOsnt3a8bfwHwv+vY1NQ2249fD2K5VgW+Zzrnb5vu5\nX4Flpn1rlNmz783q72f37lalpMTct3nMY6PlnGWeQw6jhF7HBuHGug5hFb/t81Fw9dXUb9hAdV0d\nLVu3WvZxZmCAtbW1ScfllDvvpPHAgaScUjlvHi1bt7LvkUeSjk29la+3adtIKbhvAg91d1Pd2irG\nqravq6szWKa1tWs5etRqZIUIwSZlOXo8i+nsVHj55YeT+3y+kS14sNbFw+Efc/hwI/39Z7jzzin8\n9rfWlrsdMunhdtLNq68OGnRpYzutrxW4XOcpLHQSj4cYGLDqqRPhINZjE11dt484bm3sc+e20GLw\nI787NJwj8xxyGCVqt2yhcds2Bjs7eeutt4jJMmc8HgpmzGCRSuIaalatoiEQMJD/PWVljOvs5OGX\nX07uawgEqN2yJamHP796teGchkCA4by8VL+oGjl69VVtq75WIxyxn1u4inXXHErqxwDnz/fg8y0k\nHJ6q9laNx7OEvLxuenvvQnPHejx/xukcJhj8K4S0INqK89JhlkgyOV8DgY389rdL6OubRE+PiOXp\n6YHVq9NlkWxhJ92Ew9PYtm1vss9UO/1UWMJ110V56aWfqhEuD5gco2sQDuKUxKJ9H+Xl5VmPMV0q\nshPqRoccmeeQwyihkfXzq1fzH5r5NjREw/jxtm31Tkz/uXM8airipY9+sdLZNwUC1E+YkOpXfd2B\nlZ0I9bp2oe6ptLY2AdDe/gDQRzD4pO6MbwLPEI3exfjxzzFnzjMUFk5hYOAMnZ0zTYSmTRU1+Hzf\nJBxO6cpWUS32zldB+idPDiSJXIOIRW/MGEpo5+BctaqGtjbjuAQJ32IIH1y1qob29kUEg2Xop8LO\nzgeSFvwTT8Add3yJgYEiRNR/DLgcEdtu/D70DthsUFTURUnJHYCHkpIIw8PmiWMMGJM4k0MOlyi0\nf5mGmhpLzXptbe2IfYyko9sdv3f27DRt/W6fz7ovUNaoev6nqTUdXmtxSmqfpnvb6c9a25kzv6JM\nmHCbUly8UJkw4TZl/fodafdqpZnDd1SdWrHQj8U2f/56y+/Oqr+qqjUG3fyGGxapY1yvvrZa6vnZ\n6P7z56/PoG2Lze+/bUSdP/nbr9+h+P23qWNrUEDcz/r1O5J+gLHScs4yzyGHMUBLAjJjsLPToIWb\nk3i0mHMraMlAca/X8vikykoWrFxpsPILzp0Di1K9Z4DJwBYcODlHAc0Moo3DKkklta+zc5Da2rU2\nIXuibVnZPQwPT6O7O2VN7tzZwNy5bWnx3gDr1q3g1VcHCYenoSXaVFWtoaiowCqQxzaixT6rNGXJ\nb9x4txqpsjHZxuqpwS4M8ODB
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 78
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"init(200)",
|
||
|
|
"update(100000,1.0)",
|
||
|
|
"subplot(121); plotgrid(forward)",
|
||
|
|
"subplot(122); plotclass()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"err= 0.437"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAD9CAYAAABOd5eOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXt4VOW99/3JkRlJIIEASQgWiCcOQolF8d17J2yfTcZu\nKrW124CCaSUIEgTl3dVuQiCCWIt724cAHrbaFkEtfXu11hIvCJbHTJ8tFLayjUFECIiEZCDEHGEm\nx/X+sdaa3GvNveYE0VbX97pyTTKz5l73WpP53r/7+zvFKIqiYMOGDRs2/qYR+2VPwIYNGzZsXD5s\nMrdhw4aNrwBsMrdhw4aNrwBsMrdhw4aNrwBsMrdhw4aNrwBsMrdhw4aNrwBsMrfxtcH999/PqFGj\nuPHGGy2PWb58Oddeey1Tp07l8OHDX+DsbNi4PNhkbuNrgx/96Efs3r3b8vW33nqLEydOcPz4cf7z\nP/+TBx988AucnQ0blwebzG18bfAP//APpKamWr7+5ptvUlhYCMAtt9xCS0sL586d+6KmZ8PGZSH+\ny56ADRt/LTh79ixjxozx/52VlUVdXR2jRo3yPxcTE/NlTM3G1wzRJObblrkNGwLMXyIZeSvCz9q8\nPBRFCflTtWsXq7KzKTG9XwHWSp5TgPtSU6XHrXa5wjqn+LN17VrudjpZC5QAVcCq7GwevvHGgPOa\n56ifd25qKiX5+dK5yuakX7N43KrsbKp27Qo4dm1ennTcvG98I+DYByZNokSbl34t4mcRyRytftau\nXRswluyzU4C7nU7pNek/lvPRrsH8PxQtbDK3YUPD6NGjOXPmjP/vuro6Ro8eHfQ9vQ4HAO6KCla7\nXJTNnMlqlwt3RYXhuMrycjbU1kq3wj0WY3dZPB/n8wWdkxnuigqqN25kp9dLGfAEsAdw1dZSX18f\ncHw+sMT03Cogadw44js7w56Tfs0iNtTWsnfzZsPcVrtcnKquZjXgNo2hxBvvmLuigpiTJ3kCDNfi\npv+zyF++nJLsbOP8s7OZ9dBD0rnr4xbn5FA4bBhzhw3jzRdewF1RYbheq/tS7PUarsmM/OXLWeJ0\nBrxvFtALNLa3W743Etgyiw0bGubMmcOWLVuYO3cuBw4cICUlxSCxmLHE4aDl+HHuys5mZH09zwmE\nVqKRWO7s2QB+UpARdyawCHhReG5xfDxdqanQ3BxwvE5aoJJQZXk58Z2d9AwaRP7y5f5z6qgsL+d5\nr9fw3AagFEhJT2fJpUuG13cCp7XX44D/A2Snp/PDdeuoLC+X3gtxTjpCEb+7ooI9K1YYCL9Ee8xF\nJeDxt9wS1rUUOJ0Ua2StX3/p5s3E+Xz0Ohzc/tBDAfdFh7uigjeKitjq8fifywW2FRURk5FheO5V\n+u9LL3C79vy+IAts7uzZ1Dz6KN9bt46piuJ/327Ao411JWCTuY2vDebNm0dVVRUXLlxgzJgxPP74\n43R3dwOwePFi/vmf/5m33nqLa665hsGDB/PLX/5SOk6hw8HVPh/3+HzknjxJAfCc6ZgNtbWUbt5M\n7uzZuCsqOFpTQxnql3cl8IxwbBVQjJEk7u3pYWdKCiXZ2X6ym4lKcLdrpPVsWRlVGzcyweulB9Vy\n3GNaRCAIqQItHg99I0bw/bo6pvT10Qj4gH8D9mrHxSYm8q3Fi/1jltTWGghYnJOInkGDpOfViV9q\nuQPzUlPZe/PN3P7QQ/QNHmx43epaMq+5xnDNubNnW5K3GZXl5TwjEDnAOuBVj4fzra3cHRtLbF8f\nSdprs4BKVPKsNF2TFZaWlfHBb39L45EjdAAbtfePAxpOncJdURH2fK1gk7mNrw1ef/31kMds2bIl\n5DHbTFbYBIvj4nw+v/W5s6nJ//xCoBDojokhW1HIRLXuck3v35eczG3r1kktTFE60VGCKp3s1RYR\nHVakWg083NREblMTbqA8MZGu7m7e1HRb/3y6uig9cEB9TrB6z9fV0eLxkHnVVX6LXTxv/vLlQYnf\nipivnzKFMosQUqtrScrM9P8ezm5FPK7uL38JeC0WlRz/YLq/R2Nj2awo/H+Ctr0kPp4pM2ZI5yVi\nxOjRPHHkCG5UaWiD/kJzMwuLitiZkcGIIUNCjmMFm8xt2LhMWGnevQ6H1Pp8GZiblkZucTENBw7Q\ncPCgpZxitjB1jfnEoUP82kI6MevXMlK9PzaWf+rr8xN2LpDb1UWhxbWIY+rz2bNiBS80NUFTE3z4\nYYC0pD8uWrOG9lOnGAQkCWQVynKXIdQCIZVuJLsV8bjVkvNUItltAaV9fXhQnZfxqJ/9PT097NUW\nu2DQ5x5TW9tP5Kh6f7rHwwZtd/B4yJEsoNiwYSNsAIpi+qkCZbHpuX/Lzlaqdu1S1ublBRyvgPq8\nhqpdu5RV2dnS94sQj1srGVMB5WFQ7h4+XFmbl6eU5Of7x6jatUtZ7XIpa/PylNUul/LApEnS9xdY\njLs0J0cpyc/3j7t02jTpcatdLss56z9LHA5l4bRpyta1a8O6bjPM1yIeX5Kfbzl/EeJxVaA8Yjr+\nviD3d4npuVWg3s8wULVrl3Jfaqrh/SWm8aKlZdsyt2EjQog6NqiOrCmoVvFnDgdJEydSsG4dubNn\nWzoMz7a3s9rl8ksBo+fPp/TAAb+ckjVjBpXl5ex7+mm/VCBa+eJuwE2/hnsceLSpidyqKnWuglUq\nWqarXS44ciRgXsmocoJoORYlJqLU1/PE++/7nyu0sJ7NuwLZzuQ5n4/Sw4c529bG6PnzWfTHP0ot\ndysE08OtpJuOjz4y6NLicfrupBhojI8nLjkZb08PSKJM6lEdxCI2AHPDTC7LnT2byunTobLS/9yV\nImGbzG3YiBCuTZso3byZjvp6Pv30U7r7+jibmEjSuHEs1Ehch1TiSE8nxUSOJbW1uDZt8uvhMqng\n0lVX9Y+LppFj0l8xRoS4amvZWljIvsmT/YsCQHNjI4UOB2N8PvK1YxclJtJ01VUsaGnxO2P/JzGR\nS3Fx/D8eD6u18+YCYyyiN8wSSTDn6/raWhb98Y+MbG3lRV1mam6mZMUKdf5ROAStpJurfT6DL0E/\nTlwIU4GuKVN48b33cFdUsLKoyOAYXQU0YpRY9PuRIUS9hIL5f8JKposUNpnbsBEhRM3497r1dvEi\nJcOGWR4rOjGd58/zjKmIlxj9YhWfXTB8eP+42uNW5JZiAVADnAXV+apZ6iurq2kFXhZI6kFgO7Cg\nq4s3hg1je04Oo5OTOdvezvj6egOh6QtFPvCgw2EIx5RFtVjq4tpj+6lT/UQuuRcyBHNw5i9fzoNu\nt3FeqKGAYvhg/vLlLKyuVrVqYeyV9fX9FvxLL/G9efMY0t5OF9ANXIMa226+H6IDNhycGzKEeamp\nJAKdqamsvHQpIKImYkQlztiw8TWF/pWx0mbNmrEMoXR0q9cfmDQpQGO+z+GQj6Xp+FWyOYZ4Tr8G\ny2vUHu8aP165e/hwpXDoUOXu4cOVrWvXBlyr1B8gzMusH8t8CqHGW2XS2RdOm6as1u7BauFc5s8m\nHN1f/CzM2rb+c7fTGVLn17F17VrlbqdTWauNV6XNf+vatX4/QLS0bFvmNmxEAUtttr7eoIWbw+L0\nmHMZdInCypodmZXFrIceMlj5SefPg6RU71lgFGrkTCX9cgDIk1TE5/RrkIXs6cfen57O1Zcu8YwQ\nclmyYwfu6dMD4r0BitesoeOjj7ja5/Mn2qzKzlY18hCJUSKsdi2iJX/f+vXsWbGC9SFi4a3CAD87\neNBvnYufhRVZmmPcrRA0nPTAAdZr4ZiPR1n/xyZzGzaigBXh1p84wc4PP/T/LTogdS28uKkpwMko\nkk2o8DtFUfyPk+bMYcnHHxuyIu8HUpDLAbn0SxwijqLqx7nCNchC9gA+TktjeEZGUKlIhO6wdFdU\nsHfzZvb5fOzVYuYBSkz+AaskJAidVaqfD0JngFrq683N7NF0e/GzsNK2w5VYrLJXi4EREZZokMEm\ncxs2ooCMcBc7nRSbv6xBtHDd
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 77
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"Training Curves",
|
||
|
|
"===============",
|
||
|
|
"",
|
||
|
|
"For actually training neural networks with stochastic gradient descent, it's useful to plot the error rate of predictions as the network is being trained."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"def err():",
|
||
|
|
" pred = zeros(len(data))",
|
||
|
|
" for i in range(len(data)):",
|
||
|
|
" z = forward(data[i])",
|
||
|
|
" pred[i] = argmax(z)",
|
||
|
|
" return sum(pred*1.0!=labels*1.0)/float(len(labels))"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 79
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"def training_curve(runs,updates,eta):",
|
||
|
|
" errs = []",
|
||
|
|
" for i in range(runs):",
|
||
|
|
" e = err()",
|
||
|
|
" errs.append(e)",
|
||
|
|
" update(updates,eta)",
|
||
|
|
" pylab.clf(); plot(range(len(errs)),errs); savefig(\"temp.png\")",
|
||
|
|
" return errs"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 80
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"init(10)",
|
||
|
|
"training_curve(50,2000,1.0)[-1]"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 81,
|
||
|
|
"text": [
|
||
|
|
"0.036999999999999998"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD9CAYAAABDaefJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X1YlFX+P/D3uFDmY+VTNoNhDsLgA1KYtrbbqBVlhZrV\nYvs1M7asXSv7bq3VfiuwtqStq0y2IsvWzJBdq8VKRzOZ0iwwn2gVjTJyJDXxKVQQGD6/P85vGIaH\nmxmYB7zn/bouL5m5D/c5HPEz5/6cc5/bICICIiLShU6hbgAREfkPgzoRkY4wqBMR6QiDOhGRjjCo\nExHpCIM6EZGOtBrUbTYb4uLiEBMTg8zMzGbL2O12JCYmYujQobBarf5uIxERecmgtU7d6XQiNjYW\na9euhdFoxMiRI5GTkwOLxVJf5tixYxgzZgxWr14Nk8mE8vJy9O7dOyiNJyIiT5oj9cLCQpjNZkRH\nRyMyMhKpqanIy8vzKPPuu+9iypQpMJlMAMCATkQUQhFaB8vKyhAVFVX/2mQyoaCgwKNMSUkJampq\nMHbsWFRUVOCBBx7AtGnTPMoYDAY/NpmIKHz4etO/5kjdm2BcU1ODLVu2YOXKlVi9ejWeeuoplJSU\nNNsw/hE8+eSTIW9DR/nDvmBfsC+0/7SF5kjdaDTC4XDUv3Y4HPVpFpeoqCj07t0b55xzDs455xz8\n9re/xfbt2xETE9OmBhERUdtpjtSTkpJQUlKC0tJSVFdXIzc3FykpKR5lJk6ciA0bNsDpdOLUqVMo\nKChAfHx8QBtNRETN0xypR0REICsrC8nJyXA6nUhLS4PFYkF2djYAYObMmYiLi8O1116L4cOHo1On\nTrjrrrsY1DVwyacb+8KNfeHGvmgfzSWNfqvEYGhzfoiIKFy1JXbyjlIiIh1hUCci0hEGdSIiHWFQ\nJyLSEQZ1IiIdYVAnItIRBnUiIh0JWlA/ejRYNRERha+gBfU33ghWTURE4StoQf3YsWDVREQUvoIW\n1I8fD1ZNREThiyN1IiIdYVAnItIRpl+IiHSEI3UiIh1hUCci0hGmX4iIdCRoQf3UKaC2Nli1ERGF\np6AF9R49gF9+CVZtREThKWhBvWdP5tWJiAItaEH93HOZVyciCrSgBnWO1ImIAovpFyIiHWH6hYhI\nR5h+ISLSEaZfiIh0hOkXIiIdYfqFiEhHWg3qNpsNcXFxiImJQWZmZpPjdrsdPXv2RGJiIhITE/H0\n0083ex6mX4iIAi9C66DT6cSsWbOwdu1aGI1GjBw5EikpKbBYLB7lrrzySqxYsUKzIqZfiIgCT3Ok\nXlhYCLPZjOjoaERGRiI1NRV5eXlNyolIqxUx/UJEFHiaI/WysjJERUXVvzaZTCgoKPAoYzAYsHHj\nRiQkJMBoNOL5559HfHx8k3MtXpyOH34A0tMBq9UKq9Xqlx+AiEgv7HY77HZ7u86hGdQNBkOrJ7jk\nkkvgcDjQpUsXrFq1CpMmTcK3337bpNwTT6TjnXdUUCcioqYaD3gzMjJ8Podm+sVoNMLhcNS/djgc\nMJlMHmW6d++OLl26AACuu+461NTU4MiRI03O1bOnyql7kakhIqI20gzqSUlJKCkpQWlpKaqrq5Gb\nm4uUlBSPMgcPHqzPqRcWFkJEcP755zc5V0QE0LkzcOKEH1tPREQeNNMvERERyMrKQnJyMpxOJ9LS\n0mCxWJCdnQ0AmDlzJpYvX45XX30VERER6NKlC5YtW9bi+VwrYLp39+8PQUREikG8WbrS3koMBogI\nhg4Fli0Dhg4NdI1ERGc+V+z0RdDuKAV4AxIRUaAFNajzBiQiosAKelDnSJ2IKHCYfiEi0hGmX4iI\ndITpFyIiHWH6hYhIR5h+ISLSEaZfiIh0hOkXIiIdYfqFiEhHmH4hItIRpl+IiHQkqEG9c2f1kIyq\nqmDWSkQUPoIa1A0G5tWJiAIpqEEdYAqGiCiQgh7UOVInIgqckAR1jtSJiAKD6RciIh3hSJ2ISEeY\nUyci0hGO1ImIdIQ5dSIiHWH6hYhIR5h+ISLSEaZfiIh0hOkXIiIdYfqFiEhHWg3qNpsNcXFxiImJ\nQWZmZovlNm3ahIiICLz//vua52P6hYgocDSDutPpxKxZs2Cz2bBz507k5OSguLi42XJz5szBtdde\nCxHRrLBbN6CyEqitbV/DiYioKc2gXlhYCLPZjOjoaERGRiI1NRV5eXlNyi1YsAA333wz+vTp03qF\nnYAePYBffml7o4mIqHkRWgfLysoQFRVV/9pkMqGgoKBJmby8PKxbtw6bNm2CwWBo9lzp6en1X599\nthXHjllx/vntaDkRkc7Y7XbY7fZ2nUMzqLcUoBuaPXs25s2bB4PBABFpMf3SMKjn5XEFDBFRY1ar\nFVartf51RkaGz+fQDOpGoxEOh6P+tcPhgMlk8iizefNmpKamAgDKy8uxatUqREZGIiUlpcXzcgUM\nEVFgaAb1pKQklJSUoLS0FBdeeCFyc3ORk5PjUWbPnj31X8+YMQM33nijZkAHuAKGiChQNIN6REQE\nsrKykJycDKfTibS0NFgsFmRnZwMAZs6c2aZKeQMSEVFgGKS1NYj+qOT/59tdZs8GoqPV30RE1LzG\nsdMbQb+jFGD6hYgoUEIS1Jl+ISIKjJAFdY7UiYj8j+kXIiIdYfqFiEhHmH4hItIRpl+IiHSE6Rci\nIh0Jyc1HtbVA585ATQ3gxZ5hRERh6Yy5+SgiQgX1EydCUTsRkX6FJKgDTMEQEQVCSIM6J0uJiPwr\nZEGdK2CIiPyP6RciIh1h+oWISEeYfiEi0hGO1ImIdIQ5dSIiHWH6hYhIR5h+ISLSEaZfiIh0hOkX\nIiIdYfqFiEhHmH4hItIRpl+IiHQkZEG9c2dABKiqClULiIj0J2RB3WBgCoaIyN9CFtQBpmCIiPyt\n1aBus9kQFxeHmJgYZGZmNjmel5eHhIQEJCYm4tJLL8W6deu8rpwrYIiI/EvzwdNOpxOxsbFYu3Yt\njEYjRo4ciZycHFgslvoyJ0+eRNeuXQEA33zzDSZPnozvvvvOs5IWHp56zTXAQw+pv4mIyJPfHzxd\nWFgIs9mM6OhoREZGIjU1FXl5eR5lXAEdAE6cOIHevXt7XTnTL0RE/hWhdbCsrAxRUVH1r00mEwoK\nCpqU+89//oNHH30U+/fvx5o1a5o9V3p6ev3XVqsVVquV6Rciogbsdjvsdnu7zqEZ1A0Gg1cnmTRp\nEiZNmoT169dj2rRp2L17d5MyDYO6C1e/EBG5uQa8LhkZGT6fQzP9YjQa4XA46l87HA6YTKYWy//m\nN79BbW0tDh8+7FXlHKkTEfmXZlBPSkpCSUkJSktLUV1djdzcXKSkpHiU+f777+sT+Vu2bAEA9OrV\ny6vKmVMnIvIvzfRLREQEsrKykJycDKfTibS0NFgsFmRnZwMAZs6ciffeew9vv/02IiMj0a1bNyxb\ntszrypl+ISLyL80ljX6rpIVlOR99BLz2mvqbiIg8+X1JY6Ax/UJE5F8hDepMvxAR+VfIg3pbRuq7\ndgHbtvm/PUREZ7qQ5tR/+QUwGoGKCt/ON3Wq2rr3rbf81EAiog7ojMupd+sGVFYCtbXef09VlZpY\nLS4OXLuIiM5UIQ3qnToBPXqoEbu31qwBBg5UQT3w1xhERGeWkAZ1ADjvPODgQe/LL18O3H03cPbZ\nwIEDgWsXEdGZKORBPTkZ+Pe/vStbXa1SLzfdBFgsTMEQETUW8qB+993AG28ATmfrZT/9FIiPBy68\nEIiLY1AnImos5EF9xAigf3/AZmu97PLlwJQp6muO1ImImgp5UAfUaP3117XL1NQAeXkM6kREWjpE\nUE9NBdavB8rKWi5jtwODBgEDBqjXFou6CYmIiNw6RFDv2lUF9jffbLnMe+8BN9/sfm0yqS0GuM0A\nEZFbhwjqgPaEqdMJfPCBO/UCqDXusbEcrRMRNdRhgrrWhOn69Wo7gYsv9nyfeXUiIk8dJqgDLU+Y\nLl/umXpxYV6diMhThwrqrgnT
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 81
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"init(10)",
|
||
|
|
"training_curve(50,2000,0.1)[-1]"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 82,
|
||
|
|
"text": [
|
||
|
|
"0.043999999999999997"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD9CAYAAABDaefJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XtYlGXeB/DvILN5yPUISDO8ooIczIBEXdcsTA1PIKYl\ntlkZb8u6q13Wvrtmu61SrUIHXYsuJd80Kxdoyw23FVLTedczrqlUWqJCjkgqIh5RYLjfP+4dzgxz\nPjzz/VwXVwzzMM/P5+r6zs3vvud+VEIIASIiUgQfVxdARET2w1AnIlIQhjoRkYIw1ImIFIShTkSk\nIAx1IiIF6TDUCwoKEB4ejtDQUGRkZLR6XqfToUePHoiJiUFMTAxeffVVhxRKREQd8zX1pMFgwPz5\n87F9+3ZoNBoMHz4ciYmJiIiIaHbcAw88gM2bNzu0UCIi6pjJkXphYSFCQkIQHBwMtVqN5ORk5OXl\ntTqOn18iInIPJkfqZWVlCAoKanis1Wpx4MCBZseoVCrs3bsXUVFR0Gg0eOONNxAZGdnqGCIispyl\ng2aTI3Vzwvjee++FXq/H0aNHsWDBAiQlJbVbGL8ElixZ4vIa3OWL14LXgtfC9Jc1TIa6RqOBXq9v\neKzX66HVapsd0717d3Tt2hUAMGnSJNTW1qKystKqYoiIyDYmQz02NhbFxcUoLS1FTU0NcnNzkZiY\n2OyY8+fPN7yjFBYWQgiB3r17O65iIiJql8meuq+vLzIzMxEfHw+DwYCUlBREREQgKysLAJCamopP\nPvkEq1evhq+vL7p27YqcnBynFO6p4uLiXF2C2+C1aMRr0YjXwjYqYW3jxpKTqFRW94eIiLyVNdnJ\nT5QSESkIQ52ISEEY6kRECsJQJyJSEIY6EZGCMNSJiBSEoU5EpCAMdSIiBXF5qFdVuboCIiLlcGmo\nV1YC/fsDJSWurIKISDlcGuorVgBXrwLnzrmyCiIi5XBZqF+6BKxeDURHAxcuuKoKIiJlMblLoyOt\nXAnMmCG/Z6gTEdmHS0bqxlH6iy8CAQEMdSIie3FJqBtH6cHBgL8/cP68K6ogIlIep7dfjKP0Q4fk\nY39/YNcuZ1dBRKRMTh+pNx2lAzLU2X4hIrIPp47UW47SAYY6EZE9OXWk3nKUDjDUiYjsyWn3KK2o\nEBg8WI7Sm4a6wQB07gxUVwO+LltgSUTkftz6HqVtjdIBoFMnoHdvoKLCWZUQESmX08bGLXvpTRlb\nMP36OasaIiJlctpIva1RuhH76kRE9uG0UE9La/85hjoRkX04LdQDA9t/jqFORGQfLr9JBsBQJyKy\nF4Y6EZGCMNSJiBTELUKd2+8SEdmHW4Q6t98lIrIPtwl1jtSJiGznFqHerRsgBHDjhqsrISLybG4R\n6ioVR+tERPbgFqEOMNSJiOyBoU5EpCAdhnpBQQHCw8MRGhqKjIyMdo87ePAgfH19sWnTJqsKYagT\nEdnOZKgbDAbMnz8fBQUFOHbsGLKzs3H8+PE2j1u0aBEmTpxo8YbuRgx1IiLbmQz1wsJChISEIDg4\nGGq1GsnJycjLy2t13Ntvv42ZM2fCz8/P6kIY6kREtjN5k4yysjIEBQU1PNZqtThw4ECrY/Ly8rBj\nxw4cPHgQKpWqzddaunRpw/dxcXGIi4tr9ry/f/s30SAi8gY6nQ46nc6m1zAZ6u0FdFMLFy5Eenp6\nw7302mu/NA31tnCkTkTeruWAN83UjSjaYTLUNRoN9Hp9w2O9Xg+tVtvsmEOHDiE5ORkAUFFRgfz8\nfKjVaiQmJlpUCEOdiMh2JkM9NjYWxcXFKC0txV133YXc3FxkZ2c3O+b06dMN38+dOxcJCQkWBzrA\nUCcisgeToe7r64vMzEzEx8fDYDAgJSUFERERyMrKAgCkpqbarRA/P6CiAqivB3zcZvU8EZFnUQlr\n1yBacpL/9Ns70qcPcOKE/C8RkbczNzubcqsxMbfgJSKyjduFOvvqRETWY6gTESkIQ52ISEEY6kRE\nCsJQJyJSEIY6EZGCMNSJiBSEoU5EpCAMdSIiBXGrUO/ZE7h5E7h929WVEBF5JrcKdZVKbux18aKr\nKyEi8kxuFeoAWzBERLZgqBMRKYjbhXpAAHdqJCKyltuFOkfqRETWY6gTESkIQ52ISEEY6kRECsJQ\nJyJSEIY6EZGCqISlt6q25iQW3BG7ulpuF3DrlvyEKRGRt7IkO43cbqTepQtwxx3A1auuroSIyPO4\nXagDbMEQEVmLoU5EpCAMdSIiBWGoExEpCEOdiEhBGOpERArilqHO7XeJiKzjlqHOkToRkXUY6kRE\nCsJQJyJSELfb+wUADAagc2e5D4yvrwMLIyJyY4rY+wUAOnUCevcGKipcXQkRkWdxy1AH2IIhIrJG\nh6FeUFCA8PBwhIaGIiMjo9XzeXl5iIqKQkxMDIYNG4YdO3bYpTCGOhGR5Uz21A0GA8LCwrB9+3Zo\nNBoMHz4c2dnZiIiIaDjmxo0b6NatGwDg66+/xvTp03Hy5MnmJ7GiLzR7NpCQADz2mEW/RkSkGHbv\nqRcWFiIkJATBwcFQq9VITk5GXl5es2OMgQ4A169fR9++fS0qoD0cqRMRWc7k2pKysjIEBQU1PNZq\ntThw4ECr4z777DMsXrwY5eXl2Lp1a5uvtXTp0obv4+LiEBcXZ7IwhjoReRudTgedTmfTa5gMdZWZ\n95NLSkpCUlISdu3ahTlz5uD7779vdUzTUDeHvz/QxvsHEZFitRzwpqWlWfwaJtsvGo0Ger2+4bFe\nr4dWq233+DFjxqCurg6XLl2yuJCWOFInIrKcyVCPjY1FcXExSktLUVNTg9zcXCQmJjY75tSpUw2N\n/K+++goA0KdPH5sL8/fnpl5ERJYy2X7x9fVFZmYm4uPjYTAYkJKSgoiICGRlZQEAUlNT8emnn+KD\nDz6AWq3GnXfeiZycHLsUFhYGHD8O3LolP11KREQdc8ttAox+/nMgLQ2YMMEBRRERuTnFbBNgNHEi\nUFDg6iqIiDyHW4f6pEkMdSIiS7h1qA8bJlfAnDnj6kqIiDyDW4e6jw8QH8/ROhGRudw61AHZV8/P\nN//4o0eBmzcdVw8RkTtz69UvAHDxIhAaKtswP/mJ6WOrq4G77gJiYoDPPwe6drXqlEREbkFxq18A\nwM9PhvrevR0f+/HHwMiRQFAQMHWqfUbstbXyTkxERJ7A7UMdMH8VzJo1wK9/DaxbZ59gP3UKGDRI\nfr36KnDuXMe/c/IksHw5sHgxcPmy9ecmIrKGR4S6OX31I0eAs2eByZPl7fBsDfZTp4AHHwT++Edg\n0yb52kOGANOny1qajt6NQX7vvcB998ljKyuByEhg/Xqgvt7y8xMRWcPte+oAUFcn94L55hvZM2/L\nvHlAYCDwpz81/sxgAJ5+GtDrLeuxGwP9D38AfvnLxp9fuwbk5ABZWbLXP20asHu3HMHPmAE88ggw\nZox8UwGAf/8b+M1v5ON33pG9fiIic1mTnR4R6gAwa5Ycsc+d2/q5a9eA//ovGfoaTfPnLA329gK9\npUOHgC1bZIg3DfKW6uvlaP3FF2Xov/IK0KuX6RqIiACFTpQaTZrUfgsmOxsYO7Z1oAPNWzHjxwOf\nftp+O8bcQAfkB6NeegmIi2s/0AG51j4lRW5OZjAAERHAvn2mX5uIyFoeM1IvL5c97QsXAN8me0sK\nIQM2PR146KH2f99gAN5/X7ZPDh6Uo/5HHpFvFl27WhbotvjoI9mK2bsXMPMeJETkpRQ9Ug8MBPr3\nb303pIMHgStX5CjclE6d5Ih52zaguFgG+Jo18nVnzXJOoAPyRtpXrwLt3PWPiMgmHjNSB+QyQV9f\n2Zc2evppuff6okXWvebFi8Df/w707Qs8/LDNJZrl44+BlSs5Wici0xQ9UQoA//oX8PzzclUJAFRV\nAcHBwIkTcnWMp6ivB4YOBVaskHvbEBG1RdHtFwAYNUquCTfeu/TDD2VP3JMCHZCTp0uWAEuXyjkB\nIiJ78ahQV6uBceOAL76QYbhm
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 82
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"Cross-Validated Error",
|
||
|
|
"=====================",
|
||
|
|
"",
|
||
|
|
"The problem we have been using has been quite easy: we have lots of training data and the classes are easily separated. Also, we didn't look at the performance on new data.",
|
||
|
|
"",
|
||
|
|
"Let us now change that. Let's change the training set to contain 100 samples and then measure the performance on the remaining 900 samples (samples that the classifier hasn't seen)."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"if len(data)>100:",
|
||
|
|
" test_data = data[100:]",
|
||
|
|
" test_labels = labels[100:]",
|
||
|
|
" data = data[:100]",
|
||
|
|
" labels = labels[:100]"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 83
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"Using a smaller number of training samples seems to result in better performance... we're getting an error rate of (close to) 0."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"init(10)",
|
||
|
|
"training_curve(50,2000,1.0)[-1]"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 84,
|
||
|
|
"text": [
|
||
|
|
"0.01"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAD9CAYAAABdoNd6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X9clFW+B/DPCFO2Wv4KWZxhG3UI0AApkGvF7pQoZTaJ\n273Rut7dloitZcs2t827PwLbWsnc1hXrxbrZZnWRzF3xZrDk4myGwfgDF0stVFhHEvNnKpjjDOf+\ncWRkgHmYgRlmYD7v14uXPPOcc+bMA36fwznnOUclhBAgIqJBbYi/K0BERL7HYE9EFAQY7ImIggCD\nPRFREGCwJyIKAgz2RERBoMdgX15ejpiYGERFRaGgoMBluu3btyM0NBTr1693vKbT6RAfH4/ExERM\nnTrVOzUmIiKPhSqdtNvtyM3NxebNm6HRaJCcnAyj0YjY2Ngu6X7xi1/grrvucnpdpVLBZDJh9OjR\n3q85ERG5TbFlbzabodfrodPpoFarkZmZidLS0i7pVqxYgfvvvx9hYWFdzvGZLSIi/1Ns2Tc1NSEy\nMtJxrNVqUVNT0yVNaWkpKisrsX37dqhUKsc5lUqFtLQ0hISEICcnB9nZ2U55O6YlIiL3edqQVmzZ\nuxOMFyxYgCVLlkClUkEI4VSBqqoq1NbWoqysDCtXrsTWrVu7rTC/BJ599lm/1yFQvngteC14LZS/\nekOxZa/RaGCxWBzHFosFWq3WKc3OnTuRmZkJADhx4gTKysqgVqthNBoREREBAAgLC0NGRgbMZjNS\nU1N7VVEiIuo9xZZ9UlIS6uvr0djYCKvVipKSEhiNRqc0hw4dQkNDAxoaGnD//ffj1VdfhdFoRGtr\nK86dOwcAaGlpQUVFBeLi4nz3SYiIyCXFln1oaCgKCwuRnp4Ou92OrKwsxMbGoqioCACQk5PjMm9z\nczPmzp0LALDZbJg3bx5mzpzpxaoPLgaDwd9VCBi8FlfwWlzBa9E3KtHbDiBvvPnlfn4iInJfb2In\nn6AlIgoCDPZEREGAwZ6IKAgw2BMRBQEGeyKiIMBgT0QUBBjsiYiCAIM9EVEQYLAnIgoCDPZEREGA\nwZ6IKAgw2BMRBQEGeyKiIOD3YH/+vL9rQEQ0+Pk92F/e5IqIiHzI78H+7Fl/14CIaPDrMdiXl5cj\nJiYGUVFRKCgocJlu+/btCA0Nxfr16z3Ke3nnQiIi8iHFYG+325Gbm4vy8nLs3bsXxcXF2LdvX7fp\nfvGLX+Cuu+7yOC9b9kREvqe4B63ZbIZer4dOpwMAZGZmorS0FLGxsU7pVqxYgfvvvx/bt2/3OG9z\ncx7y8uT3BoOB+0wSEXViMplgMpn6VIZisG9qakJkZKTjWKvVoqampkua0tJSVFZWYvv27VCpVG7n\nBQCb7UqwJyKirjo3hPPz8z0uQ7Ebpz1wK1mwYAGWLFni2AC3fRNcd/ICgBDAxYtuJSUiol5SbNlr\nNBpYLBbHscVigVardUqzc+dOZF6eP3nixAmUlZVBrVa7lRcArrtODtJefXWfPgcRESlQDPZJSUmo\nr69HY2Mjxo0bh5KSEhQXFzulOXTokOP7hx56CPfeey+MRiNsNluPeQHg2mvlIO3113vpExERUReK\nwT40NBSFhYVIT0+H3W5HVlYWYmNjUVRUBADIycnxOG9n113HGTlERL6mEu2d7P54c5UKt98u8Pzz\nwLe/7a9aEBENLO1jpJ7w+xO0bNkTEfkegz0RURAIiGDPJROIiHzL78G+fTYOERH5jt+DPbtxiIh8\nj8GeiCgIMNgTEQUBBnsioiAQEMGes3GIiHzL78Ges3GIiHzP78Ge3ThERL7HYE9EFAQY7ImIgoDf\nV7202QSuugq4dAkY4vdbDxFR4BuQq16GhADf+AbQ0uLvmhARDV49Bvvy8nLExMQgKioKBQUFXc6X\nlpYiISEBiYmJuOWWW1BZWek4p9PpEB8fj8TEREydOtXle3BGDhGRbyl249jtdkRHR2Pz5s3QaDRI\nTk5GcXGx045TLS0tGDZsGABgz549yMjIwIEDBwAA48ePx86dOzF69Oju3/zynyIxMcDf/gZ0s5EV\nERF14vVuHLPZDL1eD51OB7VajczMTJSWljqlaQ/0AHD+/Hlc32kzWXcqxEFaIiLfUtyDtqmpCZGR\nkY5jrVaLmpqaLuk2bNiARYsW4ejRo6ioqHC8rlKpkJaWhpCQEOTk5CA7O7tL3ry8PJw4AaxcCVy4\nYIDBYOjDxyEiGnxMJhNMJlOfylAM9iqVyq1C5syZgzlz5mDr1q2YP38+PvvsMwBAVVUVIiIicPz4\nccyYMQMxMTFITU11ypuXl4e6OmDOHIBxnoioK4PBuSGcn5/vcRmK3TgajQYWi8VxbLFYoNVqXaZP\nTU2FzWbDyZMnAQAREREAgLCwMGRkZMBsNnebj904RES+pRjsk5KSUF9fj8bGRlitVpSUlMBoNDql\nOXjwoKNffteuXQCAMWPGoLW1Fecur3DW0tKCiooKxMXFdfs+nI1DRORbit04oaGhKCwsRHp6Oux2\nO7KyshAbG4uioiIAQE5ODtavX481a9ZArVZj+PDhWLt2LQCgubkZc+fOBQDYbDbMmzcPM2fO7PZ9\n2LInIvItvz9BK4RAQQFw8iTw4ov+qgkR0cAxIJ+gBdiyJyLytYAJ9tzAhIjIdwIm2LNlT0TkOwER\n7Dkbh4jItwIi2LNlT0TkWwz2RERBgMGeiCgIBEyw52wcIiLfCYhgf/XVQFsbcPGiv2tCRDQ4BUSw\nV6nkjBy27omIfCMggj3AfnsiIl9isCciCgIBFezZjUNE5BsBFezZsici8o2ACfZcMoGIyHcCJtiz\nZU9E5Ds9Bvvy8nLExMQgKioKBQUFXc6XlpYiISEBiYmJuOWWW1BZWel23o4Y7ImIfEdxpyq73Y7o\n6Ghs3rwZGo0GycnJKC4uRmxsrCNNS0sLhg0bBgDYs2cPMjIycODAAbfydtxtZfFi4NIl4LnnfPVR\niYgGB6/vVGU2m6HX66HT6aBWq5GZmYnS0lKnNO2BHgDOnz+P66+/3u28HXE2DhGR7yhuON7U1ITI\nyEjHsVarRU1NTZd0GzZswKJFi3D06FFUVFR4lDcvLw8AUFsLWK0GAAbPPwUR0SBmMplgMpn6VIZi\nsFepVG4VMmfOHMyZMwdbt27F/PnzsX//frcr0B7s160DSkrczkZEFDQMBgMMBoPjOD8/3+MyFLtx\nNBoNLBaL49hisUCr1bpMn5qaCpvNhlOnTkGr1XqUlwO0RES+oxjsk5KSUF9fj8bGRlitVpSUlMBo\nNDqlOXjwoGOgYNeuXQCAMWPGuJW3IwZ7IiLfUezGCQ0NRWFhIdLT02G325GVlYXY2FgUFRUBAHJy\ncrB+/XqsWbMGarUaw4cPx9q1axXzusJgT0TkO4pTL33+5h2mD1kswK23yn+JiMg1r0+97E9s2RMR\n+U7AtOztduCqq+SDVUMC5hZERBR4BnTLPiQEuOYaoKXF3zUhIhp8AibYA+zKISLyFQZ7IqIgEHDB\nnuvjEBF5X8AFe7bsiYi8L6CCPXerIiLyjYAK9mzZExH5BoM9EVEQCLhgzwFaIiLvC7hgz5Y9EZH3\nMdgTEQWBgAr2nI1DROQbARXs2bInIvINBnsioiDQY7AvLy9HTEwMoqKiUFBQ0OX822+/jYSEBMTH\nx+O2225DXV2d45xOp0N8fDwSExMxderUHivD2ThERL6huC2h3W5Hbm4uNm/eDI1Gg+TkZBiNRqft\nBSdMmIAPP/wQI0aMQHl5OR555BFUV1cDkGsum0wmjB492q3KsGVPROQbii17s9kMvV4PnU4HtVqN\nzMxMlJaWOqWZNm0aRowYAQBISUnBkSNHnM57ssA+gz0RkW8otuybmpoQGRnpONZqtaipqXGZ/rXX\nXsOsWbMcxyqVCmlpaQgJCUFOTg6ys7O75MnLy3N8P22aAWfPGjyoPhHR4GcymWAymfpUhmKwV6lU\nbhe0ZcsWrF69GlVVVY7Xqqqq
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 84
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"If we plot the classes, however, we can already see that the sparse samples simply can't contain enough information about the decision boundary: in the white, empty regions, the classifier simply doesn't know where to \"draw\" the decision boundary."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"plotclass()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"err= 0.0"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD9CAYAAABHnDf0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X9sVFXeP/B3lxJK/BpEcUHafoPcVn6IFAwEWCMd13Qu\n0A1/rCZP3egjLBAev9JBn+wTl067Lbqwoskm7fDNF/yi5Lu6EnbXTWpmYCy4tk0UKCsG8ggxdVLW\n0ic1dheCj25rW8/3j8LQae/8vj/OPff9Shro9HbmcJn53HPP+ZzPKRBCCBARkVJ+4HQDiIjIfAzu\nREQKYnAnIlIQgzsRkYIY3ImIFMTgTkSkoJTB/ec//zlmz56NBx54IOkxgUAA5eXlqKiowCeffGJ6\nA4mIKHspg/vmzZsRjUaT/vzYsWP4/PPP0d3djddeew3PPPOM6Q0kIqLspQzuDz/8MGbOnJn05+++\n+y6efvppAMCqVatw7do1fPnll+a2kIiIslaYzy/39fWhtLQ0/n1JSQmuXLmC2bNnJxxXUFCQz8sQ\nEXlWrkUE8p5QnfjCyQK5EIJfQqCxsdHxNpj55fcHAYhJX7pe77lzwfeF9ecin/eb3V/hcAf8/iAq\nKxvh9wcRDndk/Rz5yKvnXlxcjN7e3vj3V65cQXFxcV4NIncJBPyIxYKIxfbEH9O0OtTWrnOwVaSq\noSHjkDU4OMXmlqQWiXRi5873Ej4XsVgQAFBdvdaWNuQV3Ddu3Ij9+/ejpqYGp0+fxh133DFpSIby\n1xmJoK2lBYVDQxiZNg3+QABrq6udbhaAW2/UUKgBg4NTUFQ0itradba9gclbpk0bMXy8qGjU5pak\n1tLSlhDYASAW24NQqEGO4P7EE0+go6MDAwMDKC0txe7duzE8PAwA2L59OzZs2IBjx46hrKwMt912\nGw4fPmxLo93M5/NldXxnJIL3du7Enlgs/ljwxt9lCvDj37CRSCd0vR5DQ4WYNm0EgYDf8A2d7blQ\nGc/FLanOhVvuFKW4wxA2sOlllBT0+4UAJn3V67rTTTMUDncITatLaK6m1YlwuMPpppEiwuEOoev1\norKyUeh6vZTvLb8/aPSxFbpen9Xz5BM78xqWIesVDg0ZPj5lcNDmlmRGhttRUtvEO0UZyXCHweAu\nuZFp0wwfHy0qsrklmZHidpTIYTLMRTG4S84fCCAYiyWMuddpGtbV1jrYquTcMuFFZDWn7zAY3CV3\nc9K0IRTClMFBjBYVYV1trTSTqRPJcDtKREDBjUF7a1+koCDvhHxyj0ikE6HQiXG3o1XSj5ESySif\n2MngTkQkqXxipyuHZWRe1EOJIpFOtLS0pc15JyJzuS64u2FRD42RYQk2kVe5biemtpaWhMAOAHti\nMZwIhRxqESWTPOf9hEMtIvIO1wV3ty3q8TLmvBM5x3XB3W2LeryMOe9EznFdcPcHAghqWsJjdZqG\nKkkX9XhZIOCHpgUTHhvLea9yqEXedrOgm8/XBF2vRyTS6XSTyEKum1B126IeL5NhCbaXpMpM4uS2\n9zDPnUgBRsFb04JobtZRXb0Wul6PtrZfT/o9XW9ANPqSnU2lLOQTO103LENEk6XLTOLktvcwuBMp\nIF3w5uS29zC4kytxcjBRuuDNyW3vcd2EKhEnBydLV42Tk9vewwlVch1ODhpjNU71eK5wGHkbJweN\nOb05BMmFY+7kOpwcJEqPwZ1ch5ODROlxzJ1ciePL5AXciYmISEFcoUpERAkY3ImIFMTgTkSkIAZ3\nIiIFcRETkYlS1VQnshODO0nNTcGSNW9IJgzuJC23BcvkNdUbpGyvW9l1wXdTx8IIgztJK9NgKcuH\nkDVvrGfXBd9tHQsjDO4krUyCpUwfQtlq3shy0TOTXXdHKtyF2Rbcm3w+jEybBn8gwM2sKSOZBEuZ\nPoTpaqrbSaaLnpnsujtS4S7MvuDe0QEACMZiAMAAT2llEixl+hDKtCGGTBc9M9l1dyTbXVgu0gb3\naDSK5557DqOjo9i6dSteeOGFhJ8PDAzgySefRH9/P0ZGRvCLX/wCmzZtSvp8e2IxNIRCSgb3zkgE\nbS0tKBwa4l2KCTIJlrJ9CGWpqS7TRc9Mdt0dyXQXlquUwX10dBQ7duzAyZMnUVxcjJUrV2Ljxo1Y\ntGhR/Jj9+/dj+fLl+M1vfoOBgQEsWLAATz75JAoLkz/1lMFB8/4FkuiMRPDezp3Yc+POBOBdihnS\nBUsVPoRWkO2iZxa77o5kugvLVcrg3tXVhbKyMsybNw8AUFNTg9bW1oTgfs899+DChQsAgOvXr+Ou\nu+5KGdgBYLSoKM9my6etpSUhsANq36XIQoUPoRVUvujZdXcky11YrlJG4b6+PpSWlsa/LykpwZkz\nZxKO2bZtG3784x9j7ty5+Prrr/GHP/zB8Lmabvz5/syZeNzny6fNUiocGjJ8XMW7FNm4/UNoBV70\n3Km9vR3t7e2mPFfK4F5QUJD2Cfbu3Ytly5ahvb0dsVgMVVVVOH/+PG6//fbEAysrMVpUhD21tUr2\nZEemTTN8XMW7FHIHXvTcx+fzwTeu87t79+6cnytlcC8uLkZvb2/8+97eXpSUlCQc89FHHyEYHEux\n0jQN9957Lz777DOsWLEi4bgmk65GsvIHAgjGYglDM3WahnW1tQ62ioi8KmVwX7FiBbq7u3H58mXM\nnTsXR48exZEjRxKOWbhwIU6ePImHHnoIX375JT777DPMnz/f0kbL6ObdSEMohCmDgxgtKsI6Re9S\niEh+abfZO378eDwVcsuWLdi1axcOHjwIANi+fTsGBgawefNmfPHFF/j++++xa9cu/OxnP0t8EW6z\nR0SUNe6hSiQ5FUsBkPXyiZ2sLUNkMVVLAZDcuBMTkcWSlwI44VCLyAvYcydDHEYwj6qlAEhuDO40\nCYcRzKVqKYBcseNgD88Edxb1ypyqFQWdonIpgGyx42AfTwR3FvXKDocRzKVqKYBceuDsONjHE8Gd\nRb2yw2EE86lWCiDXHjg7DvbxRLYMi3plJxDwQ9OCCY+NDSNUOdQikk2uGUDsONjHEz13FvXKjqrD\nCGSeXHvgnH+wjyeCO4t6ZU+1YQQyV649cHYc7OOZ8gOdkQhOjCvqVcWiXkQ5Mxpz17Q6NDczUJuJ\ntWUUwFRNcptIpBOh0IlxPfAqBnaTMbi7nGGqpqZBb27OOcDzYkHkfiwc5nJmp2oyr5+IPJEKKTuz\nUzWTXSxOhEI5PR8RuY+tPXfWlDBmdqom8/qJyLbgnmxF29mz/4lTp/7L0wHf7FRN5vUTkW3BPdmK\ntlde+Rf8859Hxz3mvSJCZu+/yrx+IrItuCdb0fbPfy5K+N6rRYTWVlebNtnJzbqJyLbgnmxFGzB5\nRRuLCOXPzIsFEbmPbdkyRsWopk/fDmByMSoWESIiyo9tPXejmhKrV1fgrbfeQyx2awiGRYSIiPLn\n+ApVLmEmIjLG8gNEpAyWzriF5QeISAksnWEe9twlxJW85FX1uo5ft7VNerxB1/FSNOpAi5zFnrtC\nuDs8eRlLZ5iHhcMkk+velEQqYOkM8zC4S4a7w5OX+QMBBDUt4bE6TUMVS2dkjcMykuHu8ORlLJ1h\nHk6oSoZ7UxLRTcxzVwwXdhERwOBORKQkpkISEdlM9pW0DO454CIjIm9zw0paBvcscZERESXbhL4h\nFJImuKfNc49Go1i4cCHKy8uxb98+w2Pa29uxfPlyLFmyBD6fz+w2SoWLjIjIDStpU/bcR0dHsWPH\nDpw8eRLFxcVYuXIlNm7ciEWLbm2Nd+3aNTz77LN47733UFJSgoGBAcsb7SQuMiIiN6ykTdlz7+rq\nQllZGebNm4epU6eipqYGra2tCce8/fbbeOyxx1BSUgIAmDVrlnWtlQAXGRGRG1bSpuy59/X1obS0\nNP59SUkJzpw5k3BMd3c3hoeH8cgjj+Drr7/Gzp078dRTT016rqampvjffT6fa4dvAgE/YrHgpEVG\n3D2KyDusWknb3t6O9vZ2E1qY
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 85
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"Let's now evaluate the error rate on previously unseen data."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"def cv_err():",
|
||
|
|
" pred = zeros(len(test_data))",
|
||
|
|
" for i in range(len(test_data)):",
|
||
|
|
" z = forward(test_data[i])",
|
||
|
|
" pred[i] = argmax(z)",
|
||
|
|
" return sum(pred*1.0!=test_labels*1.0)/float(len(test_labels))"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 86
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"cv_err()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 87,
|
||
|
|
"text": [
|
||
|
|
"0.066666666666666666"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 87
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"That's obviously much higher than the error rate on the training data. So, performance on new data can be much worse than performance on the training data.",
|
||
|
|
"",
|
||
|
|
"Let's look at this with training curves again."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"def cv_training_curve(runs,updates,eta):",
|
||
|
|
" errs = []",
|
||
|
|
" cv_errs = []",
|
||
|
|
" for i in range(runs):",
|
||
|
|
" e = err()",
|
||
|
|
" cv = cv_err()",
|
||
|
|
" errs.append(e)",
|
||
|
|
" cv_errs.append(cv)",
|
||
|
|
" update(updates,eta)",
|
||
|
|
" pylab.clf()",
|
||
|
|
" plot(range(len(errs)),errs)",
|
||
|
|
" plot(range(len(cv_errs)),cv_errs)",
|
||
|
|
" savefig(\"temp.png\")",
|
||
|
|
" return errs"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 88
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"init(10)",
|
||
|
|
"_=cv_training_curve(50,2000,1.0)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAD9CAYAAABdoNd6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlcVFX/B/DPCJhbgrKJDIqCChgCClmaRslSpiguiflY\njyGRpWVP/SqfFtFKRS03XMjUcudRStQUl3RcUAG3MHEhAR1JEhSQTWBmvr8/bowgMAvMMAN8368X\nr7hzz7n3Ozf8zp1zzj1HREQExhhjzVorQwfAGGNM/zjZM8ZYC8DJnjHGWgBO9owx1gJwsmeMsRaA\nkz1jjLUAapN9fHw8XFxc0KtXL0RGRtZZLjk5GaampoiNjVW+5ujoiH79+sHLywtPP/20biJmjDGm\nNVNVO+VyOaZPn47Dhw/D3t4ePj4+CAoKgqura41yn3zyCV566aVqr4tEIkgkEnTu3Fn3kTPGGNOY\nyjv7pKQkODs7w9HREWZmZggJCUFcXFyNcitWrMC4ceNgbW1dYx8/s8UYY4an8s4+KysLDg4Oym2x\nWIzExMQaZeLi4nDkyBEkJydDJBIp94lEIvj5+cHExATh4eEICwurVrdqWcYYY5rT9kZa5Z29Jsl4\n5syZWLBgAUQiEYioWgAJCQm4cOEC9u/fj5UrV+LEiRO1Bsw/hNmzZxs8BmP54WvB14Kvheqf+lB5\nZ29vbw+pVKrclkqlEIvF1cqcO3cOISEhAIDc3Fzs378fZmZmCAoKgp2dHQDA2toawcHBSEpKwpAh\nQ+oVKGOMsfpTeWfv7e2NtLQ0ZGZmory8HDExMQgKCqpWJj09HRkZGcjIyMC4ceOwevVqBAUFoaSk\nBIWFhQCA4uJiHDx4EO7u7vp7J4wxxuqk8s7e1NQUUVFRCAwMhFwuR2hoKFxdXREdHQ0ACA8Pr7Nu\ndnY2xowZAwCQyWSYNGkSAgICdBh68+Lr62voEIwGX4tH+Fo8wteiYURU3wYgXZxcJMK4/43DjvE7\nDBUCY4w1OZV9pNow+BO0WQ+yDB0CY4w1ewZP9vdK7xk6BMYYa/YMnuxzS3INHQJjjDV7Bk/2BQ8L\nIFfIDR0GY4w1awZP9hZtLJD3MM/QYTDGWLNm8GRv2c6Sm3IYY0zPDJ7srdpZcbJnjDE9M4pkf6+E\nR+Qwxpg+GTzZW7blZhzGGNM3gyd7bsZhjDH9M45kX8rJnjHG9Mngyd6yrSW32TPGmJ4ZPNlzMw5j\njOmfwZN9O3CyZ4wxfTN4sv/mc0ueDI0xxvTM4Mm+9B7f2TPGmL6pTfbx8fFwcXFBr169EBkZWWe5\n5ORkmJqaIjY2Vqu6xbmdeDI0xhjTM5XJXi6XY/r06YiPj0dqaiq2bduGK1eu1Fruk08+wUsvvaR1\n3YJ8E54MjTHG9EzlGrRJSUlwdnaGo6MjACAkJARxcXFwdXWtVm7FihUYN24ckpOTta57924EnpQA\nEbkRGDd8HK8zyRhjj5FIJJBIJA06hspkn5WVBQcHB+W2WCxGYmJijTJxcXE4cuQIkpOTIRKJNK4L\nAHJ5BHqPPYQQ/xA81+25Br0Zxhhrjnx9favdCM+ZM0frY6hsxqlM3KrMnDkTCxYsUC6AW7kIriZ1\nAaBjR8DCjCdDY4wxfVJ5Z29vbw+pVKrclkqlEIvF1cqcO3cOISEhAIDc3Fzs378fZmZmGtUFAAsL\noH0rngyNMcb0SWWy9/b2RlpaGjIzM9G1a1fExMRg27Zt1cqkp6crf58yZQpGjhyJoKAgyGQytXUB\nwNycH6xijDF9U5nsTU1NERUVhcDAQMjlcoSGhsLV1RXR0dEAgPDwcK3rPs7CAmijsMI9ngyNMcb0\nRkSVjeyGOLlIhNGjCV1HrEOpVQLWj1pvqFAYY6zJqOwj1YbBn6A1NwdMHnIzDmOM6ZPBk72FBSAq\nteL5cRhjTI+MItnLi3g0DmOM6ZNRJPuKAm7GYYwxfTJ4sjc3B0rzeDI0xhjTJ4MnewsLoLDABOZt\nzHkyNMYY0xOjSPb5+bw8IWOM6ZPBk725OSd7xhjTN4MnewsLoKBASPY8GRpjjOmHUST7/HzAsi0P\nv2SMMX0xeLLv2BF48ACwbMvNOIwxpi8GT/ampkDbtsCTJvwULWOM6YvBkz0gNOW0BTfjMMaYvhhN\nsn9Cxs04jDGmL0aR7M3NAdNybsZhjDF9UZvs4+Pj4eLigl69eiEyMrLG/ri4OHh4eMDLywsDBgzA\nkSNHlPscHR3Rr18/eHl54emnn67zHBYWQKsybsZhjDF9UblSlVwux/Tp03H48GHY29vDx8cHQUFB\n1Vac8vPzw6hRowAAly5dQnBwMP78808AwgT7EokEnTt3VhmEhQVAxdyMwxhj+qLyzj4pKQnOzs5w\ndHSEmZkZQkJCEBcXV61M+/btlb8XFRXBysqq2n5NVlMxNwcqHnTCg7IHPBkaY4zpgco7+6ysLDg4\nOCi3xWIxEhMTa5TbtWsXZs2ahTt37uDgwYPK10UiEfz8/GBiYoLw8HCEhYXVqBsREYFLl4ArV4C2\nvdoi72EerNpZ1SjHGGMtlUQigUQiadAxVCZ7kUik0UFGjx6N0aNH48SJE5g8eTKuXbsGAEhISICd\nnR1ycnLg7+8PFxcXDBkypFrdiIgItG8P3L0LZHXfhtySXE72jDFWha+vL3x9fZXbc+bM0foYKptx\n7O3tIZVKldtSqRRisbjO8kOGDIFMJsO9e8KoGjs7OwCAtbU1goODkZSUVGu9qjNf8vw4jDGmeyqT\nvbe3N9LS0pCZmYny8nLExMQgKCioWpkbN24o2+XPnz8PALC0tERJSQkKCwsBAMXFxTh48CDc3d1r\nPU/lzJc8Pw5jjOmHymYcU1NTREVFITAwEHK5HKGhoXB1dUV0dDQAIDw8HLGxsdi4cSPMzMzQoUMH\nbN++HQCQnZ2NMWPGAABkMhkmTZqEgICAWs9TOfOlmKc5ZowxvRCRJsNl9HVykQhEhKQkYPp0wPeb\nj2HVzgofD/7YUCExxpjRq8yd2jCaJ2i5GYcxxvTHKJJ91QVMONkzxpjuGUWyV97Zc7JnjDG9MIpk\n36YN0KoV0JHntGeMMb0wimQPCHf3ZjJus2eMMX0wmmRvYSFMc8zJnjHGdM+okr2ihCdDY4wxfTCa\nZG9uDhQWmKDjEx2R9zDP0OEwxlizYjTJnodfMsaY/hhVsufJ0BhjTD+MLtnzU7SMMaZ7RpPsKx+s\n4mYcxhjTPaNJ9lXb7PnBKsYY0y2jSvbcjMMYY/phNMmem3EYY0x/jCbZczMOY4zpj9pkHx8fDxcX\nF/Tq1QuRkZE19sfFxcHDwwNeXl4YMGAAjhw5onHdqpTNOO24GYcxxnSOVJDJZOTk5EQZGRlUXl5O\nHh4elJqaWq1MUVGR8veUlBRycnLSuG7V09++TdS1K9GVnCvUe0VvVWExxliLpiZ110rlnX1SUhKc\nnZ3h6OgIMzMzhISEIC4urlqZ9u3bK38vKiqClZWVxnWrqtpmzw9VMcaYbqlccDwrKwsODg7KbbFY\njMTExBrldu3ahVmzZuHOnTs4ePCgVnUjIiKUv5eV+aKDyRAUlBVArpDDpJWJ1m+IMcaaG4lEAolE\n0qBjqEz2IpFIo4OMHj0ao0ePxokTJzB58mRcvXpV4wCqJvuoKKCoEMrJ0KzaWWl8HMYYa658fX3h\n6+ur3J4zZ47Wx1DZjGNvbw+pVKrclkqlEIvFdZYfMmQIZDIZ7t+/D7FYrFVdgIdfMsaYvqhM9t7e\n3khLS0NmZibKy8sRExODoKCgamVu3LgBob8AOH/+PADA0tJSo7qPqzb8ktvtGWNMZ1Q245iamiIq\nKgqBgYGQy+UIDQ2Fq6sroqOjAQDh4eGIjY3Fxo0bYWZmhg4dOmD79u0q66rCT9Eyxph+iKjyttwQ\nJxeJUPX0Y8YAkyYBv5q+icEOgxHaP9RQoTHGmNF6PHdqwmieoAUem9Oen6JljDGdMbpkX1DAzTiM\nMaZrRpfseTQOY4zpnlEl+2pP
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 89
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"What you can see here is that initially, both the error rate on the training data and the cross-validated error rate decrase together. However past the first six rounds of training, the cross-validated error rate doesn't decrease any further, while the error rate on the training data keeps decreasing. We call this overtraining. In this case, it would be a good idea to take the weights of the network not at 100000 update steps, but at around 12000."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"subplot(121); plotgrid(forward)",
|
||
|
|
"subplot(122); plotclass()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"err= 0.0"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAD9CAYAAABOd5eOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXl0XPV9//26d/ZFo32xFku2bFm28QKxA2kao4ZjC+KG\n8jRpoC0JoZDQPAFD0vyAg3ECoQlO05MGTM+vaZOTx7/mKYXneXp+EEwNuBzbacIWMDZe8CJbthZr\n32akWe/c5487V3M1lmRts2m+r3PuYTQa3flqsN73c9/fzyKpqqoiEAgEgqxGTvcCBAKBQDB/hJgL\nBALBIkCIuUAgECwChJgLBALBIkCIuUAgECwChJgLBALBIkCIuSBn+Ku/+ivKy8tZt27dlK/ZsWMH\nK1euZMOGDRw5ciSFqxMI5ocQc0HOcPfdd7N///4pv//qq69y7tw5zp49yz//8z/zjW98I4WrEwjm\nhxBzQc7wmc98hsLCwim///LLL3PXXXcBcP311zM0NER3d3eqlicQzAtzuhcgEGQKHR0d1NTUjH9d\nXV1Ne3s75eXl489JkpSOpQlyjLkU5ovIXCAwkPhHNJl4q6qaluN73/teUs67bdtOQL3iaG5+PKnv\nm87feabve7XPZrbHK68cYtu2ndx44/fYtm0nr7xy6IrXzBURmQsEMaqqqmhraxv/ur29naqqqjSu\nKDXs2LGNlpadtLT8YPy5+vrHeOCBm9O4qswgGJxcIgMB06zPtW/fYR588LUJn3NLy04Atm/fMrcF\nGhBiLhDEuPXWW3nuuee44447ePvttykoKJhgsWQih/ft4/Vnn8UcDBKx2di2Ywdbtm+f1Tl0Idmz\nZxeBgAm7XeGBB25eEIHJdmy2yKTP2+3KrM/17LOvTxBygJaWH7Bnzy4h5gLBbPjzP/9zDh06RF9f\nHzU1NTz55JOEw2EA7rvvPj73uc/x6quvsmLFClwuF7/85S/TvOKJNDU1Tfj68L59vPbgg/ygpWX8\nuZ2xx3MR9O3bt7Bv32GeffZ1fvzjN3n22dfZsWPbFe+bStL13vr7LuRdy0JG+ZOiCgSCGZNJfzI7\nt21TVbjieLy5eU7ne+WVQ2p9/WMTTldf/5j6yiuHFnjl2cUrrxxSm5sfV2+88Xtqc/Pjc/48tm3b\nOdn/LrW5+fEJr5vrvzERmQsEWYo5GJz0eVMgMKfzJdsGyFb0u5b5kuy9CSHmAkGWErHZJn1esdvn\ndL6k2wA5TrL3JoSYCwRZyrYdO9jZ0jLBM3+svp6bH3hgTudbyM0+weQsVJQ/GULMBYIsRd/k3LVn\nD6ZAAMVu5+YHHpj15qeOSFHMbqSY4S4QCGaAJEnzKuzIdPbtO8yePW8YbICtOe2Xp4O5/hsTYi4Q\nzILFLuaC9DPXf2PCZhEIFpCFKOJJN3queTBoxmaLsGPHNhGdZwFCzAWCBWIhi3jSRbJLzgXJQzTa\nEggWiNeffXaCkAP8oKWFN/bsSdOKZs/UueZvpGlFgpkixFwgWCAWuognHYhc8+xFiLlAsEAsdBFP\nOhC55tmLEHOBYIHYtmMHO+vrJzz3WH09W+dYxJMOduzYRn39zgnPabnmW9O0opmxb99hmpsfp6np\nCZqbH2ffvsPpXlLKERugAsECsdBFPOkgW9rhGjNuRkZ6uXw5QFfXL8a/n4ubtiLPXCCYBSLPPP1M\nlnEDO4FmIC7ezc272L//qVQvb97M9d+YsFkEAkFWMVnGDfwAmJhxk2ubtkLMBQJBVjFVxg1MFO9c\n27QVnrlAIJgT6aoUnSrjBuLinYsNwoSYCwSCWZPOStHJujtWVHyLykofeXlPZOymbbIRG6ACwSwQ\nG6Aazc2P8/rrfzvJ86nZdFzM3R1Foy2BQJAy0l0pmswhD9mK2AAVCASzRlSKZh5CzAUCwazJ1krR\nxYzwzAWCWSA88ziL2bdOJ2LSkECQAoSYC5KNqAAVCASCHEaIuUAgECwChJgLBALBImDOYr5//34a\nGxtZuXIlP/rRjxZyTQKBQCCYJXPaAFUUhVWrVnHgwAGqqqrYvHkzzz//PKtXr07GGgWCjEFsgC4s\n6ervksmktAL03XffZcWKFdTV1QFwxx138NJLL00Qc0mS5nJqgWDGCFFNPQspvuns77IYmZOYd3R0\nUFNTM/51dXU177zzzhWvqwXqYo/rDI+TzUGgaZLnJTRfyQw4gQJgtQR1DVByO/z6T27h//V8gf/9\nf30J3z4FPuwETgB9gA8Io3Vmm6rKbap3TgXpeu9UvW9r7NA5lIL3FBhZaPGdrC95S8sP2LNnV0aJ\n+WwvYOm625iTmM806q4jfdI2GSqaDJvQhN0EOGVwukFaCh151bSMrSByxgL9I8AAMAaEYj+pxg5B\n6qljYjggxDzVTC6+zdx11z9yzTVvzlq40t3fZSbM9gKWzruNOYl5VVUVbW1t41+3tbVRXV29YItK\nBbqYO+zgKAapHjo9S7jQX0f4ohmGA4AXCAIR4hG5EHNBbnKl+B4GXqO//wUOxa6tsxGuhejvkuwo\neLZ3D+m825iTmG/atImzZ8/S2tpKZWUlL7zwAs8///wVr6ub7+rmyEzeVxdzkx3UQhPeGisDoWKG\n2ouJ9srgDwCjxK0VlatH5jN552SRrvdO1/umjyeamojYbGzbsSOrhjXPlyvF93W0cW1xZiNck/Ul\nn81QiVREwbO9e0jn3cacxNxsNvPcc8/R3NyMoijcc889k2ay1M13dXNkJu+ri7lkg1C+mb4yD8Ot\nBQRaXDAchXAQ8KNF5TONxmfyzskiXe+drvdNH0/EwtCdLS0AOSPoV4rv/IRLF9w9e3YZ+rvMfKhE\nKqLg2d49pLOb5Jz7md9yyy3ccsstC7mWlGEyHLITAi477VI1vj43dEZAGUGLyo1euSDb2b9/Pw89\n9BCKonDvvffyyCOPTPh+X18fd955J11dXUQiEb7zne/w1a9+dcrz/aClhV179iRFzA/v28frzz6L\nORjMmLuARPE9fvwU/f1Xvm42wjWfvuSpiIJne/cw37uN+ZCzwylkYpG5AwJOO51SFaODLuiJgOJF\ni8oVIIoQ8+xHURTuv//+CbURt95664Q7yueee45rr72Wp59+mr6+PlatWsWdd96J2Tz1n4kpEFjw\ntR7et4/XHnyQH8Qif8icuwCj+Go2R3qEC1ITBc/27mG+dxvzIWfFXEL75SWHFplfZgmjwy7oj4Iy\nhrbxKRrtLxZmUhuxZMkSjh07BsDIyAjFxcXTCjmAYrcv+Fpff/bZCUIOyb0LmCvpFC5IXRQ827uH\ndE1BylkxlwELIDsg5LTSJ5XgH3XASBSiYTSvXI/Kowi7JbuZSW3E1772NT772c9SWVmJ1+vlxRdf\nnPRcT8T++1+FhXyxqWnB12oOBid9Phl3AfMlnePb0n0xWSgOHjzIwYMH532enBVzPTKXHRB2WhiU\niggE7DCqgmrMYAGRX579zKQ24oc//CEbN27k4MGDtLS0sHXrVo4ePUpeXt7EF954I4rdzg8eeCAp\nkXLEZpv0+WTcBWQ7i2EWaFNTE02GoODJJ5+c03lyWsz1bJaw3cKQlE8wZIOAUcwFi4WZ1Eb87ne/\nY+dOLbWtvr6eZcuWcfr0aTZt2jThdU8sQBQ1Hdt27GBnS8sEq+Wx+npufuCBpL6vILsRYm4FxWZi\nTHISCZu1BBYRhC86ZlIb0djYyIEDB/j0pz9Nd3c3p0+fZvny5Slfqx7t79qzB1MggGK3c3OS7gIE\ni4ecFHNdyM2AbIGoVSaIDSVqmiIgl2KHUPlsZaraiJ/97GcA3HfffTz22GPcfffdbNiwgWg0yt/9\n3d9RVFSUlvVu2b5diLdgVuSkmOtpiVY0MVetEopkJjple3f9J8QmaDYzWW3EfffdN/64pKSEX//6\n16leliCGaIc7P3JKzPWuiXqcbQKkWPWQCvFwHRkxhEkgSB2iHe78ySnF0sVcP0yAJIMkq5iIIplV\nsEgxhddlXyAQJJupS/PfSNOK
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 90
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"source": [
|
||
|
|
"Cross Validation in the Presence of Errors",
|
||
|
|
"==========================================",
|
||
|
|
"",
|
||
|
|
"Note that we have a perfectly separable case here. That is, the error rate of the classifier on the training data actually goes to zero. In this case, the cross-validated error rate simply stops changing after a while. When the training data is noisier and there are a lot of hidden units, the cross-validated error rate can actually start increasing again.",
|
||
|
|
"",
|
||
|
|
"Let's introduce some errors and see what happens (there is no guarantee that anything interesting happens)."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"labels = abs(labels-(random(len(labels))>0.9))",
|
||
|
|
"test_labels = abs(test_labels-(random(len(test_labels))>0.9))"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 96
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"plotlabels()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"41 59"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD9CAYAAABHnDf0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3X9slFX+L/B3lxJK/BpEcUHa3iBPKwWRgoEAa2TGNZ2n\n0A1/rCZbN/oVFgjXKx3xZr9x6bTbIgsrbvLNbYdvLnpRcldXwu66Sc0MjAXXtokCZcVArnC9dQJr\n6abG7kJw9dta6rl/AEOHzu95nmfOOc/7lTTS6dN5js9MP3Oecz7nc4qEEAJERKSV7xW6AUREZD0G\ndyIiDTG4ExFpiMGdiEhDDO5ERBpicCci0lDK4P6zn/0MM2fOxAMPPJD0GL/fj8rKSlRXV+Pjjz+2\nvIFERJS9lMF9/fr1iEQiSX9+6NAhfPbZZ+jr68Orr76KZ555xvIGEhFR9lIG94cffhjTp09P+vN3\n3nkHTz/9NABg+fLluHz5Mr744gtrW0hERFkrzueXBwYGUF5eHvu+rKwMFy9exMyZM+OOKyoqyuc0\nRESulWsRgbwnVG89cbJALoTglxBoaWkpeBus/PL5AgDEhC/TbHLdteD7wv5rkc/7zemvUKgbPl8A\nHk8LfL4AQqHurJ8jH3n13EtLS9Hf3x/7/uLFiygtLc2rQaQWv9+HaDSAaHRn7DHDaERDQ20BW0W6\nGhlJHLKGhyc53JLUwuEePPfcu3F/F9FoAABQV7fKkTbkFdzXrl2LPXv2oL6+HsePH8cdd9wxYUiG\n8tcTDqOzvR3FIyO4OmUKfH4/VtXVFbpZAG6+UYPBZgwPT0JJyRgaGmodewOTu0yZcjXh4yUlYw63\nJLX29s64wA4A0ehOBIPNcgT3J554At3d3RgaGkJ5eTm2b9+O0dFRAMDmzZuxZs0aHDp0CBUVFbjt\nttuwf/9+RxqtMq/Xm9XxPeEw3n3uOeyMRmOPBa7/W6YAP/4N2xMOo8k0034YZXstdMZrcVOqa6HK\nnaIUdxjCAQ6dRksBn08IYMJXk2kWumkJdYdCotEw4traaBiiOxQqdNNIE6FQtzDNJuHxtAjTbBKh\nUHehmzSBzxdI9GcrTLMpq+fJJ3bmNSxD9iseGUn4+KThYYdbkpnO9va4uwwA2BmNojkYlOZOg9R2\n652ijGS4w2Bwl9zVKVMSPj5WUuJwSzKj2ocRkR1kmIticJecz+9HIBqN6w03GgZqGxoK2KrkVPsw\nIrJLoe8wGNwld2MoozkYxKThYYyVlKC2oUHaIQ7VPoyIdFV0fdDe3pMUFeWdkE/q6AmHcWTch1GN\nxB9GRDLLJ3YyuBMRSSqf2KnksEw43IP29k6MjBRjypSr8Pt90s+euxVfK6LCUC64y7CslzLD14qo\ncJTbiSn5st4jBWoRJcPXiqhwlAvuUizrpYzwtSIqHOWCuyqFg4ivFVEhKRfc/X4fDCMQ99i1Zb01\nBWoRJcPXSi7hcA9MswlebytMswnhcE+hm0Q2Um5CVYZlvZQZvlbOSpWZxMlt92GeO5EGEgVvwwig\nrc1EXd0qmGYTOjt/NeH3TLMZkcgOJ5tKWcgndio3LENEE6XLTOLktvswuBNpIF3w5uS2+zC4k5I4\nORgvXfDm5Lb7KDehSsTJwYnSbQ7ByW334YQqKYeTg4mFwz0IBo+MC941DN6Kc13hMHI3Tg4mVujN\nIUguHHMn5XBykCg9BndSDicHidLjmDspiePL5AbciYmISENcoUpERHEY3ImINMTgTkSkIQZ3IiIN\ncRETkYVS1VQnchKDO0lNpWDJmjckEwZ3kpZqwTJ5TfVmKdurqp5wGJ3t7SgeGcHVKVPg8/uxqq5O\n2fPYhcGdpJVpsJSld8+aN/brCYfx7nPPYWc0GnsscP3fVgZep85jJwZ3klYmwVKm3r1sNW9k+dCz\nUmd7e1zABYCd0Siag0FLg65T57GTY8G91etV8taGCieTYCnTUEi6mupOkulDz0rFIyMJH580PKzk\neezkXHDv7gag3q0NFU4mwVKmoRCZNsSQ6UPPSlenTEn4+FhJiZLnsVPa4B6JRLB161aMjY1h48aN\neOGFF+J+PjQ0hCeffBKDg4O4evUqfv7zn2PdunVJn0+1W5tsqD4BI5tMgqVsQyGy1FSX6UPPSj6/\nH4FoNG7IpNEwUNvQoOR57JQyuI+NjWHLli04evQoSktLsWzZMqxduxbz58+PHbNnzx4sWbIEv/71\nrzE0NIR58+bhySefRHFx8qdW6dYmUzpMwMgoXbCUaShEJrJ96Fnlxt9SczCIScPDGCspQW1Dg+V/\nY06dx04pg3tvby8qKiowZ84cAEB9fT06Ojrigvs999yDM2fOAACuXLmCu+66K2VgB9S6tcmUDhMw\nKpJpKEQmOn/oraqrc+Rvyqnz2CVlFB4YGEB5eXns+7KyMpw4cSLumE2bNuGHP/whZs+eja+++gq/\n//3vEz5X6/X/vjd9Oh73evNps5R0mIBRlSxDITLhh56aurq60NXVZclzpQzuRUVFaZ9g165dWLx4\nMbq6uhCNRlFTU4PTp0/j9ttvjz/Q48FYSQl2KnZrkykdJmBIL/zQU4/X64V3XOd3+/btOT9XyuBe\nWlqK/v7+2Pf9/f0oKyuLO+bDDz9EIHAtxcowDNx777349NNPsXTp0rjjWi36NJKVDhMwRKSPlMF9\n6dKl6Ovrw4ULFzB79mwcPHgQBw4ciDumqqoKR48exUMPPYQvvvgCn376KebOnWtro2WkwwQMEekj\n7TZ7hw8fjqVCbtiwAdu2bcMrr7wCANi8eTOGhoawfv16fP755/juu++wbds2/PSnP40/CbfZIyLK\nGvdQJZKcjqUAyH75xE7WliGyma6lAEhu3ImJyGbJSwEcKVCLyA3Yc6eEOIxgHV1LAZDcGNxpAg4j\nWEvXUgC5Yg0mZ7gmuLMnmjldKwoWis6lALLFGkzOcUVwZ080OxxGsJaupQBy6YGzBpNzXBHc2RPN\nDocRrKdbKYBce+CsweQcV2TLsCeaHb/fB8MIxD12bRihpkAtItkk64EfCQZT/h5rMDnHFT139kSz\no+swAlkn1x44azA5xxXBnRNa2dNtGIGslWsPnDWYnOOa8gPhcA+CwSPjeqI1DF5EOUo05t5oGKht\na2OgthBry2iAub+kmp5wGEfG9cBr2AO3HIO74hJmHhgGzDx6QfywIFIfC4cpzurcXy4UISJXpELK\nzurc31zT1IhIH4723DlUkJjVub9cKEJEjgX3ZEMF/+fkSfzt2DFXB3yrc3+5UISIHAvuyYYKfvLy\nyzj4n/8Ze8yNY8NW5/5yoQgRORbckw0VzB8X2AH3FhFaVVdn2f8zF4oQkWPBPelQQYLHODacPys/\nLIhIPY5ly/j8fgQMI+6xzVOnIlEpKo4NExHlx7Gee6KhguoVK/Dum29iFceGiYgsVfAVqlzCTESU\nGMsPEJE2uB7mJpYfICItsHSGddhzlxA38ya3ajJN/Kqzc8LjzaaJHZFIAVpUWOy5a4SbeZObsXSG\ndVg4TDLJN/M+UqAWETmHpTOsw+AuGW7mTW6WaD1Mo2GghunRWeOwjGS4mTe5GUtnWIcTqpJJNOZu\nGI1oa6vlmDuRyzDPXTPczJuIAAZ3IiItMRWSiMhhsq+kZXDPgewvKhHZS4WVtAzuWVLhRSUieyXb\nWU6mjYbS5rlHIhFUVVWhsrISu3fvTnhMV1cXlixZgoULF8Lr9VrdRqkke1GPBIMFahEROU2FlbQp\ne+5jY2PYsmULjh49itLSUixbtgxr167F/PnzY8dcvnwZzz77LN59912UlZVhaGjI9kYXkgovKhHZ\nS4WVtCl77r29vaioqMCcOXMwefJk1NfXo6OjI+6Yt956C4899hjKysoAADNmzLCvtRJQ4UUlInup\nsJI2Zc99YGAA5eXlse/Lyspw4sSJuGP6+vowOjqKRx55BF999RWee+45PPXUUxOeq7W1NfZvr9er\n7PCNz+9HIBqNG5rh7lFE7mLX
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 97
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"init(10)",
|
||
|
|
"_=cv_training_curve(50,2000,1.0)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXsAAAD9CAYAAABdoNd6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlcVFX/B/DPKKRp5gKCCCgGKGgIJKBmGq64Ra6JmZWR\nD09P9stWqydTe57cTQ3KLTOXVNwhFFIE0lABV1BcSFFHBAVcWAVm5vv74zxsMswCM87AfN+vly+Z\ne8+598ydme8999xzzpUQEYExxlij1sTQBWCMMaZ/HOwZY8wEcLBnjDETwMGeMcZMAAd7xhgzARzs\nGWPMBKgN9lFRUXBxcYGzszMWLVpUY31cXBxat24NT09PeHp64r///a/GeRljjD0ZZqpWyuVyzJgx\nA9HR0bC1tYW3tzf8/f3h6upaLd3LL7+M8PDwOuVljDGmfypr9omJiXBycoKDgwPMzc0REBCAsLCw\nGumUjcvSNC9jjDH9U1mzz8jIgL29fcVrOzs7JCQkVEsjkUhw7NgxuLu7w9bWFkuXLkX37t01zssY\nY0x72k5+oLJmr0kwfuGFFyCVSnHu3Dl88MEHGDNmjFYFICL+R4Q5c+YYvAzG8o+PBR8LPhaq/9WF\nymBva2sLqVRa8VoqlcLOzq5amlatWqFFixYAgBEjRqCsrAz37t2DnZ2d2ryMMcaeDJXB3svLC2lp\nabh+/TpKS0sRGhoKf3//amnu3LlTcaZJTEwEEaFdu3Ya5WWMMfZkqGyzNzMzQ0hICPz8/CCXyxEY\nGAhXV1esWbMGABAUFIRdu3Zh1apVMDMzQ4sWLbB9+3aVeZlyvr6+hi6C0eBjUYmPRSU+FvUjobo2\nAOli5xJJndufGGPMVNUldvIIWsYYMwEc7BljzARwsGeMMRPAwZ4xxkwAB3vGGDMBHOwZY8wEcLBn\njDETwMGeMcZMAAd7xhgzAQYP9idunTB0ERhjrNEzeLCfEzfH0EVgjLFGz+DBPrco19BFYIyxRs/w\nwb6Ygz1jjOmb4YM91+wZY0zvDB7si2XFKJWXGroYjDHWqBk82Ldt3hb3iu8ZuhiMMdaoGTzYW7Sw\n4GDPGGN6Zvhg/7QFt9szxpieGT7Yt7DgHjmMMaZnhg/2XLNnjDG9M3yw55o9Y4zpneGD/dMc7Blj\nTN+MI9hzMw5jjOmV2mAfFRUFFxcXODs7Y9GiRbWmS0pKgpmZGXbv3l2xzMHBAT179oSnpyd8fHyU\n5mv3dDuu2TPGmJ6ZqVopl8sxY8YMREdHw9bWFt7e3vD394erq2uNdLNmzcLw4cOrLZdIJIiLi0O7\ndu1q3YdFC67ZM8aYvqkM9omJiXBycoKDgwMAICAgAGFhYTWCfXBwMCZMmICkpKQa2yAilQXY8eMO\nXEq9hLk35sLX1xe+vr7avQPGGGvk4uLiEBcXV69tqAz2GRkZsLe3r3htZ2eHhISEGmnCwsIQExOD\npKQkSCSSinUSiQRDhgxB06ZNERQUhOnTp9fYx9fffI29a/di7idz6/VGGGOssXq8Ijxv3jytt6Ey\n2FcN3LWZOXMmFi5cCIlEAiKqVpOPj4+HjY0NsrOzMXToULi4uKB///7V8pffoCUijfbHGGNMeyqD\nva2tLaRSacVrqVQKOzu7amlOnTqFgIAAAEBOTg4iIyNhbm4Of39/2NjYAADat2+PsWPHIjExsUaw\nb2bWDE81fQoFpQVo1ayVTt4UY4yx6lT2xvHy8kJaWhquX7+O0tJShIaGwt/fv1qaa9euIT09Henp\n6ZgwYQJWrVoFf39/FBUVIT8/HwBQWFiIgwcPws3NTel+eGAVY4zpl8qavZmZGUJCQuDn5we5XI7A\nwEC4urpizZo1AICgoKBa82ZlZWHcuHEAAJlMhilTpmDYsGFK05Y35Ti0cajj22CMMaaKhNR1l9Hn\nzv/Xzj9081B89uJnGOao/GTAGGOsUnns1IbBR9ACPIqWMcb0zSiCPY+iZYwx/TKKYM+jaBljTL+M\nI9jzzJeMMaZXHOwZY8wEGEew54eOM8aYXhlHsOfeOIwxplfGEex5BC1jjOmVcQR7rtkzxpheGUWw\nb928NQrLCiFTyAxdFMYYa5SMItg3kTRBm+Zt+CYtY4zpiVEEe4CbchhjTJ+MJtjzlAmMMaY/RhPs\necoExhjTH+MJ9jyKljHG9MZ4gj2PomWMMb0xnmDPNXvGGNMb4wr23GbPGGN6YTzBnqdMYIwxvTF4\nsP/tN/E/1+wZY0x/DB7sf/5Z/M81e8YY0x+DB/vMTPE/1+wZY0x/1Ab7qKgouLi4wNnZGYsWLao1\nXVJSEszMzLB7926t8mZlif/LR9ASkZZvgTHGmDoqg71cLseMGTMQFRWF1NRUbNu2DRcvXlSabtas\nWRg+fLjWeR89AoqLgafNn0ZTSVMUlhXq4G0xxhirSmWwT0xMhJOTExwcHGBubo6AgACEhYXVSBcc\nHIwJEyagffv2Wuft0KGyds9TJjDGmH6YqVqZkZEBe3v7itd2dnZISEiokSYsLAwxMTFISkqCRCLR\nOC8AyOVz8d//Avb2QLO8ZsgtzkXnNp3r9aYYY6wxiYuLQ1xcXL22oTLYlwduVWbOnImFCxdCIpGA\niCra3DXJCwBeXnMxahQwbhxwdNNRnjKBMcYe4+vrC19f34rX8+bN03obKoO9ra0tpFJpxWupVAo7\nO7tqaU6dOoWAgAAAQE5ODiIjI2Fubq5RXgCwsanSjMM9chhjTC9UBnsvLy+kpaXh+vXr6NixI0JD\nQ7Ft27Zqaa5du1bx97Rp0/DKK6/A398fMplMbV5AtNlXdL/kvvaMMaYXKoO9mZkZQkJC4OfnB7lc\njsDAQLi6umLNmjUAgKCgIK3zPs7GBkhMFH9zzZ4xxvRDQgbs2C6RSBAeTlizBoiIAJYfX44bD29g\nxfAVhioSY4wZvfJ7pNow+AjaGl0vuRmHMcZ0zuDB3saGp0xgjDF9M3iwt7ICsrMBhYIfOs4YY/pi\n8GD/1FNA69ZATg6PoGWMMX0xeLAHKtvt+dGEjDGmH0YR7MsHVrVp3gb5JfmQKWSGLhJjjDUqRhHs\nywdWNW3SFK2bt8aDRw8MXSTGGGtUjCbY85QJjDGmP0YR7Kt1v+S+9owxpnNGEey5Zs8YY/plFMGe\na/aMMaZfRhHsuWbPGGP6ZRTBvuqc9jyKljHGdM8ogv2zzwKlpUBREQ+sYowxfTCKYC+RVBlFy1Mm\nMMaYzhlFsAcqb9JyzZ4xxnTPaIJ91Zo9P3ScMcZ0y2iCfbWaPTfjMMaYThlNsK/WZs/NOIwxplNG\nE+zLa/YtzFsAAIrKigxcIsYYazyMJtjzwCrGGNMf4wz23JTDGGM6ZTTBnh88zhhj+qM22EdFRcHF\nxQXOzs5YtGhRjfVhYWFwd3eHp6cnevXqhZiYmIp1Dg4O6NmzJzw9PeHj46NyP1ZW4jm0cjlPmcAY\nY7pmpmqlXC7HjBkzEB0dDVtbW3h7e8Pf3x+urq4VaYYMGYJXX30VAJCSkoKxY8fi77//BgBIJBLE\nxcWhXbt2agtibg60acMPHmeMMX1QWbNPTEyEk5MTHBwcYG5ujoCAAISFhVVL07Jly4q/CwoKYGlp\nWW09EWlcGH7wOGOM6YfKmn1GRgbs7e0rXtvZ2SEhIaFGun379uHLL79EZmYmDh48WLFcIpFgyJAh\naNq0KYKCgjB9+vQaeefOnVvxd7NmvsjM9IXFsxa4lX+rLu+HMcYanbi4OMTFxdVrGyqDvUQi0Wgj\nY8aMwZgxY3D06FFMnToVly9fBgDEx8fDxsYG2dnZGDp0KFxcXNC/f/9qeasG+/T0/9XsO1gg+W6y\nlm+FMcYaJ19fX/j6+la8njdvntbbUNmMY2trC6lUWvFaKpXCzs6u1vT9+/eHTCZDbq5ogrGxsQEA\ntG/fHmPHjkViYqLKwlRrxuE2e8YY0xmVwd7LywtpaWm4fv06SktLERoaCn9//2pprl69WtEuf/r0\naQCAhYUFioqKkJ+fDwAoLCzE
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 98
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"subplot(121); plotgrid(forward)",
|
||
|
|
"subplot(122); plotclass()"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"err= 0.27"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAXMAAAD9CAYAAABOd5eOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvXmUHOV5//up3rfp2RfNIo002lcEEqsN4/gnDVgx4caO\nTRxsTMAmuQaBHf+Ag5ADJhgR5zhGIucXJ+b66MYJgT9yLhgRIRQiyTGWEJLQjpaRZjT7vvfeXfeP\n6pqu7ukZzdbb9Ps5p8701PRUvd0z/a2nvu/zPK8ky7KMQCAQCDIaXaoHIBAIBIKZI8RcIBAI5gBC\nzAUCgWAOIMRcIBAI5gBCzAUCgWAOIMRcIBAI5gBCzAVZw5//+Z9TWlrKmjVrxn3O1q1bWbJkCevW\nrePEiRNJHJ1AMDOEmAuyhoceeoi9e/eO+/P33nuPy5cvc+nSJf7pn/6Jv/zLv0zi6ASCmSHEXJA1\nfP7znyc/P3/cn7/zzjs8+OCDANxyyy309/fT0dGRrOEJBDPCkOoBCATpQktLC1VVVaPfV1ZW0tzc\nTGlp6eg+SZJSMTRBljGdwnwRmQsEGmI/RPHEW5bllGx//dd/nZDjbt68DZDHbHV1zyX0vKl8zZM9\n7/Xem6lu7757kM2bt3HXXX/N5s3bePfdg2OeM11EZC4QhKmoqKCpqWn0++bmZioqKlI4ouSwdetm\n6uu3UV//0ui+mppnefzxu1M4qvTA640vkR6PfsrH2rPnEE888X7U+1xfvw2ALVvunN4ANQgxFwjC\n3Hvvvbz22mvcf//9HD58mLy8vCiLJR05tGcP+3buxOD1EjCb2bx1K3du2TKlY6hCsmvXdjwePRZL\nkMcfv3tWBCbTMZsDcfdbLMEpH2vnzn1RQg5QX/8Su3ZtF2IuEEyFP/3TP+XgwYN0d3dTVVXFCy+8\ngN/vB+DRRx/lS1/6Eu+99x6LFy/Gbrfzq1/9KsUjjqa2tjbq+0N79vD+E0/wUn396L5t4cfTEfQt\nW+5kz55D7Ny5j5/+9EN27tzH1q2bx5w3maTq3Op5Z/OuZTaj/LjIAoFg0qTTR2bb5s2yDGO25+rq\npnW8d989KNfUPBt1uJqaZ+V33z04yyPPLN5996BcV/ecfNddfy3X1T037fdj8+Zt8f5ccl3dc1HP\nm+7/mIjMBYIMxeD1xt2v93imdbxE2wCZinrXMlMSPTchxFwgyFACZnPc/UGLZVrHS7gNkOUkem5C\niLlAkKFs3rqVbfX1UZ75szU13P3449M63mxO9gniM1tRfjyEmAsEGYo6ybl91y70Hg9Bi4W7H398\nypOfKiJFMbORwoa7QCCYBJIkzaiwI93Zs+cQu3Z9oLEBNmW1X54Kpvs/JsRcIJgCc13MBalnuv9j\nwmYRCGaR2SjiSTVqrrnXa8BsDrB162YRnWcAQswFglliNot4UkWiS84FiUM02hIIZol9O3dGCTnA\nS/X1fLBrV4pGNHXGzzX/IEUjEkwWIeYCwSwx20U8qUDkmmcuQswFgllitot4UoHINc9chJgLBLPE\n5q1b2VZTE7Xv2ZoaNk2ziCcVbN26mZqabVH7lFzzTSka0eTYs+cQdXXPUVv7PHV1z7Fnz6FUDynp\niAlQgWCWmO0inlSQKe1wtRk3g4NdtLV5aG9/ffTn2ThpK/LMBYIpIPLMU0+8jBvYBtQBEfGuq9vO\n3r0vJnt4M2a6/2PCZhEIBBlFvIwbeAmIzrjJtklbIeYCgSCjGC/jBqLFO9smbYVnLhAIpkWqKkXH\ny7iBiHhnY4MwIeYCgWDKpLJSNF53x7Ky71NePkxOzvNpO2mbaMQEqEAwBcQEqEJd3XPs2/c3cfYn\nZ9JxLnd3FI22BAJB0kh1pWgiF3nIVMQEqEAgmDKiUjT9EGIuEAimTKZWis5lhGcuEEwB4ZlHmMu+\ndSoRKw0JBElAiLkg0YgKUIFAIMhihJgLBALBHECIuUAgEMwBpi3me/fuZfny5SxZsoRXXnllNsck\nEAgEgikyrQnQYDDIsmXL2L9/PxUVFWzcuJE33niDFStWJGKMAkHaICZAZ5dU9XdJZ5JaAfrxxx+z\nePFiqqurAbj//vt5++23o8RckqTpHFogmDRCVJPPbIpvKvu7zEWmJeYtLS1UVVWNfl9ZWcmRI0fG\nPG8BUB1+XK15nGgOALVx9utQmmTmAostcHsBXP27Jby3+F5+8urzDH/UAFfPAc2ACxivO9tUz5wM\nUnXuZJ23IbypHEzCOQVaZlt84/Ulr69/iV27tqeVmE/1Apaqu41piflko+5qUidt4yEBRsCSA44l\ncDL/Jj4auBX/YR20DwEDgA8IpXScgliqiQ4HhJgnm/jiW8eDD/4Dq1d/OGXhSnV/l8kw1QtYKu82\npiXmFRUVNDU1jX7f1NREZWXlrA0qUUgokbkVsOQC6+BT2w183HMT/gE3eIcANyBu3wWCWMaK7yHg\nfXp63uRg+No6FeGajf4uiY6Cp3r3kMq7jWmJ+YYNG7h06RINDQ2Ul5fz5ptv8sYbb4x5XvVMRzdN\nJjqvBNgAewFwK/QNF9L9WR4hTyeEhgA/SpP76Qj6RGdONKk6d6rOmzqer60lYDazeevWjFqseaaM\nFd99KMu1RZiKcMXrSz6VRSWSEQVP9e4hlXcb0xJzg8HAa6+9Rl1dHcFgkIcffjhuJkv1TEc3TSY6\nrwQ4AFuBDv/NegIHzYSOh8DfAQyjCLkQ8/Q/b+p4PhyGbquvB8gaQR8rvjMTLlVwd+3arunvMvlF\nJZIRBU/17iGV3SSn3c/8nnvu4Z577pnNsSQcdQI0TwKsDs4Uz6e/IxcuByA4CHgQXvncZe/evTz5\n5JMEg0EeeeQRnn766aifd3d388ADD9De3k4gEOCHP/wh3/72t8c93kv19WzftSshYn5ozx727dyJ\nwetNm7uAWPE9c+Y8PT1jnzcV4ZpJX/JkRMFTvXuY6d3GTMiqxSlUz9xuhYA9l9OWVfR25UNLEEIj\niInPuUswGOSxxx6Lqo249957o+4oX3vtNdavX8/LL79Md3c3y5Yt44EHHsBgGP9jovd4Zn2sh/bs\n4f0nnuClcOQP6XMXoBVfxeZIjXBBcqLgqd49zPRuYyZkn5hLYM2BkZxcTkrr6O3Nh64gSlTuR4j5\n3GQytRHz5s3j1KlTAAwODlJYWDihkAMELZZZH+u+nTujhBwSexcwXVIpXJC8KHiqdw+pWgUp+8Qc\nkHJg2OngEksYxAJ4mb5PLsgEJlMb8Z3vfIc/+IM/oLy8nKGhId566624x3o+/PW/8vP5am3trI/V\n4PXG3Z+Iu4CZksrl21J9MZktDhw4wIEDB2Z8nOwUcwe4HDaaqGJENgJDCDGf20ymNuInP/kJN9xw\nAwcOHKC+vp5NmzZx8uRJcnJyop94110ELRZeevzxhETKAbM57v5E3AVkOnNhLdDa2lpqNUHBCy+8\nMK3jZKWY4wCvzUxHqBSPrEeJzAMIi2XuMpnaiI8++oht25TUtpqaGhYuXMiFCxfYsGFD1POen4Uo\naiI2b93Ktvr6KKvl2Zoa7n788YSeV5DZZJ+YS6AzQ0BvYGTAgd+vCrkgM5CYTrPPydRGLF++nP37\n93PHHXfQ0dHBhQsXWLRo0SyNe/Ko0f72XbvQezwELRbuTtBdgGDukH1iDkgmCOl1+DwmQkEx6ZkZ\nSOENpiPm49VG/OIXvwDg0Ucf5dlnn+Whhx5i3bp1hEIh/vZv/5aCgoLZewlT4M4tW4R4C6ZE1om5\nDpRXbRizV5C2jF6GZ3SUeLURjz766OjjoqIifvOb38zoHILpI9rhzoysEXMd0ZKt14cw2zwE9XqC\nWAATYUddkFaofzlVyLURumCuINrhzpysCUl1mk0KgV4XwGEdxmDQobTeMhItGoLUIWk2XcxjHVkU\ng2QN45fmf5CiEWUeWfWpGL1ZD4JJ9pFn7mdYl4cXG8pboSeLrm9pTOx9lCrkes0mmC7paGdkQjvc\ndCerxHwUP5j9Hkppp8daxpDN
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 99
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 94
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [],
|
||
|
|
"language": "python",
|
||
|
|
"outputs": []
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}
|