nvidia-AI.summerschool17-no.../teaching-ncso/16-logistic-regression-grad.ipynb

830 lines
265 KiB
Text
Raw Permalink Normal View History

2017-09-22 17:38:09 +00:00
{
"metadata": {
"name": "16-logistic-regression-grad"
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Title: Logistic Regression and Gradient Descent\n",
"Author: Thomas Breuel\n",
"Institution: UniKL"
]
},
{
"cell_type": "code",
"collapsed": true,
"input": [
"\n",
"import scipy,pylab,numpy\n",
"from numpy import *\n",
"from scipy import *\n",
"from pylab import *\n",
"from urllib import urlopen\n",
"from gzip import GzipFile\n",
"from scipy.spatial import distance\n",
"from scipy.spatial.distance import cdist"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 3
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Linear Classification Problem\n",
"================\n",
"\n",
"Let's generate a linear problem, as before."
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"data = random_sample((1000,2))\n",
"labels = (data[:,0]*0.7+data[:,1]*0.4>0.5)\n",
"d0 = data[labels==False]\n",
"d1 = data[labels]\n",
"figure(figsize=(6,6))\n",
"plot(d0[:,0],d0[:,1],\"bo\")\n",
"plot(d1[:,0],d1[:,1],\"ro\")\n",
"print len(d0),len(d1)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"440 560\n"
]
},
{
"output_type": "display_data",
"png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAFtCAYAAAD8jSoDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvX9wFMedN/zBXsPuGUksEvEiwLEtUg6WZSLdwcFVnaTi\nnmjt40ne/LiTsIMhBoywZH65ktwFSd4NNqkU1OO3kJEPX+y7/OBJjlzV85zvpAuIO46VrwrZXOI3\nBJvkjMzFgJAhOtlge4UR7veP1axmZ7tnunu6Z2YVfapcWLMz3T09M9/+9uf7axohhGAKU5jCFKYw\nqXCT3wOYwhSmMIUpqMeUcJ/CFKYwhUmIKeE+hSlMYQqTEFPCfQpTmMIUJiGmhPsUpjCFKUxCTAn3\nKUxhClOYhHAU7uvWrcNtt92Gqqoq5jlbtmzBpz71KSxevBivvfaa0gFOYQpTmMIUxOEo3B955BEc\nOnSI+fs///M/48yZM3jzzTfx13/913jssceUDnAKU5jCFKYgDkfh/sd//MeIRqPM3//xH/8Ra9eu\nBQD84R/+Id59912888476kY4hSlMYQpTEIZrzv3ChQtYsGBB9u/58+fj/PnzbpudwhSmMIUpuEBI\nRSPWDAbTpk3LO4d2bApTmMIUpuAMmSwxrjX3efPm4dy5c9m/z58/j3nz5lHPJYQI/Vda2giA5P1X\nVtYk3FaQ/kskEr6PwXZ8dXWUWUfmuOXcVHc32hoakKirQ1tDA1Ld3XnntDU0UNtrj8c9nQuR+7Ib\ns5sx2M2XdS5S3d3YUVGRGSP1S6CP3e1/vPduHp/x346Kipx76kok0BwK5ZzTHAqhy+a5tzU0UO/X\n2n+irs5xXnjeT55nZr3P5lAIKZv7lmmT1YYsXGvun//857Fv3z6sWrUK/f39mDVrFm677Ta3zQIA\nYrFZGB5uA7DLdHQHbrutREn7U6BjbMYM6vEb4XDO3309PTi8dSt2DQxkj7WN/3/typXZY6Fr16jt\n3Tw66naoQuC9L0DPmHnny0BvZ2f23DFGm7SxuwXvvZvHZ2DXwAA6nn02ez+pfftwcCx39PvHxrCq\nqwstyaSr/lnPE8jMi+h8s0C7z/1jY+gAUDv+t/W+ZdrcNTCABx9+GL1LlqBhyxahMdLgqLk/+OCD\n+KM/+iP8+te/xoIFC/A3f/M3eP755/H8888DAP70T/8Ud911FxYuXIjm5mY899xzrgZkxrx5cwDE\nAXQASI7/ez/mz/+Esj6CjL6eHrTH40jW16M9HkdfT48n/TZs2YK2ioqcYzsqKvDZzZtzjrFe0CPP\nPptzTESoqgJt7njvC9AzZt75MmAWcg0A2iy/s8buFrz3ziOEI2P0ZSl8/brr/hu2bMHFWCxvXrbH\nYvjs5s3C880C8z6tfwss/Kw27x4ZwdO9vTi8davr791Rc//xj3/s2Mi+fftcDYKFLVsaMDBwGAMD\nE5p7RcUObN58v5b+nNDT04fdHV2Y8V//gZkYw5w7bsNXnkoIr7D19fWO56jSOmRgtN/x7LO4eXQU\nN8Jh3L95c/Z4X08Pejs7cf6VV6jXW1/yhi1b0DYwkHMvOyoqcP/mzfj41lvzrjfaD127hrEZM4S1\nGNbcxffuRXzvXuZ98Y5ZBn09PTh34gT1N2O+rO+FWcgZGmIHgLejUdy+dClz7G7Be+88QjgdoouY\n0Vtuse3/r0+dAgYHbfuvXbkSeOEF/PDJJ/Hg2bOYDmDmnXeiaedO1K5ciaN79lDbF919Me/T+rfA\nwu/UpuhOgAriEWS76u5OkXi8ndTVJUg83k66u1OKR2bfd0NDG6mrS5Dq6vVkwaz/hzSighAg+9/W\nWDlJdXcr77utoSGnH+O/9nhceV8iSHV3kx0VmTloo4yPNcZUdzdpj8dJoq6OtMfjzDkzt2/8t6Oi\nQmiOVc0d75h52tlRUSE0X+brzOd+U3AuZMFz7zzj60okSHMolHPOxlCIdCUSrvt3gsr3wHqfG0Mh\nknLxXKhzB+S0mairI4TIy87AC3cDZkHb0NCmXch3d6dIRcUO09y3kT+AdwI3UVdH7ct44H7B/MGk\nALLDMj63wkfFBxm0uTPuSWa+VC0wusAzvq5EgjSVlZG1JSWkqazMUbCrHJuqxdF6n12JhOvnYrS5\nJhol7RbBbn7nZWWnEldI3ejp6cPWrbn0zMBAhmlbubKWdZkrdHb25vQHhHArvDMM+sFT88DMFZqp\ngnMlJViwbJlrqoDFRb796qvo6+nhajtoc2fck3m+bgbw62gUj+3da3tPtStXaqfh3IBnfC3JJNN4\nasAtFccaG8CmF0XbUv0cjDYNGrFWEQVooCCEe76gBQYGduHZZzu0Cfdr16xTM4YPICc0ZF5c1Zyv\nKlgFZ+34fx3LluEpmzQVsu0buH1kBIe3bs30WWBzZ+XOs0J+6dJAC26voNO+FPTFEVC7CJlREMI9\nX9BmMDpqtVez0dPTh87OXly7FsKMGWPYsqXBdmGYMcNq5W/Ar/CfaEIFDmLiJfzKrNmYdekSkvX1\nVMEt++LqeuBuoVtwUtsHcD+AWgcjk3kRfae4GK01NZhTVOT73AVtsQkaeFwq3YBHudKxcxCBjkWo\nIIR7vqDNIBy22qvpkKF1Mp46baZrajEz9j38+vfuRnwEuBVjCEWnY/6HV/GMKROmVXC7eXGDonVY\nX/x5q1ejo79fy6JjtPPgww/j7pER3MC4YB//nUWBURfRigqsGPec8BOiC7XfgsYriHpdyfbhpFz5\n6ZmmFVJMvQTcdJVv3CSkouKb3EbVhoY2mn2NxOPtjv3aeerwGP82VlaSNoAkxr1LDKOJ34ZRXrC8\nV7oSCdLW0EASdXWkraHB1vvFfB7vdaKG1aB6F4lChbdQIUDW60oUPO+F1UnA+F4bS0sDMe+ysrMg\nNHdDu3722Q6Mjt6McPgGNm++n5tvl6V1Vq6ste3DKYijr6cH0956C0+bfjMCLowIOhkNzcvraDuP\n+MAAfrR7N/an0xP3RdF0rBpRH4Af9vbiu6a2WBqSKJURlChYt9BNUQQF5vs0ArRy4tAV0VY874Vx\nTh+AfwDwjPHD8DCe2LABeOGFgpz7ghDugLOgtYNbWocFJ6+M3s7OHAEIZF7gpkgEdcuWZQVfH4Be\nAC/29eHgokVoeuop6svU19ODgx0deP/0aSwYHcUKZOgKni2k7NaT9nH0Avn3RRFAVkHVC+QIdtZ1\n5jHxUhlB85CRhZ0wmkx0jW6vKwM874VxzkEAXZbznhkaQuuTTxbmPCveQTDhYVd5cEvrsODkR8vy\nt95WVWXr+0zbhlO36yaax2kLK0tb0K5LMLbRVqrJev+818nAz4AflWA9pw01NZOKrvGKRuN5L4xz\n1jDezzXRqKv+eWhIO8jKzoLR3HlheMVcvPgBBgcHEYvNwrx5c7B69Tz098vROiw4aZcsrWFmeXl2\nW9iL3O0owKcFY/w6I3mRE/0gS1vQ6JHTkQhg0dyBfC3Zev86k18F1btIFLT53hQO4/Kbb+K7V6/m\nnGt+TwpNq/fKg4jnvTD+/3994QsAJRfOR5J9uzHUmp+nLCaVcKd5xQwPt+H11zM5avbujSvzi891\nrfwDbPl6vmul3Qvc29kJgP0ArELXKXmRk4CUpS1oH0fdsmVoO3DA8cO03n8DgE0A9pvOaY5E8BVF\nH7Qb7yIVwtHcxvkrVzADwJziYqH2jHNan3wS77/xBm4fHcVDo6M4yliEDbqm0Lw9vF6MMwrwxL+0\n8fygqgptr71myUGbyVcjA1n7ifV5fkuqd0wuWoblFQO0c3nH8IJG80QizSSR6Mo7lxWeLZpvhLmN\n5aQfVNMWIrli2uNxsq2qijRGIqRrfMwJIPO3xlB03i2xeW4Mb4k14TBpqa7mnh9rGzxUmx2sz9vu\nPWG9G42lpVx0gArqIKgQ8T5KdXeTdbFY9v1sB8gjsZj0fMimwbA+T1nZOamEe11dgiHcNxKgjZSU\nrFWSl4a1iEQijUJtp7q7yYaa
}
],
"prompt_number": 1
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"augmented = concatenate([ones((len(data),1)),data],axis=1)\n",
"augmented[:3,:]"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "pyout",
"prompt_number": 2,
"text": [
"array([[ 1. , 0.52174249, 0.93138903],\n",
" [ 1. , 0.80899277, 0.6778396 ],\n",
" [ 1. , 0.3524324 , 0.24216655]])"
]
}
],
"prompt_number": 2
},
{
"cell_type": "heading",
"level": 1,
"metadata": {},
"source": [
"Sigmoid Functions"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"We have considered two ways of finding discriminant functions so far:\n",
"\n",
"- perceptrons: $H(w\\cdot x)$, via the perceptron criterion function\n",
"- linear: $w\\cdot x$, via least square regression\n",
"\n",
"Advantage: both have simple algorithms to compute the weight vector\n",
"\n",
"Disadvantage: the weight vectors aren't very good (at least in the non-separable case)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(gradient descent)\n",
"\n",
"Let's consider a general class of predictors of class labels $f_\\theta(x)$, parameterized by some parameter vector $\\theta$, with a least square error measure:\n",
"\n",
"$$\\hbox{error} = E(\\theta) = \\sum_i (f_\\theta(x_i) - c_i)^2$$\n",
"\n",
"This is a _least mean square_ error (LMS) error function.\n",
"\n",
"We can optimize these via _gradient descent_. That is, we want to find $\\hat\\theta$ such that:\n",
"\n",
"$$ \\nabla E(\\hat\\theta) = 0 $$\n",
"\n",
"The set of all such points constitutes local maxima/minima.\n",
"\n",
"The way we find such local minima is iteratively:\n",
"\n",
"$$ \\theta_{i+1} = \\theta_i - \\nabla E(\\theta_i) $$"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(error gradient descent)\n",
"\n",
"By the chain rule, we can compute this gradient:\n",
"\n",
"$$ \\nabla E(\\theta) = \\sum_i 2 (f_\\theta(x_i) - c_i) \\nabla_{\\theta} f_\\theta(x_i) $$\n",
"\n",
"Putting this together with the previous equation gives us an iterative scheme for finding local minima of the error.\n",
"\n",
"A necessary condition for this to work is for the function $f$ to be differentiable in $\\theta$."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(smoothed Heaviside function)\n",
"\n",
"To make $f$ differentiable, we replace the Heaviside function with a smooth approximation in the form of the _logistic function_, a kind of _sigmoid function_:\n",
"\n",
"$$\\sigma_q(x) = \\frac{1}{1+e^{-q\\\\,x}}$$\n",
"\n",
"We will see later that this function has other nice properties.\n",
"\n",
"This function looks non-symetrical, but it actually is. Let's plot it."
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"import matplotlib.pyplot as plt\n",
"xs = linspace(-6,6,100)\n",
"def sig(x,q): return 1.0/(1.0+exp(-q*x))\n",
"plot(xs,sig(xs,1.0)); plot(xs,sig(xs,2.0)); plot(xs,sig(xs,4.0)); plot(xs,sig(xs,8.0))"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "pyout",
"prompt_number": 3,
"text": [
"[<matplotlib.lines.Line2D at 0x397ba50>]"
]
},
{
"output_type": "display_data",
"png": "iVBORw0KGgoAAAANSUhEUgAAAXIAAAD9CAYAAAChtfywAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlcVPX++PEXAi644L4BiggBmrtoZiWaptkVM7s3s1+r\nec0Ws7K89r2VerupLbbZYpbaomZ1VayEDBX3fTdFQUAQFR0ERdlmOb8/PoogM8MAA8MM7+d9nAfO\nzOec86GLb9+8z2dx0zRNQwghhNOq5egOCCGEqBgJ5EII4eQkkAshhJOTQC6EEE5OArkQQjg5CeRC\nCOHkSg3kTz31FK1ataJLly4W20yaNImgoCC6devG/v377dpBIYQQ1pUayJ988kmio6Mtfr5mzRoS\nEhKIj4/nq6++YuLEiXbtoBBCCOtKDeR33nknTZo0sfj56tWrefzxxwHo27cvWVlZpKen26+HQggh\nrKpwjTwtLQ0/P7/C176+vpw+fbqilxVCCGEjD3tc5OZZ/m5ubiXamHtPCCFE6UpbSaXCgdzHx4fU\n1NTC16dPn8bHx6dcnXFm06dPZ/r06Y7uRqVx5e+vIt9bQm4uM5KT+ePiRV7x82NcmzY09/S07eT8\nfFi1Cr77DrZsgfbtYcAA6N0bQkIgOBgaNy5Xv4q6+fszaSZOXjzJvrP7OHLhCHG6OOJ0cSRcTKBR\nnUa0925P+8bt8WnoQ5sGbWjTsA0t67ekhVcLmnk1o1m9ZjSo3aDaJGe2/P+Xmwtnz8KZM5CefuO4\ncAF0uhvHxYvqMBigaVPw9lb/F3h7q6NhQ3U0aqS+Nmigjvr1bxxeXuqoV099rVtX/dnDA8rzn8yW\n/84VDuQRERHMmzePMWPGsGPHDho3bkyrVq0qelkhqr3DV64w8OBBJvn48FnfvjTysPGvU1oafPwx\nfPst3HorPP20+nPz5pXST6PJyMbkjcQmx7IpZRN7z+zFu643vdr0okurLowOHU1I8xCCmgZRv3b9\nSulDZdI09Z80OVkdSUmQkgKpqXD6tDpycqBNG3W0bg2tWqmjUydo0UL9p2/WTB1Nm6oAXE3+nbJJ\nqT95Dz/8MBs3bkSn0+Hn58eMGTPQ6/UATJgwgeHDh7NmzRoCAwOpX78+ixYtqvROC+Fo2QYDD/71\nF3M7duSx1q1tO8lkgvnz4c034bHHVBYeFFQp/Tt/9Tyrj69mZdxKYrbFsLbdWsL9w3ml3yuEtQ2j\nRf0WlXLfypSdDceOQVyc+nr8OCQkqNdffgkBAeDvDx06QM+eMHIk+PqCj48K0M4UmMvKraqWsXVz\nc3Pp0kpsbCzh4eGO7kalceXvr6zfm6ZpPHzsGA3d3VkQHGzbSXFxKvM2meDrr1UqaGf5hnxWxq1k\nwb4F7D2zl6GBQxkVMoqGZxpy3z332f1+lUXTVBa9dy/s2weHDqkjPV1Vm0JDVeUpJET9O3jmTCzD\nh4c7utuVxpbYKYFciDL6LC2Nr8+eZVuPHtRzdy/9hJ07YcQIlYk/+yzUsu+E6rTLaXy882MWH1hM\nl1ZdmNBrAhHBEdT1qGvX+1SWq1dh927Ytg22b4ddu9T7vXqpo1s36NoVOnYEW/5zuxoJ5ELY2ZGr\nVxl04ADbevYksF690k/YvRvuuw8WL4bhw+3al7PZZ5m9dTbfH/yex7o9xnNhzxHUrHJKNfaUm6uq\nSrGxsGEDHDyoAnW/fnD77dC3ryqJuHIppCxsiZ12GX4oRE3x7blzTGjb1rYgvncv/O1v8M03dg3i\nufpc3t78Nl/s/oLHuz/O0eeO0rqBjXV6Bzl5En7/HaKjVRDv0gUGDYL//EcFcC8vR/fQuUkgF8JG\nmqaxSqdjuS317ZMnVfD+6itVVrGTmMQYnvntGXq17cWRZ4/QtmFbu13bnjQNDhyAlSvVodOpX0zG\njYOlS+0yqlIUIYFcCBsdy8kh32SiR4MG1htqGjzzDEyZooZO2MGVgis8v+Z5YpNj+fy+zxkeZN8y\njb3Ex8OyZSpYFxTA6NFqoM5tt9n90YAoQgK5EDaK1OkY2bx56RM0fvhBpaAvvWSX+57IOMEDyx+g\nj08fjjx7hAa1S/mHpIpdvQo//6wG48THw0MPqWHxffpInbuqSCAXwkaRGRm83aGD9UY6Hbz6Kvz2\nm5rKV9F7xkUy/tfxvD3obcb3HF9tZlOCGlH56acqA+/fX33bw4eDrRNbhf1IIBfCBmfy8zmRk8MA\nb2/rDV9+GcaOVdPsK+ijHR8xd/tcfn34V/r69q3w9exB0+DPP+Gjj9Sz3AkT1BhvX19H96xmk0Au\nhA1+zcjg3qZN8bRW6I2JgU2b4MiRCt9v9pbZfLP/G7Y8tYV23u0qfL2KMpkgMhLeflstEfPyy7Bi\nhVpHRDieBHIhbBCp0/FEaVPxZ8yAd99VqyhVwMyNM1l6eCkbn9jo8FEpmqYC9vTpULs2vPEGRETI\ng8vqRgK5EKXINhjYcukSP1obdnjokFqt6YEHKnSvmRtn8tNfP7HxiY20auDYxec2boTXXlOjT+bM\ngXvvlYeX1ZUEciFKEX3xIv29va2vbvj55/DPf1boAeeSQ0tYdGARO8btcGgQT0hQA27++kuVUsaM\nkQy8upNALkQpfs3IIKJZM8sNLl2C5cvh6NFy32PH6R289MdLrH98vcOCeE4OvPOOWklw6lT45Reo\nU8chXRFlJP/OClGKw1evEtaokeUG330H99yjFrsuh5RLKYz+aTSLRi7i1pa3lrOXFRMVpRZkPHlS\nrX3y6qsSxJ2JZORCWKFpGgm5uXS0NDxD01RZ5csvy3X9XH0uEcsieKXfK9x3S9UvNZuZqcoomzap\nCT2DB1d5F4QdSEYuhBXn9Xrq1KpFE0uzXDZsUGur3nVXua7/7w3/Jrh5MC/dZp9ZoGWxZo1avKph\nQ/WsVoK485KMXAgrTubmWl/p8PPP1Rrj5RjOsTF5Iz8e+ZFDzxyq0hmb+flqNMqqVbBkidomVDg3\nCeRCWJFgLZBfvgx//AELF5b5upfzL/NE5BMsGLGAZl5WHqTa2fHjahRKQIBanbBJkyq7tahEUloR\nwgqrgXzdOrWsn7UHoRa8/MfLDAkYUqWrGP7yC9xxh5pW/8svEsRdiWTkQliRkJvLcEtDD6Oj1SyZ\nMoqKj2J90noOPnOwgr2zjdGoZmQuWaJ+gejZs0puK6qQBHIhrLCYkWuaGrM3eXKZrldgLGDyH5P5\nbPhnNKzT0E69tCwrS63hlZOjdp1r2bLSbykcQEorQlhhcejhsWPqAWdISJmu9/nuz+nYpCP3BpU9\nky+rlBS1vGxAgFqxUIK465KMXAgLLur1GDWN5uaGHkZFlXnxkYycDN7Z/A4bn9hox16ad+CA2i70\nlVfstr+FqMYkIxfCgutlFbNDA6OjYdiwMl1v+sbpPHTrQ4S2CLVTD83780810fSjjySI1xSSkQth\ngcX6+JUrsGOHWt/VRscuHGP5keUce+6YHXtY0sqVarvQFSvUCBVRM0ggF8ICi4E8NlbtANTQ9oeV\nr/75KtPumFapY8aXLVMbPkRFyciUmkZKK0JYYDGQX6+P22jf2X0cOHeAZ8OetWPvilu4EKZMUWUV\nCeI1jwRyISwwG8ivDzssQyCftWUWU26fQh2PyllOcOFCeOsttezLrY5ZPFE4mARyISwwG8jj49WW\nOTZGzDhdHJtObWJ8z/GV0EP48Uf497/VdqG33FIptxBOQGrkQphx2WAgx2Side3axT/YuBEGDbJ5\n2OGcrXN4oc8L1K9d3+59XL1azUdauxaCg+1+eeFEJJALYcbJaxOBSgw93LMH+vSx6Rqnsk6x+vhq\nEl5IsHv/1q2Dp5+G33+Hrl3tfnnhZKS0IoQZCbm5dDT3oHPPHjVixQbvbXuP8T3H06SefVenOngQ\nHn4Yfv4ZwsLsemnhpCQjF8IMs/XxvDw1Nb9bt1LPP3/1PEsPL7X7uPHUVDVjc948WUdc3CAZuRBm\nmA3khw+rJ4rWNpq45pt93/BA
}
],
"prompt_number": 3
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(derivation of the gradient)\n",
"\n",
"Using the sigmoid function, the discriminant functions become:\n",
"\n",
"$$g_c(x) = \\sigma(w_c \\cdot x)$$\n",
"\n",
"As before, we compute the gradient of the error:\n",
"\n",
"$$\\hbox{err} = \\sum_i (\\sigma(w\\cdot x_i)-c_i)^2$$\n",
"\n",
"Expanding this yields:\n",
"\n",
"$$\\nabla_w \\hbox{err} = 2 \\sum_i (\\sigma(w\\cdot x_i)-c_i) \\sigma'(w\\cdot x_i) x_i$$\n",
"\n",
"Note that\n",
"\n",
"$$\\sigma'(x) = \\sigma(x) (1-\\sigma(x))$$\n",
"\n",
"\n",
"\n",
"Let's implement this. Note that the class labels are zero and one again, $c\\in\\\\{0,1\\\\}$, matching the output of the sigmoid function $\\sigma(x) \\in [0,1]$."
]
},
{
"cell_type": "code",
"collapsed": true,
"input": [
"# sigmoid function\n",
"def sigmoid(x): return 1.0/(1.0+exp(-x))"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 8
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# random initial vector\n",
"w = random_sample((3))\n",
"w"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "pyout",
"prompt_number": 6,
"text": [
"array([ 0.94899947, 0.20670301, 0.74203598])"
]
}
],
"prompt_number": 6
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"# gradient descent\n",
"eta = 0.01 # learning rate\n",
"for iter in range(500000):\n",
" i = iter%len(data)\n",
" if iter%100000==0: print iter,eta,w\n",
" x = augmented[i]\n",
" c = labels[i]\n",
" s = sigmoid(dot(w,x))\n",
" sprime = s*(1-s)\n",
" delta = s-c\n",
" w -= eta*(s-c)*sprime*x"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"0 0.01 [ -7.97938626 11.37764969 6.17891424]\n",
"100000"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 0.01 [ -8.53862331 12.13940127 6.63656511]\n",
"200000"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 0.01 [ -9.03479587 12.81716619 7.04190882]\n",
"300000"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 0.01 [ -9.48282345 13.43059416 7.40735779]\n",
"400000"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 0.01 [ -9.89269465 13.99289566 7.7412153 ]\n"
]
}
],
"prompt_number": 9
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"d,a0,a1 = w\n",
"d0 = data[labels<=0]; d1 = data[labels>0]\n",
"figure(figsize=(6,6)); ylim((0,1)); xlim((0,1))\n",
"plot(d0[:,0],d0[:,1],\"bo\"); plot(d1[:,0],d1[:,1],\"ro\"); plot([0,-d/a0],[-d/a1,0],\"g\")\n",
"print len(d0),len(d1)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"440 560\n"
]
},
{
"output_type": "display_data",
"png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAFtCAYAAAD8jSoDAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsfXt8FcXZ/zcQ4BwlhEOCHAKoGLxADJhUKNiapFQTLGqt\nbZOgXJRbMNy9SxJPRLGK1pZALLzVXtTqS++2SQtBkRPaEqWWnxSlvhpoBUIEYxSQBAjM74+TPdmz\nZ3Z3ZnZ2z54038+HD7BnduaZ2d1nnnmuCYQQgh70oAc96EG3Qq9YE9CDHvSgBz2Qjx7m3oMe9KAH\n3RA9zL0HPehBD7oheph7D3rQgx50Q/Qw9x70oAc96IboYe496EEPetANYcrcZ8+ejSFDhiAzM1O3\nzZIlS3DppZdi3Lhx2LVrl1QCe9CDHvSgB/wwZe533nknNm3apPv7n/70J3z44Yf44IMP8D//8z+4\n6667pBLYgx70oAc94Icpc7/22mvh8/l0f//DH/6AWbNmAQC+/OUv47PPPsPHH38sj8Ie9KAHPegB\nNyzr3A8dOoQRI0aE/z98+HAcPHjQarc96EEPetADC0iU0Yk2g0FCQkJUG9q1HvSgBz3ogTlEssRY\nltyHDRuGAwcOhP9/8OBBDBs2jNqWEML1JyWlEACJ+pOaWsTdl8w/D2x5AHNenSN8fyAQiCn9pvTl\n5lJWHaHrmrbBmhqU5ecjkJuLsvx8BGtqotqU5edT+ysvKHB0LXjmZUSzFRqM1ku7FsGaGqxITw/R\nSP0S6LRb/cM6dzV9yp8V6ekRc6oOBFCSmBjRpiQxEdUGz70sP586X+34gdxc03VheT9Znpl2niWJ\niQgazFukT70+RGFZcr/55puxbt06FBcXo6GhAQMHDsSQIUOsdgsA8PsHoqWlDMAq1dUVGDIkWUr/\nonjoqw/hsnWXYffHuzF2yNiY0mIHOvr1o14/6/FE/L++thably7FqsbG8LWyzn/nTJ0avpZ46hS1\nv97t7VZJ5QLrvAB7aGZdLwV1VVXhth06fdJotwrWuavpU7CqsREVa9eG5xNctw4bOyKpX9/RgeLq\napRWVloaX+95AqF14V1vPdDmub6jAxUAcjr/r523SJ+rGhsxbcYM1I0fj/wlS7hopMFUcp82bRqu\nueYavP/++xgxYgR+8pOfYMOGDdiwYQMA4Bvf+AYuueQSjBo1CiUlJXj22WctEaTGsGGDARQAqABQ\n2fn3FAwffoG0MUSQ7ElGRU4F7t9yv63j1NfWorygAJV5eSgvKEB9ba2t4ynIX7IEZenpEddWpKfj\n+sWLI67pvaBb1q6NuMbDVGWBtnas8wLsoZl1vRSomVw+gDLN73q0WwXr3FmYsLeDvi15zpyxPH7+\nkiU47PdHrctyvx/XL17Mvd560J2n9v8cG79en5e3tuKxujpsXrrU8vduKrm/8sorpp2sW7fOEhF6\nWLIkH42Nm9HY2CW5p6evwOLFU2wZzwy1tfVYXVGNfv/+O87rdQYNdx7B070exb23VXD1k5eXZ9pG\nltQhAqX/irVr0bu9HWc9HkxZvDh8vb62FnVVVTj45pvU+7Uvef6SJShrbIyYy4r0dExZvBjnzj8/\n6n6l/8RTp9DRrx+3FKO3dgVr1qBgzRrdebHSLIL62loc2LmT+puyXtr3Qs3kFAmxAsBHPh8unDBB\nl3arYJ07CxNuS6SzmPY+fQzH/589e4CmJsPxc6ZOBZ57Di8+/DCm7d+PvgD6jxyJopUrkTN1KrY+\n9RS1f97Tl+48tf/n2PjN+uQ9CVBBHILoUDU1QVJQUE5ycwOkoKCc1NQEJVNmPHZ+fhnJzQ2QrKw5\nZMTAb5JCpBMCEAKQ310BkrI4kWz946vSxy7Lzw+Po/5TXlAgfSweBGtqyIr00BqUUejTozFYU0PK\nCwpIIDeXlBcUkGBNjWn/yp8V6em67WmQtXasNLP0syI9nWu91Pep2z7EuRaiYJk7C33VgQApSUyM\naDM/MZFUBwKWxzeDzPdAO8/5iYkkaOG5UNcOiOgzkJtLCBHnna5n7grUjDY/v8x2Jl9TEyTp6StU\na19Grkbky3IOINfeCXLj9Azp4wdyc6kvpvLAYwX1BxMEyAoNfVaZj4wP0m1rp8xJZL1kbTB2gYW+\n6kCAFKWmklnJyaQoNdWUscukTdbmqJ1ndSBg+bkofc70+Ui5hrGr3/luzdyjGS0h6ekrbGXw+fll\nGt4QILnIjWIYbw4DSXqoLzlx6oTU8d0quWsZZxAg5QCZlZwshfnoMeaZPh9z325bO/WclPUKAKSY\nY07dHcGaGlKWn08CubmkLD9f2rq4fXMkxHwT6tbMPZrRhv4UFJRLpDASubkBzXjRkrvyJ2Ohnzyy\n7RHdvkRe3FgeyY1gN+PU7R/s6hm3rZ3bNhu3QYYqLt5htAl1a+YezWhDf3JzA8x98Kp1ojeUIOmP\n70bo3AlAbhs4iEy/ZgzxrkgkS6bmRr2QVl5cN0oddjNOM12kEUNUb6JzsrJIaXa2K9bObZuN22D3\n5sciXNl1cpABUd4pJULVbvTrp+NO5dHaq+mora3H0qWRXjeNjSEHqqlTc6j3hDx1ylT35KC//2d4\n/7zLUdAKnI8OJPr6YvjJ43jmb+9h6PnAZ54gNi8NpV5QrNwsvsB6yJk61XbPGBZovVeGTZ+OioYG\nU48TESj9TJsxA5e3tuIsgCno8hbR83Sgesikp2Nyp+dELGHmfaSFVW+heAGv15XoGGZeZ7H0TLMV\nkjcZXVgZiq5zf4hZ5y6q1jHz1FFLHK0ekAvuBfnnBZESx/yMDFLWqWMtU0mgsTaMskLv5FEdCDBJ\nOlqJiPU+Xmmuu6g+/ltUFKJeV7xgeS+0TgLK91qYkuKKdRflnXEhuSvS9dq1FWhv7w2P5ywWL56i\nK3VrceqUjq9tuzYMIXpcozHUgQgD24Gy7cD91wMTDoYkjvraWiTs24fHVPcoARdKBJ2IhObkfbST\nR0FjI15evRrr29q65kWRdLQSUT2AF+vq8GNVX3oSEq+fuVuiYK3CykkvnqCepxKgFRGHbiGmQA2W\n90JpUw/g9wCeUX5oacHdc+cCzz0Xl2sfF8wdMGe0RrCq1tGDNhBhwd+BtRMA35kTAEIvsJoBAqEX\nuMjrRe7EiWHGVw+gDsDz9fXYOHo0ih59lPoy1dfWYmNFBU7s3YsR7e2YjJC6guUIKXr0pH0cdUD0\nvCgMSMuo6oAIxq53n5omVlVGLKJg7YARM+pO6hr1PNUBWgeSkzFi4kRpqj6W90JpsxFAtabdM83N\nWPjww/G5zpJPELpwcKgoWFXr6IF2hL41bwhJf3Ik6TjboevWtywz09D3mXYMpx7XGQ2NhIirLWj3\nBXSO0VpVk3b+rPeJoLsYLfWe09zs7G6lrnFKjcbyXihtZuq8nzN9PkvjWzXUivLOuJHcWVFbW4+q\nqjocPvwFmpqa4PcPxLBhgzF9+jA0NIipdfRAky6XLFqEsiNP4MXdL+pKDf3T0sLHwjpEHkcBNikY\nnfcpyYvM1A+iaguaemSv1wtoJHcgWkrWzt/O5Fe8kr5bQVvvBR4Pjn7wAX58/HhEW/V7Em9Svez0\nDnpgeS+Uf3//llsASi6c04JjWzHUqp+nKLoVc6d5xbS0lOHdd0M5atasKbDM0NVjVVXV4dSpRPTr\ndzWW3Jcf7vv7Bwfj27/8Nn6ycI3uC1xXVQVA/wFoma5Z8iIzBimqtqB9HLkTJ6LspZdMP0ztB5wP\nYAGA9ao2JV4vbpf0QVvxLpLBHNV9HDx2DP0ADB4wgKs/pc3Chx/Giffew4Xt7bitvR1bdTZhRV0T\nb94eTm/GIQG4628aPS9kZqJs1y5NDtpQvhoRiNpPtM/zEaHR0b3UMnpeMUA5k3cMK2hqHq+3hAQC\n1eE2Rb8qIo8GH9X1VefNN2IU3MOifpCttuDJFVNeUECWZWaSQq+XVKMrQrPQ67U1FJ31SKxeG8Vb\nYqbHQ0qzspjXR9sHi6rNCNrn
}
],
"prompt_number": 10
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"That looks quite good, but not perfect. Let's look at the value of the discriminant function."
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"xs = linspace(0,1,100)[:,newaxis]\n",
"ys = linspace(0,1,100)[newaxis,:]\n",
"imshow(sigmoid(a0*xs+a1*ys+d).T,origin='lower')"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "pyout",
"prompt_number": 16,
"text": [
"<matplotlib.image.AxesImage at 0x4cdbbd0>"
]
},
{
"output_type": "display_data",
"png": "iVBORw0KGgoAAAANSUhEUgAAAP0AAAD5CAYAAADspDPqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvUuMbGl21/vb73ceaOM6p5FBR3DV1+7GmLawkBCSG1qn\n0UW3rRY2PQCZEs8pRkw8uzChS2IA5jFCLVTCA6tHrRLy5NagLQQYgYwEQpZKQi7JtqoOFxcnMyJ2\nPPbrDr5vxV7x5d6RearqPDLPXtJWZETskxW5K/57vf7/tbxhGAYWW2yxN8b8V/0BFltssZdrC+gX\nW+wNswX0iy32htkC+sUWe8NsAf1ii71htoB+scXeMAtf5C/3PO9F/vrFFlvsjM11418o6I39X8D/\nDQT2P6ePyB4xkNojs0dhDj+BIII0gAr4fcAXgB8Cftgej4Avgv/FluiLW/I/sOKL6cc8Sj7iER/z\niI/4Ih+bn4ePeTg85Q8cfo/q97aUv7eDj4CPgf9ljr///8Lf/yPAM+DSHN3aHJs9rA9Q91ADW2AH\n7IED0ADtxNHZo3eOYeIQ0+ecsx8AX7vd/4zXxn7A3frMP+Bufd5/cOa9lwB6MF9b9+veY2DgYbIM\ngUfDeIMIoPegD8ALzP1hAyT2kPvDCsihz332ZcqQD6z8kiKpKNhQsLaPGwpvQ+7VZOGOJOsYqh2s\nwVtj0LvD/HcyTtAcNOAfIO4gbqHpzWka0HL4E0d/5upIPLSwpBZ7GfYSQC+Ah1Owwwh4eU1A73Ma\nEcQG+I0He8+AcYu5AWRAbo/SgyufPo/YxTmXPCCjJrdHoX7O/S1puicjJNz0hJseT0AfYwINcd/2\n8PYQtBA1kAzQ9tAN5lOHnHrunmnwT10d18OLydU5d85iiz2vvQTQ/xGug15MvtYC+MA+9zhNAToY\nemh82CnQ18AaA/gCWHlwFTAUEbsq55LWgnzrPNbkfk2ebSiSgHQN4XqA7QA7+Nr/aX/nFgN8ORLw\nG4hD6HpoB3OIx5+KZW4C/bnw3bPHTSH+4zPvva72+FV/gOe0x6/6A3yO9hJA/38wfmW11wfzlW4Z\nIRGonxuMi43NYx9AF8LBN2CMGMP8NaPHL6AvfA5lCsXAOl6zSmxYTz2G+azJvQ2ZX+OnW5KLHd52\ngC187ccx+b2AXof5LUStAn0/hvjy182F+Br0A2NYL1fCc97X753z+I9nr/3ra49f9Qd4Tnv8qj/A\n52gvKaeH08BXTIf1U6APOcbWQwithc3OvqVz+5Qj6IfCoykSujxkU61ZRxtWwQj6kjUl66P3T5Ke\nstqP0cPGPuaMUYX9GH5jwvu+hbY9LdS5f81NoHfz/Dlgz90MFlvs09hLAL0EuVNfc53ju3CR/F48\nfghDAF1gwvytZ4AuIFWgJ/fo84A+99mFGau8IvNrcm8EfcHG5PteTRYfKMotyRaCdY9XG49/LM3v\n7MfYg3cwR9gYjx/3pqgXDtd7E1MhPoxe3lOv6SujowABuaceF+Av9lnsJYH+3Hsa+C7oBfgW9AL8\nNoR9YECpu30bjpV86fgd8oRVW5EGWwrfhPlXXJAL6KnJ4x15sIHtQFo1BPVgbiY1I+gP6nEPXmyB\n3xnwh9086DXwNWDdHF+ulNwQluLdYi/CXlL1fiqXd9t4un0XcAp6AX5kPnLnGfDvGUPyjDG3P7b5\nPZoypt4WrL2Kq3hDHtTkbE4r+kFNEawJsp6o6gi2/fh7JbyX3N6C3m9Gbx/1Bvxy2xLQd/Yv0cDv\nOfXw2uu7VKZzBb7lZrDYp7WX2LLTX2nt46Zq3trzt4ye3ub2fWB+rXhfAb1u4a2AAroiYig8tpSs\ngoosUG07p50Xxx1FuR9/pwa9BfvR0zemdx+1BvBxc/qJXU/venx9dQT4Ogm6qYV3G9LOYotN2Usk\n58hXW3ex5XXt5d0cX8g6KsTvI/PvDp55S3r2mtSXm8cuD+nykG2Us84q0mhL4Z327Y/9+2hPU2yI\nDw3eZsATyp2AXqr4NrcPDjA0EB+g9eyndVp4moWnq/pibvwjj8PEz3PnLLbY89hLyumnglft/fXN\nwM3tG/Wz7t3H0Pmwt737mhH0UtRTbN42i9iUBUlYUYYbCn8s6Ek1fxXU5EmNl/UkZUNUdyPodetO\njoNh6UUJpC20nanoy1/ggl68s/bSbqi/VO0Xe9H2ksL7qWxV5/lCQXFzeznkuWLo0UAXweCPBT2X\noptyBH1ThgzbnCipKP1p0OdBTR5sCPOGoOyJtt3YHZAwX44U4/H3poXntdDsoelM/75hnm8vr4np\nwh0zry/h/GKfl73E8F5/fV3ToD9XzZeC3h5Tycfk96035uEuL78AchjKgK5KOMQ5G7/kKrygYOzd\n59RkbMmoicMDSdEQHw74W0yYL7/fVdgcDOD9ZsztWwzw3b9GSpQhtwPyOXquS9hZbLHb2ksk58D1\ngp42zzm0h5cwP+BEjDMEMMQG9HvPADJhbN+Jp8+hLwOG0mefZmzSioSdKuJtVFFvQx5uKfMNXQfU\nEAjYN5yC3np6z96LwhZi37L0htPKhHxyHeILi+/c1Zp7fynoLfZp7SWG94Pz2k2H6/Gljaf79rF5\nrfMMPXfnnbLqUkZu/spjuPRos5g6M2F+EWwoA6XAE0WeX1PEG5LsQFK0BFVnfp+E+fqwHl8IO0kE\nXWtBP5xW9DXo5SYAp0Qc99bnqdflHNS/WyYWLPa89pKr9+cG9ZyT32qyjgB/z5GX33kwROB7xtML\n8DXor8xjlwdsixQ/rSjSEfTlUX67NtX9oCaJG/ysJik7o+XXLTzN1lOg923rTuf2um8vf5HEK2I3\nxUDulVpssU9rLxH08qh9luvZp3J7t4XntvEOp/RcTdiRMH+NAX1mNPdNmbDNOzZByVVycfTyuTx6\nhp6bhnvivCVr9ja3H05JO5qtt7ea+8aQdeLGgl4BX4Neh+ZyO9R9juch7Cx6/MWex15yTi92rkR1\njqyjge+G+hH0tnfvFvROBvJ4DFVAlydsk5xLLmwRbyTqZFaIkwVbsmxHzoZwOxDWA54LekXYobWk\nndaAvhnMDUB4+bqSr+Ma3aeYuipS+ryJsDN3zmKLaXsFoNdfS/FtU3n8VGFP5/Ya9LH5N31onvqq\nqCe5vQI9ZUBfROzKnKuuMeG8r4t5Wwpq09ZL1uzDEG/TEW76sV6gQ3zp29uPEzQQHYwY5zBANFxv\n38nzgOsxz1Ruf46scxvN/WKLib1k0M9RT9xz5AbQOudL08udriMtPM9q7sPTQRtC0ZUwv4Ch8Nnn\nCUNesUo25GlN7lmw6xzf25D5O7ykJq46vM0AJZOtOwG934zhvajwhJrrau2nwvmbcvupVt0yZWex\n29orAL0u6N1E2pn6Wk9N15EWnmc09wNjbq/HaimKbp97HPKUtghZX9Rkyc7k8ieDNgT0W+K0oyx3\nUA3m5iEhvnj53fjfDGIIEkhaODSjVKjheiPy08zTm/LsC5FnsdvaK6Th6vfFpr7ybp7v8vIVJ19y\n+yE08/S2Kr9fM+b4Mlpr5TFcBezjlHVRsvKqo/RWPL2E+2l8IK+2xHVDuO7xt8MIdp3fbzlmH35j\nhmimvZm00/XXS5f6588yT2+xxW5rL3GIxhTob/MVnirutfZ1PTl3Yp4etoUXc1rYswo8rnwoPA5l\nwrovyQcD+pK11dzXo7ePa8oowqt7gs1wnKd3Qs3d2d8vYX5iwvzecvLb/vpfMkz8/Gnm6S222G3t\nFVbvxXQhz6XpukU+8fASILseP8K428Dw8nFaeCeeHihhKDzaMmZbFWzyilW8oQgchh41mW+Gavop\nxGV/qrnXwBfQ2/ZdmFh6bj/y8bXm3hXlzHl6Ke65rT33anLDe4st9hqAHq7ryVyqiibp6Nxeg14X\n9ALofUPYEUmsruavsZ4eyKEtYvoiYDPsWAW10txvVBvPgD6JWzNPb8/I0pNiYc5YxRf5rZ2pFzfj\nIgxN2LmN/FZX85l4b6o8uvDy
}
],
"prompt_number": 16
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Blue represents zero and red represents one. From this picture, we can see that the transition region between the two values is quite fuzzy; the decision boundary isn't at all sharp.\n",
"\n"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"xs = linspace(0,1,100)\n",
"ys = xs\n",
"cla(); plot(xs,sigmoid(a0*xs+a1*ys+d)); savefig(\"temp.png\")"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "display_data",
"png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD9CAYAAABHnDf0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAG6xJREFUeJzt3X10VPWdx/E3kFQDKk9BhJlUIEmTQEhwGwzIqkHF8NDG\nVmw31N21SCPLUVncs7v22LPHxLY8ePa0UrNu0QquUAK1WoM1RAgyIA9J3IaGBxEDSzAZJRohBaEl\nyeTuH7cEAnmYTGbmztz5vM6Zkwz5dfj6O/HTn9/7u7/bzzAMAxERsZX+VhcgIiL+p3AXEbEhhbuI\niA0p3EVEbEjhLiJiQwp3EREb6jHcH374YUaOHMnEiRO7HLN48WISExNJT09n3759fi1QRER6r8dw\nnz9/PqWlpV3+vKSkhKNHj1JTU8OLL77IokWL/FqgiIj0Xo/hfvvttzN06NAuf75p0yYeeughADIz\nM2lqaqKhocF/FYqISK/1uefudruJi4trf+90Oqmvr+/rx4qISB9E+eNDrjzBoF+/fleN6ezPRESk\nZ76cEtPncHc4HNTV1bW/r6+vx+FwdDpWx9iY8vPzyc/Pt7qMkKC5uKQvc9HaCh99BAcPwoED5vdH\nj5ovgLg48+V0wk03wciR5is2FoYPh2HDYOhQGDgQQmEdpt+LS3xdGPc53HNycigsLCQ3N5fy8nKG\nDBnCyJEj+/qxItKNTz+F996D8nLz9cc/gsMBEydCaircdx8kJJivYcOsrlas0GO4z5s3jx07dtDY\n2EhcXBwFBQW0tLQAsHDhQmbPnk1JSQkJCQkMGjSINWvWBLxokUjT0gI7dkBpKbzzDrjdcPvtMHUq\nLF0KGRlw3XVWVymhpMdwLyoq6vFDCgsL/VJMpMjKyrK6hJChubjkyrnweMxA37gR3ngD4uNhzhz4\n1a/MMB8wwJo6g0G/F33XL1jnuffr1089dxEvnDoFa9bACy/ADTfA974H3/0u3Hyz1ZWJFXzNTr/s\nlhGRvnO74ac/haIi+MY3YP16yMy0uioJVzpbRsRijY3wr/9qXgy97jr48ENYu1bBLn2jcBexSFub\n2XpJTobz581tjM8+a25RFOkrtWVELPDhh/CDH5gBv3MnjB9vdUViN1q5iwSRYcDKlfC3fwu5ubBr\nl4JdAkMrd5EgOXcO8vLg8GF4/30YO9bqisTOtHIXCYJjx8wbjqKiYM8eBbsEnsJdJMD27TPbMI88\nAv/zPxATY3VFEgnUlhEJoIoKyMmB//5vuP9+q6uRSKJwFwmQ996DuXPNu03nzLG6Gok0CneRACgv\nN1fqRUVwzz1WVyORSGfLiPjZ8eMwbRq89JJW7NJ3vmanLqiK+FFTkxnoTz2lYBdraeUu4ictLTBr\nlnlT0i9+YXU1Yhe+ZqfCXcRPFi+G//s/KC6291nrElw68lfEQiUlZqhXVyvYJTQo3EX6qKHBPARs\nwwYYMsTqakRMasuI9IFhmA/WmDTJfNCGiL9pt4yIBV54AT77DPLzra5EpCOt3EV8VFsLX/867N0L\nX/ua1dWIXWnlLhJkTzwBS5Yo2CU06YKqiA9KS+HAAfN4AZFQpJW7SC9duGDuaV+5Eq691upqRDqn\ncBfppZ//3HyotY4XkFCmC6oivVBfb257rKyEceOsrkYigY4fEAmCRx6B4cNh2TKrK5FIoXAXCbDj\nxyEjA2pqYNgwq6uRSKGtkCIB9pOfwKOPKtglPGgrpIgXjh41DwarqbG6EhHvaOUu4oUf/xgefxyG\nDrW6EhHvaOUu0oOPPjKP9D161OpKRLynlbtID37yE/jnf4bBg62uRMR72i0j0o1PPoEJE8ydMjqr\nXayg3TIiAfDCC/Dggwp2CT9auYt04c9/hptvhl27dPKjWCdgK/fS0lKSk5NJTExkxYoVV/28sbGR\nmTNnMmnSJFJTU3nllVd6XYRIKFq/HiZPVrBLeOp25e7xeEhKSqKsrAyHw8HkyZMpKioiJSWlfUx+\nfj4XLlxg2bJlNDY2kpSURENDA1FRHTfiaOUu4cQwIC0NfvYzmDHD6mokkgVk5V5ZWUlCQgJjxowh\nOjqa3NxciouLO4wZNWoUZ86cAeDMmTMMHz78qmAXCTfbt0NbG9xzj9WViPim2xR2u93ExcW1v3c6\nnVRUVHQYk5eXx1133cXo0aM5e/Ysv/nNb7r8vPzLHjSZlZVFVlaWb1WLBNhzz5nbH/v1s7oSiTQu\nlwuXy9Xnz+k23Pt58Zu9dOlSJk2ahMvl4tixY8yYMYPq6mquv/76q8bm6ynCEgZqa2HPHtiwwepK\nJBJdufAtKCjw6XO6bcs4HA7q6ura39fV1eF0OjuM2bNnD9/5zncAiI+PZ+zYsRw5csSnYkRCwSuv\nwPe+BwMHWl2JiO+6DfeMjAxqamqora2lubmZjRs3kpOT02FMcnIyZWVlADQ0NHDkyBHG6SkGEqba\n2mDNGnj4YasrEembbtsyUVFRFBYWkp2djcfjYcGCBaSkpLBq1SoAFi5cyFNPPcX8+fNJT0+nra2N\nZ599lmE6E1XC1Lvvmkf6TppkdSUifaObmEQu8+CDMGWKeQKkSCjQk5hE+uj0aRg7Fo4dMx+lJxIK\ndLaMSB9t2AD33qtgF3tQuIv81erVupAq9qFwFwEOHIBPP9VRA2IfCncR4NVX4R//EQYMsLoSEf/Q\nBVWJeIYBY8bAW2+Zh4WJhBJdUBXxUXm5eTfqxIlWVyLiPwp3iXgbN8Lf/Z0OCRN7UVtGIlpbG8TF\nQVkZXPaYApGQobaMiA927YLYWAW72I/CXSLaxZaMiN2oLSMRq7UVHA7z7Pb4eKurEemc2jIivbRj\nh9lvV7CLHSncJWKpJSN2praMRCSPB0aNgspK8wYmkVCltoxIL+zZA6NHK9jFvhTuEpGKi+G++6yu\nQiRwFO4ScQxD4S72p3CXiHP4MFy4ALfcYnUlIoGjcJeIc3HVrrNkxM4U7hJx3nxTLRmxP22FlIjy\nySeQmgoNDRAdbXU1Ij3TVkgRL7z1FsycqWAX+1O4S0TRLhmJFGrLSMQ4e9Y8KKy+Hm64wepqRLyj\ntoxID7ZuhcxMBbtEBoW7RIzNm2HOHKurEAkOtWUkIhiGebzvtm2QlGR1NSLeU1tGpBsHDsA118DX\nvmZ1JSLBoXCXiFBSArNn665UiRwKd4kImzfDrFlWVyESPOq5i+396U9mv/3kSRg40OpqRHpHPXeR\nLmzdCtOmKdglsijcxfY2bzb77SKRRG0ZsTXDMO9K3bkTEhKsrkak9wLWliktLSU5OZnExERWrFjR\n6RiXy8Utt9xCamoqWVlZvS5CJFCqq+G66xTsEnm6Xbl7PB6SkpIoKyvD4XAwefJkioqKSElJaR/T\n1NTEtGnTeOedd3A6nTQ2NhIbG3v1X6SVu1hg6VLzeN+VK62uRMQ3AVm5V1ZWkpCQwJgxY4iOjiY3\nN5fi4uIOY9avX8/cuXNxOp0AnQa7iFXeecc84lck0nQb7m63m7i4uPb3TqcTt9vdYUxNTQ2nTp1i\n+vTpZGRksHbt2sBUKtJLZ89CVRXceafVlYgEX1R3P+znxe18LS0tVFVVsW3bNs6fP8/UqVOZMmUK\niYmJV43Nz89v/z4rK0v9eQkol8s8BVJbICWcuFwuXC5Xnz+n23B3OBzU1dW1v6+rq2tvv1wUFxdH\nbGwsMTExxMTEcMcdd1BdXd1juIsE2pYtcO+9Vlch0jtXLnwLCgp8+pxu2zIZGRnU1NRQW1tLc3Mz\nGzduJCcnp8OY++67j127duHxeDh//jwVFRWMHz/ep2JE/EnhLpGs25V7VFQUhYWFZGdn4/F4WLBg\nASkpKaxatQqAhQsXkpyczMyZM0lLS6N///7k5eUp3MVytbXQ1ARpaVZXImIN3cQktvTSS7BjB6xb\nZ3UlIn2js2VELqOWjEQ6rdzFdjweGDECDh2CUaOsrkakb7RyF/mr//1f8zwZBbtEMoW72I5aMiIK\nd7EhhbuIeu5iM2fPwujR5mFh
}
],
"prompt_number": 12
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's update a little longer and with a higher training rate."
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"eta = 0.1\n",
"for iter in range(500000):\n",
" i = iter%len(data)\n",
" if iter%100000==0: print iter,eta,w\n",
" x = augmented[i]\n",
" c = labels[i]\n",
" s = sigmoid(dot(w,x))\n",
" sprime = s*(1-s)\n",
" delta = s-c\n",
" w -= eta*(s-c)*sprime*x\n",
"\n"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"0 0.1 [-40.23615769 56.53045166 31.6657019 ]\n",
"100000"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 0.1 [-40.38455406 56.79383207 32.0603326 ]\n",
"200000"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 0.1 [-40.6134952 57.1124 32.24526009]\n",
"300000"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 0.1 [-40.83933274 57.42943186 32.42569789]\n",
"400000"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 0.1 [-41.06277951 57.7431406 32.60419749]\n"
]
}
],
"prompt_number": 20
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Now the decision boundary is a lot \"sharper\":"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"d,a0,a1 = w\n",
"xs = linspace(0,1,100)[:,newaxis]; ys = linspace(0,1,100)[newaxis,:]\n",
"imshow(sigmoid(a0*xs+a1*ys+d).T,origin='lower')"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "pyout",
"prompt_number": 17,
"text": [
"<matplotlib.image.AxesImage at 0x4d02c50>"
]
},
{
"output_type": "display_data",
"png": "iVBORw0KGgoAAAANSUhEUgAAAP0AAAD5CAYAAADspDPqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvUuMbGl21/vb73ceaOM6p5FBR3DV1+7GmLawkBCSG1qn\n0UW3rRY2PQCZEs8pRkw8uzChS2IA5jFCLVTCA6tHrRLy5NagLQQYgYwEQpZKQi7JtqoOFxcnMyJ2\nPPbrDr5vxV7x5d6RearqPDLPXtJWZETskxW5K/57vf7/tbxhGAYWW2yxN8b8V/0BFltssZdrC+gX\nW+wNswX0iy32htkC+sUWe8NsAf1ii71htoB+scXeMAtf5C/3PO9F/vrFFlvsjM11418o6I39X8D/\nDQT2P6ePyB4xkNojs0dhDj+BIII0gAr4fcAXgB8Cftgej4Avgv/FluiLW/I/sOKL6cc8Sj7iER/z\niI/4Ih+bn4ePeTg85Q8cfo/q97aUv7eDj4CPgf9ljr///8Lf/yPAM+DSHN3aHJs9rA9Q91ADW2AH\n7IED0ADtxNHZo3eOYeIQ0+ecsx8AX7vd/4zXxn7A3frMP+Bufd5/cOa9lwB6MF9b9+veY2DgYbIM\ngUfDeIMIoPegD8ALzP1hAyT2kPvDCsihz332ZcqQD6z8kiKpKNhQsLaPGwpvQ+7VZOGOJOsYqh2s\nwVtj0LvD/HcyTtAcNOAfIO4gbqHpzWka0HL4E0d/5upIPLSwpBZ7GfYSQC+Ah1Owwwh4eU1A73Ma\nEcQG+I0He8+AcYu5AWRAbo/SgyufPo/YxTmXPCCjJrdHoX7O/S1puicjJNz0hJseT0AfYwINcd/2\n8PYQtBA1kAzQ9tAN5lOHnHrunmnwT10d18OLydU5d85iiz2vvQTQ/xGug15MvtYC+MA+9zhNAToY\nemh82CnQ18AaA/gCWHlwFTAUEbsq55LWgnzrPNbkfk2ebSiSgHQN4XqA7QA7+Nr/aX/nFgN8ORLw\nG4hD6HpoB3OIx5+KZW4C/bnw3bPHTSH+4zPvva72+FV/gOe0x6/6A3yO9hJA/38wfmW11wfzlW4Z\nIRGonxuMi43NYx9AF8LBN2CMGMP8NaPHL6AvfA5lCsXAOl6zSmxYTz2G+azJvQ2ZX+OnW5KLHd52\ngC187ccx+b2AXof5LUStAn0/hvjy182F+Br0A2NYL1fCc97X753z+I9nr/3ra49f9Qd4Tnv8qj/A\n52gvKaeH08BXTIf1U6APOcbWQwithc3OvqVz+5Qj6IfCoykSujxkU61ZRxtWwQj6kjUl66P3T5Ke\nstqP0cPGPuaMUYX9GH5jwvu+hbY9LdS5f81NoHfz/Dlgz90MFlvs09hLAL0EuVNfc53ju3CR/F48\nfghDAF1gwvytZ4AuIFWgJ/fo84A+99mFGau8IvNrcm8EfcHG5PteTRYfKMotyRaCdY9XG49/LM3v\n7MfYg3cwR9gYjx/3pqgXDtd7E1MhPoxe3lOv6SujowABuaceF+Av9lnsJYH+3Hsa+C7oBfgW9AL8\nNoR9YECpu30bjpV86fgd8oRVW5EGWwrfhPlXXJAL6KnJ4x15sIHtQFo1BPVgbiY1I+gP6nEPXmyB\n3xnwh9086DXwNWDdHF+ulNwQluLdYi/CXlL1fiqXd9t4un0XcAp6AX5kPnLnGfDvGUPyjDG3P7b5\nPZoypt4WrL2Kq3hDHtTkbE4r+kFNEawJsp6o6gi2/fh7JbyX3N6C3m9Gbx/1Bvxy2xLQd/Yv0cDv\nOfXw2uu7VKZzBb7lZrDYp7WX2LLTX2nt46Zq3trzt4ye3ub2fWB+rXhfAb1u4a2AAroiYig8tpSs\ngoosUG07p50Xxx1FuR9/pwa9BfvR0zemdx+1BvBxc/qJXU/venx9dQT4Ogm6qYV3G9LOYotN2Usk\n58hXW3ex5XXt5d0cX8g6KsTvI/PvDp55S3r2mtSXm8cuD+nykG2Us84q0mhL4Z327Y/9+2hPU2yI\nDw3eZsATyp2AXqr4NrcPDjA0EB+g9eyndVp4moWnq/pibvwjj8PEz3PnLLbY89hLyumnglft/fXN\nwM3tG/Wz7t3H0Pmwt737mhH0UtRTbN42i9iUBUlYUYYbCn8s6Ek1fxXU5EmNl/UkZUNUdyPodetO\njoNh6UUJpC20nanoy1/ggl68s/bSbqi/VO0Xe9H2ksL7qWxV5/lCQXFzeznkuWLo0UAXweCPBT2X\noptyBH1ThgzbnCipKP1p0OdBTR5sCPOGoOyJtt3YHZAwX44U4/H3poXntdDsoelM/75hnm8vr4np\nwh0zry/h/GKfl73E8F5/fV3ToD9XzZeC3h5Tycfk96035uEuL78AchjKgK5KOMQ5G7/kKrygYOzd\n59RkbMmoicMDSdEQHw74W0yYL7/fVdgcDOD9ZsztWwzw3b9GSpQhtwPyOXquS9hZbLHb2ksk58D1\ngp42zzm0h5cwP+BEjDMEMMQG9HvPADJhbN+Jp8+hLwOG0mefZmzSioSdKuJtVFFvQx5uKfMNXQfU\nEAjYN5yC3np6z96LwhZi37L0htPKhHxyHeILi+/c1Zp7fynoLfZp7SWG94Pz2k2H6/Gljaf79rF5\nrfMMPXfnnbLqUkZu/spjuPRos5g6M2F+EWwoA6XAE0WeX1PEG5LsQFK0BFVnfp+E+fqwHl8IO0kE\nXWtBP5xW9DXo5SYAp0Qc99bnqdflHNS/WyYWLPa89pKr9+cG9ZyT32qyjgB/z5GX33kwROB7xtML\n8DXor8xjlwdsixQ/rSjSEfTlUX67NtX9oCaJG/ysJik7o+XXLTzN1lOg923rTuf2um8vf5HEK2I3\nxUDulVpssU9rLxH08qh9luvZp3J7t4XntvEOp/RcTdiRMH+NAX1mNPdNmbDNOzZByVVycfTyuTx6\nhp6bhnvivCVr9ja3H05JO5qtt7ea+8aQdeLGgl4BX4Neh+ZyO9R9juch7Cx6/MWex15yTi92rkR1\njqyjge+G+hH0tnfvFvROBvJ4DFVAlydsk5xLLmwRbyTqZFaIkwVbsmxHzoZwOxDWA54LekXYobWk\nndaAvhnMDUB4+bqSr+Ma3aeYuipS+ryJsDN3zmKLaXsFoNdfS/FtU3n8VGFP5/Ya9LH5N31onvqq\nqCe5vQI9ZUBfROzKnKuuMeG8r4t5Wwpq09ZL1uzDEG/TEW76sV6gQ3zp29uPEzQQHYwY5zBANFxv\n38nzgOsxz1Ruf46scxvN/WKLib1k0M9RT9xz5AbQOudL08udriMtPM9q7sPTQRtC0ZUwv4Ch8Nnn\nCUNesUo25GlN7lmw6xzf25D5O7ykJq46vM0AJZOtOwG934zhvajwhJrrau2nwvmbcvupVt0yZWex\n29orAL0u6N1E2pn6Wk9N15EWnmc09wNjbq/HaimKbp97HPKUtghZX9Rkyc7k8ieDNgT0W+K0oyx3\nUA3m5iEhvnj53fjfDGIIEkhaODSjVKjheiPy08zTm/LsC5FnsdvaK6Th6vfFpr7ybp7v8vIVJ19y\n+yE08/S2Kr9fM+b4Mlpr5TFcBezjlHVRsvKqo/RWPL2E+2l8IK+2xHVDuO7xt8MIdp3fbzlmH35j\nhmimvZm00/XXS5f6588yT2+xxW5rL3GIxhTob/MVnirutfZ1PTl3Yp4etoUXc1rYswo8rnwoPA5l\nwrovyQcD+pK11dzXo7ePa8oowqt7gs1wnKd3Qs3d2d8vYX5iwvzecvLb/vpfMkz8/Gnm6S222G3t\nFVbvxXQhz6XpukU+8fASILseP8K428Dw8nFaeCeeHihhKDzaMmZbFWzyilW8oQgchh41mW+Gavop\nxGV/qrnXwBfQ2/ZdmFh6bj/y8bXm3hXlzHl6Ke65rT33anLDe4st9hqAHq7ryVyqiibp6Nxeg14X\n9ALofUPYEUmsruavsZ4eyKEtYvoiYDPsWAW10txvVBvPgD6JWzNPb8/I0pNiYc5YxRf5rZ2pFzfj\nIgxN2LmN/FZX85l4b6o8uvDy
}
],
"prompt_number": 17
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"xs = linspace(0,1,100)\n",
"ys = xs\n",
"cla(); plot(xs,sigmoid(a0*xs+a1*ys+d))"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "pyout",
"prompt_number": 18,
"text": [
"[<matplotlib.lines.Line2D at 0x4a351d0>]"
]
},
{
"output_type": "display_data",
"png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD9CAYAAABHnDf0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAGO5JREFUeJzt3X9wFOd9x/GPQEqqgMNvqHV3jYCTkbCQcCoHsGsjkjjC\nJJangczI7R82VlUNNfY4cWec2m0t+Q8PkLQTj9V2oGPjxjYauTaJSAPnDpSzE/FDqQVixpLhRCT7\npBqKACEMtiWO7R9XhATS3el0d3u7+37NaKTzLcuXZ6SPH3332WczDMMwBACwlUlmFwAASDzCHQBs\niHAHABsi3AHAhgh3ALAhwh0AbChquD/66KOaN2+elixZMuYxTzzxhPLy8lRcXKwjR44ktEAAwPhF\nDff169fL5/ON+f7u3bvV0dGhQCCgbdu2acOGDQktEAAwflHD/Z577tGMGTPGfH/Xrl16+OGHJUnL\nli1TX1+fTp8+nbgKAQDjNuGee09Pjzwez9Brt9ut7u7uiZ4WADABmYk4yY07GGRkZNx0zGj/DQAQ\nXTy7xEx45u5yuRQMBoded3d3y+VyjXqsYRh8GIaee+4502tIlw87jkVfn6FXXzX0p39qaN48Qzk5\nhtatM/T884YaGgwdPWrowgVnjAXfFxP/iNeEZ+7l5eWqq6tTRUWFDh06pOnTp2vevHkTPS1gKYYh\nvfOOtG2btG+ftGqVtHat9LOfSR6PxC+uSLWo4f7QQw/p3XffVW9vrzwej2prazU4OChJqq6u1po1\na7R79255vV5NmTJF27dvT3rRQDrZv1/627+V+vqkH/1IeuUVafp0s6uC00UN9/r6+qgnqaurS0gx\nTlFaWmp2CWnDymPR0yM9+qh08qRUWytVVEiTJ8d/PiuPRaIxFhOXYUykqTOevygjY0L9IyCd/Nd/\nSX/+59Jjj0lPPy1lZZldEewq3uxMyGoZwCmuXpW2bJFefFF67TXp2982uyJgdIQ7MA5PPSU1NUnN\nzeELpUC6ItyBGP3sZ+EVMU1NUoSbtoG0QLgDMdi5U/rJT6QDBwh2WAPhDkRx8KBUXR2etX/ta2ZX\nA8SG1TJABJcuSYsXS3V10gMPmF0NnCje7CTcgQiefVbq7JR27DC7EjgV4Q4k2IkT0l13SceOSTk5\nZlcDp4o3O3nMHjAKw5Aef1x65hmCHdZEuAOj2LkzvL3A44+bXQkQH9oywA0++0xatCh8B+rKlWZX\nA6ejLQMkSH29VFhIsMPaCHdgGMOQXnqJdgysj3AHhjl4UPr0U6mszOxKgIkh3IFh6urC2/hO4icD\nFscFVeD/ffJJ+G7Uzk6epIT0wQVVYIK2bQs/TYlghx0wcwckDQxIubnSf/5neKUMkC6YuQMT8Mtf\nhte2E+ywC8IdkPTmm9LDD5tdBZA4tGXgeJ9/Ls2bJ3V0SHPmmF0NMBJtGSBO+/dLRUUEO+yFcIfj\nNTZKDz5odhVAYtGWgaNdvSp5POHZ+223mV0NcDPaMkAc3n9f+upXCXbYD+EOR9u1SyovN7sKIPEI\ndzjarl3022FPhDscq7NTOnVKWrbM7EqAxCPc4Vi/+pX0ve9JkyebXQmQeIQ7HIt+O+yMpZBwpC++\nkGbNCm/ze8stZlcDjI2lkMA4tLSElz8S7LArwh2O1NQk3X232VUAyUO4w5EId9gdPXc4jmGEd4Fs\naZHcbrOrASJLWs/d5/MpPz9feXl52rx5803v9/b2avXq1Vq6dKkKCwv16quvjrsIIJUCASk7m2CH\nvUUM91AopI0bN8rn86mtrU319fVqb28fcUxdXZ3uuOMOHT16VH6/X0899ZSuXLmS1KKBiaAlAyeI\nGO7Nzc3yer3Kzc1VVlaWKioq1NjYOOKYW2+9Vf39/ZKk/v5+zZo1S5mZmcmrGJggwh1OEDGFe3p6\n5PF4hl673W4dPnx4xDFVVVX65je/qZycHF28eFFvvvnmmOerqakZ+rq0tFSlpaXxVQ1MQFOTtHGj\n2VUAo/P7/fL7/RM+T8Rwz8jIiHqCF154QUuXLpXf79fJkyd13333qbW1VbeMsoB4eLgDZujtlf7n\nf6QlS8yuBBjdjRPf2trauM4TsS3jcrkUDAaHXgeDQblvuAp14MAB/eAHP5AkLVy4UPPnz9fx48fj\nKgZItgMHpOXL2U8G9hcx3EtKShQIBNTV1aWBgQE1NDSo/IbNOPLz87V3715J0unTp3X8+HEtWLAg\neRUDE0C/HU4RsS2TmZmpuro6lZWVKRQKqbKyUgUFBdq6daskqbq6Ws8884zWr1+v4uJiXb16VVu2\nbNHMmTNTUjwwXk1NUpy/5QKWwk1McIzPP5dmzw7v4T51qtnVALFh4zAgiiNHpEWLCHY4A+EOxzh6\nVPr6182uAkgNwh2O0doqFRebXQWQGoQ7HOPoUcIdzsEFVThCKCRNmyb19IQ/A1bBBVUggpMnpblz\nCXY4B+EOR6AlA6ch3OEIXEyF0xDucITWVmnpUrOrAFKHcIcjMHOH0xDusL2zZ6X+fik31+xKgNQh\n3GF7ra1SUZEUw+MJANsg3GF79NvhRIQ7bI9lkHAiwh22x8VUOBHbD8DWBgak6dPDF1Wzs82uBhg/\nth8ARvHhh+FVMgQ7nIZwh63Rb4dTEe6wNfrtcCrCHbbW1iYVFppdBZB6hDts7cMPpfx8s6sAUo/V\nMrCty5elWbOkTz+VJk82uxogPqyWAW5w4oTk9RLscCbCHbbV3k5LBs5FuMO2PvxQKigwuwrAHIQ7\nbIuLqXAywh22RVsGTsZqGdhSKCRNnSr19kpTpphdDRA/VssAw3z0kTR3LsEO5yLcYUu0ZOB0hDts\niZUycDrCHbbEzB1OR7jDlpi5w+kId9gSa9zhdIQ7bOfMmfBSyLlzza4EME/UcPf5fMrPz1deXp42\nb9486jF+v1933HGHCgsLVVpamugagXG51pLJyDC7EsA8mZHeDIVC2rhxo/bu3SuXy6U777xT5eXl\nKhjWzOzr69Njjz2md955R263W729vUkvGoiElgwQZebe3Nwsr9er3NxcZWVlqaKiQo2NjSOO2bFj\nh9auXSu32y1Jmj17dvKqBWLAShkgSrj39PTI4/EMvXa73erp6RlxTCAQ0Llz57Rq1SqVlJTotdde\nS06lQIxYKQNEactkxNC0HBwcVEtLi/bt26fLly9rxYoVWr58ufLy8m46tqamZujr0tJS+vNICmbu\nsDK/3y+/3z/h80QMd5fLpWAwOPQ6GAwOtV+u8Xg8mj17trKzs5Wdna17771Xra2tUcMdSIbPPpM+\n+USaP9/sSoD43Djxra2tjes8EdsyJSUlCgQC6urq0sDAgBoaGlReXj7imAcffFC//e1vFQqFdPny\nZR0+fFiLFy+Oqxhgok6elHJzpcyI0xbA/iL+CGRmZqqurk5lZWUKhUKqrKxUQUGBtm7dKkmqrq5W\nfn6+Vq9eraKiIk2aNElVVVWEO0wTCEi33WZ2FYD52M8dtrJli3T6tPQP/2B2JUBisJ87IOnECWmU\nyz2A4xDusBXaMkAY4Q5bCQSYuQMS4Q4buXhR6uuTXC6zKwHMR7jDNjo6JK9XmsR3NUC4wz64mApc\nR7jDNriYClxHuMM2uJgKXEe4wzZoywDXEe6wDdoywHWEO2zh3DlpYIDnpgLXEO6whWuzdp6bCoQR\n7rAFLqYCIxHusAUupgIjEe6wBS6mAiMR7rAF2jLASDysA5ZnGNK0aVJXlzRzptnVAInFwzrgWP/7\nv9KXvkSwA8MR7rA8WjLAzQh3WB4rZYCbEe6wPFbKADcj3GF5J04Q7sCNCHdYHuEO3IylkLC0q1el\nqVPDK2amTjW7GiDxWAoJR+rulmbMINiBGxHusDRaMsDoCHdYGssggdER7rA0lkECoyPcYWm0ZYDR\nEe6wNNoywOhYCgnLGhyUbrlFunBB+vKXza4GSA6WQsJxOjsll4tgB0ZDuMOyaMkAYyPcYVmslAHG\nRrjDslgpA4yNcIdl0ZYBxhY1
}
],
"prompt_number": 18
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's do more of that still.\n",
"\n"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"eta = 1.0\n",
"for iter in range(500000):\n",
" i = iter%len(data)\n",
" if iter%100000==0: print iter,eta,w\n",
" x = augmented[i]\n",
" c = labels[i]\n",
" s = sigmoid(dot(w,x))\n",
" sprime = s*(1-s)\n",
" delta = s-c\n",
" w -= eta*(s-c)*sprime*x"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"0 1.0 [-18.92549311 26.57268748 14.97405465]\n",
"100000"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.0 [-26.38035611 37.06943269 20.67225609]\n",
"200000"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.0 [-31.05291324 43.63732416 24.36693909]\n",
"300000"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.0 [-34.65012316 48.69053925 27.22045794]\n",
"400000"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.0 [-37.64181362 52.89046892 29.59935134]\n"
]
}
],
"prompt_number": 19
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"d,a0,a1 = w\n",
"xs = linspace(0,1,100)[:,newaxis]; ys = linspace(0,1,100)[newaxis,:]\n",
"imshow(sigmoid(a0*xs+a1*ys+d).T,origin='lower')"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "pyout",
"prompt_number": 21,
"text": [
"<matplotlib.image.AxesImage at 0x3ff27d0>"
]
},
{
"output_type": "display_data",
"png": "iVBORw0KGgoAAAANSUhEUgAAAP0AAAD5CAYAAADspDPqAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvUmsZNlZtvvsPnb0/Y44nfMKqWSXMcaWfZEQVwZKaSR0\nbZWwqQHIpGinGDHx7Oqf4JQYgGlGqIRSeGDVqFRCHtXACAFGILg/XOR76xdy1emi7/uIHbHvYMXK\n2GfnOSezqrI5mbkeKZR50scndlXlG2utb33v+2lBEAQoFIqXBv1ZP4BCoXi6KNErFC8ZSvQKxUuG\nEr1C8ZKhRK9QvGQo0SsULxnmk/zhmqY9yR+vUCiu4arb+CcqesGXgJ//EN9vIDYg8ldz+7IAB4gB\nKSAJFMUrkYCsC4cWfBqMn/RJfG5A8rMDvuD8K19w/oXPLv6Dz0z/C++DFvb/7WP+Pxv4XxD8Nww6\n4tVdQwf4K+D/BObAElgB/va1Cb2C7etZ8wM+3L/hm8APeL6e+Qc8X8/7P675356C6D8sUkab0K9r\nxAfAGiG9BeJDYQzYsNRhYEPMgnNYZw2GBxlGgzhn6X3KdgPPbXDoHpNeDNArAWZzA12gDfYUEgbM\nNzANxL8Ue/tO69C7R89CUvgKxfPEDRW9lBnb3wNoCKEbiLV3iViLJ7BxYJmAyQa6GtQ1ONGhZDI8\nzHKWOMAzGng0cJwV1UKL2MEKBqB1wZqDO4HEBKZLMNdiT7Fi9zEjRR8WuRb6MyV+xfPCUxD9rQ/5\n/WEJaVxcT8NbfwOYiV/XLmxWW9HrQvSnOuQ1hoksp9UDCrQp0yDtjMgWx+SmQ7GXb4E5AXMAizXE\nffjft6JfIISvh16b0JNp2yd71iv+rWf43h+VW8/6AT4kt571AzxGbqDow8gPACkvue5KKVqIf4Qp\nBGNYGTCMQcuGGgR5jWkhScfzqOX2OY0dUrB6FDI9UuURjrfCbq7QRkAfrBXE5/B/+NANxF5Cbu9B\niDss/puyut961g/wEbj1rB/gQ3LrWT/AY+QGbu+vQopebqo1dmf7KTCClQkjE3QhetIwLSdYVh3q\nxj6nZpOi3aaaqVEI2qTLU2xvBX2gI1b8+ACWmjg4LILdR8yGy7f5N0X4CsWjcoNFH1zykkU9HyH6\nJeIfYYY429uwjMHEFtv8ms763GTtmQxiWc6TexRiLap6jXy8i1lokdqfCtF3wRxDbCi2+NMVzP1d\n9WC9fafwdn7DbpuvhZ5aobjJ3GDRw+4EHUWu9CvEar8t6AUOrONC+AMLGjqcAUWYZBLUvQoZelSp\nUTQ7pPIzvKPO/ZXeGIEzAteH+EiIfrl9B/lRE76ui6IKeorngRss+vAGOrzSw26bLzffC+6v9kEC\n/BiMAMOEOlCAecmlc1Ckltjj1DqgbLQopnrsBXVMb43prdEHAXofnBm4C0gsYRnAfLvNN3lQ9AEP\n3jMoFDeZGyz6KOH7e/mSZbYVYrWfASPYWDA1YONCE8jBqmwz8dJ0rRLn+QOK8TYlt01Za5ApjElX\nJmj9AHpgTCAxhfVcVPRn691HS1j0V+1DFIqbzHMg+uCS38uVX266l4j1dgrY4mw/j8FiA20N0hp+\nycIvW/QTBWruHtlkl4pTp2qdoRcCkpUZen8jCnoDiA1gM4HZAqbrXVdA+OMmfG3HJV8rFDeR50D0\n8GDJLNyl57Nr2Amd74kDU5hY4grvXIMiLLIO7VKR4+CICnU8rYGbXlE+6GIOgTZoAzBHYM/ADSCx\n2hX0wq1Dl3XpgdoFKG42z4How3f14f638HorC3uykm8h7u6nMI1Dy4KsBnlYlB3a/1sRfbOiotfw\nqFNOd1hbOsEAtKbo0jP6YA/F2T452X2kyCbg60QvP6LUiq+4iTwHog8TXUMvO9vLevsUGMPCAM0V\nffYN2JwZLE5cxskMrazHcfoIz2pToUEh2yPlTbC6PvRBH0JsAekpLNcw3Vbzl1ws5q1DT6dQ3HSe\nI9GHV3zYraXyg0BW8sOit2AVA38DPUOI/tRgVY4xSWZoaUL0FbNJxaihZQJi3hKr50MH9B64YzCH\n4mw/XouGHbniy48cUFt6xfPDcyT6KFJm4W1+2IU3ByzYzMTvpwF0TIK6TnBqsMw5dNMFrPInqBgN\nPKOOm5yTqQywxwv0ToDeDTBGwBBiAcS3V3jh9tw1u2u7cLeeatZR3FSeY9HDxb44Kf4VF8w4TICR\nuGzvxUXDzjn4BZOul2cxNfBiDcpGk1yyT8WskViMsZsb9HYAQ1HYs1fChSf3EdJfv+bBar4eeTol\nfMVN4jkU/WWtueFVPmq93Z7tVyaMbegYUNdYFwymewkWbZNGrsKJfUjZbrBvnxIvzEhXppitJfRF\nYc+eQ3IEy6Xw3csVX9pvLyvsySeTT61Q3ASeQ9HD1Sdo6bnXEZKUK/4EfAeIb4UPnGsEJwZB3mag\n5TjLHFA2mlRoEIstsQtN3P0l9Lain4E+gsUEpguYry+K3ufiR5FEbvfViq+4KTzHor9MQrI1V+fi\nNt8S3XnL5Vb0JiQ0KOpscibDVJaz6j5Fo0VFq5PZeu4zkxFaO0BrBVhjsAbgrsFd7US/ZLe/iFpv\ntciTKdErbgLPqeiv4jLP/RzxjzkGXFhpMIhDwxERezmNSTFFc2+P82yHE7dB3u5RyHVILwc4zRVO\nayVKA9uzfXIGKx9WwYMrvRS+StVR3FReUNGHy2nm9veOeK0MkaeHAzkI0hrjvTSTToqs1adoNyk5\nTfadU0p6E725wWmvYAh0wJ5Asg++Jgp6l3nu5burLb3iJvKciz7qvpOEPfeRol4Qg3UCFg4MdKgL\n+21Q1piYSeqJKsfOER4N0uYIPdcgeTCHAcJ+O93abwORshP13EfP9uGuArXyK24CL4Doo6dn2FXz\npegN7rvwgjlsFrB0YWCKK7xTIA+TdIJaVXjuPRoUrC6Z3AQO2qKjryU893of3KXw3bv+xWs8ueLL\nin7YNaAKeoqbwHMueri4nsqvw/bbcKeeXO0n4NswckHfeu7zMC/F6O0VaNhVzmIHlIw2hVSf0rqN\nU17iVFZowwCtv03QvcRzL1f8aLNO9CkVimfFCyD6MA/z3MuwjTGsLZiYsHKhAWRh6cXYVA3aMY+z\nwgGFRJtyoknZqJErjbEr/s5zPxYtuv5UOHhn26Keya6aH64uqBVecVN4QUR/2Uk5LPpowo4DwTZP\nz49DT5zt1+cm64rJMJmlEa9wkjykYtXw9BpmPiC9N0bf5ukZA3AGQvTuAtyNWOXlkAwp/Gier8rT\nUzxrXhDRw+70LC/MokGach2WwreBuDjjS8/9mSaitTIxWuUiCQ6pUKestUhlFuwdtGCwhrY411uD\nbbRWAPHV7idHR2Bd5blXolc8C14Q0UfTdcJV/bDww+m5NqJF1xWe+7YlhmTkYVmy6X0ij72Yc2oe\nUNZblJNdZqaD1tlgNDZo3QCjB/YIYktIzMS5fhq5wrtqW68894pnxQsi+su4rKAn23NDZ/vAEp77\nwBXtuQ1Yn5vMT+KMEjka+SrH2Q5lq0lVq1HIdsl4Y+zuSpzth+DORV7HwhdXePLdosMulfVWcRN4\nwUUfXe1ll54ci2WJXxeu6LLZBm2sz0RW/jAd0LQrJLMDKlaNfesEI+uTqMyw+yuxze+COxS5erNA\n5OmtQvbbcClRobgJvICij46hiK744YSdObtt/liEaXZt0bBzCn7OpJvJYZUPqRo1qkaNeHJBvtrD\nHi8w2gFaD7SpaNqJDSG53JUM5YhrGZ0t+wWjT6vO94qnyQsm+mie3lWil+d7GbYho7UQCTt1HfKw\nypv09nIsFzpVu05Vr1NIdji0j0nOQGuA0RbuOwbgLCE5FJuGKTszzmUpOxJ1lad42rxgopdc1p77\nkIYdHOG5n7hi4k0TgnOd5VkMrbyhnS9xUjikbDSpOjXM9JpMeYpbXYigjZ6o5G+GMF9t8/S2d/fS\nlCOv8KLpOgrF0+QFFT08uGkO
}
],
"prompt_number": 21
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"xs = linspace(0,1,100)\n",
"ys = xs\n",
"cla(); plot(xs,sigmoid(a0*xs+a1*ys+d)); savefig(\"temp.png\")"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "display_data",
"png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD9CAYAAABHnDf0AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAFopJREFUeJzt3V9wVOX9x/HPQmKNFMPfUtjN/AJsJln+JKhhgHa0oR0b\nimM6U+hMvLI0jSkVqb3o2LEXJnZGCe1FLfEiOIr1Xya1tcZpca0gi1MhpBomTAExUtImi42GJKKg\nJCzP72JLYkiyu9ls9uw5+37NZCTd0823Z8Lbp8+ePesyxhgBABxlmtUDAAASj7gDgAMRdwBwIOIO\nAA5E3AHAgYg7ADhQ1Lj/8Ic/1IIFC7Ry5cpxj9m+fbvy8vJUVFSko0ePJnRAAMDERY37li1b5Pf7\nx3187969ev/999Xe3q7du3dr69atCR0QADBxUeN+6623avbs2eM+/sorr+juu++WJK1Zs0b9/f3q\n7u5O3IQAgAmb9J57MBhUTk7O0Pcej0ddXV2TfVoAwCRkJOJJrr2DgcvlGnXMWP8ZACC6eO4SM+mV\nu9vtVmdn59D3XV1dcrvdYx5rjOHLGD300EOWz5AqX6lyLl580ej//s/olluM6uuNPv44fc9FKnxx\nLoa/4jXpuJeVlemZZ56RJDU3N2vWrFlasGDBZJ8WSJqnn5Z++lOpoUF6+23pnnukG2+0eipgcqJu\ny9x11106ePCgenp6lJOTo5qaGg0ODkqSqqqqtHHjRu3du1der1czZszQnj17pnxoIFF275Z+9Svp\njTek/HyrpwESJ2rcGxoaoj5JXV1dQoZJFyUlJVaPkDKsPBePPy79+tdSICAtXWrZGEP4vRjGuZg8\nl5nMps5EfpDLNan9IyCRPvwwvFJvbZUWL7Z6GmB88baT2w8gLdXXS5s3E3Y4Fyt3pJ2BASk3V3rt\nNSnCXTWAlMDKHYjRiy9KPh9hh7MRd6QVY6THHgtf+gg4GXFHWjlyRDp3TrrjDqsnAaYWcUdaeewx\n6b77pOnTrZ4EmFq8oIq0cfastGKFdOaMlJ1t9TRAbHhBFYji4EFp/XrCjvRA3JE2WlulW26xegog\nOYg70kZrq3TzzVZPASQHe+5IC8ZIc+ZI774rcdNS2Al77kAEHR3SjBmEHemDuCMtsCWDdEPckRbe\neYcXU5FeiDvSAit3pBviDsczhrgj/RB3OF4wKLlc0qJFVk8CJA9xh+NdXbW7XFZPAiQPcYfjsSWD\ndETc4XjEHemIuMPxiDvSEXGHo3V3SxcuhD8zFUgnxB2OdvQoL6YiPRF3OBq3+UW6Iu5wtBMnpOXL\nrZ4CSD7iDkc7e1byeKyeAkg+4g5HO3uWd6YiPRF3OFowSNyRnog7HOuTT6QrV6Qbb7R6EiD5iDsc\n6+xZye3mMkikJ+IOx2JLBumMuMOxeDEV6Yy4w7GubssA6Yi4w7HYlkE6I+5wLLZlkM6ixt3v96ug\noEB5eXmqra0d9XhPT482bNigVatWacWKFXr66aenYk5gwtiWQTpzGWPMeA+GQiHl5+dr3759crvd\nWr16tRoaGuTz+YaOqa6u1qVLl/Too4+qp6dH+fn56u7uVkZGxsgf5HIpwo8CEm7xYmn/fmnJEqsn\nAeIXbzsjrtxbWlrk9XqVm5urzMxMlZeXq6mpacQxCxcu1Pnz5yVJ58+f19y5c0eFHUg2Y6QPPpAW\nLrR6EsAaESscDAaVk5Mz9L3H49GRI0dGHFNZWalvfvObWrRokT755BP94Q9/GPf5qqurh/5cUlKi\nkpKS+KYGojh3TpoxQ8rKsnoSYGICgYACgcCknydi3F0xvLXvkUce0apVqxQIBHT69Gndfvvtamtr\n08yZM0cd+8W4A1OJF1NhV9cufGtqauJ6nojbMm63W52dnUPfd3Z2ynPN/VMPHTqk73//+5KkpUuX\navHixTp16lRcwwCJwmWQSHcR415cXKz29nZ1dHRoYGBAjY2NKisrG3FMQUGB9u3bJ0nq7u7WqVOn\ntIRXsGAxVu5IdxG3ZTIyMlRXV6fS0lKFQiFVVFTI5/Opvr5eklRVVaUHH3xQW7ZsUVFRka5cuaKd\nO3dqzpw5SRkeGA+XQSLdRbwUMqE/iEshkUQ//rFUWCj95CdWTwJMzpRcCgnYFdsySHfEHY7EtgzS\nHXGHI3G1DNIde+5wnMuXw29e+uwziTdLw+7Ycwf+p7tbmj+fsCO9EXc4Di+mAsQdDsR+O0Dc4UBc\nKQMQdzgQ2zIAcYcDsS0DEHc4ECt3gLjDgdhzB4g7HIhtGYB3qMJhBgelG26QLl2SprF0gQPwDlVA\nUl+fNHs2YQf4KwBHOXdO4rNiAOIOh+ntJe6ARNzhMMQdCCPucBTiDoQRdzgKcQfCiDschbgDYcQd\njtLbK82da/UUgPWIOxyFlTsQRtzhKMQdCCPucBTiDoQRdzgK71AFwog7HIWVOxDGXSHhGJcvS9df\nLw0McOMwOAd3hUTa6++XsrMJOyARdzgIWzLAMOIOxyDuwDDiDsfg3anAMOIOx2DlDgwj7nAM4g4M\nI+5wDOIODIsad7/fr4KCAuXl5am2tnbMYwKBgG666SatWLFCJSUliZ4RiAnvTgWGZUR6MBQKadu2\nbdq3b5/cbrdWr16tsrIy+Xy+oWP6+/t177336rXXXpPH41FPT8+UDw2MhZU7MCziyr2lpUVer1e5\nubnKzMxUeXm5mpqaRhzzwgsvaNOmTfJ4PJKkefPmTd20QATEHRgWMe7BYFA5OTlD33s8HgWDwRHH\ntLe3q7e3V+vXr1dxcbGeffbZqZkUiIK4A8Mibsu4XK6oTzA4OKjW1lbt379fFy9e1Lp167R27Vrl\n5eWNOra6unrozyUlJezPI6GIO5wgEAgoEAhM+nkixt3tdquzs3Po+87OzqHtl6tycnI0b948ZWVl\nKSsrS7fddpva2tqixh1INN7EBCe4duFbU1MT1/NE3JYpLi5We3u7Ojo6NDAwoMbGRpWVlY045rvf\n/a7+/ve/KxQK6eLFizpy5IiWLVsW1zBAvK5ckT7+WJo1y+pJgNQQceWekZGhuro6lZaWKhQKqaKi\nQj6fT/X19ZKkqqoqFRQUaMOGDSosLNS0adNUWVlJ3JF0H38szZwpTZ9u9SRAauB+7nCE06elb387\n/E/ASbifO9IaL6YCIxF3OALvTgVGIu5wBFbuwEjEHY5A3IGRiDscgbgDIxF3OAJvYAJGIu5wBFbu\nwEjEHY5A3IGRiDscgbgDIxF3OAJxB0Yi7nAE4g6MxL1lYHvGSNddJ128KGVmWj0NkFjcWwZp6/x5\nKSuLsANfRNxhe2zJAKMRd9gecQdGI+6wPd6dCoxG3GF7vb3S7NlWTwGkFuIO22PlDoxG3GF77LkD\noxF32B5xB0Yj7rA94g6MRtxhe8QdGI24w/aIOzAacYftnTtH3IFrEXfYHit3YDTiDlszhjcxAWMh\n7rC1CxfCd4O8/nqrJwFSC3GHrbElA4yNuMPWiDswNuIOWyPuwNiIO2yNuANjI+6wNeIOjI24w9aI\nOzA24g5bI+7A2Ig7bI0P6gDGRtxha6zcgbFFjbvf71dBQYHy8vJUW1s77nH/+Mc/lJGRoZdeeimh\nAwKREHdgbBHjHgqFtG3bNvn9fp04cUINDQ06efLkmMc98MAD2rBhg4wxUzYscC3uCAmMLWLcW1pa\n5PV6lZubq8zMTJWXl6upqWnUcbt27dLmzZs1f/78KRsUGAsrd2BsEeMeDAaVk5Mz9L3H41EwGBx1\nTFNTk7Zu3SpJcrlcUzAmMDbiDowtI9KDsYT6/vvv144dO+RyuWSMibgtU11dPfTnkpISlZSUxDwo\ncK3PPgvf8jcry+pJgMQJBAIKBAKTfh6XiVDj5uZmVVdXy+/3S5IeffRRTZs2TQ888MDQMUuWLBkK\nek9Pj2644QY98cQTKisrG/mD/hd/IFGCQWn1aunsWasnAaZOvO2MuHIvLi5We3u7Ojo6tGjRIjU2\nNqqhoWHEMf/617+G/rxlyxbd
}
],
"prompt_number": 22
},
{
"cell_type": "heading",
"level": 1,
"metadata": {},
"source": [
"More About the Logistic Function"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(log likelihood ratio)\n",
"\n",
"Our justification of the logistic function above has been somewhat heuristic: we have justified it as a \"smooth analog\" of the Heaviside function.\n",
"\n",
"The logistic function actually comes from the log likelihood ratio, used as a discriminant function:\n",
"\n",
"$$ D(x) = \\log\\frac{P(\\omega=1|x)}{P(\\omega=2|x)} = \\log\\frac{p}{1-p} $$"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(discriminant functions)\n",
"\n",
"Remember from Bayesian statistics where the discriminant functions come from.\n",
"\n",
"Minimum error rate classification under a 0-1 loss is given by maximizing the posterior:\n",
"\n",
"$$C(x) = \\arg\\max_i P(\\omega=i | x) $$\n",
"\n",
"This maximization can be done using many different kinds of functions, including the log likelihood ratio above."
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(log likelihood ratio for Gaussians)\n",
"\n",
"Let's compute the discriminant function for a simple Gaussian and assume priors $p_1$, $p_2$:\n",
"\n",
"$$ G_1(x) = c e^{-(x-\\mu_1)^2/2} $$\n",
"\n",
"$$ D(x) = \\log \\frac{p_1 G_1(x)/(p_1 G_1(x)+p_2 G_2(x))}{p_2 G_2(x)/(p_1 G_1(x)+p_2 G_2(x))} = \\log \\frac{p_1 G_1(x)}{p_2 G_2(x)}$$"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(linearity of log likelihood ratio)\n",
"\n",
"$$ D(x) = \\log \\frac{p_1 c_1 e^{(x-\\mu_1)^2/2}}{p_2 c_2 e^{-(x-\\mu_2)^2/2}} $$\n",
"$$ = \\log C e^{-(x-\\mu_1)^2/2 + (x-\\mu_2)^2/2} $$\n",
"$$ = \\log C e^{-\\alpha - \\beta x} $$\n",
"$$ = -\\alpha - \\beta x + C $$"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"(justification of logistic regression)\n",
"\n",
"Look at the equation again:\n",
"\n",
"$$ \\log \\frac{p}{1-p} = D(X) = -\\alpha -\\beta x + C $$\n",
"\n",
"Or:\n",
"\n",
"$$ p = \\frac{1}{1-e^{-\\alpha-\\beta x}} $$"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Using scikit-learn\n",
"=================="
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Logistic regression, like most other learning algorithms we're using, has good implementations in the `scikit.learn` Python package. Let's look at how to use that.\n",
"\n"
]
},
{
"cell_type": "code",
"collapsed": true,
"input": [
"from sklearn import linear_model,datasets"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 27
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"All `scikit` learning algorithms start with allocating a model and optionally giving some learning parameters.\n",
"\n"
]
},
{
"cell_type": "code",
"collapsed": true,
"input": [
"logreg = linear_model.LogisticRegression(C=1e5)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 33
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Next, the model is trained on the data and the labels. Data is always given as a 2D array, with rows giving the input vectors $x$. Classes are given as a 1D array of labels.\n",
"\n"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"logreg.fit(data,labels)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "pyout",
"prompt_number": 34,
"text": [
"LogisticRegression(C=100000.0, class_weight=None, dual=False,\n",
" fit_intercept=True, intercept_scaling=1, penalty='l2',\n",
" tol=0.0001)"
]
}
],
"prompt_number": 34
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"After a model has been fit, it can be used to predict labels.\n",
"\n"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"z = logreg.predict(data)\n",
"print 1*labels[:10]\n",
"print z[:10]\n",
"print \"errors\",sum(z!=labels)"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"[1 1 0 0 0 0 1 1 0 1]\n",
"[1 1 0 0 0 0 1 1 0 1]\n",
"errors 1\n"
]
}
],
"prompt_number": 35
}
],
"metadata": {}
}
]
}