mirror of
https://github.com/saymrwulf/nvidia-AI.summerschool17-notebooks.git
synced 2026-11-20 09:52:58 +00:00
1231 lines
2.4 MiB
Text
1231 lines
2.4 MiB
Text
|
|
{
|
||
|
|
"metadata": {
|
||
|
|
"name": "38-pca"
|
||
|
|
},
|
||
|
|
"nbformat": 3,
|
||
|
|
"nbformat_minor": 0,
|
||
|
|
"worksheets": [
|
||
|
|
{
|
||
|
|
"cells": [
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Title: PCA\n",
|
||
|
|
"Author: Thomas Breuel\n",
|
||
|
|
"Institution: UniKL"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"from pylab import *\n",
|
||
|
|
"from scipy.spatial import distance\n",
|
||
|
|
"from scipy.ndimage import interpolation\n",
|
||
|
|
"import tables\n",
|
||
|
|
"import random as pyrandom"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 1
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"def showgrid(images,rows=None,cols=10,size=1,cmap=cm.gray):\n",
|
||
|
|
" if rows is None: rows = (len(images)+cols-1)//cols\n",
|
||
|
|
" figsize(cols*size,rows*size)\n",
|
||
|
|
" for i in range(len(images)):\n",
|
||
|
|
" subplot(rows,cols,i+1)\n",
|
||
|
|
" imshow(images[i],cmap=cmap)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 2
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "heading",
|
||
|
|
"level": 1,
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"The MNIST Database"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"# loading the MNIST database\n",
|
||
|
|
"with tables.openFile(\"mnist.h5\",\"r\") as mnist:\n",
|
||
|
|
" print mnist\n",
|
||
|
|
" images = array(mnist.root.deskewed,'f')\n",
|
||
|
|
" labels = array(mnist.root.labels,'i')\n",
|
||
|
|
" test_images = array(mnist.root.test_deskewed,'f')\n",
|
||
|
|
" test_labels = array(mnist.root.test_labels,'i')"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"mnist.h5 (File) ''\n",
|
||
|
|
"Last modif.: 'Tue Jun 5 10:16:56 2012'\n",
|
||
|
|
"Object Tree: \n",
|
||
|
|
"/ (RootGroup) ''\n",
|
||
|
|
"/deskewed (EArray(60000, 28, 28), shuffle, zlib(9)) ''\n",
|
||
|
|
"/images (CArray(60000, 28, 28), shuffle, zlib(9)) ''\n",
|
||
|
|
"/labels (CArray(60000,), shuffle, zlib(9)) ''\n",
|
||
|
|
"/test_deskewed (EArray(10000, 28, 28), shuffle, zlib(9)) ''\n",
|
||
|
|
"/test_images (CArray(10000, 28, 28), shuffle, zlib(9)) ''\n",
|
||
|
|
"/test_labels (CArray(10000,), shuffle, zlib(9)) ''\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 3
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"# characters from the (deskewed) MNIST database\n",
|
||
|
|
"for i in range(16):\n",
|
||
|
|
" subplot(4,4,i+1)\n",
|
||
|
|
" imshow(images[i])"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAVUAAAD8CAYAAADHaDe8AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsfWlspFeV9lP7vu+L7fK+dbs3h+5AmkkgC2hEhkwQAwME\nsgBCAoSCRjAaAYEfQ/gRIT6IRmjIjwBSBIMERJqZFstM0kmnk+7E7V68b2W7yrXv+/796Lk3b5XL\nrnK5tgY/ktVu+3XV68fnPffcc55zLqtUKpVwhCMc4QhHaArYnb6BIxzhCEf4S8KRUz3CEY5whCbi\nyKke4QhHOEITceRUj3CEIxyhiThyqkc4whGO0EQcOdUjHOEIR2giGnaqFy5cwNjYGIaHh/GDH/yg\nmfd0BBzx20occds6HHELoNQA8vl8aXBwsLSxsVHKZrOlEydOlObn5xt5qSNUwRG/rcMRt63DEbe3\n0VCkeuXKFQwNDcFms4HH4+ETn/gEfv/73zfb3//V4ojf1uGI29bhiNvb4DbyQ06nEz09PfT/VqsV\nb731Vtk1LBbrcHfWJSh1oOHsr4XfI25bi3bze8TtbTTkVOslRiAQQCgU7ntNOp0+9DXNeI1q10Qi\nkX2vbxXq5Vcmk0Eul+/5/Wg0uu/3yTUymQzAu4bCfH8Wi1Xzdep9H+Y1Tqdz3+tbhXq5FQqFEIlE\n+16TSqX2vabW95t1TbXvh0KhfV+zFaiXW4VCAYVCse81kUikrmuITZVKJfr+5N96X+Og12xtbe17\nfUNO1WKxYHt7m/5/e3sbVqu1kZc6QhV0gl8WiwU2m00/L5VKHYkkW40j220dOmW35IOg03bbUE51\nenoaKysrsNvtyGaz+NWvfoWHH3642ff2V4t28cuMTMkHm82uaqiVP3On4i/NdrtpO91Obpm2y2az\n6Qfho5N22lCkyuVy8ZOf/AQPPfQQCoUCnnzySYyPj1e9rp7XavQaQhyHw6m6dWUSu9c1B72XdqBe\nfgUCwb6vU/l95haJgM/nAwCKxSIKhQLy+Tzy+TzYbDY4HA64XC69BqjOXa37qPeadqCdtttK+2fu\nJP7S7Laea1gsFk3V5XI5aruFQgEsFgscDgc8Hq+qzTf7XnbdW6lFLp3FYtXMVbQLtZzpXohEIl0b\nmbFYLFgslrqv34sDptGVSiXqTDkcDthsNgqFAgqFAorFYlO5cDqdXc2tSqXq9G0cCqFQqCv5ZbFY\n6O3tbcrrkIWF2CabzQaXywWPx6OprHw+j1wud+j3Y2Jra6v5hSoCm80GuVxOV4UrV64c5uUo6jGG\nyu0p+bxYLJZ9v5oTuRPyhc3mdq8FhXydx+NBoVDAYDDAarWCzWYjm81ie3sbHo8H8Xgc+Xy+zAF3\n09bzIGiF3TLtqdW8sFisMsdRKBSa7jgOg1b5BQLyXJPoVCQSQSqVwmw2Q6/XQy6XI51Ow+12w+v1\nwufztfWZP5RTZbFYeOWVV6BWq5t1P7siqmo5k0KhAB6PB41GA7FYDB6PB5FIBKFQCJfLhXA4DJlM\nBo1GA4PBALFYDDabjXA4jEAgAKfTCb/fj1wu17XOoRXcVoJwLRAIoFarMTw8jMHBQQwMDIDP56NQ\nKGBtbQ1bW1twuVzwer3wer0olUpl6ZQ7Dc3klmmn7eCDy+VCLpdjYGAAIpEI8XgcHo8HTqeTOptO\n/11abbuVtQCRSASNRoPR0VGMjIzAaDQil8vB4/Hg+vXrmJmZQTKZRCaTacn9VOLQyZjD/AGr/SzT\nSMkHn88Hn8+nq3M6nYZcLsfx48dhMpkgl8uh1+uh0+nw6quvYmlpCTabDcePH8c999wDo9EIgUCA\nhYUFzM7O4o9//CMSiQTNwXA4nK4wxko0834qIykWi0WjepFIBLPZjPe85z0YHx+H1WqFQqGAQCCA\nw+GA3W7H0tIS3nnnHbjdbgAAm81GsVhEsVi8I6PXZnBbuRtq5S6IvCafz4fZbMaHP/xhaDQabGxs\n4MqVK3A4HABAo9dOo9XPErE1Ho8HsVgMjUaD4eFhnDlzBgMDAxAKhdRPuFwuuN3uO8Opslgs3H//\n/eBwOPjiF7+Iz3/+82XfT6fT774Rl1uWUCfGR3Ig5EMgEEAikUCj0UAqlUIoFEIul0OlUkGhUEAs\nFiObzYLP51OHKhAIIBaLIRQKkUgkoNPpYDAY0N/fD71eD5FIRHMrmUwGhUKB3j/zfkiRphtQi1vg\ntvaTQCAQ1JVQZ/7OJAdlsVgwNjaGkZER9Pb2Qi6Xg8fjgcViQa1Wg8fjQafToVQqYXV1FaFQCNls\n9kAPcCaTaZtR10I93KZSKfo5sVEmisUixGIxFAoFJBIJBAIBfD4fIpFI03dAZAFks9nQarWwWq2Q\nSCTI5XIIBoNIJBJ0K9wNwUEtfpn673q07Hu9B8mnkrw/sWfg9t9MKpWir68Pp0+fxjvvvINoNIpC\noXBgftLp9IFs91BO9dKlSzCZTPD5fHjggQcwNjaG8+fP0+/XQxaPx4NcLqdCdplMBq1Wi4GBAej1\neshkMuj1ephMJhgMBqjVauTzeRplEYKIw0ylUtBoNJBIJFAqlSgWiwgGgwiHw3A4HHC73dQIgXIn\nU+n4O+kEanELoKbgvhKVURWXy4VEIkFfXx/GxsboIsRisZDNZpFOp8HlcmEwGGCz2RCLxXDlyhUU\ni0X4fL5dEW+tKivT6cdisQPdezNRD7d7ie2ZBRKxWAyz2QytVguZTIZisUhzzySCP6yTY/7NWCwW\n9Ho9rFYrOBwOIpEInE4nQqEQ3ckRMAOadqMWv4cpYFcWVvP5PLXVXC6HXC6HVCpFAzS9Xo+pqSk4\nnU7Y7Xb6MwdZ8IRC4YGagg7lVE0mEwBAp9PhkUcewZUrV3YZ514gv5RKpcLQ0BDOnj2L06dPg81m\ng8/nQy6XQygUgsPhQCQSQSQSgcfjIZPJlG2zyOqUSCQQCAQwPz+PhYUFiEQi8Pl8cLlcpNNpxGIx\neDweuFwuOJ1OpNNpuvp347b1MNxWgvx+zGppsViEWq2mDpXk6Jgrfz6fx87ODtLpNJRKJSQSCf7+\n7/+ecux2uxEMBpFKpe6oNMBhuGXK+NRqNcbGxqBSqcDlcrGysoJsNkvtknl9o2DauUAggNFoRG9v\nL1QqFeLxOOLxeEcdaDU003YrUWlXpKAaj8cRi8WQSCSQzWaRyWSQz+eRTCZpcCCVSmm02ko07FST\nySQKhQJkMhkSiQT+8Ic/4Dvf+c6BXoPkRCQSCUZHR3HfffcBKH8oSRREkMvlkM1mkcvlUCwWwefz\nIZPJkM/nkU6nsb6+jqtXr4LP54PFYpVFsOQjHo8jl8t1Tf6pEs3gthJMx0qixp6eHoyNjWFwcBBm\nsxkCgWCXE4hGo/D7/fD5fFCr1bjrrrugVquhUqlw69YtrKys0HxVN2w9a+Gw3BaLRXA4HAiFQhiN\nRkxMTIDNZtPIO5vNgsvlNpULEhVrtVrYbDb09vaiUCggFoshHA4jmUw25X2agVbYbjVUFquAd7XW\nwG1FRDabRSwWQygUQjqd7v7qv8fjwSOPPALgthbsU5/6FB588MEDvUapVEI4HMby8jLsdjt8Ph/N\nTwHVt5OFQgE+nw+hUAi5XA4KhYIOcRCJRAgGg1haWiozbBKZEb1lZfjfbVFVM7gF3nWkzAecx+PB\nYDBgYGAAIyMjGBkZQU9PD6RSKdWo8vl8KqiWSqUIBoNYWVmByWRCX18fRkdHYTQaIZFIAADhcLjr\noqW90Ci3xEby+Tx4PB7UajVsNhumpqZgt9uxurpK00rNcqrM3YVer8fExASOHz+Onp4evPnmm1ha\nWsLOzg7i8Tg4HA4AlO0YOoFm2W69YLPZNEWoUqkglUqpIiiTySAUCmFrawvb29twuVzgcrl0d9oq\nNOxU+/v7MTs7e6g3Z7FYSKVS8Hq9mJ+fx6VLl2A2m6FQKJBKpWg0IBaLwefzkc1mEY1GaYQUj8ch\nlUphsVhoXisWiyGbzSKZTNLt
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 6
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Eigenvector Computation\n",
|
||
|
|
"========================="
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Let's start by looking at what the \"average\" character looks like:\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"mu = mean(images,axis=0)\n",
|
||
|
|
"gray()\n",
|
||
|
|
"imshow(mu)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 4,
|
||
|
|
"text": [
|
||
|
|
"<matplotlib.image.AxesImage at 0x45f3750>"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAPwAAAD5CAYAAAADZljUAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnVuMX1X1x9evM8MTKJjQaUNJikDlXhoJGGMRAkVMhH9J\njYEoEihqeDEEYuRBY33B8kAMIA9EkfSJyAuXByBIDAgkpg+2iQEjJraxadqiIAmFwFx6/g+6hjVr\nvuuyz+86v9/+Jidnn31u++yzP3uttfeZ33SapmmoqqpqIrRm2AWoqqoanCrwVVUTpAp8VdUEqQJf\nVTVBqsBXVU2QKvBVVZOkpqVeeOGF5gtf+EJzzjnnNLt3716xn4jqUpe6DHFBagX8wsJCc/bZZzcH\nDhxo5ubmms2bNzdvvfVWBb4udRmhBamVS793714655xzaOPGjTQzM0M333wzPfvss20uVVVVNUC1\nAv7w4cN05plnLm1v2LCBDh8+3LNCVVVV9UetgO90Or0uR1VV1QDUCvgzzjiDDh06tLR96NAh2rBh\nQ88KVVVV1Se1GbSbn59vPv/5zzcHDhxoPvnkkzpoV5e6jOCCNE0tND09Tb/61a/oa1/7Gi0uLtLO\nnTvp/PPPb3OpqqqqAarzP2vc+wvXOL+qaqhCaNcv7aqqJkgV+KqqCVKrGL6qSqs0hOtTJFkVqAJf\ntUxtxl76PV5TO4feqQI/wSoB1Tu218BrwOX1K/zdqQI/YcrCaR2H8nsFPMPsAc77KvjtVIGfEHUD\nus6Ltq3zPUibpoGgdzodeJ6VX+WrzsOPubpx271tKx3dm5ubbHalaXS9qpVCdVMt/BirrVXPgO4B\n37azl1ae05GlrypTBX6CFbnvHuhRJ9CNJOwR9LUjKFMFfozUdiTdstAZ4DudTtqlt0BGKoG+Kq8K\n/IipWyvZzeh6FnBee5Y9it8t6PXgnczTo/i1EyhXBX6I6tU8eHRcaR6COQI+Uz4Gnc+JgEUxPafR\nNeS+KqwK/AA1iA9dup1Ws2DvJfAW+NY5Eejy/Aq+rwr8ANQt6CVuus5vOwLvpbNuPUvDicD3hKx4\nHcxrpwp8n9UN7NnprgzE0b4SwK1tS2juPTreGwPIDvxV6FeqAt9HtY2724AfgZxJZ+G28jJxOFpO\nnDgBz0PPaFnyCndOFfghqi240XYGbGt/BmyvE0Cj7jKNAF+zZo0Jvr6WB3h17WNV4PugNgNuHoxt\nYG6zjhZ9nN7WrrsFOx974sSJZfkMvmf5LdXR+pwq8D1Umxhbb6OBsFJoZboEaGtZs2ZNqkPS8+yW\n+y4BP3HixFIn4HUAFvzevD3aP+kdQQW+B2oDemSd26RLAUbbXjp6Rm21kRXnhWHn62vo5fWIaBn8\nnqIRfNQ5TJIq8F2oG9BLrbQHN+d7IPM253lpa03k/9WatuA6rRf9HBJ6eS9t4TXEKLbXFt4q96SF\nABX4lsrAboFvue29dL/ROrtYHUIUo1tg87K4uLhiWz+3BJuvzx0BHxPNBug89PGO9d7GHfwKfAuV\nwt7Gmpe65RpqBO3U1NSyNcrzOgf9sYzeZqCt9Zo1a2hxcXEJdF1POk5n0KV3UeLS67W8nwf+OENf\nge+RItjlNoJcbiOY0T6djhYJt1x0J4DO0cCjeF0CLhcGXbrxDP3i4mJYt9LCe/Jg1yBHbv64Ql+B\nLxRqeBnYLbhLrHgUZ2to9fb09PQK2KemppbyvXOnpqZWDMDp9OLiIi0sLCyDfWFhYeka0oVHsMv6\nQcpM0/UK+nFVBb4LIeuN1py2Ft6PrDey4pb7rq21XjPY1tpz9eXHMdbgHMMu11NTU2FHwM+gp+hQ\njG95F3J+3xqoQ1N0louvjxsXVeAL1CZGl2vPVY8seDTIpt117boz8HJh2HmxOgtORyPwDLJeZD6n\np6enYccgQwLO47AAzeEj2K0xhih+l7Km8lY7/BX4hCJLrvO8tBWje5bbi6tlXrRo4FEH4HUcaORd\nQo9gX1hYoPn5eRN8CTtaZDig7ydddWnxieyPf5CQpc9YfrR/1FWBd+TF6x7smTjdct+zg25otF27\n5zpvenqaZmZmlq1l2vIOpIX3FgS5tOZ6kdePXH6GHk3/cT1akHPsb8XyrBKXX7/71QJ+Bd5QBnYv\nRkd5JW57ZMUtl92L0WdmZpYBr9Me8NrC6zn1EydOLEE+Pz9PMzMzS9sSeNkR8LaGnmHntaw7PfAn\nFxTXE9lf6UnwLStfMp3n7R8VdQX8xo0b6TOf+cxSY9q7d2+vyjVUlcCeseZE5IKuB928WLrEbdfb\nDDaCXQKPvAMZY1sf0TDAvDD009PTS6Az2BJ0Db2EfWFhYRnQcnBPzufzO7Bcex3nW0Jf6XFavu/V\nOqXXFfCdTodeeeUV+tznPter8qwqZSy9F6dblj0agNNuO4IbLQz2SSedtAx03vbuwRYeQc9rCTsv\n8v4Sfgm6FcJI2DudT+fu+Z76HXjf47OVzwp9sDMO8/hdu/Sj+mBt1caV53XWlbcG5iyrzfsil13H\n5nqbwT7ppJOWpXkddSjRl3Rzc3PLYJ+bm1vR6UhrztCjMMYLkaRbr+fyJehyjb7Ft6btvK/0Vjv0\nXVv4a6+9lqampugHP/gBfe973+tVuUZGugPIAl5i1UtH2a00gly77hp2DbwFu7bwPLimP531xhlk\nfnb2QcLvfcTD70GO2hPhv8nPTst1C/0oqivg33jjDVq/fj3961//om3bttF5551HW7du7VXZBi4L\nbk5nYG87KGdBptMe9Ah0mc5aeMvSy/lwLjPPl8uRcF0f2Q5Ox/VoXEMP7Ml3gf4KT/8xjh7Q09Ij\n9TLPiuNXE/RdAb9+/XoiIjr99NPppptuor17965q4KWQ287pEthLB+UsmJElz1p3FL9ngEeWmaHi\ntLT8sk4s0DPQy33S5fcG9Iho2dgCCgkk7BzTe5/rZgbwPNBHsSNoDfxHH31Ei4uLdMopp9CHH35I\nL730Ev3sZz/rZdkGKg9wncfb2Vhdx+mW62uBnBmQy4AvB+sQ8NE4ggReu/hTU1PwuXWeZdEt8LMD\nehzPyz/Kke/Mgp2fSaoN6KMIN1Jr4I8dO0Y33XQTEREtLCzQt7/9bbruuut6VrBBCg3UoXwvVi9x\nYy2wSqHOgq6n3xD4epQedUroYxsGX/4pq64HeT1kwTMxvtWZIO9Lj+ATLf/TWjlN571/eS4fp9Or\nBXRWa+DPOuss2r9/fy/LMhR5g3LeMd2Cj6a99Ag7GnXP7LdG6j3Y0Yc3eu19ZceNH8GJrqXn3jXs\nMq29J+SqW5KgsyVnyy7TyMrrdx8Bvhrgn+gv7aKeXR6DLIrO91x5z6pr6+59/toGdG3hEewMvDfW\n4H1HLy28NTiprbpea+uOxkWs96DfB1LTNMtA1+Mx1jny2tHUHSrPKHUCEws8esH65VvH6EaHLJDV\n8DOwa+BlXjcdgQf7zMyMCTsvEnCULglhGHL59Z0HvAe7Naeu98tySkn33lMG8lEDXGsigc+8XH08\nAhw1yMyotDUYZ33yGln8bCigOw69WGWWA3b8/NJCWn+eatUhqise9LOO9ayxvme0X0/deV4Bn8/X\nXa2DdayJA95zBfU6aoCeRbesujVI57nfJRY+s1gDZBYEGhj9FZtOI8+H68H6I5bIRS9x5zMegN6P\nztHPkwVf7x+lTmGigI9gR9s6HVkqy6pbI/EZ0C0rHwGPpvc82CX0SBH0sr5k3cipMIYe1al3ncgr\nQ4NsnM+uPIKO43r9bPI6FryrKXZnTQzwGdiRVZdpZN2taShvgM4CP7NkYI/m9KPRcMtKyrWVh+qL\ngeJlenoaHuNdwwPfsuDel3Wcz2WzOnkLfgT+augAJgL4rGWX+Zl4Uo/Ee9Y+GqzLjKa3hV3nISuf\ngast9NqCyrrVMbznQXmwo0E7Cbu28HK/9ETQRzdt3fdRdO3HHvg2brzMixbPnbdcemuwzhqw82J4\nC3h0P71Pl9WzoDptxe+63hh2
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 4
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"If you think of each character image as a vector in a high dimensional space, the characters form a cloud of points in that space.\n",
|
||
|
|
"\n",
|
||
|
|
"That cloud is not in general centered at the origin. Let us center it at the origin.\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"centered = images - mu[newaxis,:,:]"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 11
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"In terms of images, this has an odd looking effect.\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"showgrid(centered[:50])"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAkwAAAEzCAYAAADD+FxKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsfXtwY+d13w8EQOJNvAG+ubskd1e7K62ktR6WJcd2FLtx\n68T1REmmdjyy07SZaaetO2090+lzJhM3MxnHTTNO0zipnKaepA8/WteSLduSbMXSytZqLe2b7xf4\nAEk8CIAkQNz+wfzOHlxegCCJlW2ZZwazJIHFvb97vnO+8/5shmEYOKIjOqIjOqIjOqIjOqK61Paj\nvoEjOqIjOqIjOqIjOqIfdzoymI7oiI7oiI7oiI7oiPagI4PpiI7oiI7oiI7oiI5oDzoymI7oiI7o\niI7oiI7oiPagI4PpiI7oiI7oiI7oiI5oDzoymI7oiI7oiI7oiI7oiPagAxtMTz/9NE6dOoXh4WH8\nh//wH1p5Tz82dITxrUFvdYxvdXzAEca3Cr3VMb7V8QE/HRjrknEAqlQqxokTJ4yJiQlja2vLuOee\ne4yrV68e5Kt+bOkI41uD3uoY3+r4DOMI41uF3uoY3+r4DOOnA2MjOlCE6eLFixgaGsLg4CCcTid+\n5Vd+BV/+8pdbbcv9SOkI41uD3uoY3+r4gCOMbxV6q2N8q+MDfjowNqIDGUxzc3Po6+uT33t7ezE3\nN1fzGZvN9hP9evvb3/6Wx/jTwMe3OsafhnX604Dxrb5Ofxow/jSs058GjI3I0fDdOrTXl5LOnj2L\nc+fONfzM66+/3vAze71/p77jC1/4QsPPky5cuIALFy7Uff/73/9+w/f5mfvvv3/X3/mcm/2O/dzH\nH/7hHzbNxwsXLuBtb3tb3fdfeeWVhu8385k78R2f/exnm8b40EMP4eGHH677/ve+972m3jf++qQh\nwzB2Xfull17CQw89BMBahva6htVnPv3pTzf8POn++++3XGOafvCDHzT8zF7vt+I7rN7/oz/6o4bX\nJLVKFt/s79iPLD744IOyhqxIr7H9foZr9+WXX8aDDz4IoL6u3+s65vc/85nPNI3x/PnzuPfee+u+\nf+nSpUO9f6e+40//9E8bfp70yCOP4JFHHmn4mRdffLHhZ/Z6vxXfYfX+7/zO7zS8JunRRx/FY489\nVvf9F154oeH7jT5TrVYBAN/5znfw2GOPNVxXe13H/P5v/dZvNbynAxlMPT09mJmZkd9nZmbQ29t7\nkK86EBnq+DvDMGAYhjxEAE1Zis3Qm4XRvLnyZ41te3tb3jNjOyjWHzc+kodWz+Kg9GZi1Hxsa2ur\n+Xu1Wv2JX6dWZMXD7e1twfWTitGoc8Sm5qXG9pOwTs28avS5O7FOf1z1DYCW8fHHBR9//0nXN43W\nKd/Xn2nl3mFFB0rJXbhwAbdu3cLk5CS2trbwF3/xF/jABz7Q6nuzJO3BV6tVUV7b29s1v5sf5EHo\nzcCo8egXN57t7W3BZ8Z4WPpR8dGMVWM04zsszh/1WtV8rFarqFQqqFQqgrMVvPxx4CFxmtfqT5Is\nArsVsBmf/tesbw5Dd3qdWm2me/Gw1ev0Ry2LVvpGr1N+7jD0o5RF/mxeqz+psghg1/o046A+tcLY\nij3STAeKMDkcDvyn//Sf8N73vhfb29v4+Mc/jtOnT+/6XDwe3/O76n2GYOPxuFjJZuOCDzAcDqNS\nqdRY0NrDP+h9NIOxu7u74fdavW9WyMlksmZxcwHYbDa0tbUhGo0KPsMw0NbWJviI9yD30SwfD/Ld\nZqxdXV2WfOQij8fjgpH42traBLPNZqv7LBu93yzGvbwk/b6VIPb09NQIcblcrhHktrY2xGIxlMtl\ntLW17eIjr2GVytvrPpvB19XV1RBfo88QL9cp/8a/6+hZLBZDpVKpwab5uNd91Hv/zZBFXt9qA+Lf\niI+6xszDeutwr/t4M9Ypf+/u7t5lJGi9EwwGUSqVYLfb5WXG2NPT0/R97BdjMpls+N3Nvm/mo+al\n1qlWenWva9T7TDP4dA1QPar3GeLp7e21lEXuIdvb24jH46JvrGRxr/uo934zGAcGBhp+t/l9q7Xa\n19dnyUMAqFQqiEajKBQKsNls6OjogMPhQLVarbED9nsfe5HNuBNmGHaE6ld/9VcP9H+tPB9tIGnr\nmddqa2sTwXY4HHA4HLsKubhQmqEvfOELe37WZrPh7//9v38gbPpns5dqt9vlniuVCra2tkQ4iI1Y\nD5oW+MM//MOmnoXNZsNv/uZv7gsjYO31mA1d7akDEB7qV6OCPP27FZbPfvazTWP8J//kn+wLm8ao\nDQetsDQOs9dut9st+bjfsPmnP/3pptbpb/zGbzT9nY1wmiO65qigzWbbxUONkfezH/qjP/qjOy6L\n/L2ertEGBjcgK3x3Whb/0T/6R/vCp3HWkz+v1wuPxwOv14u2tjZUq1Vks1nk83nhqcPhONQ6/cxn\nPtM0xieffLLp762HVUfGNNZKpVLDRy2HxKjX6n7oT//0T5tap//8n//zQ+Orh5F/1/evMVKnmh3u\nZul3fud3msL4L//lv9w3vnpOmBljpVIRXOFwGIFAAJVKBZubmygWi7JeD5Ke+63f+q2G+A4UYSIN\nDg4iEAjAbrfD6XTi4sWLh/k6AGj4kGg8bG1t1XjuDocD7e3tcDqdcDqdorS1F2i+xpuJ0ayUzeFF\nbrAdHR3weDyIxWLwer3Y2trCysoK5ufnBXNHR0eN8Jg9XGB/QtBqHpqVs1lBk5dbW1sShQF2lJfT\n6azho2EYlt4tiQqP19XK8E5g3IuP5XIZGxsbaGtrQ0dHB+LxOMLhMDweDwqFAhYXF7G2toZSqYT2\n9vYaPhKHmZrlZSv5aKWczSH+crksssjIBBVze3u78FFjMEcM90Nv1jrVkRauUx3x5aajdY42kn7U\nsmheo/yX61Sn2TY2NrC1tYVYLIaTJ0/izJkz6OzsRLVaxcWLF/Haa69hfX29ZoOy0jf7wXun9gwz\nRh3l5WZKnUMsHR0dcLlc6OjoQEdHx64N9qA1P2+mLGp8m5ubck2Px4O2tjasr6/DZrPB4/HUOOSU\nwR9HWTQ7nuVyWXi5ubmJ9fV1BAIB9PT04J577sHw8DCmpqYwNTWFXC4HYCewYN4LWlHTdCiDyWaz\n4bnnnkM4HD7UTVgJtt5cKQTlchmlUgnr6+twu93wer1wu93w+XwIh8PY3t5GJpNBZ2cnIpEIuru7\nEQ6H4XQ6sbGxgUwmg8XFRSwtLaFcLu+yxO8kRuIzK+ZyuSyGj9/vR29vL8LhMLxeL6rVKoLBILxe\nL7LZLHK5HLLZLDY3N+F2u+F0Omsia2YBb2aBtBIfv88s2FqBUWmVSiVRYtyIXC6XvLjpOhwO2ZSI\nkYLa29uLeDyOQqGA1dVVLC4uolwuW0aiWslDAHWNeRpDTqcTgUAA3d3diEQiqFQq6OnpwRtvvIF8\nPo9isShK2+l07vIAdargzeRjPVk083BzcxOlUgmlUkmMChoRLpcLbrcbbrcbHR0dwke94e53Q7oT\nclgPo66L2NzcRLVaRXt7u+icUqmEjY0NUeZMBzAlYJVOfjMwWhn0Zn1D+eMLAMLhMHp6euD3++F2\nu0XWNjc3USgUUC6XRd/UW6f6uo3wtnqdUh/odIzP50MgEEAwGESlUkGxWBSebW5uwmazwel0wuVy\nweFwoFAoIJfLYW1tTZwd7Wzv13BqNUYdbeGa07K4sbEh+2RXV5ekCvP5PObm5mSNa2fUHE37Ucii\nVQbCrGu4R2xsbGBjYwPFYhG5XA6VSgXxeBzt7e0IhULY2tpCoVDAzMxMjfFYL1p4UOPpUAYTwbaC\nrDx2HWbkdSqVCkqlEvx+P6LRKDo7OxGNRtHf349cLoebN28ikUjgxIkTeNvb3oahoSG4XC6srKxg\nfHwcr776KjKZjCywZsKvh8FoFSrWyqtUKokHEA6Hcfz4cbjdbrhcLjidTkSjUUSjUSwvL2Nubg6r\nq6tiRW9vb4tBYfb+9qOsW8FD
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 12
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "heading",
|
||
|
|
"level": 1,
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Image Covariances"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"We're treating images as data vectors.\n",
|
||
|
|
"\n",
|
||
|
|
"Our images are stored in rank 3 arrays `centered[N,height,width]`.\n",
|
||
|
|
"\n",
|
||
|
|
"To treat these as data vectors for machine learning, we need to reshape this into `data[N,height*width]`\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"def make2d(a): \n",
|
||
|
|
" \"\"\"Make an array of images into an array of data vectors.\"\"\"\n",
|
||
|
|
" return a.reshape(len(a),prod(a.shape[1:]))"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 13
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Let's look at the covariance matrix of this data now.\n",
|
||
|
|
"\n",
|
||
|
|
"This tells us how much each pixel \"co-varies\" with any other pixel.\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"covariance = dot(make2d(centered).T,make2d(centered))/len(centered)\n",
|
||
|
|
"print covariance.shape,amin(covariance),amax(covariance)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"(784, 784) -0.114308 0.208461\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 14
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Let's look at the entire covariance matrix. Here, green is zero, red is high, and blue is low."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"figsize(12,12)\n",
|
||
|
|
"imshow(covariance,cmap=cm.jet,vmin=-0.2,vmax=0.2)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 15,
|
||
|
|
"text": [
|
||
|
|
"<matplotlib.image.AxesImage at 0x5bb77d0>"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAsEAAAK7CAYAAAATCFLTAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvcvPJNlVL/qLyMzvVdVvsAE3dp9zjIA5SEzM6QGPPwGY\ngIAx0yNd6Q58pSvRniLGTC0zMxISAwa8JniK5HsFOkf28W1kY/e7qr5HZkbcwY5frLXXXnvHjsj8\nqr6ic0l21xcZsfZ7vdfaTd/3PU5wghOc4AQnOMEJTnCCzxG0L7oDJzjBCU5wghOc4AQnOMHzhpMQ\nfIITnOAEJzjBCU5wgs8dnITgE5zgBCc4wQlOcIITfO7gJASf4AQnOMEJTnCCE5zgcwcnIfgEJzjB\nCU5wghOc4ASfOzgJwSc4wQlOcIITnOAEJ/jcwb0IwX/zN3+DX/qlX8Iv/MIv4Bvf+MZ9NHGCE5zg\nBCc4wQlOcIITLIbm2HWC9/s9fvEXfxF/+7d/iy996Uv41V/9VXzzm9/EL//yLx+zmROc4AQnOMEJ\nTnCCE5xgMRzdEvyd73wHX/3qV/HOO+9gs9ngd3/3d/Htb3/72M2c4AQnOMEJTnCCE5zgBIthfWyE\n77//Pn7+539+/Pvtt9/GP//zP0fvNE1z7GZPcIITnOAEJzjBCU5wAhe8wIejC8G1Au5X/vuX8ZV3\nvxz+/e6X8ZV3v3LsrjwY6NGCU9+gR4PlESj9YLxv0KmnDTo0+Mev/wO+9vVfRxv9Nh9/6J/0sTMO\ng0Pwe3AM/P2II53fHk30rPSuhX/4+j/i17/+teg7O/f9+P8Wf10b3ncNumhe9L7pzXNU4O/NHMsX\nAXOL/WL8FneHVs1RM7QQr+nff/2f8Otf//Xhjen1ZhsdGjTjCNj3LsJh+zOFn+/3Az7dd4D7Md1T\n/L0efwoePejGfVbf/x4Yz7/0Wcaknwl+oUqlNnh+NB1r0aNDSuu5Nv04N3Xrq8fhQavWRLd7DPya\n3ui54Zj/8et/P9KAFGdMW8LeD7/k5l3j93DEENMX4m+cPiNa630BZ9x/aSXgatEnbWr8DXq02E/i\n1m3YvvzD1/8RX/v6fx+fWto3b13lLGJcN/0shUadj/p5qpNtavvtt1Nyzpd5ieZV0+2kc3ZsYBuH\nzMch8P2/+z6+/3f/e/z7H/6vf3LfO7oQ/KUvfQk/+MEPxr9/8IMf4O23307e+8q7X65esEPAI3yH\nCqLH6cX94J/bDr8sCZ7HFHrJlDVO/e8ebSQE1AoA/LcnsGjhrocQmrmEvBu/0/unj0hjh3bA3w/M\nRAttdfhDn9PfGwQhUITx2n3cDMwz/34/sj5RhJpB0JnaU9O4G7U2qdIzBR1Wwyh0L1P8gL9XO9W2\nj78Z17Qx+GVdeL46iDBQh5/A3WeFin7Y9VBP+2QH+KAF92bA06kzJG3nzlu5HcHfG/GzTGs8har0\npoc/1yPd7jR+3ZKPnxgbHE7r7DrmfjsGfu+Zpg21EK9pl30GhLMYVmo1i3767cp67NU6hjHU7IUY\nhD4HHqAVwFy7S/BPQcwf5uBfFc8jz/ihIDRNhODVgWvpARWqFwVfefcrkXH1uQnBv/Irv4J/+7d/\nw/e+9z383M/9HL71rW/hm9/85rGbqQb/IBxnM9X3gUR2XrsUpoIAETapJxRSSAj/DqQcqBPy+oj9\n9Ech1GUgq24y/evRj+OZFgVoM1hhh34kfCv1Rq8Iex8JVGyhtCbh60DwZQ36aM7Dsy4Sotrhb81g\n82NYJevqMfd+sO402EcjmIYwCu7B0ObavEFhN+BvBsw9VuOs5iDgji2oaf8FYwPZdTUgQl6+71D7\nyeIN7+fbSi1rPn7+am270/in1jZmy3qfTYHMY1kwJH575qYs/b4YwXaaQWHK07XpNdbqxDRY69IU\n/joFRcZoxzSF/xh8pIyjVh0irnr6LcpjnVAYaF7KI3IWP62cigch3n+xMCYgVuhpYdtaaamU52ZO\nzze5Xq03pwTCSZftCSuc94M4TdyHCpTkWf2AMW55NYsm10B7wFw8Tzi6ELxer/Hnf/7n+O3f/m3s\n93v88R//sVsZgqEQ9w0t9omG1Q8ijXUZUehcunB7rAcdK95ILfajMAUAK+yq8AnTL5MoEokvv/sV\n9GgHra5uDC322GM9EpK9Oey1fZ2CvStYyHO6tIMIuxutDVMESjTYJhFMiVe7KDkesWaklmMN77z7\ntiuUtIMguh+JRz++I23QdVleC+LPzbW17oR9FtrcY53sYw8sgfXmQQuxe2cu63HvI9wyRunj2+/+\nl2hPlPaZ/c32XfrRu8+nwDJZ3R49E3G4SIx/hV2RWdf0RzNPD38O9BzaOSWkdK4ef+ldUcjq3k/7\nvhrOp+yRqX0s79bR6aCglem6Dgvhvtf7eYpX5fDb53pt9FinBD16l3L7foWdu+5TtJsK0JyzomeL\n+HOKlDbiWPjKu1+GeM3ivnZYzeSVqapWExayxxor7KrWQeOfEw6SW/PcKOK/51lnS/tU9yPIKPvo\nN/HQHs9L/jIIwACOXyKtqtGmwf/Z/x+zvvHc6LUw5WbQ7gAx4S+fFjJOTxitEbws5A5nGBeivtNq\nPGcDegRUg3foSUCW4qdyIn+LEJYe2JqY0WlBg/OyBD/bsBYPK+hZhmfjVZfAlCA2d70tePtmLn4v\nrCZl1nIe5ghL+TZ9HN5emBL0uLakE1oRA/y11VBijL6iEWBq7qf6vgT0Pt5jdXRXqLdfj4c73VMM\ncfLmv2SlTNe2rPzZ9RMPjh/3aIXgqTZztEKHiNUqRFPverjnglUWY0VAwojmhqxQMaoRNLUgPY07\nPaO6nVKbOR5p93ipP/dxHmpgWv4R4B4M472fMIkXBf9386fPJzHuPuH+akpI3NsxwgGaQZf1fqFL\neA7kDmbuOV0bdURk2gqg8fBAlUMI5HcPv3UZ5dwwEk6Q104DA5lWjjQBstb/fGgGHKYlz2PCksYh\n96BrLo+fxDW2hMwj7B1WWYYXC/zCjGtxh/fbrFXCMgc6kWtwM4yiBFMKWg53EGLbSfw55Skdl8yx\nXStk1peOfp5HHapj+2UtX5oxsy/2TNcqoqlFPjwV7w/xi0I4dca9NthHfmWt9Lrvc/HnQCy3/vnP\nWSn1HrE+wjmW0VI4SUn5rWmDlnIJLCjTc/a91q3N8AZC3hOl90+9Um/DFIT+dmDeQWfe0WNJcS33\n9vCb4DnLzWnsLakJk/DC6/JA/v/8YI4AHN5vIZVzGSbxcoQ1LIWXRgieil07FKjJhrbuJy6WsaE1\nh8v2bW6fat01pdgpab+BMPEYvyVOGIlHvn2ralhXbcBPa3ndAS5ZMxrEQjTDU6TFUrzrfhCmpIIA\n3UkMfym1Ya2jOdBCgRYk+DdnRuIVRTCvYQaa4HuEUfeffZlyx1klgDh6pHF++vyK1ag+QYUWt5o4\nv1r8tILFscx1qVnCJsuCTjsIls0o8DMwx3oUWP1C9hqFmTmWNF+41PGSdYJDSbHKtcv/1vR3Lv4c\ncH7SONU6mllPV1P+I2PVZ2dZSJ0fSlEv6EnugR+XmtLpVEj18eo+HMaDY+G8g0cnMPYpNSocC3QC\nnV7/NDm1n9zP/F4n/ebeD97h51cpocYIMI0jqIsvyor9POClEYIPgUAoy8REXKLLsztrQNxacw7D\n/L7UZqyHQ5yW7tEaOrXBPnrWQpK/GtVenRUCALRGDsSuuSA8ilAVrLV1eLXwY4Vjwd86loAUPKLW\nICakhDR1YY4GLhp3SkRlTWi5aisEyPpkDlmDbpj1GjeYrWYh8Zex9ccqCFJNYnruiYOx3Z5wbS1f\nFGynzrBdQ4vf9nsu/gDx/ham1Ju904Mx8NI+94O8VW6pVX0TBt0Mq+onc8YYlgJd2HHspMZ/HHpq\n54cJn1boPabAIYmfdg93Ed2T5/7e1t6PKWWf+HuUK6/o9z0KFLefhzqaTYFwmWUwXhNf4RReUrYE\nHwPYn1x1G1mrPNx/IvkJLKTn
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 15
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"The diagonal of this matrix is just the variances of each pixel. We can transform these back into a 2D image. Necessarily, these are all positive."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"d = covariance[arange(784),arange(784)]\n",
|
||
|
|
"imshow(d.reshape(28,28),cmap=cm.jet,vmin=-0.2,vmax=0.2)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 17,
|
||
|
|
"text": [
|
||
|
|
"<matplotlib.image.AxesImage at 0x5eb9590>"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAArsAAAK4CAYAAAB09zUsAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvTuvLE9S7h1ZvccEAwkDMTCDBBL4uByNwUU4SHhImAjh\n8AVe6ZXgSEc6wuc7IEy8EdYIDs5YWGDgwBmNgbDBYHZnHCMzMp+IjMzK6u619/6vHY9Uq/JWVVmX\nrv71s6KyEjMzhUKhUCgUCoVC71DH5+5AKBQKhUKhUCj0VgrYDYVCoVAoFAq9WwXshkKhUCgUCoXe\nrQJ2Q6FQKBQKhULvVgG7oVAoFAqFQqF3q4DdUCgUCoVCodC71Ye3WnFK6a1WHQqFQqFQKBQKKc1G\n030z2CUi+v/5/9tu+3d//vf0P/78N96wN6HPqTi/71txft+v4ty+b8X5fb/62s7t/0r/e1oXYQyh\nUCgUCoVCoXergN1QKBQKhUKh0LvVFwO73/neL37uLoTeUHF+37fi/L5fxbl934rz+34V57Yr8Sya\n99kVp3QpZjcUCoVCoVAoFHpE/yv97+kDal+MsxsKhUKhUCgUCr1aAbuhUCgUCoVCoXerh2H3+9//\nPv3qr/4q/cqv/Ar9xV/8xSv7FAqFQqFQKBQKvUQPwe79fqc//dM/pe9///v0T//0T/RXf/VX9M//\n/M+v7lsoFAqFQqFQKPSUHoLdH/7wh/TLv/zL9N3vfpe+9a1v0R/8wR/Q3/zN37y6b6FQKBQKhUKh\n0FN6CHZ//OMf0y/8wi+0/Le//W368Y9//LJOhUKhUCgUCoVCr9BDrwtOKW21+7s///uW/s73fpG+\n873vPLK5UCgUCoVCoVCo6d9+8G/0bz/4v1ttH4Ldn//5n6cf/ehHLf+jH/2Ivv3tbw/tvqZ3ModC\noVAoFAqFPo2+873vKBP17/7n/5m2fSiM4dd//dfpX/7lX+hf//Vf6b//+7/pr//6r+n3fu/3HllV\nKBQKhUKhUCj0ZnrI2f3w4QP95V/+Jf3O7/wO3e93+qM/+iP6tV/7tVf3LRQKhUKhUCgUekoPwS4R\n0e/+7u/S7/7u776yL6FQKBQKhUKh0EsVb1ALhUKhUCgUCr1bBeyGQqFQKBQKhd6tAnZDoVAoFAqF\nQu9WAbuhUCgUCoVCoXergN1QKBQKhUKh0LtVwG4oFAqFQqFQ6N0qYDcUCoVCoVAo9G4VsBsKhUKh\nUCgUercK2A2FQqFQKBQKvVsF7IZCoVAoFAqF3q0CdkOhUCgUCoVC71YBu6FQKBQKhUKhd6uA3VAo\nFAqFQqHQu1XAbigUCoVCoVDo3SpgNxQKhUKhUCj0bhWwGwqFQqFQKBR6twrYDYVCoVAoFAq9WwXs\nhkKhUCgUCoXerT587g6EQqFQ6NMqtRR/xl68WmWvuKbe056FQqHnFLAbCoVCn0ifGzI7Dr5Hlf1K\n9e/bAu/7PpKh0HtTwG4oFApNlM6bVH0a7NH9eWybr8K08di88hg808uCueLwvs12Eaz9deiWoVDo\ncypgNxQKfXG6Dimv1KfFk9cB7PnyV5zlt/QuvfPLTm5slzZBdPcKsnu5hti9o6FbJZMK+A2FPr0C\ndkOh0GeXDxfvDwseCWN4JaA+W++3Rz1+zpKTG9fGLUzhbGvrMAZZfg7V4/r5SXC129qF51Ao9KwC\ndkOh0GfR2/4b/PPqEXi/4vCeAfAMxVbLrfDtFedqvY6Vw+pBYXFvV0A793fTBHKxD2dgKvXPRGFz\nW/79XPmh0JepgN1QKPRJ9R4iGs//Sb7rwq7b7reZ/ev8Svle23kfUFfPq4ZbvYaZm9uB09uaD8IF\ndH3IXQHuug/Sj7H0TOt9CIVCr1HAbigU+iT6JkLuK6DWX9+Zg7pyZcd6b7l52Rp09/vgt3l8GQ86\n0xbc7o284LXyXF5Zvwe42tH13F6sH9vMxAG8odAbKmA3FAq9ub4poPtquB3XO4PU19T54DoH2R0g\nfiy84rx9Glp5IOn9q9+C4c4Dad4IDdblnUGu3y8p8/dQSjv2frlXfSj0/hWwGwqF3lTPgO7nHZWB\n6HX9fUWYwXn5Y3kfdF/h8F5rN8LlHCxHJ/TM3T0H3TPInfdjvncaetdXU7i7odBbKWA3FAq9mR4F\n3c8LuXv9fMQFXocbzMp3246QuoLb63X+tv2+XW2TXLjUoQyjy9sBcsfdxV5oTNagKw7wHL5Hlxcx\neQ695zB7ZT9CodCuAnZDodAn0KsA8i119V/0++vwIPVtynbSJb+Tnm1LdAa4u/WsALKjJsLlCLW7\nYIhwa8MZNOhqsF1B7thH3KO5i3s+AsNe/HEoFLqigN1QKPQmuvqIzm4IwKfUMzG8M9C7DrR7bfbT\nV+F45izrurN6/1iim9sRcgaaY9pD2bVWoKvT6DB7kGuBF/d3Fasb7m0o9KkVsBsKhT67ViD0qfUo\n4K6XuxpT+8gyPsy+rl5vd+zvc3UIuhY055A7wqensUYjsg+6M/j2gVfvoQ5t8IB3/XBbwHAo9EoF\n7IZCoZdr19X9ktzc52Jwz5c5h9R5fr99B9ZXlGkcOwfZR+EX4dEDyhnkXtPowWpfdg66I3z7kL1y\neb0rOkIWQqFPo4DdUCj0WbTr5n5ej+s5wB3br2Jir0Gt39YDVVu3Atwzp1f35zXlpc7CpQ+8GnKv\ngu8s8EBD7ZiyIQvjdlfhC95We7mngOBQ6LUK2A2FQm+kq1/XZ/Gfn1pn8aZ+23n7WSztK+quQO7Y\ndg+MS97fv+fKPdAdwZPIA1CrHVD00NUi7By+xz71PVq5vL7DG2AbCr29AnZDodAn1/pf2p8bdB+D\n3HXbnQfI9tvZ8II11O7OZb0+OPv76Mf0XofeEXR9r3eGqDNoXNf66/H7ogHbznELa+D1exAKhd5S\nAbuhUOilOvvq/iaA7i7kXm93HWZn6ZVD68+fh2Fv22dlZfvrskfg8jGN61pte3e+D7w+dl8JcQiF\nQtcVsBsKhd5Au/+Y/VJA960ht5Rdg94raT2/Ar17bb1tXyvzxh7QL+gt+DjC5BxRr0DwuSe7A8Kz\n1mvgtb04d3xDodArFbAbCoU+mVZf6fO6t49ofA3krtqcg6vv4O639SD1VXV78Poc9CJWarjUdStM\n3QPH+VIe2u705Qx4A25Doc+rgN1QKPSZtBPv6gHUp9Z5H85eArsPtx5o7tZ7oPq6sq7HwHfVRgPk\nysG9CrbrJUeMnQHvWV/OHd51OEMAcCj0lgrYDYVCn0Re3OZanxt0n4fcMRZ3XM5zT9dl69CD87IR\nbM+W9/bF2//HoTcpyLTu6YikFhZpWuYLUdQGKHjwfQV+fRi2+2w/Eft9D4VCVxWwGwqFPqvO/+3/\n6XXu1F6vXzm218v8h9OeSZ9B8Gr/ns33eN0RLqUXHnzatC8PbH1hvZf2t9/737fmA3AAbSj0eRSw\nGwqFnpZ84fvlY93eQ07S9tPjwWx/ev2qzl927piu6nyo9ZZfA20H2UeBeLbvz4Nu31N8dM46vLtp\n2cZjYHwOvF66r7ukOuDaPAKvRW+d3+lrKBTaU8BuKPQO5MHj55AHeh7weiEN3tf6W4HuznrXQHut\n7hHQvQq+Nv3qurP9fLRNj871eyN7Pqu3bcb2+2Bs8yvgtIEIiLI9TWQxF+v0lqTnAbyh0KsVsBsK\nfWH6UsCV6LG+WOA9B1vv63z/K/4VQDxbxyvKHwHbR+Zn0HoOtWv3t2gnBne3rK/Lwumry1aQO8vP\ncBb3aP7I2fh4Gvq+Oi09ngPvqNn/TEKhkKeA3VDoM2juFX2q7b12Wz7gesD7POw+Are7y+yC25W2\nbwm4Mt8DWw2zV2B4fhz2z93q2HhwOnNxz6C2oGJxTVfAO3Nz+z6cXzW4TouqNqRBu7k6fQa83pZ7\n33Uq4DcUGhWwGwp9Ar013L4SZq+PmoDL+cC7htuz/HOwerWdrXvcuTx3da+BLSKeX3cOt7tlRIl1\nvm5suq9XINe2ncPurIfnU5qs
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 17
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"We can also look at the covariance of pixels with the pixel at (14,14). We see that pixels above and below that pixel are likely to be on when it's on, while pixels to the left and right are likely to be off when it's on. We get an entirely different picture when looking at the pixel at (14,20). Nearby pixels have similar covariances.\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"d = covariance[14+14*28,:]; subplot(131); imshow(d.reshape(28,28),cmap=cm.jet,vmin=-0.2,vmax=0.2)\n",
|
||
|
|
"d = covariance[20+14*28,:]; subplot(132); imshow(d.reshape(28,28),cmap=cm.jet,vmin=-0.2,vmax=0.2)\n",
|
||
|
|
"d = covariance[21+14*28,:]; subplot(133); imshow(d.reshape(28,28),cmap=cm.jet,vmin=-0.2,vmax=0.2)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 21,
|
||
|
|
"text": [
|
||
|
|
"<matplotlib.image.AxesImage at 0x6fdd610>"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAArsAAADfCAYAAAD2mRvnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvT2sNclRN/7rmXPux+76i7/kxdjeZy1ZGAIHlky4sBLY\n4AAECcJ6AwtDQooTkF7QLv9XwgkRJAg5MMkKEj4SLOTgWdsEOLFjSPaxhYyFxbvG3ue5954z02/Q\nU9NV1dU9M+fM+br31NXc6ZkzH/1VVb+qru5x3nuPM53pTGc605nOdKYznekeUnXoDJzpTGc605nO\ndKYznelMu6Iz2D3Tmc50pjOd6UxnOtO9pTPYPdOZznSmM53pTGc6072lM9g905nOdKYznelMZzrT\nvaUz2D3Tmc50pjOd6UxnOtO9pTPYPdOZznSmM53pTGc60/0lvyH90z/9k//Yxz7mP/rRj/ovfvGL\nye8Aztt5O29qOySdefa8nbfp26HozK/n7bxN33LkOqaZRE3T4GMf+xi++tWv4oMf/CB+7ud+Dm+8\n8QZ+9md/tr/GOYf/7f+wP/7aa1/Hz7/2SuaJDj6zj0e+O5JXhLPyeroyR19/7U288tovZH+XuZDv\nm5vK9bJfOlRevGi3mP76a2/i51/7eQB2W8R2duoZ8xPPi84xRvaN/+P+FBuw2yw0lmf/0P8RgGEe\nGaISz04l6pe5fjIuL3T19n1ou7pJ+06so7IcnD8v89I8fQaQdbNdn9mWDsWzU/kV4DJqet+ak1+B\nUP+vdPLS4tdNdez++XVeuk952Q2/Wv2U2rckOwOV+HWjMIZvfvOb+OhHP4qXX34Zy+USv/Vbv4V/\n+Id/2ORRZzrTmfZAZ54905lOh878eqYzzUuLTW76j//4D3z4wx/ujz/0oQ/hX//1X5PrvvbaN/r0\n22/9EDkPxThbwMHDw8FBWqq5K4fIvsIN/D4/Rav38LT/vMRaduyME/ZcvCq0vbTk0/t25d2dSk8e\nP8GTx989dDYAjOfZr7/2JgDgO4+f4Mnjt/Do1UeT35Wv/U3bxQmPztQ7A43pFbwPjbl2OtkeCf0s\nz3rymNwcZrTAplJejoMvA5EekRR49jv7z46iqfwKAD98623k6z/0batvzc+vMPg18pbM0zY0hV83\nff4pPXcTOhV+tSnw65NR124Edp0bVwmvsGGkJ4+fDDSxHrIrvYN3b36d749Ldz969dFAM+rhnDlJ\nqrmXXn3paLr+4fJi1/BLk4FWKlDnasFpeQk5eOnVl/HSqy/3Z7/2+te3zsemNJZnKVTjyeMnGwHd\nSLwnkZGyGb306kv9c2S4kv3etKR5fo6QhwNLt6X8KFGpFqL84rS7vMxLY+SqTO0u54/6PsNJO0i4\nrvDdfY9Ev//a69/AIWgqvwLAk8dvhXsB5A0pTp719Pn4FaD6L5VhMx27Ob/SXUNv262r5MyvubxY\n/Drmvkfi3hK/bgR2P/jBD+K7340eq+9+97v40Ic+ZFwZK+cRU/oWlSqTQ1jfX5mr+GEWDZWTv253\nTZp6o4fqZZ906LxwQQbk2mmodaQRFMX2dq0a6maob1mjDcch2sbzbKnut6HN64H6pQ1VOWlFNUZE\n874xzpO6Td3o0Yp83uIoVulN87fT5jQuL/PyZT4v3FDL8aUEQLual7EJTeVXgIOX/KglB4ohlR8f\n3Ya4LtE1vd3TN+VXnpecUZmOsMxNZ37N5WUbx8o42gjsfvKTn8S///u/46233sJP/dRP4W/+5m/w\nxhtvJNdNa9JSAAEfcrKZdN4m0Ew5T+e0cnwc3f44qBREMTTYksIe6eXfJaPGd+pcHU/7juXZuWto\n3vKXZITOuxUGY7eKHNDm3q5dUFmh6rwMea1OjVK+3Mc7OcUekHo195WjYdqcX0uAUgLFcI1lHs5L\ntlzfXMduwq98JNjSB1Z4xX3iu01p3/y6y+dvBHYXiwX+4i/+Ar/8y7+MpmnwO7/zO2KWaKTp3SXX\nzXxX1bx78jv0E+ageeGK9Eid2YmTVcfSuJGUa/t4jg9x7Z9cUeUcgsbz7C76u3W8Odm8I5VVDtTm\n+owV/LJryo1jHSIv+6KUL/fJo5IvY7zqvszh8bQtv+b6uutKmfavXfBrSa7zt0zt49N5RPvt88DN\nNpIfKh2WX+eljcAuAHzmM5/BZz7zmeI1m3p2S9fIgSf6v4vB41119DFxhw+H7DbU6dzQtVZSGvDI\nmK7d1rUNyo+pfcfw7HwqzhaK29VHPndcQfG+MF5xWYb0ruiY8rI/4kPo++dLS4Yc03TWlDbj1zG1\nyUH+bvi1JNfnAZPjecTqd9YVtsH8cOkw/Lo72hjsjqNddRc+FGUx0/E1R86f9NBpqA19z15xMEyu\n1SjZT9+3/0i88WukHidtX1tpnct22z1tKweOSX4cU162ozg2dwi+1L3CIXo55So/p1Xn2wFGYDf8\nul/dXH5m2u/y3klZ8lPqB/PTofl1btox2N0tlRYfO66Oyj0J/PhMkaTSsZcXGyZ935y9IOcTcMgB\n3IfazhE0SOWymyGwacuLnQ7leu9cijhfW7uCJJovJPfIq+bJwVAN2vz80Gh+fj02aGQhAgl7h5YD\nTJ8Xacy1m9THmFrcHSyXH/A4fTpCsDtVYaWROMcJKOfoMPMq8zH1NPWN25YyHSzJh31EQaW9M3Pk\nxKKoACQwj/k7dbLaex5xqttnbtrlcoFDNL2v5WMq5fm0TLm0/bxSHm1lpof2t2n9qfWyb2Mlx8/3\ng5e3ozn49ZA8mSNrybJxZSxda4HfYX61ciefmo5Z6t/1M+fm1/sTdnlkYNdeOmgaHQdLabLV1/i7\n54uminmgnOSE+yZCX38Octw9MsZruhjizynBhTmJK8h9vG9fxNvcs//TBrKimhyzzNH2dBiu32Sp\nuaE6zSvK9AOpqRc0Bb62wpSfFOD+LZ2rTVpfv1fmTFI+7/s3WHa9Esfx0i741XZPHIq4l1ouWTam\n3+V7c/58nl+HcyqNTv4REJ4jS9vOy6+nq8dSOiKwm86CnCJ2jr9RdGnGly5vNW5G6Qc50uGq1Owo\nv1GzyrQoH+mRtfNolVzPqLbvs483I1lTeo3H01eUsjROnZ3i6ZHKRT5vF7R/c2MT5VCqU8fOaAUZ\neCkqUH2sc6XfkofJ/AklYzOVESXKe6j42fJd+2rDPD8/JNoFv1qGzOFISi9t2ORBuTPTZaBc4teh\nXPpkz03TFJTaTpe5+PWwJuicdDRg167SMYLHtsOOlbbrMPMsixIZJqo1yerTgC6/ZtPggfzScvR7\neo6us4Zdh+/blMrv280790m8h223NFMKjXcPJA7jjxi/1Fy5Ti1oGpVlbpP3QBz7PuVYWoLceF3M\nP/94z3RTRxuk/Jz1hEP6lIb5+aHQrvj1eHyEw0uW2a2fAtuh9DC/5nPoxRNKbiNqMe6lnptfKV/p\nuVOjowG7nKYxh1bBx9cg29m3Xu11ehOyGMiGMtMjqLeZt8lBuDy/m/s2Jfm+1Dd3X0gDgXBuCm9q\nELELlTeHAbj5W6fKHqtOrSeUgG6LSijPvMJNr5I+p/iWysxJWdnaxOWdNRpjybLDtCGRBED3jYvH\n03z8etj2LJEeCSyXNK8Ry4C3zK+l3CG5m3SNHfQUVxUJx/Px63H55benIwG7qUcDGOMlzMWZ3I/G\n0aQHILd5Eh8ayduzNjuXnqrv3wz4btp++2734/FZzEWyneXw2XYlvV/mgAXXxpSQFFN+uStSlrRF\nlVehNdPWwKfMa6o8uRrmpeD+oZTnp/r2Y33IkR/ZlzY3jndBx5Wbw9H94teUppVP615uUPI9pcfw\nq52rMT5hyT9+h/x6n+hIwK50x49f83CTxjw1GgdMpz9Tp3PsfD87/pmGyWr1h8BxY8iSTMPmgB2m\nEO5NVVuVHLfmnu6P+ZC5aBV85i3IwyBC21J4hQNXo5u0+hT//64gpt1O5xUXzjSWUv+mDlWgNN9r\nHtX8un2udtV/SQpsxiPb8nKe
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 21
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "heading",
|
||
|
|
"level": 1,
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Eigenvector Computations"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Returning to the view of images as data vectors in a high dimensional space, we can ask the question: what is the major direction of variation of these data vectors? This question is answered by computing the eigenvectors of the covariance matrix.\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"evals,evecs = linalg.eig(covariance)\n",
|
||
|
|
"top = argsort(-evals)\n",
|
||
|
|
"evals = evals[top]\n",
|
||
|
|
"evecs = evecs.T[top] # NB: eigenvectors come out backwards\n",
|
||
|
|
"print evals.shape,evecs.shape"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"(784,) (784, 784)\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 23
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Let's look at these eigenvectors.\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"figsize(12,12)\n",
|
||
|
|
"for i in range(36):\n",
|
||
|
|
" subplot(6,6,i+1)\n",
|
||
|
|
" imshow(real(evecs[i]).reshape(28,28),vmin=-0.1,vmax=+0.1,cmap=cm.jet)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAArsAAAK6CAYAAAA5P5QnAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvcmPJEl25vkT3U1t8S32jK2qsqaSjR5wGsNjZzEOJIE6\n8kaAR4Lgldfq7gGj0GwS/EsIHnkiwVNWZvHAwhyI7gY5zcqqjH31xXbTTUT6IKqmora5uUd4ZFaO\nP4SGmpvpIiJP5Mkn33siIrTWmku5lEu5lEu5lEu5lEu5lG+hOF93Ai7lUi7lUi7lUi7lUi7lUi5K\nLsHupVzKpVzKpVzKpVzKpXxr5RLsXsqlXMqlXMqlXMqlXMq3Vi7B7qVcyqVcyqVcyqVcyqV8a+US\n7F7KpVzKpVzKpVzKpVzKt1Yuwe6lXMqlXMqlXMqlXMqlfGvl3GD37//+7/nkk0/4/ve/z1/91V+9\nzzRdygeWS11+e+RSl98eudTlt0cudfntkEs9/hqLPocURaG/973v6a+++kpnWaZ/8zd/U//Lv/zL\neR51KV+zXOry2yOXuvz2yKUuvz1yqctvh1zq8ddbvPMA5J///Od8/PHH3L9/H4A/+IM/4G//9m/5\njd/4jfk1Qoj3AsYv5fyit9gv5FKXvx5yqctvj1zq8tsh2+gRLnX56yCXbfLbI+t0eS6w+/z5c+7c\nuTP/+/bt2/zTP/3T0nX/Wf8nPn/4BT98+On8O1MVdPm/AER5Xi9fPPy8fIZGsLlSLj/T3LOYjvPI\nuz5DI/ji4Rd8+vCHZ7gHWFE+Xzz8KT98+EMEqlEmVb7/m/iLrZ5/Vl3+9sP/2HhXVd4mlXqe4uV8\nmOu+ePj5mfK/Sr54+Dn/8eEPWVUui1Klya538O66PN8zhFVm5n+jx0/nKWWh3sLF6XJT2m29rpPF\ndqkb+dvO6H/x8Kd8+vC31/4u5uWwvt2bvNh16uybQm6jy/X2qqnLxbpWX/H+dflf9I/naV9nR1e1\n0dpmmE9VGW5+xnoR6LJdLurytDubchH2YTnPp4uxdaZur5I/F3+59bO21eWnf/appcf1ad9Uou9i\n16p6e/ozhJWi9Wl5X7rc/Izati+2u8oWffHwi7X1QaC31uVZ2uRPH/6sTPciFqGRhmURVror23i2\nNmTL191XbqpTi/Z0m1yuKhP7vr8U/3XtvecCu9uOXj5/+AWPP3vM5w/h7oN73H9wr+wSq85eo9lG\nDbYSTkkblO9Y/PZdqoydkvM9ozZf2xtdUzarc7Qojz97zKPPnp49XWfU5U8fwv0Hd7j34J65f02n\n3ngH9iDnbJ3Oaqmbx2nloq0SXKxpH74+iIXPJk1VGqs8Pf7sMY8/e3zmtJy3XVa6tFO2CrAty/bt\n8l1k/Rtq86wa0w/sO7ava6fnZF3NrXWJZdVEqcuLb5dP+OnDn3HvwT3uLuiySl1dDnYulwHLqhxW\nOVtdPnbnWNv05h1nqSPv3z7UNuCsTwFbj48/e3Ku1Gyry6pNAtybt8vV+tqUl/O3yO16Y7tkNpfq\n+9Ll6c/Q1G3Qvn4RwGkEjz97zJPPHlm/bydnbZM8/Jy7D+5z98F96z2rQZr1loW/ttH4Jvm6+8r1\ndcpGNrWNYe319X11WRpdPtmqdM4Fdj/66COePq0N+NOnT7l9+/bSdZ8+/G2wRmZ1hpojnG0Sur2y\n9UITtJmmd+2Wz/+MCtbUf233NnP16e+89+A+dx98Z37tFz/5Yqt3nE2XFZtsg9Zag+sYvTrvZ8v/\nJhGc3njr91cj+uYTPnR9WGzINjtS50Zz78Fd7j24d8G6XM+YLOtrvbx7Z7ZZahZ80fhWbxdrwe7p\nPiNbTtflpqfV/LPNHl18u/zhw0/56cOfbWRlsXTZZAirb8T8l1X6FA0rvY5GsDvz6lmLv20r798+\nrLYBpz1BWHpsDgo//8nPtn7Strq89+A+n1r2dZnRrdO1Kd3vp4/aLNV128DQd5dNz2jCwWWgW7fH\nqtyMLu/Orce2ujxLm9QNZtdOz+l95WrAe/5y/Dr7ys11ahnoii1rYFUm98u+smoXP/vJ52vvORfY\n/a3f+i1+8Ytf8OjRI27dusXf/M3f8Nd//dcrs3LH6rTrhJpfmwlfL/cf3Nn4+yqp3apGVqXjrLL+\nGafzBrXRvHsm478uzYsMjm583j6fZ9Hl3Qf3V3R5TTiyiuuz83v/wZ13BkimPizzR5tk8YpVDNji\nHael8n3UqbvlM8xT1NpOeht513ZZie1x2VQK69vl9mnepIdlp18T3CrtoLTLtR9+wkS2541AAEIo\nHEebc1mui0f95IvQ5TIYuAhdgrEp696x3Pk3pfrm7oN7C6XdvKaZq6ZUXeH9B3eXLKFd1qulCZPv\nLuhh1Vu36y+aqdjuzlrulB3o+wBr2+ryTtkWNrU9e9iyLt3nqcf2u+z6tM3wbzGVFZCrdLmKm66f\nvf5vOL2/ML84Vrtr/mIAUbO/sM9nkfO2ycXBy6a+svmMe437ziPnwU6L9X6Vjd62jlVXbbLzi2Wy\nSernLJflqWnR20bZL8jf/d3f8ad/+qdIKfmjP/ojfvzjHzcfLAQ/1v/P8gsbKr5YVqhpas9Wvc8L\nOKq3Ln5unrfNt33Xchqqbr85pjXf/oX4860nUGyvS5sDWmSKbM0upnXZrfyugNdOyWbdLkKm9Z9t\nw9ysoe/KANupWf28D6/Lzalc1PM2chY+9TSWtDrb4FbikuNT4JHpgLSIzCFDtHRAAQocoXCDAs8v\ncIOcQGQEmMMnx0XiUeAigaaVeB9i69IuE43gL8V/fa+6/C/6x6xqe4sM87Iuly2wYJ1d3vz8uvzs\n/C4zu02WsqlbPR+SNFfDrH4V8wHL4jNXpfcs9mGdLL/LrrP/TfzF1nqEd7OxzRTZ7avZiuy8blef\nl23jqv55s3Vt3mWGlqatVpqrdGvXFadxtT2UrZ+6SZo6WV3flnPa7K2q6/9c/OV7bZP/Wf8nFjXQ\nTNumvrKZavve88kyhlg3IG5e2ewf1+GhZgpX92v1+bRB7/rnLF959r7yXMwuwI9+9CN+9KMfnfm+\niiV43yBis5yXgl8EPqcrYfFvjVg6nx14b3tVxXOerWFsr0vz/LqjA7vp1ppd3VlWn5v3nU9s9nFT\nCdmaW9WY13XAze70/QAhUwe2yfWH0OUmsfV8EXXVvn75+ba2qk5T4pISkhCREDHTLSZFl3HWY5x2\nUYUDhYACHEcSxCl+nBIECTFTOoxpMyEiISCbt8fTmM+vU7bXZdOqNmvPJruz3G2t1uD65y9K00Is\nM2mLA0sDd1wkLkWpa1vcBnRSls4qZme1Vd7WPpwmyxzz2dsEnNfGrmobi2mzGS5bQ9sNVu2hmP2W\nRXu5TmrQVkPWAo8CD4nXaL/OHNwua13Mhzq11ta9sdlmV9e35XyujkQ9q2yjRzNg20TwbOorbbFr\n77vt/7X4FnuwqhENXTuoxt822D0rIXRW3HNWYvEsfeW5we755ZvWpSzL4ijGPjbJIoiyK9Hicy9C\nPkTJLnIJ9l/r3l83hqpxvJ+S2Ca/TV5tEcY2/655Bpt1er+ybc6/7lZykXV1leFc/L06THdoOs2E\niAltxnQY6S59ecBJekB/uo/MPMiBDFxHEjIl8qdEesKOGJARNHQr0HgUDcP+6yyLPNIydFl1T/Xr\nNqBo/fObsu5ti91d1TKbjL3EbdxlGHgHKNCIOVDaJiXvU6+2DbtoO77d8202tMk8i4XfN4nNtK77\n7TRtK5hDydoDEzR0WuBZAFeWcLgo22KOKHWrT7G4Nae9yEKfLh+qnZs8mU/rdLBtWvR7rnOrGNrF\nAb8NeNeTQebq5eHkOvLi4mTbsvwawO6HkfWMUfPzKjC7GeAuf7MKPFUVZtV1p6X4NNZ7cYT4IZrw\n6gZ7lvvrBnHafYvvWnRP2Zqyr19nVJrOtGZDtjuKylDXPJLT4JOq684qp/MOi9d9OFnkgt6nbBo0\nVuVtn2umr9ZDpgJz6IBZ2mI2
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 24
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Let's look at the corresponding eigenvalues.\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"figsize(6,6)\n",
|
||
|
|
"plot(sorted(evals,reverse=1))"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 44,
|
||
|
|
"text": [
|
||
|
|
"[<matplotlib.lines.Line2D at 0x64b0950>]"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAW8AAAFtCAYAAADI9OsfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAF5ZJREFUeJzt3X1wVNXhxvHnxgCKCBYmJJjEXzDlJRtCshBMS6EsIjIV\ngiDYASw4SPtHHdviWMcyoy12prz4MhWw05nSSpkyA52h0woMIETYgoCmmGCrtEAh0fASpkADJLxs\nXs7vjzUpIWGzJndv9oTvZ2ZH2b177gORZ45n7z3rGGOMAABWSejsAACAL4/yBgALUd4AYCHKGwAs\nRHkDgIUobwCwUMTyPnLkiPx+f9OjT58+WrlypVfZAAC34ER7nXdDQ4NSU1NVXFys9PT0WOcCAEQQ\n9bJJUVGRMjMzKW4AiAOJ0R64YcMGzZkzp9lzjuO4HggAbgcdvbk9qpl3KBTS5s2b9cQTT7QaIN4f\nP/vZzzo9AznJSU4yNj7cEFV5b9u2TSNHjlRSUpIrJwUAdExU5b1+/XrNnj071lkAAFFqs7xrampU\nVFSkxx9/3Is8MREIBDo7QlTI6S5yusuGnDZkdEvUlwq2+mbHcW39BgBuF250J3dYAoCFKG8AsBDl\nDQAWorwBwEKUNwBYiPIGAAtR3gBgIcobACxEeQOAhShvALAQ5Q0AFqK8AcBClDcAWIjyBgALUd4A\nYKEOl/dPfuJGDADAl9HhL2OQjPg+BgCIXtx8GQPlDQDeorwBwEIdLm/HobwBwGuUNwBYqMPlnZBA\neQOA11yZeTc0uBEFABAtlk0AwEKUNwBYiPIGAAtR3gBgIcobACxEeQOAhShvALAQ5Q0AFqK8AcBC\nlDcAWIi9TQDAQuxtAgAWYtkEACxEeQOAhShvALAQ5Q0AFmqzvKuqqjRz5kxlZWXJ5/Ppgw8+aPY6\n5Q0A3kts64Af/ehHevTRR7Vx40bV1dWppqam2euUNwB4zzHm1tV78eJF+f1+nThxovU3O44GDDD6\n29+k1NSYZQSALsVxHEWo3qhEnHmXlZUpKSlJ8+fP18cff6yRI0dqxYoV6tmzZ9Mx1dWL9cYbUu/e\nUiAQUCAQ6FAgAOhqgsGggsGgq2NGnHkfPHhQX//617V//36NGjVKCxcuVO/evfXzn/88/GbHUWqq\n0YEDUnq6q7kAoMtyY+Yd8QPLtLQ0paWladSoUZKkmTNnqqSk5KYQrHkDgNcilndKSorS09N19OhR\nSVJRUZGys7ObHUN5A4D32rzaZNWqVXryyScVCoWUmZmpNWvWNHudjakAwHsR17zbfLPjKCPD6L33\npAcecDMWAHRdMV/zji4EM28A8BrlDQAWorwBwEKUNwBYiPIGAAtR3gBgIcobACxEeQOAhShvALAQ\n5Q0AFupwebO3CQB4z5WZd0ODG1EAANFi2QQALER5A4CFKG8AsBDlDQAWorwBwEKUNwBYiPIGAAtR\n3gBgIcobACxEeQOAhShvALAQG1MBgIXYmAoALMSyCQBYiPIGAAtR3gBgIcobACxEeQOAhShvALAQ\n5Q0AFqK8AcBClDcAWIjyBgALsbcJAFiIvU0AwEIsmwCAhRLbOiAjI0O9e/fWHXfcoW7duqm4uLjZ\n65Q3AHivzfJ2HEfBYFB9+/a9xeuUNwB4LaplExOhnSlvAPBem+XtOI4efvhh5efna/Xq1a28TnkD\ngNfaXDbZt2+fBgwYoP/85z+aOHGihg4dqrFjxza9fvToYq1fL5WWSoFAQIFAIJZ5AcA6wWBQwWDQ\n1TEdE2lN5CavvPKKevXqpeeffz78ZsfRY48ZPfWUNH26q7kAoMtyHCficnQ0Ii6bXLlyRZcvX5Yk\n1dTUaMeOHcrJybkpBMsmAOC1iMsmZ8+e1fQvptR1dXV68skn9cgjjzQ7hvIGAO9FLO+BAwfq0KFD\nEQegvAHAe9xhCQAWYmMqALAQG1MBgIVYNgEAC1HeAGAhyhsALER5A4CFKG8AsBDlDQAWorwBwEKU\nNwBYiPIGAAtR3gBgIfY2AQALsbcJAFiIZRMAsBDlDQAWorwBwEKUNwBYiPIGAAu5cqkgV5sAgLc6\nXN533EF5A4DXmHkDgIUobwCwEOUNABaivAHAQpQ3AFiI8gYAC1HeAGAhyhsALER5A4CFKG8AsBDl\nDQAWorwBwEKulHd9vRtRAADRYldBALAQyyYAYCHKGwAsRHkDgIWiKu/6+nr5/X4VFha2HIDyBgDP\nRVXeK1askM/nk+M4LQegvAHAc22W98mTJ7V161Z997vflWnla+IpbwDwXmJbBzz33HN67bXXdOnS\npVZf37p1sc6ckRYvlgKBgAKBgMsRAcBuwWBQwWDQ1TEjlveWLVvUv39/+f3+W5546tTFKi4OlzcA\noKWbJ7avvPJKh8eMuGyyf/9+bdq0SQMHDtTs2bO1a9cuzZs3r/kALJsAgOcc09pCdiv++te/6vXX\nX9fmzZv/92bH0dtvG+3ZI61ZE7OMANClOI7T6meIX8aXus6bq00AID60+YFlo3HjxmncuHEtnqe8\nAcB77CoIABZiV0EAsBB7mwCAhShvALAQ5Q0AFqK8AcBClDcAWIjyBgALUd4AYCHKGwAsRHkDgIUo\nbwCwEOUNABaivAHAQuwqCAAWYuYNABZiS1gAsBAzbwCwEOUNABaivAHAQpQ3AFiI8gYAC1HeAGAh\nVy4VrKtzIwoAIFodLu9u3aTaWjeiAACi1eHy7t6d8gYAr7ky8w6F3IgCAIgWyyYAYCGWTQDAQiyb\nAICFmHkDgIVcuc67vp4bdQDASx0ub8fhQ0sA8FqHy1sKL52w7g0A3nGlvJl5A4C3XJt5U94A4B3X\nZt4smwCAd5h5A4CFmHkDgIUilve1a9dUUFCgvLw8+Xw+LVq0qNXj+MASALyVGOnFO++8U7t371bP\nnj1VV1enMWPG6P3339eYMWOaHceyCQB4q81lk549e0qSQqGQ6uvr1bdv3xbHsGwCAN6KOPOWpIaG\nBo0YMULHjx/X97//ffl8vmavL168WJWV0m9+I127FlAgEIhVVgCwUjAYVDAYdHVMxxhjojnw4sWL\nmjRpkpYtW9ZU0I7jyBij8eOll1+WHnrI1WwA0CU1dmdHRH21SZ8+fTR58mQdPHiwxWuJieHNqQAA\n3ohY3ufOnVNVVZUk6erVq9q5c6f8fn+L4/gGeQDwVsQ17zNnzuipp55SQ0ODGhoaNHfuXE2YMKHl\nIMy8AcBTEcs7JydHJSUlbQ7CzBsAvOXKHZbMvAHAW66UNzNvAPCWazNvyhsAvOPazJtlEwDwDjNv\nALAQM28AsBAzbwCwEDNvALAQM28AsBAzbwCwEDNvALAQt8cDgIW4PR4ALMTMGwAsxMwbACzEzBsA\nLMTMGwAsxMwbACzEzBsALMTMGwAsxMwbACzEzBsALMTMGwAsxMwbACzEzBsALMSWsABgIVfKu3t3\nKRRyYyQAQDRcKe8ePaTr190YCQAQDcobACxEeQOAhShvALAQ5Q0AFqK8AcBClDcAWIjyBgALUd4A\nYCHKGwAs5Fp5h0KSMW6MBgBoiyvlnZAQ3lmwttaN0QAAbYlY3hUVFRo/fryys7M1bNgwrVy58pbH\nsnQCAN5xjLn1YkdlZaUqKyuVl5en6upqjRw5Un/5y1+UlZUVfrPjqPHt/fpJ//qXlJTkTXAAsNWN\n3dleEWfeKSkpysvLkyT16tVLWVlZOn36dKvHMvMGAO8kRntgeXm5SktLVVBQ0Oz5xYsXS5KuXpX2\n7AlozpyAi/EAwH7BYFDBYNDVMSMumzSqrq5WIBDQSy+9pGnTpv3vzTdM/bOypD/9SfL5XM0HAF1O\nzJdNJKm2tlYzZszQd77znWbFfTOWTQDAOxHL2xijBQsWyOfzaeHChREHorwBwDsRy3vfvn1at26d\ndu/eLb/fL7/fr+3bt7d6LOUNAN6J+IHlmDFj1NDQENVAlDcAeMeVOywlyhsAvER5A4CFKG8AsBDl\nDQAWorwBwEKUNwBYiPIGAAtR
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 44
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"figsize(6,6)\n",
|
||
|
|
"plot(sorted(log(evals),reverse=1))"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 43,
|
||
|
|
"text": [
|
||
|
|
"[<matplotlib.lines.Line2D at 0x3a27ed0>]"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAFtCAYAAADrr7rKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xt81NWd//F3bggqV+VmggJJSEiIuQiEWKoBjKgQykUt\nweKWtqyWah9qK9X6s8JWAqylK7sVS1UUXRVqXURdbkENsAIGTUDuBEkkIYACghDQkXB+f5wSQBBD\nMjPf7+T7ej4e8yAzk5nzEfU9Z873XMKMMUYAAE8Id7oAAEDwEPoA4CGEPgB4CKEPAB5C6AOAhxD6\nAOAhkYF6486dO6tFixaKiIhQVFSUioqKAtUUAKCOAhb6YWFhKiwsVJs2bQLVBADgAgV0eId1XwDg\nLmGBWpHbtWtXtWzZUhEREbrrrrs0duzYU42GhQWiSQBo9Boc2SZAqqqqjDHGfPbZZyY1NdUsX768\n9rkANutXjz32mNMl1Al1+hd1+lco1BkKNRrjn+wM2PBOx44dJUlt27bVsGHDuJALAC4QkNA/evSo\nDh8+LEmqrq7WkiVLlJKSEoimAAAXICCzd/bu3athw4ZJko4fP6477rhDN954YyCaCqjs7GynS6gT\n6vQv6vSvUKgzFGr0l4BdyD1vo2FhzOwBgAvkj+xkRS4AeAihDwAeQugDgIcQ+gDgIYQ+AHgIoQ8A\nHkLoA4CHEPoA4CGEPgB4CKEPAB5C6AOAhxD6AOAhhD4AeAihDwAeQugDgIcQ+gDgIYQ+AHgIoQ8A\nHkLoA4CHEPoA4CGOhf7s2U61DADeFWYaerR6fRoNC9NFFxkdPCg1bRrs1gEgNIWFhamhke1YT79T\nJ+nTT51qHQC8ybHQ79pV2rHDqdYBwJscC/34eGnrVqdaBwBvciz0r75a+vhjp1oHAG9yLPTT06XV\nq6XgX0YGAO9yLPSvuUY6elRav96pCgDAexwL/fBwqX9/6f33naoAALzH0RW5mZmEPgAEk2OLs4wx\n2rNH6t5dqqqSmjULdhUAEFpCenGWJHXoIPXoIS1f7mQVAOAdjm+4NmAAoQ8AwRKw0F+0aJESExMV\nHx+vqVOnfufv9eghbdoUqCoAAKcLyJh+TU2NEhIStHTpUkVHR6tXr1569dVX1b17d9voaeNSmzdL\nffpIGzbY/XgAAOfm2jH9oqIixcXFqXPnzoqKitLIkSM1f/78c/5uYqL0059Kjz0WiEoAAKeLDMSb\n7tq1S51O67bHxMTogw8+OON3JkyYUPvztddm69e/zpYxUlhYICoCgNBTWFiowsJCv75nQEI/rA7J\nfXroS9If/iD98Y/2TwCAlJ2drezs7Nr7EydObPB7BmR4Jzo6WhUVFbX3KyoqFBMTc97XLFki/ed/\nsgkbAARSQEK/Z8+eKi0tVXl5uXw+n+bOnashQ4ac9zVXXWXH9b/1BQAA4EcBCf3IyEj95S9/0cCB\nA5WUlKQf//jHtTN3zudnP5OWLZMWLQpEVQAAR7dhOJcFC+xsng0bpHbtglsXALiZP6Zsui70JemB\nB6SaGmn69CAWBQAu12hDv7LSnqz16adS8+ZBLAwAXMy1i7MaKiZGGjZM6ttXWrrU6WoAoPFwZU9f\nsscozp8v3XWX9OKL0sCBQSoOAFyq0Q7vnO7996WhQ6Wnn5ZuvTXAhQGAi3ki9CVp7VrpppukX/1K\nevTRABYGAC7mj9APyDYM/paWJhUXS7162cPUH39ciohwuioACD2uvJB7LldcIX30kbRmjT1Q/dNP\nna4IAEJPyIS+ZI9XXLxYuv566brrpHfecboiAAgtITGmfy5z50oPPyz16yf97W8M9wBo/DxzIfe7\nHDkiDR4stWpld+i88ko/FAcALtVoF2fV1aWX2s3ZMjLs7c47pa1bna4KANwrpENfkpo2tQevlJRI\nCQn2vN3775eOH3e6MgBwn5Ae3jmXqiq7S+exY9LUqdK11wakGQAIOs8P75zLFVfY7Zl/9jNp5Egb\n+rt2OV0VALhDowt9SYqMlMaMkUpLpUGDpPR0u5K3qsrpygDAWY0y9E+66CLpkUek5culAwek5GTp\nt7+1s34AwIsadeiflJgoPfWUndnz2Wf2/sSJ0s6dTlcGAMHlidA/qV07u03z229Ln39uh31uukl6\n7TXp66+drg4AAq/Rzd65EMeOSf/zP9Jzz9lvAc88Y/ftZ3UvADfy/IpcfyoosGfz7t8vjR0r3XOP\n1Lat01UBwClM2fSjnBxp/Xq7wnfbNqlLF+nmm6U33pBOnHC6OgDwD3r638Hnk55/3g79bN9uz+wd\nPVrKyrKzggAg2BjeCZLdu6VZs6Q335QqK+2xjUOHStnZUliY09UB8ApC3wEbNkh//7v0j39Il1wi\n/cu/SD/+MeP/AAKP0HfQ8ePS0qV2CGjhQik1VRoxQrr9dqljR74BAPA/Qt8ljh6V/u//pL/+VVqx\nwn4DGD1aGjJEuuYaKZzL5QD8gNB3IWOktWulV16xi8AOHrQHvdxwg71ddpnTFQIIVYR+CNi+XXrr\nLendd+0eQOnpUlqaXQuQlMQwEIC6I/RDzKFDUlGR9P770tNPSzU19lrA4MHSgAFSSgofAgC+G6Ef\nwoyR9u6V1qyx3wQKCuy1gbQ0qVcv6brrpOuvZ00AgFMI/UbEGLsGYO1a6YMPpPfek9atkzIz7ayg\na66Rrr5aatbM6UoBOMWVoT9hwgQ9++yzavvPieuTJ0/WTTfddGajhH6dHDokLV5svwV89JG0ZYsU\nF2c/AK65xp4HnJHB7CDAK1wZ+hMnTlTz5s31wAMPfHejhH69fP21XRz20Uf2tmyZHSLKyLAfAj/8\nob02cPHFTlcKIBD8kZ2RfqrlDAR6YFx00ale/kl790rFxfZD4M9/lvLy7LkB8fFS7972+kBsrNSh\nA9NFAQSop//888+rZcuW6tmzp6ZNm6ZWrVqd2Sg9/YA5csTuFbRli50pVFQkVVTY84Fbt7ZTRjMz\n7Z+pqVL79k5XDKCuHBveycnJ0Z49e856fNKkSerTp0/teP6jjz6q3bt367nnnjuz0bAwPfbYY7X3\ns7OzlZ2dfaFl4AKcOGEPii8pkVavtheJ162TmjSx4X/6LTHRHi4PwFmFhYUqLCysvT9x4kT3jemf\nrry8XLm5uVq/fv2ZjdLTd4WTM4Y+/vjUh8DatfabQY8edvFYp0724nFcnHTllXZfIT4QAGe48kLu\n7t271bFjR0nSf/zHf2jNmjV65ZVXzmyU0He1o0ft+oFPPrGHx2/fbm87d9oZRf37S92721tiov3z\nWyN4AALAlaF/5513au3atQoLC1OXLl00c+ZMtf/WwDGhH7o++8yuKN6yRdq82d62bJGaNrUXj9PS\n7PWC9HT7baFpU6crBhoPV4Z+nRol9BuVk6uLt261w0PFxfbawZYt9uJxZqaUnCwlJJy6tWnjdNVA\n6CH04WonTki7dtkVxps32w+FrVvth0GzZjb8s7LsN4KsLDu1lIVmwHcj9BGSjLHTSjdvtucQbN4s\nrVxph446d7aLza6+2m5Al5xsLyDzYQAQ+mhkjh2zF4yLi6X16+1t40bpyy/tVNIePewis/797RYU\nUVFOVwwEF6EPTzhwwE4n3bDBTiddskQqK5O6dbMfBJmZ9paSwnRSNG6EPjxr/35p2zb7YfDBB/a2\nc6cdDkpOttcH+vaVfvADPgjQeBD6wGm+/PLUkFBpqT24vrxc6tdPuvFGKSfHfhgAoYrQB77Hnj02\n/JcssX82bWo/BLKz7bWB6GinKwTqjtAHLoAxdqbQe++dunXsaGcKDR4s3XILK4vhboQ+0AA+n/0Q\nKCqyR1YWFtqtqIcMkW66yV4o5sxiuAmhD/hRdbU9peztt+1w0PHj9jrAoEH220BiotMVwusIfSBA\njLHTQufNswvIiorsReBBg+xF4bQ0vgUg+Ah9IEh8PmnRInsx+H//1z72r/8q3XOPdMklztYG7yD0\nAQcYY3canT7dfggMHSo98IBd
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 43
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "heading",
|
||
|
|
"level": 1,
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Interpretation of Eigenvectors and Eigenvalues"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Why do we perform these odd eigenvalue computations?\n",
|
||
|
|
"\n",
|
||
|
|
"Assume that we have a diagonal matrix \n",
|
||
|
|
"\n",
|
||
|
|
"$$ D = \\left( \\begin{array}{ccc}\n",
|
||
|
|
"a & 0 & 0 \\\\\\\\\n",
|
||
|
|
"0 & b & 0 \\\\\\\\\n",
|
||
|
|
"0 & 0 & c \\end{array} \\right) $$\n",
|
||
|
|
"\n",
|
||
|
|
"What are its eigenvectors and eigenvalues?"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Simple: the eigenvectors are the Euclidean basis vectors, and the corresponding eigenvalues are the diagonal elements of the matrix $D$:\n",
|
||
|
|
"\n",
|
||
|
|
"$$ \\left( \\begin{array}{ccc}a & 0 & 0 \\\\\\\\0 & b & 0 \\\\\\\\0 & 0 & c \\end{array} \\right) \\cdot \n",
|
||
|
|
"\\left(\\begin{array}{c}1\\\\\\\\0\\\\\\\\0\\end{array}\\right) = a \\left(\\begin{array}{c}1\\\\\\\\0\\\\\\\\0\\end{array}\\right)$$\n",
|
||
|
|
"\n",
|
||
|
|
"or\n",
|
||
|
|
"\n",
|
||
|
|
"$$ D \\cdot e_1 = a ~ e_1 $$\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Now assume that we rotate the coordinate system with a rotation $R$.\n",
|
||
|
|
"\n",
|
||
|
|
"The matrix $D$ becomes $M = R D R^T$, and the vectors $e_1$ becomes $v = R e_1$. Then:\n",
|
||
|
|
"\n",
|
||
|
|
"$$ R D R^T \\cdot R e_1 = a ~ R e_1 $$\n",
|
||
|
|
"\n",
|
||
|
|
"or\n",
|
||
|
|
"\n",
|
||
|
|
"$$ M \\cdot v = a v $$"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"We see:\n",
|
||
|
|
"\n",
|
||
|
|
"- the normalized eigenvectors form a basis for a rotated coordinate system\n",
|
||
|
|
"- in that coordinate system, each eigenvector is a Euclidean basis vector"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"There's a second important point to the interpretation of these matrices.\n",
|
||
|
|
"\n",
|
||
|
|
"Consider a multidimensional Gaussian density: \n",
|
||
|
|
"\n",
|
||
|
|
"$$ p(x) = \\frac{1}{\\sqrt{(2\\pi)^k|\\boldsymbol\\Sigma|}}\n",
|
||
|
|
"\\exp\\left(-\\frac{1}{2}({\\mathbf x}-{\\boldsymbol\\mu})^T{\\boldsymbol\\Sigma}^{-1}({\\mathbf x}-{\\boldsymbol\\mu})\n",
|
||
|
|
"\\right) $$"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Now assume $\\mu=0$ and $\\Sigma = \\hbox{diag}(\\sigma_1,...,\\sigma_d)$.\n",
|
||
|
|
"\n",
|
||
|
|
"$$p(x) \\propto \\exp\\left( -\\frac{1}{2} {\\mathbf x}^T\\Sigma^{-1}{\\mathbf x} \\right) = \\exp(\\sum r_i \\sigma_i^-1)$$\n",
|
||
|
|
"\n",
|
||
|
|
"We can get an idea of the shape of this distribution by looking at $p(x) = \\hbox{const}$:\n",
|
||
|
|
"\n",
|
||
|
|
"$$p(x) = \\hbox{const} \\Leftrightarrow \\sum \\frac{x_i^2}{\\sigma_i} = \\hbox{const} $$\n",
|
||
|
|
"\n",
|
||
|
|
"But this is just the equation of an ellipse; in 2D:\n",
|
||
|
|
"\n",
|
||
|
|
"$$\\frac{x^2}{a}+\\frac{y^2}{b} = 1$$\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"(summary PCA intepretation)\n",
|
||
|
|
"\n",
|
||
|
|
"- PCA finds a rotation of space that diagonalizes the covariance matrix\n",
|
||
|
|
"- the diagonal elements in this rotated coordinate system give us the distribution of the data along the corresponding axis"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Projection and Reconstruction\n",
|
||
|
|
"============================="
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"PCA allows us to \"compress\" data. In particular, on average, if we leave out some direction given by an eigenvector, we lose as much variance as was contributed by that eigenvectors. The best vectors to leave out (for data compression) are those with the smallest variance.\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"ys = dot(make2d(evecs[:20]),make2d(centered).T).T\n",
|
||
|
|
"print ys.shape"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"(60000, 20)\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 25
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Now we only have 20 numbers (instead of 784) stored per digit. What does the reconstruction look like?\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"reconstructed = dot(ys,make2d(evecs[:20])) + mu.ravel()[newaxis,:]\n",
|
||
|
|
"reconstructed.shape = (60000,28,28)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 26
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"showgrid(reconstructed[:50])"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAkwAAAEzCAYAAADD+FxKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsfWmMZFd1/6/2vbqqt+pllp7xeDzGxsZgBULif4zAmCwg\nEBIiCcTYJkGRSBQ5QhAREpIPsfmASIB8QIk/GCIhSCQCAsmK88EQAY4T7/bYs3b39FrVXUvXvr//\nh87vznm3X1W9WnoM4zpSqaurXr13z73nnnvO75x7rsMwDANjGtOYxjSmMY1pTGPqSM7XuwFjGtOY\nxjSmMY1pTL/oNDaYxjSmMY1pTGMa05h60NhgGtOYxjSmMY1pTGPqQWODaUxjGtOYxjSmMY2pB40N\npjGNaUxjGtOYxjSmHjQ2mMY0pjGNaUxjGtOYetDABtPjjz+OM2fO4MYbb8SXvvSlUbbpF4bGPF4f\ndL3zeL3zB4x5vF7oeufxeucPeGPw2JGMAajZbBo33HCDsby8bNTrdeP22283zp49O8itfmFpzOP1\nQdc7j9c7f4Yx5vF6oeudx+udP8N4Y/DYjQZCmJ5++mmcOnUKS0tL8Hg8+OhHP4rvf//7o7blXlca\n83h90PXO4/XOHzDm8Xqh653H650/4I3BYzcayGDa2NjA0aNH1f9HjhzBxsaG6RqHw/FL/XrnO995\n3fP4RhjH653HN4KcvhF4vN7l9I3A4xtBTt8IPHYjd9dvO1Cvm5JmZmYwOzvb9ZpUKtX1ml7fH9Y9\nXnnlla7XkxKJBObm5jp+v7293fV7O9ccxj1eeOEF2+N4+vRpnD59uuP358+f7/q9fo3Vc8+dO4eb\nbrrJ9JmhndrT6zn69z/84Q9t8xgIBBAIBDp+X6lUhvr+sO6RyWS6Xk+an5/HwsJC12s2Nze7XtPr\n+1Hcw+r7Z555puszSfPz85ifn+/4/dbWVtfv7VxzGPd49tlnbctpKBRCKBTq+H2pVOr6vZ1rDuMe\nqVTKNo/RaBQTExMdv9/b2+v7e12X5PN5RKNRy987HI6ez7B6ztraWtfrSdPT05ienu56ze7ubtdr\nen0/intYff/aa691fSZpamrqdW//IPc4d+5c1/sNZDAtLi6ahGNtbQ1HjhwZ5FYdSQq41Xtagy6X\nCy6XC36/X33fbrfVS58o/dBh8yjJMAyTQuF7tl/yPSo6rHHs1EZ+Lv86nU64XC54PB4EAgEYhoFW\nq9Vx/OS97YztYcqqzucox6YfupZyCljzqc9RfZ4OS4fNo9Xck//z5XQ64Xa7TTx20lX90LXQqTp1\nG5du3/0y8aiPEdeHVquFRqOh9C7HdlhZvdb8SbIzZr9Mc1HXHVbzzUq/WF03KhooJHfnnXfiwoUL\nWFlZQb1ex3e+8x184AMfGFmjJMN8Ucj5arfbAAC32w23241gMIhgMAi/3w+PxwOXyzX0BDhMHklW\nvOr/04gg36MShMMYR7v9LRcfr9cLj8eDYDCIQCAAr9cLl8s1ksl92LLajfQxBQ7HqLpW/HUzfhwO\nh2muUk6t+mAQOkwedQVNI97lcin94vF4lFx6PB54PB643W44nc6RLLTXWk6t2qsbhuRrVDL7es1F\nyh7XkGaziVarhVqthnq9jkajoT6T+nUQmX29dE036uTMDErXai7yL2XRap7p81Vecxi6diCEye12\n4+tf/zruvfdetFotPPjgg7j55psPXNcL1tWvsTIY/H4/Go3GASXs8XjgdDrh8XgwPT2t7tNsNlGr\n1VCtVtV9wuFwX+0g2eGx172tvtd5DAQCaLVa6ns50K1WCx6PB9VqFQ6HA263WwmHvG6Qdtgdx6mp\nqa73npqa6imcU1NTHRWRy+XC4uIivF6vSanp18t7dHrGoDy63d2nAr/vxKfH41HvdUOBk9rr9cLp\ndB64xm4bOl1jh79IJNLz3vIaHXFxOByYmJiAy+Uy/UbnNRgMotlsqt/pi26vdnT6/rDmom4kxGIx\n+Hw+NcekIgb2w36hUAjNZlMtss1mUzlwdngcZi5KObMi+X0nWfV6vSYDV6IsLpcLgUAAHo8HDofj\nAFrPa71er+129Mujz+frem+r7yWvgUDANM+AfT1Kx5PIEtcIGsXyXuRPR/57tcMOf8FgsCt/dq6R\n33dyzPx+v2ldkd/1+wxJo+DR6ns9lygSiSidKSMulMVoNKpkWX4HQF3TywaxMxamNhqHgVthn/lb\nbrnF9vW64pUdwwkrB9/v9yMUCiEWiyEUCsHn88EwDDQaDVQqFVSrVZPn0AlC70SvvPJKz+scDgdu\nv/122zxa8cr7OBwO5cV6vV643W60221Uq1WUSiXlCdGadrvdthPVrOiFF16w1Q8OhwO/8zu/0/X7\nTp9ZfSeVr9PphM/ng9/vh9/vV2NcqVQUv3bu2Yl++MMf2uZxcnLS1nWdPpOTlt6Qz+dT6Fm73Uaj\n0VB86YvQoJTJZGzJ6dve9jbb99QVlzR6uoU46vW6mm8ATJ5hJ8TCzng+88wztnh861vfaos/PTzB\nttE4IoIkkSQ6KIZhoNlsol6vqxf1zKCo2rPPPmtbTnvlYurX6/9bhcMDgQD8fj+8Xq+Jv0ajoQxC\nnT/Zj3YolUrZ5lEmFdvhj3IljVsdkSBP1WpVOdRElCSSOAxSsba2ZktOz5w5Y+t+vUjKMdcOop7k\ns16vKyPCaj73S6+99potHvV81F6kI5tyTNheaQ/oyBMjMNKRoY6VfdWLzp071/XagRAm0tLSEqLR\nqFIyTz/99ED3sTKWrKB94OpE9/v9CIfDCIVCyqOg4iaEzuulp9TvZB8Vj52IYSm/34+JiQnE43FM\nTk4qTz2bzSKZTCKTyWBvb89yQRpkERoFf3Zgffk/lbH0Wl0uF2q1mlpw6/W6+l8qdV3ZWz2705iO\nYgx78Sqv8fl8CAQCWFhYUOhnqVRCKpVCOp1GPp83TexR0LA8Sv44bux73WCSzgxw1WjqNHY6UtOp\n3w6TP5LVgi/5YrvdbrfJ4JXt5+JbqVQUkuFwOEyyLZ9nlw5TTvlX9j916JEjRzA3N4doNIpyuazk\nNJfLoVarmfiQC5B0Eq4lj+RFypgcL/mXOrJWq6FSqaBQKKBcLiunrF6vd0WRBjEqDnvNAK7OOUZZ\n4vE45ubmEAqF4Ha7sbm5iVQqpYx5AKqvuvFrh0bNn9VYct5x7jkcDpNj4vV6EQgElEHVaDRQrVZR\nrVbVuLKfBpVVKxrKYHI4HHjyySdteed27qUrUH3xZVgjEAggFoshHo8jFArB4/Gg1WopNKZSqcDj\n8Zi8P7lAEXGSiu1a8AiYFTZRiHA4jMnJSbWTaWZmBqFQCI1GA5lMBpubm9ja2kIymcTe3h5KpZLy\n/mTfyL6S/XjY/HXy1OmxsW2GYShh5mTXkcFCoaCUGJWgXLjlvfTYvERARsmjlafOv1Yon8fjQSgU\nwtGjR3HjjTdibm4OxWIRKysruHDhAq5cuYJSqaSuHzZvgm0ZxThaGTrS4ZDziH1Oz44efLPZVEYH\nQzuyjyif+rMPmz8r9IfvpUFEHRMMBk0KWzc4ms0mGo0GXC7XgZC6bozZoVHrGv3e+v8ejwfhcBjH\njh3D6dOnsbCwgGKxiNXVVVy4cEGFxTmeVohov/I6KjmVm0X8fj+CwSAikQgikYjJkebYVSoVlMtl\nFItF06Jaq9VQLpcVIiPzYyV//czPwxxHHVBgX0SjURw/fhyJRAKRSAQXLlxQqRzkSzoE+px8vfmT\njgqNXRpMcu2gwURELRKJIBgMKoOpWCyiXC6bojKdxnQQGspgGvbhJN0w0qFx2YmRSASxWAxTU1OI\nx+Mq1lypVJDP55HL5VAsFlWH1Wo1FQphB1LZSSXSjY9RRS2lsBNCDgQCmJycxNGjR3HixAmcPHlS\nGUz1eh2ZTAbT09OIx+MIh8NYX19HKpVCNptVsXgdimafyr+92jUsyXFjWJEvCanWajXUajXFv8/n\nQ61Ww97eHqrVKvL5vPre7/er
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 27
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"showgrid(clip(reconstructed[:50],0,1))"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAkwAAAEzCAYAAADD+FxKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsfVmMZFd9/lf7vm9dXd09vc+MZ+zxMoEoxAEExuQhyAgJ\nyEIMNosikShyFEFEojh5iM2DhQLkAf2DIodIKCQSAYFicBKMFTCy8Sz2eDzTnp7unt6quqprX3up\n+3+YfGdO3a7qruVW9zCuTyrNdHfVrfvdc87v/PajUxRFwQADDDDAAAMMMMAALaE/6hsYYIABBhhg\ngAEGuN0xUJgGGGCAAQYYYIABDsBAYRpggAEGGGCAAQY4AAOFaYABBhhggAEGGOAADBSmAQYYYIAB\nBhhggAMwUJgGGGCAAQYYYIABDkDXCtNzzz2HEydOYGZmBl/+8pe1vKfbBgOOdwbudI53Oj9gwPFO\nwZ3O8U7nB7w9OLaE0gV2dnaUqakpZWFhQdna2lLOnDmjXL58uZtL3bYYcLwzcKdzvNP5KcqA452C\nO53jnc5PUd4eHPdDVx6ml19+GdPT0xgfH4fJZMLHP/5xfO9739NalztSDDjeGbjTOd7p/IABxzsF\ndzrHO50f8PbguB+6UphWV1cxOjoqfh4ZGcHq6mrDe3Q63a/06zd+4zfueI5vh3G80zm+Hebp24Hj\nnT5P3w4c3w7z9O3AcT90pTAddNEBfjXwdhjHtwPHAX718XaYp28HjgPc2TB286FYLIbl5WXx8/Ly\nMkZGRjS7qXah0+lgNBqhKApsNhsAoF6vY3d3Fzs7O6jX61AUBUqXx+UdJUdZuHR7/wfhqMZR1uYN\nBgOMRiN2d3dhsVigKAq2t7exs7OD3d3dnrnfLnO1n7jT+QH958j1pigK9Hr9nr/xVa/XYTAYhFyR\nX/x8NzjqeaqWN/2QP0fN8TBwVPzk8dLpdD3tewfhsDnKa7PZ32TPkBZrcV90k/i0vb2tTE5OKgsL\nC0qtVmua+AWg7y+dTqc4HA7FbrcrIyMjysjIiBKJRBSv16vYbDbFZDIper2+6+sfFUedTqfodDpF\nr9crOp2ub99zFONIXgaDQTGbzYrD4VACgYDicrmUWCymRKNRxev1KlartaexO0qO6nHs1/WPYp6S\nk/zqN79+c5TnpF6vV0wmk2I2mxWLxaLYbDbF4XAoLpdL8Xg8itVqVdxut+J0OhW73a5YLBbFZDIp\nBoOhp+dxO8xTWeYYDAbxPLQa58PgyPs87Pl5GPP0oPHT6/UN8/hXlaM8dpyH8nzkS6fTKUajUaxV\ns9msGI1GsRa7/f790JWHyWg04utf/zoefvhh7O7u4vHHH8fJkye7uVRP0Ol0sFqtMJlM8Hq9AICt\nrS2USiXxHkVlAXaCo+Io32s3990ujmIcFUn7p2VgNBphtVphs9mEd3Bra0uT7zsMjrTo1JaQPHZ8\nj16vF54Krca2X/zIR7bg+H/53uv1Our1uibf2Qr94EhOer0eer0eBoMBAGCxWGAymWA0GmEymcT/\nAaBUKsFoNKJWq2F7e7vBG7q7u9v1uB7WPAVurb1ma9FgMECn08FkMgFAAy+Ocbfztl8cW81TQlEU\nTdfbfjiKPUPNq988+ylv+KKcNJlM4v9A41ju7u7CaDSKvzG6JI+11s9Cp/Tp6fY7Xm00GmE2mxEM\nBuH1emGz2aAoCmq1GorFIorFIqrVKra2tkR4rtMFf9D7tOKo1+thNBphsVhgs9lgs9lgMplQr9dR\nrVZRKpVQqVQ0UyKIdp5DpxzVAkudSCePg8FggNVqhd1uh9PpRL1ex/b2NsrlMqrVKmq1Ws8Tvh8c\n1Z9T/8vN1+l0CoV+Z2cH5XIZW1tb2NraaljY/ebYzRjKgosCq1mYhoJrZ2en8xvvAP1Yi+rxokyx\nWCywWCywWq2wWq1CgaKSUa1WUalUxIvKExWnbsa1n/OUn1XPU6PRCKPRCJfLBbvdLkLiW1tbqNVq\nYq6qQ+TdzlmtOcpzlONHJddoNArlD4DgwX+3t7fbvqdOcFh7hmyk6XQ6sXdwnlKGcu/TEv1ai1TY\nDQaDMFRMJpMYR1kh4pjzb7u7u3vGl3tNp2O83/u78jAR4+PjcLvdguDLL7/cy+U6gsVigdvthtfr\nFffARW2xWLC7uysWEx8gHzbQ/kI5DI5GoxEOhwORSATDw8OIxWLweDzY2tpCPB7H9evXsb6+js3N\nTc2/Wyt+sjDmBsvNVs4J4aQGbo7Bzs4OKpWK8CptbW11PdFbQesxVCuD6ji61WqFw+HA5OQkhoeH\n4fF4kMvlcOPGDWxsbCCdTgtFXos8LS05yuNGgUQrjkJarRAchuXej3XI+67X6w0bED2eDocDNpsN\nVqsVZrNZbMK1Wg2VSgW5XE4IbHUOye7ubsN3HBVH3o/actfr9XA4HPB4PJiZmcHY2BgCgQAKhQJW\nVlawvr6OVCqFcrncwK3X+dqPeUol1263w263w+FwiLGjAlEqlZDP55FOp5HP55HP5/vmdTqMPYP3\nTS+M3+/H6OgonE4nTCYTlpaWEI/HUSgUNFeY+iFPOZ40WmRjhWusWq1ie3sbiqLAYrHA5XLBbDbD\nYDCgWq2iXC4L5wIAYcT16hmV0ZPCpNPp8MILL8Dv9/d8I7yeWtOUX0ajETabDQ6HA8FgUHiXrFYr\ntra2UCwWkcvlUCwWYbFYhCUoW0s7OzsdeZy05ihDr9fDYrHA7/cjFothZmYGx48fx/j4OPx+P6rV\nKlZWVjAyMoJr165hYWEBm5ubKBaLYuL0Ci34NbP0mlkI9ABWq1UANz1MVG7lEEczyJ4OeaNux+LV\ncgybKUtUCPk3s9kMj8eD6elpnDlzBqOjo8hms7h69Spee+01zM3NoVAooFwuN2zavd6XFuOotta5\n9uRiCvWm2e9wHO+tX+sQuCUD6PV0uVxCtnDTlZVIGmL0xnDetgoJtYN+c5S/R6fTiXk6OzuLe++9\nFyMjI8jn85ifn8fFixeFJ3R7e3vfxNtOv7sXjvL+QMWWY8U9IRAIwOfzwePxiJBOPp9HNpvF5uYm\nCoWC8NjXajXk83kUCgUUi0WxV6gNg048a4c1jvwug8EAr9eLqakpxGIxeL1e+Hw+vPnmm7h27VpT\nmbqfx7id79SaH9cWlSW73S4UJgBiLOg5okfN7/fD5XKJVJxsNotsNot8Pi/2fjms3C7HVuhJYer1\ny4G9G5Ber4fJZGpwjcsPMRAICC9MNBqFzWaDTqdDPp9HMplEPB5HJpNBNptFuVxGuVxGpVJBtVoV\nL51OJ7TPdiZKvyxovV4Pl8uF4eFhnD59Gvfffz8eeOABjI+Pw+fzoVKpYHV1FceOHcPQ0BC8Xi8u\nX76MlZUVFItFzbwTvVxDDmvIIQ3mJHHDAW5apwxjADc9a3a7HeVyGbVabd/v4AbeLF7dzkLQSrnk\nv7K1rhY+JpMJTqcTU1NT+LVf+zXcfffdyOfzGBkZwe7uLjKZDOr1OnZ2doSXTYu8tV7HkUopFV6z\n2SyUQSpLtVptT/hNrTQ2u4+j5tfOdanAO51O+P1+hEIhoSzRQOEGpdfrUa1WG55RL+Eq9b30G4qi\nwGQyweFwYHx8HPfddx+mp6dRKpUQiURQLBaxurqKUqm0Z35r8d3dQvZE2Gw2eDwehEIhRKNRjI+P\nY2xsDLFYDJFIBH6/H2azGTqdTmyk2WxWbKLVahX5fB4rKytYWVnB8vIyMpkMCoVCy9y0dr1SWo1j\nO4qqwWCAx+PBxMQETp8+jZGREdhsNuzs7GB9fR3FYnFPPqUsv9Tz9jD5yfcke5fkvWN3d1c4Oxhm\npPcwGAwiGo2iXq+jWCwimUzCarU2hOvkvaVXr3jPHqb3v//9MBgM+NznPofPfOYzHX1WttpMJpPQ\nGl0uFzweD1wuF9xuN5xOp1gYsVgMx44dg9vthsPhECXp+Xwe8XgcS0tLWF1dxfr6OtLp9J6EMfXr\noAXQC8eDYDAY4Pf7cezYMZw8
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 28
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"ys = dot(make2d(evecs[:100]),make2d(images).T).T\n",
|
||
|
|
"print ys.shape\n",
|
||
|
|
"reconstructed = dot(ys,make2d(evecs[:100])) + mu.ravel()[newaxis,:]\n",
|
||
|
|
"reconstructed.shape = (60000,28,28)\n",
|
||
|
|
"showgrid(clip(reconstructed[:50],0,1))"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"(60000, 100)\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAkwAAAEzCAYAAADD+FxKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsfVlso+d19sNd3Fdx0TKipNGMZvd4i5HWWZo6SVskcBAg\n6ZY6S5ugFwUKF0UCtGndXjTOhRE07X8RIEbhtkCA3AQNkMJtltpxEsf2jGfGM5pFGu0bSXFfRVLk\n919MnndefiIliqTGzpgHIEYjkR+/873nPe9ZnnOORlEUBX3qU5/61Kc+9alPfWpJ2rf7BvrUpz71\nqU996lOf3unUN5j61Kc+9alPfepTn/ahvsHUpz71qU996lOf+rQP9Q2mPvWpT33qU5/61Kd9qG8w\n9alPfepTn/rUpz7tQ32DqU996lOf+tSnPvVpH+rYYHrxxRcxPT2NqakpfP3rX+/lPb1jqM/j/UH3\nO4/3O39An8f7he53Hu93/oB3B48tSemAdnZ2lMnJSWVxcVGpVCrKuXPnlOvXr3dyqXcs9Xm8P+h+\n5/F+509R+jzeL3S/83i/86co7w4e96KOIkyvv/46jh49inA4DIPBgN///d/Hf/3Xf/Xalntbqc/j\n/UH3O4/3O39An8f7he53Hu93/oB3B497UUcG0/r6OkZHR8X/R0ZGsL6+3vAejUbza/1673vfe9/z\n+G5Yx/udx3eDnL4beLzf5fTdwOO7QU7fDTzuRfo9/9qC9rvovSSNRgNFUaDV3rH9FEWB0p/20ha9\nk9bxsOjdwGOffv3pnSan6vvphU59p/HYpz4dlDoymIaHh7G6uir+v7q6ipGRkZ7dVDNSbzaNRgOt\nVgudTodarSb+rdVqPfvOe81jMyLfRqMROp0OwB3lRV7r9XrHyuxer6O8hrTmtVotNBqNWEPgrtGr\nfnVCb4esNiPyTsMeaDTuuzmQ3gn8HTYdNo9cn2brIHuf/Hu9Xu/p978T5FSj0Yg9aDQaAdzhs16v\nd61rgHcGj4dNb9e5qF4X/r6VTu2G3olreM8CJp0An6rVqjIxMaEsLi4q5XK5KfALQM9eGo1m18tg\nMCgDAwOKzWZTjEajYrFYFKPRqGg0mp5972HzSF72eo9Wq1W0Wq1is9kUr9er+Hw+xePxKDabTTGZ\nTIpWq+2Y53u1jrw/8qvVahW9Xq8YjUbFbDYrVqtVMRqNis1mU2w2m2K1WhWz2ayYTCZFr9e39Zze\nbh73e3EdjUajeBkMBkWn03Uts+8E/uQ1Uu/VXlz/MHmkTHIvqV86nU4xGo1CHnU6Xc+f39stp+TT\nbDYrAwMDQtc4nU7FYrEoBoNB0Wq1v9Y87ie7vZDXe6lPKbPNzgCdTifWk2vKNfx14LHdNdPpdOIs\nMRqNPdGne1FHESa9Xo9//dd/xUc+8hHUajV84QtfwIkTJzq5VFukKIqwmOXIkk6nE55epVLp2gNS\nU695lHmQoy173bead4PBILy+er2OnZ2djnk+jHVsFnanZy7fJ3/ms9DpdNDr9cJTIG98r/rz7dK9\nltW9SKvVioiamp9uZPft5k/Nl1arbYjE9GJfHoacNnvV63Vx/4qiCD2j1+uhKAqq1WrLSLYcgToo\nvd1yqtFoYDQaYbFYoNPpMDAwINaNOqbbNXy7eWxG1D1arVbw+07ci3KEmnKm0+mEzNZqtQadSl1q\nNpsBALVaDdVqVfwM4B3HYzvE85/8DQwMQK+/Y8bUajVsb29jZ2enZzK76/uVXl+RF+5xvpoKzWAw\nwGQywWq1wmw2o1qtolwuI5/PC6PJYDA0pLBqtRp2dnZEWJkbYj/W9/v7QXiUU1DyASPTwMAALBYL\nHA6HEPRyuYxCoSAUNfmpVCqoVCqoVquCp4NSO0vfCY974R+4NgMDAzCZTOJwqtVqDc+kXq+jUqmg\nXC6jXC5jZ2enI0XWax6bfU5tCJM/t9sNp9OJgYEB7OzsoFAoIJ/PI5/Po1qtNpXJw1jHbvmTf5Zl\nWK/Xw2g0wmAwCEOee40/V6tVIaPdqJpe80hZk9PCza5pMpnEvuThlMvlUCgUGpwVOTUi65Z2eT5M\nOW12Da1WC5PJBLPZDL/fj8HBQbjdbrEnE4kEtra2kEgkkMlkUCqVxKHc6ToeNo/yetLIpWzygOX9\ny04njcJyudxgDHfC52HuRTlQwD3Ie9dqtTAajTCbzTCbzQ187uzsoFqtYnt7G5VKpWNdSjpsfSPv\nSfmspHE7MDAAp9MJt9sNs9kMvV6PQqGATCaDeDyOYrGI7e3tjh3tvd7fUYSJFA6H4XA4oNPpYDAY\n8Prrr3dzuX2JFrPb7UYwGITP58P29jYymQxWV1eRzWZRrVbFwxwYGIBGo0GxWBSvcrksDKt2hKYX\nPKoPGb1eLwSfnoLBYEAwGMTY2BhOnjyJ4eFhaDQaxGIx3L59G4uLi4LHcrks7r8ZvuIg1Ks1VAs3\nsPvw0Ol0sNvt8Hg88Pl8sNlsACAMXm5kAEKBMYrYqTHRSx6b8Ss/f66px+NBKBTCww8/jJMnT2Jo\naAjJZBI3b97EtWvXcPv2bXEI8VoAxHq+neso8ye/yCsPIqPRCLvdDrfbDbvdjoGBARQKBbHPqJyL\nxWLbDsq95I8kr6EsrzycqG/8fj8GBgagKAqi0Sii0ahwZNQRqk4xP4fFo9rwpR7yer0YHR3Fhz70\nITz66KOYmJhAvV5HIpHAL37xC7z++uuo1WoolUpC58j0TpBTmS/uPzosVqtVnAVWqxVGo7HBeS6X\ny9je3kY6nUY6nUalUulJVOIw9qLaCNTr9ULeAMButyMQCGBkZASBQACJRAKJRAKpVAr5fB61Wq1n\nQYzD0qetjF2+NBoNBgYG4PV6MTY2hmPHjsFms0Gj0WBpaQnz8/PCES2Xy4eCZerKYNJoNHjppZfg\n8Xi6uob8s1arFVEIWssmkwlGoxEmkwkulwuBQABHjhzByMgIdDodyuUyNjY2EIvFkEgk4PF4MDg4\nCLvdDkVRsLW1hY2NDayuriIejyOVSqFcLouozV6bpFseZUGgscTF5ya32Wzw+/148MEH8dhjj2F0\ndBRerxcAkM1mcebMGczPz2N2dhZXrlzB8vIy0um0iLjxGcqefbvWda/4o5DTk9NoNEJ4gTvheKvV\nCq/XKzxap9MJvV6P7e1tZLNZlEolbG9vi+iSwWAQHhSfoUyyIbXX4dQLOW3Fr7y5KaOjo6OYmprC\nY489hrNnz8LlcqFQKGBsbAyBQAA+nw+3bt1CJBJBsVgUhyuVOdfxIAbiYfBImSVvPIgYcXE6nfB4\nPDCbzTAYDMhms+JVKBRQKpUEH/Rs5dTBQe+pV/wBdw97puCod0wmk+DN5XJhcHBQHEQ2mw0GgwEL\nCwuYm5vD/Pw8tra2UKlUxDW4hpTjt4NHtaErv+T1HBoawtTUFM6dO4fz58/D5/OhXq/D7/cjk8kg\nk8kgl8uJA1eOZndqFPbqzGDaiTJpNpvhcDjgcDjgdrvh8/mEnvF4PDCZTNDr9bvgDMViESsrK5if\nn8f169cRi8WQSqU6vrde8diKX0Y8zWaz0Ie1Wg1HjhzBQw89hKNHj2J4eBiJRAKrq6t46623sL6+\njlQqJfQV9bL8LAC0bSz2Wk4BNOxBg8EgdIzNZhN2AHAnQ+F2uzE1NYVHHnlEnCGTk5MYHBxEqVRC\npVJBPp8XAQn1qxvqymACOs+DNgu/0WKlZ+Dz+eDz+eB0OmG328WBGwwGMTExgfHxcbjdbuh0OiST\nSayvr2NxcREulwt+vx9utxv1eh2Li4uYmZnBxYsXMT8/j1qthlwuJ+59PyXe7UOmMMiHrBx1cblc\nmJqawvve9z58/OMfFx6ETOvr67hx4wYMBoOIyCiKIg4yGihM1VGZtXP/3fDHdaOxIKdCt7e3sb29\nLSKDXq8Xg4OD8Pv98Pl8cLlcsFgsKJfLDcq5Uqlge3sber1eKGoah7LRxFAzc/P7YcG6JdlYkg0k\nGhFmsxlWqxUTExM4c+YMzp8/
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 29
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Note the best approximation property: if we use the eigenvectors corresponding to the largest\n",
|
||
|
|
"eigenvalues, then we get the best linear approximation of the inputs for that many inputs.\n",
|
||
|
|
"\n",
|
||
|
|
"Why? Again, think about this in diagonalized space."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Classifiers after PCA\n",
|
||
|
|
"=====================\n",
|
||
|
|
"\n",
|
||
|
|
"Many classifiers are invariant under isometric linear transformations of the input\n",
|
||
|
|
"and/or just have a linear transformation right at the input. Among these are:\n",
|
||
|
|
"\n",
|
||
|
|
"- nearest neighbor\n",
|
||
|
|
"- MLPs\n",
|
||
|
|
"- support vector machines\n",
|
||
|
|
"\n",
|
||
|
|
"Performing PCA speeds them up greatly, and, as long as the PCA reconstruction is\n",
|
||
|
|
"close to the original, doesn't change the performance.\n",
|
||
|
|
"\n",
|
||
|
|
"Some degree of \"throwing away\" data by PCA can actually improve classifier performance\n",
|
||
|
|
"by removing noise (which usually shows up in small PCA components)."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"def nneval(training,ltraining,testing,ltesting,metric='euclidean',p=2):\n",
|
||
|
|
" ds = distance.cdist(make2d(testing),make2d(training),metric=metric,p=p)\n",
|
||
|
|
" closest = argmin(ds,axis=1)\n",
|
||
|
|
" predictions = [ltraining[i] for i in closest]\n",
|
||
|
|
" errors = sum(array(predictions)!=array(ltesting))\n",
|
||
|
|
" return errors*1.0/len(testing)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 17
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": true,
|
||
|
|
"input": [
|
||
|
|
"def proj(images,n):\n",
|
||
|
|
" return dot(make2d(evecs[:n]),make2d(images).T).T"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 18
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"nneval(proj(centered,20)[:1000],labels[:1000],proj(test_images-mu[newaxis,:],20),test_labels)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 23,
|
||
|
|
"text": [
|
||
|
|
"0.083000000000000004"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 23
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"for k in [5,10,20,50,100,200,500]:\n",
|
||
|
|
" print k,nneval(proj(centered,k)[:1000],labels[:1000],proj(test_images-mu[newaxis,:],k),test_labels)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"5 "
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"0.2414\n",
|
||
|
|
"10 "
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"0.1359\n",
|
||
|
|
"20 "
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"0.083\n",
|
||
|
|
"50 "
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"0.0712\n",
|
||
|
|
"100 "
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"0.0757\n",
|
||
|
|
"200 "
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"0.0753\n",
|
||
|
|
"500 "
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"0.0762"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 24
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "heading",
|
||
|
|
"level": 1,
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Stepwise PCA Computation"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"There is another way we can compute PCA based on a simple\n",
|
||
|
|
"eigenvector computation.\n",
|
||
|
|
"\n",
|
||
|
|
"For a matrix $M$, the eigenvector with the largest eigenvalue\n",
|
||
|
|
"is easy to find as\n",
|
||
|
|
"\n",
|
||
|
|
"$v_0 = \\lim_{k\\rightarrow\\infty} \\frac{M^k v}{||M^k v||}$\n",
|
||
|
|
"\n",
|
||
|
|
"\n",
|
||
|
|
"for any starting vector $v$ that is not orthogonal to the\n",
|
||
|
|
"eigenvector (a random starting vector rarely is)."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"\n",
|
||
|
|
"\n",
|
||
|
|
"Why does this work? You can figure this out just by \n",
|
||
|
|
"looking at a diagonal matrix:\n",
|
||
|
|
"\n",
|
||
|
|
"$M = \\begin{pmatrix}e_0 & 0 & ... & 0 \\\\\\\\ 0 & e_1 & ... & 0 \\\\\\\\ ... & ... & ... & ...\\\\\\\\ 0 & 0 & ... & e_n \\end{pmatrix} $\n",
|
||
|
|
"\n",
|
||
|
|
"You get $M^k$ by exonentiating each diagonal element.\n",
|
||
|
|
"If $e_0$ is the largest element, then\n",
|
||
|
|
"for sufficiently large $k$, $e_0^k >> e_i^k$ for $k\\neq0$.\n",
|
||
|
|
"As a result, only the entry of $v$ corresponding to $e_0$ will\n",
|
||
|
|
"\"survive\" for large enough $k$.\n",
|
||
|
|
"\n",
|
||
|
|
"Usually, we perform this computation incrementally."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"v = randn(28*28)\n",
|
||
|
|
"for i in range(1000):\n",
|
||
|
|
" v = dot(covariance,v)\n",
|
||
|
|
" v /= norm(v)\n",
|
||
|
|
"imshow(v.reshape(28,28),cmap=cm.jet)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "pyout",
|
||
|
|
"prompt_number": 30,
|
||
|
|
"text": [
|
||
|
|
"<matplotlib.image.AxesImage at 0xd2e3910>"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAATQAAAExCAYAAADyTF4sAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvb+PLM1V//8+Vd0zs7vXtkCCRxa2ZCRk8QkQDkg+SJYs\ngU1m4QxEYPFLJARkkGEy/wHkyCQWGRBZwBc5QR/JkUlAQIJkLLCQEOC7uzPdVXW+wakfp6qrZ2b3\n7n2e3XvnPKqnf8zs3J6e7le/z4+qImZmXOxiF7vYO2Dmoz6Ai13sYhd7KrsA7WIXu9g7YxegXexi\nF3tn7AK0i13sYu+MXYB2sYtd7J2xC9AudrGLvTM2vK0PJqK39dEXu9jF3nNbqzZ7NNC+9a1v4fd+\n7/fgvcdv/dZv4fd///cX7/n/+P9W29/42vfw1a99+rH/5LOxy/d4Xnb5Hs/L3vb3+AX6f6uvPcrl\n9N7jd3/3d/Gtb30L//AP/4BvfvOb+Md//MdHH+DFLnaxiz2FPQpo3/nOd/BTP/VT+MxnPoNxHPEr\nv/Ir+Iu/+IunPraLXexiF3uQPQpo3//+9/HpTxdJ+alPfQrf//73T/7dz37h44/5556dXb7H87LL\n93he9lF+j0fF0M4N+H/ja9/L6z/7hY/jc1/4xGP+uWdnl+/xvOzyPZ6XPfX3+O63/wd//+3/Peu9\njwLaT/zET+B73yuw+t73vodPfepTi/e9CwHOi13sYh+tfe4Ln6gg+ad/9G+r732Uy/lzP/dz+Jd/\n+Rf867/+K6Zpwp/92Z/hy1/+8mM+6mIXu9jFnswepdCGYcAf//Ef45d+6Zfgvcdv/uZv4v/8n//z\n1Md2sYtd7GIPMnpb46ER0aIO7WIXu9jF3tR+gf7famHtpevTxS52sXfGLkC72MUu9s7YBWgXu9jF\n3hm7AO1iF7vYO2MXoF3sYhd7Z+ytDR90sYs9hTFe7jBUhMuEah+2XYB2sWdljwHYRwW9U8DqHdcF\ncm/XLkC72LOwc6B03nsAvDXA1UfQB1Y5imOfcAHb27EL0C72kdsxUJ2C2NrrTwu2dQy1e9p/dw1c\nDLpA7S3YBWgX+0htHUhvvv9poNZHT9rT/rtLwK0rsgvUnt4uQLvYs7M+nE7vawFWUPEYqKW/bv9W\n/lVeUWFrALvA68OxC9Au9pHZeZB62HtaiL0Z1BDhVT5JY6lVaafA1oPaBXRPaxegXezZ2CmYnQYZ\nrayX7XOPpH6/bGu4aZV2CmynoHaxp7ML0C72LOw4rM577WmhptUZGpjVKu0U2C5Q+/DsArSLde1t\n1nY9Blan3wv0AMYr79e2dFiX6qx8Qo0tDa9jILtA7MOxC9DeUXsTIJ0ulQDW1M6pzOKbAOzYsfUT\nBMf/Zmlaky339hTbKXids36xp7ML0J65fVhK6dT+8zKKbxIDOw2zc45zeYynrY2YnXpXUmM14Mo+\n5PXz4HWB29PZBWjP1J4SZA8tTj0/ZvU066deezo7BxrFkeQuamoXNOU9OS+x2Ff/9QVeb9MuQHuG\n9hQ39JtA7NR6WQLF6Tq2r14eWz+3U9Dpd/VeXy9yPfVZCUPa1WS0MKvTBVh97WJvyy5Ae2b20Ar5\np/rcY+Bq97XQWl/2gfYmrmYPCst92j1M2/W/0dt/2mqk9Y5DK7W1b3KB3NuzC9BegD11gP98BbZc\nalhpgMl7+lBbA9yx9Z61N7/eXstnJrT0AXZeEmB9WSswdPatK7iLvQ27AO0Z2UOyek/xWecALC17\nMNMQS+9ZAq79u4fF1HrQ6oOsj+YSz0JeL6883taAtfbaxT4cuwDtGdtjMn8P/RwNmRZgvX0aZi20\nztu3VGm9Y0p2DGLHlkWZ9WNbUJ92juvZA1fvtXT8a8mAC+Derl2A9kztKWB2DjTWIHYKZC2k2tf6\n7zntdrbO43mRutb5bTOVbWwrvXJapZ0CWKvKynGU/x8D3MWe1i5AeyZ2jit2LtSOgavd1wdX2wCG\nUctj71sHXs/tlH9bXut9h+NuZvt63wleRv96Z0b/7bEjX24nOLUK7div1QNl+/rFHm4XoL1le4iy\nOgWfY5/bUznnqDDdQgdW7T69HfJ+k9f1v99bPuSc9EDXf+04yE4ju6cj62bUt+2j/NhnFKXYAnBN\nqb2JgnufYXgB2hPbQwDW/s2p5bn71m7h3r4EqQSqY8tQ3drLZQsgraMeekbS3/V01XK9D7TeNqmj\n1Xg2HTyfu+90W575NVV3jp16X+8afF8gdwHaE9k5IDv2nrUb9tTyFMROtQSztq29Vu9PSKgV2jnR\nv/YGW95wKQK1VIPtvvRvngZa+Rbl6Mt2+VbnbTNMdWTryq7/y6S9evuY6b9ZvpbOWd/KeXq3wXYB\n2hPYOaA69fopWJ27fgxcazDzsBW02u1zWwmyl2+tNVZvWa/XZyFBq0VQu17Qt55fRQOlh35jC19t\ny7EJ0pYAq6GldWTv19HXwhpwkqpbnrP0twCOvH7Ov/Eu2AVob2hrwHrI/ofAqt3uOTd1nKt1rup9\n6Vb2sKvr5bbvoaDc8rWm6uVA10tye0uAqs9f/pvtv3uqYCQsvpmASq8vlwYBAWYVbBpmaTuBjUEV\n5Hq/fc8VXbM1lbYE3Loie5ehdgHaW7Bj0Frbdwxa7fa5aqxealVTq7NTrQVfC7e0L4GlnwetNSJw\nukSXQfEY1vHjYYH87vqstXAzGWilDSvfWr83Ac3CZ0CldQ0wDS+9rr/rMYW2Zq060/vba6kHtncV\nXj27AO0N7BxwnfueU/BK+x+SiTwnLubULb22fky96VsfSEA7nj7o39RLGIUMNEGP6+DIwS5UzppC\nFAyWb5Y+sV32msa0ha9+J62seiAD1oFmEM6+3loF14PY2r73BXQXoD2hnYLZ2vaaUnuICjsGsrWm\n4ZVu83ZdL48rOgPEm3ktmH4O2HQLoPhvC2rKsZVtjwHhDKClf3fIn1K3BLR2u34Y1J+YVJoGWXot\ngWoNaOn1ALN47RhstFo7BbZTUHsX7QK0J7Jj8DrntceCrLdcg1dPnSUozBFaM8bF7b4GsfY1AIt/\nuY05abDpb1Xf7LKvHOOIuTqmepsbKKxpXBu/6bjySfocWXgEmAy1Vinr3y9BRoMsvabB1gPZuQpN\ng6zdXgPb+wi1C9AeaTWkzoPZ2g2RlrpC6hjQjrmQad/x+JcsHcdbebEcm30x2sSmjjpxDTgG5JMp\nRtlI1hf7EEBUwEVgGArlG5ICGifkjJi5WcZjDSAQR1XG8UwtlhFoNMJhxkAOIxwchrw+wMHRIIAj\np5Yenhw8HFzc78lWsF4DdwvxAMq/cBt366m0NTgduyr1e8ue8o6HfebLsgvQHmEtzHjltRZgx5Zr\nQGtdyoe4klXjTlyMLVyI0AoDfBiq7dQ8y3sDrECNLQInoJVtJgiwTGx6XW9H0BGxWlf7IwADR6Dx\nUJZhxByBK8sRIUSghQi0ICBDs20Q4MnG5uBJADeQw0wRXuQwmLK0xmEgX+0bTHR4yakIYptIqDOn\nrQuu1wPMSTc84ecct1SbvEf//92G2gVoZ1rrNoo9DGb1evr71vFqXch6X11Gsa7AcvyL+7Exx0MG\nmvcDnB/gvY1L2XbewvsBPkSABYOQgBbMYh8IMDbAGAbZIOs2wNhmewE8r2DnZYmAABJohREuDJiD\nrM/NdvAkMPMMeM7rFBjwEKB5AUgwA7xxcMZjIA9rHKzxEV4eg5F9g43NOFirtjnCzJZYW98Br7db\nbd0+hrQiO+aOS2xwOex36yNAvSddaWsga13Zl2wXoJ2wPsgeHjPrQ0y/VruMvWUvMN/bzhGi6Dbm\nqFFyI1NLMHMW3g3SZgvnBvi8z8L7CLBgEIKs+7xdGgxAA8MMoSzHZntgGBtgrReQGQ+boMYe1gYY\n9jDkwWwKvPwIF9vsh7iU7eANyEWYuQg2B5Djsu4ZhgO89bB2hDUes/Ww1sOauIzrwxAhNkizcTlw\n3NYqrkpP6KSChVdQ61fxpV+v
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 30
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"If we want more than one eigenvector, we could simply subtract $v_0$ from\n",
|
||
|
|
"all the input vectors and repeat the computation. However, that is\n",
|
||
|
|
"kind of slow.\n",
|
||
|
|
"\n",
|
||
|
|
"Instead, we can update the covariance matrix and remove the entire\n",
|
||
|
|
"subspace corresponding to $v_0$."
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"evs = []\n",
|
||
|
|
"cov1 = covariance.copy()\n",
|
||
|
|
"for j in range(10):\n",
|
||
|
|
" sys.stderr.write(\"%d \"%j)\n",
|
||
|
|
" v = randn(cov1[0].size)\n",
|
||
|
|
" for i in range(1000):\n",
|
||
|
|
" v = dot(cov1,v)\n",
|
||
|
|
" v /= norm(v)\n",
|
||
|
|
" evs.append(v)\n",
|
||
|
|
" l = dot(covariance.ravel(),outer(v,v).ravel())\n",
|
||
|
|
" cov1 -= l*outer(v,v)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stderr",
|
||
|
|
"text": [
|
||
|
|
"0 1 "
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stderr",
|
||
|
|
"text": [
|
||
|
|
"2 3 "
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stderr",
|
||
|
|
"text": [
|
||
|
|
"4 5 "
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stderr",
|
||
|
|
"text": [
|
||
|
|
"6 7 "
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stderr",
|
||
|
|
"text": [
|
||
|
|
"8 9 "
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 32
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"evs = [v.reshape(28,28) for v in evs]\n",
|
||
|
|
"figsize(12,12)\n",
|
||
|
|
"for i in range(10):\n",
|
||
|
|
" subplot(6,6,i+1)\n",
|
||
|
|
" imshow(real(evs[i]).reshape(28,28),vmin=-0.1,vmax=+0.1,cmap=cm.jet)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAArsAAADvCAYAAADy4xxKAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvcmPJEl25vkT3U1t8yX2PbOyuqqI7uE0hkdmVRxIAnWs\nGwEeCYJXXovNAYOYxjT4lxA88kSCp6zM4oGFORDdDXK6Misz9tU323UTkTmIqqmoLe7mHh4RyRx/\ngMLczXQRlSfvySffeyIitNaaC7mQC7mQC7mQC7mQC7mQ76A4H7oAF3IhF3IhF3IhF3IhF3Ih70ou\nwO6FXMiFXMiFXMiFXMiFfGflAuxeyIVcyIVcyIVcyIVcyHdWLsDuhVzIhVzIhVzIhVzIhXxn5QLs\nXsiFXMiFXMiFXMiFXMh3Vi7A7oVcyIVcyIVcyIVcyIV8Z+XMYPcf/uEf+OEPf8j3v/99/vqv//o8\ny3Qh71kudPndkQtdfnfkQpffHbnQ5XdDLvT471j0GaQoCv29731Pf/PNNzrLMv3bv/3b+l//9V/P\ncqsL+cByocvvjlzo8rsjF7r87siFLr8bcqHHf9/inQUg/+pXv+KTTz7h3r17APzhH/4hf/d3f8eP\nfvSj+TlCiHMB4xdydtEb7Bdyoct/H3Khy++OXOjyuyGb6BEudPnvQS5s8rsj63R5JrD77Nkzbt++\nPf//1q1b/PM///PSef9F/zmfP/iCHz/41PpWoMvPTeWLB5+X99Dzq9eJRpTPENY3ekU5Ti9vew+N\n4IsHX/Dpgx+f4hpYVVdfPPgFnz74CSy8aSX/TfxfG93/tLr89MGP53W8+FSjm9X6qa754sHnp3r/\nVfLFg8/53Qc/ZpM2VJVJlKWo5MO0B1G2RjEv0xcPflHeo2kX71KXf67/4kQ9HKfLSj6ft2XzTvXb\nnXytfY/j6rCur/W6rv1Dfc1pZZN2Wbel1e9WvYs5w5nXCVa9/Ffx3zYqz1nscl0d2fpY7R+r9/90\n7btt4nc/f/AFP3nwu6e8sinnZZc/KfuLqmzNdz5Z6vpg5X02tUnYXJe/+5c/Ln267Q2W29xx7/E2\n9Ve968n3EFaJjrfL8/D1x9+j9u2LPr6yui8efNHoL961Tf6F/jm/ePDLhm9slvU4/yqsci/38aeV\nD91XHtemFu1yk7d8G9xzJrC76ejl8wdf8OizR3z+AO7cv8vd+3fN9WURNxdbCSeUDdBL525emSeX\n5Gz3qLvrkwF7/SwQa97I3FPPz3z02SMef/b4VLUKp9clD77g9v273L1/r/H8VQ2vLqet883ff73U\n5nGSPrRVg4sG//7bg1j425SpKmP1Tg/fsS6/ePCLUpeau5ZdLsqmFveu9xtf/4TaXTqohW/XDxTX\n3+ukNzl9izF2+bAsyeY1dXq7/HxDXa6CTbV/XWe/mnXlXx6k1fdsQrZN5Xzs0jy3gg7L1r95KTTi\nzP4VNtfl488e8cWDXyCAO/fvcO/+XZq1sVl9nr3+Nq2lunaP78fPy9effA9ttVL7/EUAV+vyofX7\nZnI6m3wMDz7nzv173DlFX7lY/2ezIFs+dF+5vk3ZyEaD1ZqOf9ZZcc+ZwO7Nmzd58uTJ/P8nT55w\n69atpfM+ffATWBqZ6YXPzWRzZesFE7SZobd1ome/R7Oz2OxdjjeK6hxjzPfu3yk7OvO+v/yrzzd6\nxml0qR98zu+WzMPi8+tyLpdVL5x3Wt2vErGB8dbPr0b0zTu87/awaMg2O1K/zbvX5Y8ffMrnD7BG\n2sv6OIlNZV7utwcmJz2hqp/m902GZhHsnq2LOP7ss3Q4BoDemYOuz//qlxtdd1of++M5k3m8LpvQ\nqfrGvmaV/dYQdvH+dT3XPtbuwE7fRs7HLmtWvf6mLu/mUr2DAS0fzb/f1CZhc13euX+XHz/4MbVO\nFhldU5bj6/O8+qjjZfP6PI+h8HH3aNr6ss5rNreqt3dtkz9+8Cm6weza5Tm5r1wNeM9ejx+yrzy+\nTS0DXbFhCzwL7jkT2P2d3/kdvvzySx4+fMiNGzf427/9W/7mb/5m5avcvn93Awd7vNy7f/vkkxak\nUnD1lMVynEXW3+OkEW6t8MrINpV1Zb4zZ3D0/F1PLsWynEaXd5ZYo9oh289f5tXrUt27f/tMoMEW\n0x5qt7CJXhfPWH6X5StOKuV5tKk75T3MXdR70aVAH6uH43Rpy+0T6/BkuXv/zsrvbW4WdLlsTB16\ndFC4WuGg+OFPLtNWE+pLBEoIlHBQojrbTrJYDp2dR7tc9y5nkdPaZdVhLL6Drb/joi5379+xanu5\nq6sGG6taQ9UV37bKYT/z5MF6/fcq/7h4/Uk2d7vs/Jol3OTKWmof47y1jW+qy0WftKo91rBndZnO\n6pPsZ9nt+GQucNlPVUCu8m2ruOn63uv/h5Pt0vxSpQwt/0Lp6xZr7Swa3VSPsFiHldZO7iub97jb\nuO4schbstIhnVvWVm7ax6qzj+tvT9Hlvg3uE3jTLfkH+/u//nj/7sz9DSskf//Ef8/Of/7x5YyH4\nuf4/lx/YUPF5jPqOk81yi1bJSZ3ESU9d/Lv5uel721et6qg0ouzGF8/9v8V/3XgCxdl1qawatjW7\nWNZmbdpvdlapA9iseN7qZ6/Si/33KhB0lrZznKy73/vU5V/on6+52i7NpuzuqqtOLuu6c5q1XgFb\nUzceEo8cnwJfZ0Q6JVLmcJQGBShQOBSeS+555J5LRjA/cnwkLhIPiWuV+Dx1XJdaNJ5g8gPP3y5t\na2jaV7M9H2eXTUZ81Xnr7l/Xn/2+y0zkIuixBy7VcMRm6MHo0mjfadznJPt8+35m2ffabfY0Nglv\np8tK6l9E45v6d1PKxbKul2XfuKrejveuzatUqU2JO9dZNdhs6tw+e9EKN/cfdm0dR6Q1+4tmqzxv\nm/wv+s9Z1ECzbJv6V7v+z9pfLrfjdWRU88xm/7gODzVLuM4ON8E968u3+szT95VnYnYBfvrTn/LT\nn/701NeZ4p0NgJ5dzkrBLwKfk5Ww+H/V4dmfpwfem55V8ZynM4yz6nLV81c36qbp1CGLswNeU5cn\nO4JmV79szHYZFpm/uo2ez6DMtIFN3vrbrMvj5TQtexUTCU09uUjD4iIJSIlIiEiImdFVE7rFmJ4c\n4xQKIYECJA5pGJDgk7oBExEzocOENjMicgKy0hbVGif+bZDNdVlxr8L6r27neq0uV/nhVXWw/v6L\n0vTv61g/eyBjdFsdq8BuBZxq4FuDvnX21GR8zq5XvfDXWWwCzqrL5XKvA1DNuti8rHVN2uct+8t1\nUoO2GrIWeBTl0LTSncSdD2qqw+i8wEXNwYtjefV1T2z2JnbLXH9ls7+wS3862USPJg51HMGzqX+1\ne7e32//rODvUiIaunTmQbPqQ6rrTEEKnxT2n98Gb95VnBrtnl7fl9N69LI5imuzFelkEUXYjWrzv\nu5APXbPrnl8bQ2Uc51MTmzGILGiwqSP7f7tDrQ51zL3PIpu++bdVl+d179WdrP27noNdjwIXSYuE\nNhM6jOkyYlcdsSsP2cmP8HIJGZCDFC5TETL1IiZEDOhzRDZ34knZAiUeNFz7v19Z5mhrOckuNwGE\nx92/KevB5+JZNtj1yed6tqXAw0FR4CHQKBywIPFxJTlPf1t366cfgJ5WNiu3zYbaLOfpymozret+\nOx7sGh9ZQckK2OZlFKUoAW+lR3ceVzGQ2PSROQKBg0Kf4HFrTnuRhT5Z3peVVwOt43SwaVmW2fy3\nL9vi34ssrg1415NB5uxlumnVAO3bgXs+ANh9P7KqglcpehWYPR7gLn+zCjxVDWbVeSeV+KTOZ3GE\n+D5MeLXBnub62iBOum7xWYvhKVtT9vnrnEozmLacwVndvXLUNpOkrLOr804rJ/MOi+e9P1nuIM9P\nmjBq0TZW66PJ+kgCXSYg6IxWNiNOZsSzKe3ZlO54RDSa4Y1TvJLZFQqU56C3NWJH4W5LVOQiAw8Z\nuGhPzEFT1RGbrtpdKPX5vPsm
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 33
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "heading",
|
||
|
|
"level": 1,
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"\"Neural\" PCA Algorithm"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"This algorithm can be converted into a fully incremental algorithm that does not even\n",
|
||
|
|
"require the covariance matrix to be stored, but only the eigenvector candidates.\n",
|
||
|
|
"\n",
|
||
|
|
"The key idea for that is that (for input vectors $x_i$):\n",
|
||
|
|
"\n",
|
||
|
|
"$M = \\sum x_i^T x_i$\n",
|
||
|
|
"\n",
|
||
|
|
"And (without normalization):\n",
|
||
|
|
"\n",
|
||
|
|
"$v_0 \\leftarrow M \\cdot v_0$\n",
|
||
|
|
"\n",
|
||
|
|
"Replacing that\n",
|
||
|
|
"\n",
|
||
|
|
"$v_0 \\leftarrow (\\sum x_i^T x_i) \\cdot v_0 $"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"$v_0 \\leftarrow (\\sum x_i^T x_i) \\cdot v_0 $\n",
|
||
|
|
"\n",
|
||
|
|
"But if we don't have the sum, we can just do small updates and hope things will average out in the long run:\n",
|
||
|
|
"\n",
|
||
|
|
"$v_0 \\leftarrow v_0 + \\eta (x_i^T x_i) \\cdot v_0 $\n",
|
||
|
|
"\n",
|
||
|
|
"Furthermore, we can update $v_1$ in a similar way by removing\n",
|
||
|
|
"any component that points in the direction of the current $v_0$:\n",
|
||
|
|
"\n",
|
||
|
|
"$ x_0^{(1)} = x_0 - (v_0\\cdot x_0) v_0 $\n",
|
||
|
|
"\n",
|
||
|
|
"$ v_1 \\leftarrow v_1 + \\eta (x_i^{(1) T} x_i^{(1)}) \\cdot v_0 $"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"Fully implemented, this gives a \"neural network\" that allows us to compute the top $k$ eigenvector\n",
|
||
|
|
"(PCA components) without any storage of the covariance matrix.\n",
|
||
|
|
"\n",
|
||
|
|
"Algorithms are often considered \"neural network algorithms\" if they\n",
|
||
|
|
"\n",
|
||
|
|
"- can be implemented in terms of a low-depth circuit\n",
|
||
|
|
"- do not require storage of the full dataset or the full covariance matrix"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"# randomly initialized eigenvectors\n",
|
||
|
|
"d = 28*28\n",
|
||
|
|
"evs = array([randn(d) for i in range(6)])\n",
|
||
|
|
"evs = array([e/norm(e) for e in evs])"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 35
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"# incremental updates\n",
|
||
|
|
"eta = 0.001\n",
|
||
|
|
"def update(x):\n",
|
||
|
|
" x = x/norm(x)\n",
|
||
|
|
" for i,e in enumerate(evs):\n",
|
||
|
|
" # update e_i\n",
|
||
|
|
" e += eta*x*dot(x,e)\n",
|
||
|
|
" # normalize e_i\n",
|
||
|
|
" e /= norm(e)\n",
|
||
|
|
" # remove e_i from the vector x\n",
|
||
|
|
" x -= dot(e,x)*e"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"prompt_number": 36
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"# running the incremental algorithm\n",
|
||
|
|
"for iter in range(1000000):\n",
|
||
|
|
" if iter%100000==0: print iter\n",
|
||
|
|
" update(centered[iter%len(centered)].ravel())"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"0\n",
|
||
|
|
"100000"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"\n",
|
||
|
|
"200000"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"\n",
|
||
|
|
"300000"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"\n",
|
||
|
|
"400000"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"\n",
|
||
|
|
"500000"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"\n",
|
||
|
|
"600000"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"\n",
|
||
|
|
"700000"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"\n",
|
||
|
|
"800000"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"\n",
|
||
|
|
"900000"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"output_type": "stream",
|
||
|
|
"stream": "stdout",
|
||
|
|
"text": [
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 37
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [
|
||
|
|
"# final outputs\n",
|
||
|
|
"figsize(10,10)\n",
|
||
|
|
"for i in range(6):\n",
|
||
|
|
" subplot(1,6,i+1)\n",
|
||
|
|
" imshow(evs[i].reshape(28,28),cmap=cm.jet)"
|
||
|
|
],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"output_type": "display_data",
|
||
|
|
"png": "iVBORw0KGgoAAAANSUhEUgAAAksAAABsCAYAAAB6i/alAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzsvcmPXNl95/s5544xR+ScHGpgkSWWqmSpJcHofrIANmxp\n440ENAQBXrhtQ22gV14Z8MrUyrL+gF48b7pXtoW3kA0YFrx5NNpaWA9yl6yqUs1VLE45Z8x3vuct\nzp0imEkmk5kkS5U/4CCSwTvF757he76/SSilFGdyJmdyJmdyJmdyJmdyoMin/QBnciZnciZnciZn\ncibPspyBpTM5kzM5kzM5kzM5kwfIGVg6kzM5kzM5kzM5kzN5gJyBpTM5kzM5kzM5kzM5kwfIGVg6\nkzM5kzM5kzM5kzN5gJyBpTM5kzM5kzM5kzM5kwfIscHST37yE65evcqVK1f4y7/8y5N8pjM5gpzp\n/+nKmf6fnpzp/unKmf6frpzp/ymJOobEcaxeeukl9dFHH6kwDNUXv/hF9dZbbx3nUmdyDDnT/9OV\nM/0/PTnT/dOVM/0/XTnT/9MT8zgA62c/+xmXL1/mhRdeAOC73/0uf/d3f8crr7xSHCOEOBEw91kW\ndUi+0DP9Pxk50//TkzPdP1050//TlTP9Pz05TPfHAkt37tzh4sWLxb8vXLjAv/7rv9533O+r/8Hr\n1/+BL13/3Ue+x5M+72nc80Hn/S/x3w8976j6/6b6MR9c/xteuv7dR362X/fzHnbuP4lvHXreUfX/\nvT9f4XvXVx75uf7q+taxzjvuuQLF/319i//2gPMUB0/Ax33WB533m+KNQ887qu6tnQHJD/8C40//\n7JGf7Umf9zTu+aDzoqXOoecdVf//U32HH19/k29df/WRn+0kzzus386e9wbfuv7ake4hKBfS4z7n\nw879r+JHh553VP3znT+H71x/9Af70fXjnfc45z5L5/2Xw/vLscDSUZHr69f/gY0b7/L6dVi7doW1\nay8f53afCdm48S4bN9470rFH1f8H1/+GvRtvwPW/oXftNRauHW1C+CzK3o032L9x+CJdlaPq/+c3\nJnB9C4AvX2vwlWuNYz/faYlAZW12IajKURacx5Gf35jwbzcmRzr2qLpPfvgXqJ/+C8kP/wLxtd9C\nfu3rj/OIv9aS/vR/o376L0c69qj6//H1N3n7xhY/vg5Xry1z9drxwP/jSNmzHyZHO06gUIhDx8lx\n5e0bW7x9Y/tIxx6ZNXrzhgYFAK9e0+1M7pc3b+h2BDkWWDp//jy3bt0q/n3r1i0uXLhw33Ffuv67\nvH6dYzM9T1pE1ublSRTPW7v28gyY/MX3/+HQY4+q/5eufxceg3l5UqInH3XgJHTaC3UuC3Ng8oPv\n/82hxx5V/1+51jg2Q3Rakuu4+FQKqRRSpZhJUj0QACUEKQIlygXlpN/JV+aA5F99f+vQY4+qe+NP\n/+yxmJ5TF1EFp7rvS5EW/13VsVKnOwbk174OFTCZ/vAHhx57VP1/6/qr/Pg6x2ZeHlcevgWoHnv0\nPn3SQAng6rWVGTD54++/eeixR9U/r147PkP0JEXMfc7LaS++80DyR98/9NBjgaWvfvWrvPfee3z8\n8cecO3eOv/3bv+Wv//qvDzx27dqV49zixM6bHwT3LRaVIXXu2iUM4vuuOT/w5q/5pH/jo+i/d0w2\n6UHnPWjCWLj26pEmlKr+JSkr165iEc0ck+u92o56v6NMfsfVzVH1/+VjMknHOS/XwVeu1Q/UR65n\ngcIkxlAJpoqRgcIIFdf+g6K3M4QUPXFZgA2JJYgMk9g0iYUkxSBFFu/jSf5GeLS+L772W8e6x2me\nJ4TSTSoMkRQt/s9fxXbGgO67SabnNJUkqf5EifuA05P+jY+i/6vXlo91jwed96Bx/blrK6RZgPc8\nsC8/75cr19ZIkcX4OGw+mf/+uL/vcc49sv6PyyQ9DgP16rVSwVVVVQHR/N8S+OI1sCv/l5Ij2Nk2\nf90n/BuFOsyb6SHyj//4j/zJn/wJSZLwR3/0R/zZn83u4IQQ/L76H8d6qJOUKtApB4KqTPcKkU3/\nh8HY/OhykTh9xuN/if9+qKMZHE3/31Q/PvHnmgebj3sNqZcEDJKZa6aV5V3rXRYT4cPk6PT74fJP\n4luPrf+fqSdn9jxsks8l17FBiq0CHBVgqxBzlGKOFMZUQYxuEqgDDYhdA8+x8B2bSFpEmMSYj/xO\nHkV+U7zx2Lq3dgYn/lwnIUIoDDNBGimWjLBkiC3DmYU6yXQcYxIlJnGsm1LiQMB00hItdR5b//9T\nfefEn+sozOZBjNJR5uy8N+fzUHUsVeeT02CW5uW/ih89tv75f56EPWRO5oFN8UCHNKPSqkAqPaQd\nvkSfnPwXcajujw2WHiZPAywdtHfIO3o+fErIU0KffMGel3yApMURcmYRP015GFh6mJwmWDp8YX4w\nRKmek2tSL+B6eZgFS5KkmL7Kpf4od3oQE3VUeRhYepg8ObCkKn27fC9VXQpUxiQlGCrGTXxqsY8b\n+Zh7Cms3xRhUwJIBdIEORC2DadNh2nAITJsAm1DYGeyafScnJQ8DSw+TZxIsZWY3KVNMM8YyI2wj\npCY9XOlhijibWRQRltYzDkFsE0a6pUqicpbpFOVhYOlhcppgSc/csw4T1V5fnd2r5+Tn5WvBPG9t\nkmASYRHPANeD5pLT3ig/DCw9TJ4YWFJzf+ctrfyfomSQ8k+JnmOsrJmV7wGSA9qTAkwPAEvHMsM9\nK3IY0XrQp8yWX6NYhqvDbn4XAdXlpzzLQBSwKueiPhthmocxSrPfP3jhrn7mEMgkxiLCIkJkNqBc\n53EBo8p20GT4WZP7da6KnXF1Q5DzcgbaH8mMEqwowRpFWKMYc6AwNhViE9ijnIxcYFk3uaSwFmNq\nEoQLGIJUGiD0m/4sv4ejSm56kzLFNGJcw8cxAupySlOMaYgxFlHxzgIcptSYUscTNaZGHSUESazB\nqQLNMCGejEPlU5CD+lNambXL3j67qcrni6TYhpVzdz7r51cxiXEIsQlwCLIrRsWVDNIDt9ef2T4/\nbwqrgqO8xcyCm9ysnwMlq9Jyc3/OLpnZdzlAyjdvVcmvd9gznaJ8qsHSPOlaNbPlU7lRMfPkS26+\nUOdg6SATW8k3SQRptoNWkO2kVdED1K/9oJmHj9XvZ/8vn76qC7diHjxV2SSLCJsQmxCRTU4gtAki\ng1EhFhIboJj8Ssbw11//Vbm/n6sMfJbcZznZJ9myEWMnCVaQYHsJcjtFbqbIjRRxG8RtBVuUu7sm\ncB6YgowVthFj1FOEoResSJrFnWXxzj6Di8cjiJQp0kyxjAhX+jTkhLYc0mZIhwE2YbHsT6kzpolF\nhCETDZSUQYSlR5nSGzWhfr11Pv/b8nl6lnU2srminC8i7GzesDOuqNxsKUQxPlwCGoxpMEGgijmo\nvGo8sw581uce4GAGKQdIMRBRgpz8uypYctCbMRcNlGCWZZKUYCknreeNPvPA6AltGD51YGnelCMr\n3XneByZnLqoA6bBFXBQ9QEs+XHKIFVUGXJIdV96xlF/XwXMQW3Tw/i4tNGdUm0ox0qRoZhpjJhlY\nMkIsGYKERBokhpHt9XQzcJHZxCRJi0mvSo/nT/VZkXyyLv99vxNr/n4MEswkxgkj3GmsWaQ7wMfA\nraxto50sbaCTfdZBtBWGpzCilCQxMM2keOdnMicK7uNeFSAESihIFUKAKRNsEVITHi1GdOlnzIbW\nrQZOGoQmwiASmvtQqdAO3yLbsJ2y79LTkvvHtv6sMkhxttWKMbN5QuvIxyXAJVAuQWoTJg5hahMr\nk1iZKCEwZIxpJNTlhI6wSYXEIMEmzJi7qFg7St5WFLOaQM2Aps/EZmGeQcpbggZIYeUzZNaEJtFI\nw4SCc5hnh/JrKQ42ux3mD/UE5VMDlg5ikKqLcdVYkzMV+T4j94fJdxvVpdwkps6UOlMswoK01X4D\nbjb47GLhzncvBtYMzTsPuT6t
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"prompt_number": 38
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"collapsed": false,
|
||
|
|
"input": [],
|
||
|
|
"language": "python",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": []
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"metadata": {}
|
||
|
|
}
|
||
|
|
]
|
||
|
|
}
|