nvidia-AI.summerschool17-no.../teaching-ncso/old/32-rnn.ipynb

571 lines
32 KiB
Text
Raw Permalink Normal View History

2017-09-22 17:38:09 +00:00
{
"metadata": {
"name": "32-rnn"
},
"nbformat": 3,
"nbformat_minor": 0,
"worksheets": [
{
"cells": [
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Title: Multilayer Architectures\n",
"Author: Thomas Breuel\n",
"Institution: UniKL"
]
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"\n",
"import scipy,pylab,numpy\n",
"from numpy import *\n",
"from scipy import *\n",
"from pylab import *\n",
"from urllib import urlopen\n",
"from gzip import GzipFile\n",
"from scipy.spatial import distance\n",
"from scipy.spatial.distance import cdist\n",
"\n",
"def sigmoid(x):\n",
" return 1/(1+exp(-x))"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 1
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"N = 20\n",
"ni = 3\n",
"no = 2\n",
"seq = array(rand(N)*ni,'i')%3\n",
"output = 1*(roll(seq,1)==0)*(seq==1)\n",
"target = zeros((N,no))\n",
"target[output] = 1\n",
"plot(seq[:20])\n",
"plot(output[:20])"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "pyout",
"prompt_number": 2,
"text": [
"[<matplotlib.lines.Line2D at 0x3c30650>]"
]
},
{
"output_type": "display_data",
"png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD9CAYAAABDaefJAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJztnX1wVMeZ7p8RkgAhIiERBJYUvqQRCIQEJtHaGxZ5ba5j\n1w0hcWpjV8WXwuQWReHr8iZ/3NpbrrVJeV1414nLDk4uzsYuu7KLXfHexCQL2mxVVi4WB8h6xcbG\nZEYYZIQAxUhCCAmjr3P/aA4ajaZnzkf3Od193l+Vy7Y0c6Y17znPeft9T/cTsyzLAkEQBGEEeWEP\ngCAIghAHiTpBEIRBkKgTBEEYBIk6QRCEQZCoEwRBGASJOkEQhEFkFfWuri7cddddWLVqFVavXo0X\nX3wx4+see+wx1NbWorGxEe3t7VIGShAEQeQmP9svCwoK8Pzzz6OpqQnXrl3D7bffjk2bNmHlypW3\nXnPw4EGcPn0aHR0dOHbsGHbu3ImjR49KHzhBEAQxnayZ+sKFC9HU1AQAKC4uxsqVK3HhwoUprzlw\n4AC2bt0KAGhubsaVK1fQ09MjabgEQRBENhzX1Ds7O9He3o7m5uYpP+/u7kZ1dfWt/6+qqsL58+fF\njZAgCIJwTNbyi821a9fw9a9/HS+88AKKi4un/T59p4FYLDbtNZl+RhAEQeTGzW4uOTP10dFRPPDA\nA/jmN7+JLVu2TPt9ZWUlurq6bv3/+fPnUVlZyR1YWP/s328BsHDxYnhjEPnPk08+GfoYwv7n5EkW\n03/5F/ou//Ef2Xdx6VL4Y/HzfU5MWMjPt/B3fxf+36HKP27JKuqWZWH79u2or6/H448/nvE1mzdv\nxuuvvw4AOHr0KEpLS1FRUeF6ILLp7WX/TiTCHQchDorpJPZ3YH8nunL1KjA2RjH1Q9byy5EjR/DT\nn/4Ua9aswdq1awEAzzzzDM6dOwcA2LFjB+6//34cPHgQNTU1mDNnDl599VX5o/aAfbInk8DGjeGO\nhRBDakyjjv0d6C7qFFP/ZBX1L37xi5iYmMh5kL179wobkCx6e4GKCnMygJaWlrCHEDp2TP0KgAnf\nZTLJvgsVRN3P92nadRoGkVlRevkycMcd5mQAJgiRX+yY+hUA3b9Ly2Ln9R13mCHqDQ3AtWvAwIC4\nMUWJyIh6by9w553miDrBYnr77cClS8D162GPJjwuXQJmzQJqatiNTmcuXwY++1mgthbo6Ah7NHoS\nKVFvbgY6O4HR0bBHQ4jAnqovWwacPh32aMIjkQDq6oDycjUydT/09rK/o66OSjBeiZSo33Yb+6ez\nM+zRECKwBSAej/YMLJlk34FJoh71mPohUqJOGYBZpMY0ygKQTFKmTkwSCVEfGQGGhoCSEsoATOLy\n5cmsLsoCkEhQpk5MEglR7+sDysqAvDw6WUyitxeYP59iapdf5s83o1GaGlMPCyojTyRE3b77AzSt\nMwXLAvr72c06yjEdHQU+/hhYvtysTL2kBCguBtI2hSUcEDlRj3pWZwoDA8Ds2UBhIXsEbnxcf0Hz\nwtmzQGUlMHMmu8H19+ud3dK16p/IiXpVFXDlCjA4GO6YCH+kxjQWi26z1G6SAuwGN3u23ot2aFbt\nn0iIul2nA1hdvaaGFjboTmpMgeg2S+0mqc38+frOWK5fZ5t52bt7U6bujUiIeurdH6CTxQQopgy7\nSWpTXq5vs9SOqW29ENWY+iWSok7TOv2hmDLs1aQ2OjdLKaZiiKSoUwagPxRTRqZM3RRRX7YM6Opi\n60wI50RS1CkD0J/0mNbWAh99BDjYKdoYBgdZUzTVaMwkUS8sBKqrgTNnwhuTjkRC1NObarW1tLBB\nd9JjOmcOE4Sb/i2RIJlk53JeylWsc6PUXkyWSlRnYH6IhKinZwBlZey53p6e8MZE+CM9pkD0BCC9\n9ALo3Si1t31IJWoxFUEkRR2gEozuUEynN0kBs8ovQPRiKgLjRX1iYnI5eSqUAegNZer8TN0kUY9a\nTEVgvKgPDLB6a0HB1J9TBqA3vKwuSgKQuprUxjRRp+vUPcaLenpDzYYyAL3JFNcorSq1fUlra6f+\n3LRG6W23kV+pW4wX9Ux3f4BEXWeGh5moFRVN/fnixdHxK7V9SefNm/pz0xqlsRj5lbolsqJeU0N+\npbqSvpzcJj8/On6lmZqkALvRWRa78ekG71qlEow7IivqM2eSX6mu8GIKRGcGlqlJCrAbnY519bEx\ntpiqtHT676ISU1FEVtQBygB0JVdMoyAAmZqkNjqKen8/M8aYMWP67+g6dYfxos5rlAKUAehKrphG\nQQDSt9xNRcdmaaYmqQ1dp+4wXtRpqm4eFFN++QXQs1maqUlqQ36l7oi0qNO0Tk+iHtNUX9JM6Fh+\nyRZT8it1R6RFPSpZnWlki2kU/EpTfUkzYZqoA3StuiHSok5+pXqSLaZR8CvN1iQFzBT1KMzARGG8\nqGdrqpFfqZ5kiylgfrM0W5MUMK9RClCm7gajRd2yaFpnIlGPabYmKWBeoxQwP6YiMVrUh4fZdDx9\nOXkqNK3Tj6hP1XmrSW2o/BJtjBb1XCcKQBmAjlCmnjtTN03Uya/UOZEXdcoA9GJ0lO3al2k5uY3J\nfqWZfEnTMVHUya/UOUaLeq6GGkB+pbrR18d2JszLcuaa7FeayZc0ndJSduPTabO6XI1SwPwZmCiM\nFnUnmTr5leqFk5gC5gpArtILwAR/3jx2A9QBJw80AObGVDSRF3WASjA6EfWY5mqS2uhUgrl6le0N\nX1iY/XWmxlQ0JOqgDEAnoh5TJ5k6oJeoRz2moiFRh/mLVUwi6gIQdVGn6zQ3Rou6k0YpYP6ycpNw\nE1PTBMD2JXUi6jqtKnXSJAXYEz/kV5obo0U96lmdiTiNqYl+pTxf0kzotKo012pSG/IrdQaJOsiv\nVCecxtREv1KnTVLAzPILYOYMTDQk6iC/Up1wIwCmzcCcll4Ac0XdtJjKgET9JtSE0YMoCwCJOl2n\nTjBW1EdGgKEh5priBGqW6oHTRilg3lTdTfnFxEYpQNepE4wV9b4+tlo023LqVEzL6kyFMnVnrzWx\nUQqQX6kTckreI488goqKCjQ0NGT8fVtbG0pKSrB27VqsXbsWTz/9tPBBesHNxQ+Yl9WZiGUB/f3s\nZu0Ek2Kay5c0HVPLL+RXmpucor5t2za0trZmfc3GjRvR3t6O9vZ2PPHEE8IG5we3om5aVmciAwPA\n7Nm5l5PbmORXmsuXNJ2yMnYD1CGjpWtVLDlFfcOGDZiX48FYS8Ezx+2JUlXFLgLyK1UXtzGNxcwR\nADelF4Dd+GbP1mOhjhdRN2UGJoN8vweIxWJ499130djYiMrKSjz33HOor6/P+Nqnnnrq1n+3tLSg\npaXF78dzcdNQA1jt3V7YsG6dtGERPnAbU2CyBHPHHXLGFBRumqQ2drM0297zYXP9OjA2xkoqTjG9\nWdrW1oa2tjbP7/ct6uvWrUNXVxeKiopw6NAhbNmyBUnON54q6rJxe/cHJrM6EnU18RNT3UkmgcZG\nd++xm6VO6/BhYMc0FnP+nngceOcdeWMKm/SEd/fu3a7e7/vpl7lz56Lopgnofffdh9HRUfQpsJGz\nFwEwqbFmIlGOqZdMXYdmaZRjKgvfot7T03Orpn78+HFYloUyp48nSCTKWZ2pRDmmbmvqgLmiTn6l\n2clZfnnooYfwzjvv4PLly6iursbu3bsxenOTlB07duCtt97Cj370I+Tn56OoqAhvvPGG9EE7wasA\nvPiinPEQ/vES01S/UqdrFlTDiS9pJkwV9cJC9mDDmTPAihVyxqUzOUV9//79WX+/a9cu7Nq1S9iA\nROGlqZa6sMFNjY8IhsuXgdWr3b0n1a90yRIpw5KOE1/STOiwqtTNatJU7GYpifp0NM1dcuMlAyC/\nUrXxElNA/xKMl9ILoMeqUjerSVPRPaYyIVFPg5ow6hLVmHppkgLmll8A/WMqEyNFfWLC3XLyVCgD\nUBfK1N1hsqjrHlOZGCnqAwOs
}
],
"prompt_number": 2
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"xs = zeros((N,ni))\n",
"for i,n in enumerate(seq): xs[i,n] = 1.0\n",
"imshow(xs[:20,:].T,interpolation='nearest')"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "pyout",
"prompt_number": 3,
"text": [
"<matplotlib.image.AxesImage at 0x3db1a50>"
]
},
{
"output_type": "display_data",
"png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAABQCAYAAADiHjXEAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAACs5JREFUeJzt3W1IW2cfx/HvmcrAdZ0KmjqVnYqOWnA+1OErsdCmnV0J\nykYf2JiopUXWjbK9mO82xx4cDMaYZXTdGK4D1zLoA6ihLSNlONrAiFCwVAdLZ1t1K1JmLawu5H7R\n+3Y6E6M5SU6S+/eBAybnSq5/rqT/xsv/uS4jGAwGERGRtPKI3QGIiEjsKbmLiKQhJXcRkTSk5C4i\nkoaU3EVE0pCSu4hIGsqM9oEzMzPs3buXGzduYJomp06dIicnZ1k70zRZv349GRkZZGVl4fV6LQUs\nIiKRRf3NvaenB6fTydjYGNu2baOnpydkO8Mw8Hg8+Hw+JXYRkQSJOrmfO3eO1tZWAFpbWzlz5kzY\ntrpOSkQksaJO7tPT0zgcDgAcDgfT09Mh2xmGwfbt26mrq+P48ePRdiciImuw4py70+lkampq2f3v\nv//+ktuGYWAYRsjnGB4eprCwkD/++AOn08mmTZtoaGhY1i7c40VEJLxwMyMrJvcLFy6EPedwOJia\nmmLDhg1MTk5SUFAQsl1hYSEA+fn5tLS04PV6Qyb3h95eIRoPsHWlcHmb7hXPR9K9Yv+pJdJYeFh5\nNJNlLKy+p5CI1+Ih3p9NSI73JBavIxIPkUYzOcYiFqznrPCinpZxuVz09fUB0NfXR3Nz87I29+/f\nZ3Z2FoC5uTnOnz9PZWVltF2KiMgqRZ3cu7q6uHDhAk8//TQ//PADXV1dANy+fZvnn38egKmpKRoa\nGqiurqa+vp7du3ezY8eO2EQuIiJhRV3nnpeXx8WLF5fd/+STTzIwMABAaWkpIyMj0Ue3hBmj5xHQ\naMaWaXcAacW0O4A0YfkKVbfbzaZNmygvL+ejjz4K2eb111+nvLycqqoqfD5flD2ZUccoy5l2B5BW\nTLsDSCum3QGkCUvJPRAIcPjwYdxuN6Ojo/T393Pt2rUlbQYHB/nll18YHx/niy++oLOz01LAIiIS\nmaXk7vV6KSsrwzRNsrKy2LdvH2fPnl3SZvHFTvX19dy9ezdsTbyIiMRG1HPuALdu3aKkpGThdnFx\nMVeuXInY5ubNmwsXQC3lWfSziX5BExH5h/+/x2pYSu6rvfDo30X24R+31Uo4IiJpzWTpV95LK7S1\nNC1TVFTExMTEwu2JiQmKi4tXbHPz5k2KioqsdCsiIhFYSu51dXWMj4/j9/t58OABJ0+exOVyLWnj\ncrn45ptvALh8+TI5OTlhpmRERCRWLE3LZGZm0tvby86dOwkEAnR0dFBRUcGxY8cAOHToELt27WJw\ncJCysjIee+wxvv7665gELiIi4VlK7vDPomGGYfDIIw9/ETh06NDCeY/Hw4kTJygtLQUelkbW1tZa\n7VZERFZgKbn/r8794sWLFBUV8eyzz+JyuaioqFjSrrGxkXPnzlkKVEREVi/ude6gzTpERBIt7nXu\nhmHw008/UVVVRVFRER9//DGbN28O+XyNixawNFl7lXsyLAOaiCVRV8PqWKTL64iFZFmuNxniSIb3\nI1aS4TO+9vH0s7TSPXwxZNzr3Gtra5mYmCA7O5uhoSGam5sZGxsL2XarlWBERNKeyWor3eNe5/74\n44+TnZ0NQFNTE/Pz88zMzFjpVkREIoh7nfv09PTCnLvX6yUYDJKXl2elWxERiSDude7ff/89n3/+\nOZmZmWRnZ/Pdd9/FJHAREQnPUnJvb29nYGCAgoICrl69unD/4jr369ev89dff5GRkcHRo0epqamx\n0qWIiKyCpWmZtrY23G532PNay11ExB6WkntDQwO5ublhz2stdxERe1jeZm8l4dZyFxGR+LK8tkwk\nq1/LXVt1iIiszM9qt+uIa3Jf61ruW+MZjIhIyjNJyEVMkWgtdxERe1j65r5//34uXbrEnTt3KCkp\nobu7m/n5eUBruYuI2MlScu/v74/Ypre310oXIiISBcvTMu3t7TgcDiorK0Oe93g8PPHEE9TU1FBT\nU8N7771ntUsREYnA8h9U29raeO2113jllVfCttFmHSIiiWX5m3ukC5lAm3WIiCRa3Ovc17JZRzeN\ni26ZJLrSPRk2Q0gW6fI6wPr7mixjkQwbfiTLWPz//lv1kxR17rC2zTpU6S4ishKTpKhzB23WISJi\nh7gnd23WISKSeJanZSJdyKTNOkREEs9Scp+YmGBqaorc3Fzy8vI4ePAg7e3tS9q8+uqrXL9+naGh\nIYLBII8++qilgEVEJDJLyT0rK4tPPvmE6upq7t27x5YtW3A6nVRUVCy0Wbxhx5UrV+js7OTy5cuW\nAxcRkfAszblv2LCB6upqANatW0dFRQW3b99e0kYbdoiIJF7MSiH9fj8+n4/6+vol94fbsCP06pCe\nRT+baEV3EZHF/CS0zv3evXu8+OKLfPrpp6xbt27Z+dVv2LE1FuGIiKQpk4TVuc/Pz/PCCy/w8ssv\n09zcvOz8WjfsEBER6ywl92AwSEdHB5s3b+bIkSMh22jDDhGRxLM0LTM8PMy3337LM888Q01NDQAf\nfPABv/32G6ANO0RE7GIpuT/11FM0Njby+++/YxgGBw8epKmpaUkbj8fDiRMnKC0tBR6WRtbW1kbR\nmx/9gTWW/Gg8Y8WPxjKW/Gg8rYt7nTvEaj13P3rDY8mPxjNW/GgsY8mPxtO6uNe5g9ZzFxFJtJgt\nHBauzn3xeu67du1idHQ0Vl2KiEg4wRiYnZ0NbtmyJXj69Oll5/7888/g3NxcMBgMBgcHB4Pl5eUh\nnwPQoUOHDh1rPMIxghbnTObn59m9ezdNTU1hyyEX27hxIz///LOW/RURiaO417lrPXcRkcSLe527\n1nMXEUk8y9My8eZ2uzly5AiBQIADBw7w1ltv2R1SSjNNk/Xr15ORkUFWVhZer9fukFJKe3s7AwMD\nFBQUcPXqVQBmZmbYu3cvN27cwDRNTp06RU5Ojs2RpoZQ4/nOO+/w5Zdfkp+fD8CHH37Ic889Z2eY\nKSnu2+xZEQgEOHz4MG63m9HRUfr7+7l27ZrdYaU0wzDweDz4fD4l9ii0tbXhdruX3NfT04PT6WRs\nbIxt27bR09NjU3SpJ9R4GobBG2+8gc/nw+fzKbFHKamTu9frpaysDNM0ycrKYt++fZw9e9busFJe\nkv+yltQaGhrIzc1dct/iPQtaW1s5c+aMHaGlpFDjCfqMxkJSJ/dQa8HfunXLxohSn2EYbN++nbq6\nOo4fP253OGlhenp6YTE8h8OhzWhi4LPPPqOqqoqOjg7u3r1rdzgpKamTe/h13yVaw8PD+Hw+hoaG\nOHr0KD/++KPdIaUVwzD0ubWos7OTX3/9lZGREQoLC3nzzTftDiklJXVy//da8BMTExQXF9sYUeor\nLCwEID8/n5aWFs27x4DD4WBqagqAyclJCgoKbI4otRUUFCz8J3ngwAF9RqOU1Mm9rq6O8fFx/H4/\nDx484OTJk7hcLrvDSln3799ndnYWgLm5Oc6fP09lZaXNUaU+l8tFX18fAH19fSE3rZHVm5ycXPj5\n9OnT+oxGKWZ7qMZDZmYmvb297Ny5k0AgQEdHx7IVJ2X1pqenaWlpAeDvv//mpZdeYseOHTZHlVr2\n79/PpUuXuHPnDiUlJbz77rt0dXWxZ88evvrqq4VSSFmdf49nd3c3Ho+HkZERDMNg48aNHDt2zO4w\nU1LS17mLiMjaJfW0jIiIREfJXUQkDSm5i4ikISV3EZE0pOQuIpKGlNxFRNLQfwCbQMj7QDMZeAAA\nAABJRU5ErkJggg==\n"
}
],
"prompt_number": 3
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"nh = 3\n",
"ys = zeros((N,nh))\n",
"zs = zeros((N,no))"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 4
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"A = randn(nh,ni)\n",
"a = randn(nh)\n",
"B = randn(no,nh)\n",
"b = randn(no)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 5
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"def sigmoid(x): return 1/(1+exp(-x))"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 6
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"def forward():\n",
" for t in range(N-1):\n",
" ys[t] = sigmoid(dot(A,xs[t])+a)\n",
" zs[t] = sigmoid(dot(B,ys[t])+b)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 7
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"forward()"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 8
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"dzs = zeros((N,no))\n",
"dys = zeros((N,nh))"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 20
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"def backward():\n",
" for t in range(N-1)[::-1]:\n",
" dzs[t] = (zs[t]-target[t])*zs[t]*(1-zs[t])\n",
" dys[t] = dot(dzs[t],B)*ys[t]*(1-ys[t])"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 21
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"backward()"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 10
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"def update(eta=0.3):\n",
" global A,a,B,b\n",
" for t in range(N-1):\n",
" B -= eta*outer(dzs[t],ys[t])\n",
" b -= eta*dzs[t]\n",
" A -= eta*outer(dys[t],xs[t])\n",
" a -= eta*dys[t]"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 13
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"update()"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 14
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"for i in range(1000):\n",
" forward()\n",
" backward()\n",
" update()\n",
" if i%100==0: print i,sum((zs[:N-1]-target[:N-1])**2)**.5"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"0 2.08162539801\n",
"100"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.8904233524\n",
"200"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.87767422863\n",
"300"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.86765662972\n",
"400"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.86310296207\n",
"500"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.86075458681\n",
"600"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.85941584069\n",
"700"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.8585984244\n",
"800"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.85806426965\n",
"900"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.85769429465\n"
]
}
],
"prompt_number": 15
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"A = randn(nh,ni)\n",
"a = randn(nh)\n",
"B = randn(no,nh)\n",
"b = randn(no)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 60
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"ns = 3\n",
"states = zeros((N,ns))\n",
"S = randn(ns,nh+ns)\n",
"s = randn(ns)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 61
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"def forward():\n",
" for t in range(N-1):\n",
" ys[t] = sigmoid(dot(A,xs[t])+a)\n",
" states[t] = sigmoid(dot(S,concatenate([ys[t],states[t-1]]))+s)\n",
" zs[t] = sigmoid(dot(B,ys[t])+b)"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 62
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"forward()"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 63
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"dstates = zeros((N,ns))"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 64
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"def backward():\n",
" for t in range(N-1)[::-1]:\n",
" dzs[t] = (zs[t]-target[t])*zs[t]*(1-zs[t])\n",
" dys[t] = dot(dzs[t],B)*ys[t]*(1-ys[t])\n",
" v = dot(states[t],S)\n",
" dys[t] += v[:nh]\n",
" dstates[t-1] = v[nh:]"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 65
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"backward()"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 66
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"def update(eta=1.0):\n",
" global A,a,B,b,S,s\n",
" for t in range(N-1):\n",
" B -= eta*outer(dzs[t],ys[t])\n",
" b -= eta*dzs[t]\n",
" S -= eta*outer(dstates[t],concatenate([ys[t],states[t-1]]))\n",
" s -= eta*dstates[t]\n",
" A -= eta*outer(dys[t],xs[t])\n",
" a -= eta*dys[t]"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 70
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"update()"
],
"language": "python",
"metadata": {},
"outputs": [],
"prompt_number": 71
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"for i in range(1000):\n",
" forward()\n",
" backward()\n",
" update()\n",
" if i%100==0: print i,sum((zs[:N-1]-target[:N-1])**2)**.5"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"0 1.85758042421\n",
"100"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.85705193087\n",
"200"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.8567707023\n",
"300"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.85659832279\n",
"400"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.85648265595\n",
"500"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.85640002593\n",
"600"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.85633822734\n",
"700"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.85629036387\n",
"800"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.85625225887\n",
"900"
]
},
{
"output_type": "stream",
"stream": "stdout",
"text": [
" 1.85622124147\n"
]
}
],
"prompt_number": 72
},
{
"cell_type": "code",
"collapsed": false,
"input": [
"print A"
],
"language": "python",
"metadata": {},
"outputs": [
{
"output_type": "stream",
"stream": "stdout",
"text": [
"[[ -5.88549888 -9.06246784 -5.03989056]\n",
" [ -1.36378817 -0.28744102 3.41206429]\n",
" [ -6.06789449 -10.10759542 -4.22821784]]\n"
]
}
],
"prompt_number": 73
},
{
"cell_type": "code",
"collapsed": false,
"input": [],
"language": "python",
"metadata": {},
"outputs": []
}
],
"metadata": {}
}
]
}