{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\n\n\nTrain, convert and predict with ONNX Runtime\n============================================\n\nThis example demonstrates an end to end scenario\nstarting with the training of a machine learned model\nto its use in its converted from.\n\nTrain a logistic regression\n+++++++++++++++++++++++++++\n\nThe first step consists in retrieving the iris datset.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "from sklearn.datasets import load_iris\niris = load_iris()\nX, y = iris.data, iris.target\n\nfrom sklearn.model_selection import train_test_split\nX_train, X_test, y_train, y_test = train_test_split(X, y)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Then we fit a model.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "from sklearn.linear_model import LogisticRegression\nclr = LogisticRegression()\nclr.fit(X_train, y_train)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "We compute the prediction on the test set\nand we show the confusion matrix.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "from sklearn.metrics import confusion_matrix\n\npred = clr.predict(X_test)\nprint(confusion_matrix(y_test, pred))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Conversion to ONNX format\n+++++++++++++++++++++++++\n\nWe use module \n`sklearn-onnx <https://github.com/onnx/sklearn-onnx>`_\nto convert the model into ONNX format.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "from skl2onnx import convert_sklearn\nfrom skl2onnx.common.data_types import FloatTensorType\n\ninitial_type = [('float_input', FloatTensorType([None, 4]))]\nonx = convert_sklearn(clr, initial_types=initial_type)\nwith open(\"logreg_iris.onnx\", \"wb\") as f:\n    f.write(onx.SerializeToString())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "We load the model with ONNX Runtime and look at\nits input and output.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "import onnxruntime as rt\nsess = rt.InferenceSession(\"logreg_iris.onnx\")\n\nprint(\"input name='{}' and shape={}\".format(\n    sess.get_inputs()[0].name, sess.get_inputs()[0].shape))\nprint(\"output name='{}' and shape={}\".format(\n    sess.get_outputs()[0].name, sess.get_outputs()[0].shape))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "We compute the predictions.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "input_name = sess.get_inputs()[0].name\nlabel_name = sess.get_outputs()[0].name\n\nimport numpy\npred_onx = sess.run([label_name], {input_name: X_test.astype(numpy.float32)})[0]\nprint(confusion_matrix(pred, pred_onx))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "The prediction are perfectly identical.\n\nProbabilities\n+++++++++++++\n\nProbabilities are needed to compute other\nrelevant metrics such as the ROC Curve.\nLet's see how to get them first with\nscikit-learn.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "prob_sklearn = clr.predict_proba(X_test)\nprint(prob_sklearn[:3])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "And then with ONNX Runtime.\nThe probabilies appear to be \n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "prob_name = sess.get_outputs()[1].name\nprob_rt = sess.run([prob_name], {input_name: X_test.astype(numpy.float32)})[0]\n\nimport pprint\npprint.pprint(prob_rt[0:3])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Let's benchmark.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "from timeit import Timer\n\ndef speed(inst, number=10, repeat=20):\n    timer = Timer(inst, globals=globals())\n    raw = numpy.array(timer.repeat(repeat, number=number))\n    ave = raw.sum() / len(raw) / number\n    mi, ma = raw.min() / number, raw.max() / number\n    print(\"Average %1.3g min=%1.3g max=%1.3g\" % (ave, mi, ma))\n    return ave\n\nprint(\"Execution time for clr.predict\")\nspeed(\"clr.predict(X_test)\")\n\nprint(\"Execution time for ONNX Runtime\")\nspeed(\"sess.run([label_name], {input_name: X_test.astype(numpy.float32)})[0]\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Let's benchmark a scenario similar to what a webservice\nexperiences: the model has to do one prediction at a time\nas opposed to a batch of prediction.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "def loop(X_test, fct, n=None):\n    nrow = X_test.shape[0]\n    if n is None:\n        n = nrow\n    for i in range(0, n):\n        im = i % nrow\n        fct(X_test[im: im+1])\n\nprint(\"Execution time for clr.predict\")\nspeed(\"loop(X_test, clr.predict, 100)\")\n\ndef sess_predict(x):\n    return sess.run([label_name], {input_name: x.astype(numpy.float32)})[0]\n\nprint(\"Execution time for sess_predict\")\nspeed(\"loop(X_test, sess_predict, 100)\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Let's do the same for the probabilities.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "print(\"Execution time for predict_proba\")\nspeed(\"loop(X_test, clr.predict_proba, 100)\")\n\ndef sess_predict_proba(x):\n    return sess.run([prob_name], {input_name: x.astype(numpy.float32)})[0]\n\nprint(\"Execution time for sess_predict_proba\")\nspeed(\"loop(X_test, sess_predict_proba, 100)\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "This second comparison is better as \nONNX Runtime, in this experience,\ncomputes the label and the probabilities\nin every case.\n\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Benchmark with RandomForest\n+++++++++++++++++++++++++++\n\nWe first train and save a model in ONNX format.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "from sklearn.ensemble import RandomForestClassifier\nrf = RandomForestClassifier()\nrf.fit(X_train, y_train)\n\ninitial_type = [('float_input', FloatTensorType([1, 4]))]\nonx = convert_sklearn(rf, initial_types=initial_type)\nwith open(\"rf_iris.onnx\", \"wb\") as f:\n    f.write(onx.SerializeToString())"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "We compare.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "sess = rt.InferenceSession(\"rf_iris.onnx\")\n\ndef sess_predict_proba_rf(x):\n    return sess.run([prob_name], {input_name: x.astype(numpy.float32)})[0]\n\nprint(\"Execution time for predict_proba\")\nspeed(\"loop(X_test, rf.predict_proba, 100)\")\n\nprint(\"Execution time for sess_predict_proba\")\nspeed(\"loop(X_test, sess_predict_proba_rf, 100)\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Let's see with different number of trees.\n\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "measures = []\n\nfor n_trees in range(5, 51, 5):   \n    print(n_trees)\n    rf = RandomForestClassifier(n_estimators=n_trees)\n    rf.fit(X_train, y_train)\n    initial_type = [('float_input', FloatTensorType([1, 4]))]\n    onx = convert_sklearn(rf, initial_types=initial_type)\n    with open(\"rf_iris_%d.onnx\" % n_trees, \"wb\") as f:\n        f.write(onx.SerializeToString())\n    sess = rt.InferenceSession(\"rf_iris_%d.onnx\" % n_trees)\n    def sess_predict_proba_loop(x):\n        return sess.run([prob_name], {input_name: x.astype(numpy.float32)})[0]\n    tsk = speed(\"loop(X_test, rf.predict_proba, 100)\", number=5, repeat=5)\n    trt = speed(\"loop(X_test, sess_predict_proba_loop, 100)\", number=5, repeat=5)\n    measures.append({'n_trees': n_trees, 'sklearn': tsk, 'rt': trt})\n\nfrom pandas import DataFrame\ndf = DataFrame(measures)\nax = df.plot(x=\"n_trees\", y=\"sklearn\", label=\"scikit-learn\", c=\"blue\", logy=True)\ndf.plot(x=\"n_trees\", y=\"rt\", label=\"onnxruntime\",\n                ax=ax, c=\"green\", logy=True)\nax.set_xlabel(\"Number of trees\")\nax.set_ylabel(\"Prediction time (s)\")\nax.set_title(\"Speed comparison between scikit-learn and ONNX Runtime\\nFor a random forest on Iris dataset\")\nax.legend()"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.6.4"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}