{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "%matplotlib inline"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "\n# DL85Booster with cross-validation\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "collapsed": false
      },
      "outputs": [],
      "source": [
        "import time\nimport numpy as np\nfrom sklearn.metrics import accuracy_score, roc_auc_score\nfrom sklearn.model_selection import GridSearchCV, KFold\nfrom dl85 import DL85Booster, MODEL_LP_RATSCH, MODEL_LP_DEMIRIZ, MODEL_QP_MDBOOST\n\ndataset = np.genfromtxt(\"../datasets/tic-tac-toe.txt\", delimiter=' ')\nX, y = dataset[:, 1:], dataset[:, 0]\nn_folds, n_folds_tuning, verbose_level = 5, 4, 0\n\ndepth = 1\n# params = {'model': MODEL_QP_MDBOOST, 'regulator': [.5, 6, 25, 100], 'name': 'MDBoost'}\nparams = {'model': MODEL_LP_DEMIRIZ, 'regulator': [0.5, 1, 25, 50], 'name': 'LPBoost'}\n# params = {'model': MODEL_LP_RATSCH, 'regulator': [0.25, 0.5, 0.75, 1], 'name': 'LPBoost'}\n\n\n# for maximization problems\ndef get_objective_max(estimator, X, y):\n    return estimator.objective_\n\n\n# for minimization problems\ndef get_objective_min(estimator, X, y):\n    return -estimator.objective_\n\n\n# use the minimum margin criterion to choose the best regulator\ndef get_min_margin(estimator, X, y):\n    return min(estimator.margins_)\n\n\n# use the sum of normalized margin to choose the best regulator\ndef get_sum_margin(estimator, X, y):\n    return sum(estimator.margins_norm)\n\n\neval_criteria = {\n    'obj': get_objective_min,  # use get_objective_min in case of minimization problem\n    'auc': 'roc_auc',\n    'acc': 'accuracy'\n}\nbest_reg_criterion = 'auc'\n\nprint(\"######################################################################\\n\"\n      \"#       DL8.5 boosting classifier: CV + hyper-parameter tuning       #\\n\"\n      \"######################################################################\")\nprint(\"<<=== Optiboost ===>>\")\n\nkf = KFold(n_splits=n_folds, random_state=42, shuffle=True)\ntraining_accuracies, test_accuracies, test_aucs = [], [], []\nstart = time.perf_counter()\nprint(\"Model building...\")\nfor k, (train_index, test_index) in enumerate(kf.split(X)):\n    print(\"\\n===== Fold\", k+1, \"=====\")\n    data_train, target_train = X[train_index], y[train_index]\n    data_test, target_test = X[test_index], y[test_index]\n    clf = DL85Booster(max_depth=depth, model=params['model'])\n    grid = {'regulator': params['regulator']}\n    gd_sr = GridSearchCV(estimator=clf, error_score=np.nan, param_grid=grid, scoring=eval_criteria, refit=best_reg_criterion, cv=n_folds_tuning, n_jobs=-1, verbose=verbose_level)\n    print(\"Hyper-parameter tuning: grid search cv...\", end=\" \")\n    gd_sr.fit(data_train, target_train)\n    print(\"==> DONE!!!\")\n    print(\"Best regulator chosen:\", gd_sr.best_params_['regulator'])\n    print(\"Prediction...\", end=\" \")\n    pred = gd_sr.predict(data_test)\n    print(\"==> DONE!!!\")\n    training_accuracies.append(round(gd_sr.best_estimator_.accuracy_, 4))\n    test_accuracies.append(round(accuracy_score(target_test, pred), 4))\n    test_aucs.append(round(roc_auc_score(target_test, gd_sr.predict_proba(data_test)[:, 1]), 4))\n    print(\"train_acc:\", str(training_accuracies[-1]) + \"%\", \"test_acc:\", str(test_accuracies[-1]) + \"%\", \"test_auc:\", str(test_aucs[-1]) + \"%\")\nduration = time.perf_counter() - start\nprint(\"\\nModel built. Duration of building = {}%\".format(round(duration, 4)))\nprint(\"Average accuracy on training set = {}%\".format(round(np.mean(training_accuracies), 4)))\nprint(\"Average accuracy on test set = {}%\".format(round(np.mean(test_accuracies), 4)))\nprint(\"Average auc on test set = {}%\\n\\n\\n\".format(round(np.mean(test_aucs), 4)))"
      ]
    }
  ],
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.6"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}