pruebas.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Automatically created module for IPython interactive environment\n",
      "Loading 20 newsgroups dataset for categories:\n",
      "['alt.atheism', 'talk.religion.misc']\n",
      "857 documents\n",
      "2 categories\n",
      "\n",
      "Performing grid search...\n",
      "pipeline: ['vect', 'tfidf', 'clf']\n",
      "parameters:\n",
      "{'clf__alpha': (1e-05, 1e-06),\n",
      " 'clf__penalty': ('l2', 'elasticnet'),\n",
      " 'vect__max_df': (0.5, 0.75, 1.0),\n",
      " 'vect__ngram_range': ((1, 1), (1, 2))}\n",
      "Fitting 3 folds for each of 24 candidates, totalling 72 fits\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[Parallel(n_jobs=-1)]: Done   1 jobs       | elapsed:    0.7s\n",
      "[Parallel(n_jobs=-1)]: Done  50 jobs       | elapsed:   10.0s\n",
      "[Parallel(n_jobs=-1)]: Done  58 out of  72 | elapsed:   13.7s remaining:    3.3s\n",
      "[Parallel(n_jobs=-1)]: Done  72 out of  72 | elapsed:   15.8s finished\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "done in 17.124s\n",
      "\n",
      "Best score: 0.940\n",
      "Best parameters set:\n",
      "\tclf__alpha: 1e-05\n",
      "\tclf__penalty: 'elasticnet'\n",
      "\tvect__max_df: 0.5\n",
      "\tvect__ngram_range: (1, 2)\n"
     ]
    }
   ],
   "source": [
    "# Author: Olivier Grisel <olivier.grisel@ensta.org>\n",
    "#         Peter Prettenhofer <peter.prettenhofer@gmail.com>\n",
    "#         Mathieu Blondel <mathieu@mblondel.org>\n",
    "# License: BSD 3 clause\n",
    "\n",
    "from __future__ import print_function\n",
    "\n",
    "from pprint import pprint\n",
    "from time import time\n",
    "import logging\n",
    "\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.feature_extraction.text import CountVectorizer\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.grid_search import GridSearchCV\n",
    "from sklearn.pipeline import Pipeline\n",
    "\n",
    "print(__doc__)\n",
    "\n",
    "# Display progress logs on stdout\n",
    "logging.basicConfig(level=logging.INFO,\n",
    "                    format='%(asctime)s %(levelname)s %(message)s')\n",
    "\n",
    "\n",
    "###############################################################################\n",
    "# Load some categories from the training set\n",
    "categories = [\n",
    "    'alt.atheism',\n",
    "    'talk.religion.misc',\n",
    "]\n",
    "# Uncomment the following to do the analysis on all the categories\n",
    "#categories = None\n",
    "\n",
    "print(\"Loading 20 newsgroups dataset for categories:\")\n",
    "print(categories)\n",
    "\n",
    "data = fetch_20newsgroups(subset='train', categories=categories)\n",
    "print(\"%d documents\" % len(data.filenames))\n",
    "print(\"%d categories\" % len(data.target_names))\n",
    "print()\n",
    "\n",
    "###############################################################################\n",
    "# define a pipeline combining a text feature extractor with a simple\n",
    "# classifier\n",
    "pipeline = Pipeline([\n",
    "    ('vect', CountVectorizer()),\n",
    "    ('tfidf', TfidfTransformer()),\n",
    "    ('clf', SGDClassifier()),\n",
    "])\n",
    "\n",
    "# uncommenting more parameters will give better exploring power but will\n",
    "# increase processing time in a combinatorial way\n",
    "parameters = {\n",
    "    'vect__max_df': (0.5, 0.75, 1.0),\n",
    "    #'vect__max_features': (None, 5000, 10000, 50000), #\n",
    "    'vect__ngram_range': ((1, 1), (1, 2)),  # unigrams or bigrams\n",
    "    #'tfidf__use_idf': (True, False), #\n",
    "    #'tfidf__norm': ('l1', 'l2'), #\n",
    "    'clf__alpha': (0.00001, 0.000001),\n",
    "    'clf__penalty': ('l2', 'elasticnet'),\n",
    "    #'clf__n_iter': (10, 50, 80), #\n",
    "}\n",
    "\n",
    "if __name__ == \"__main__\":\n",
    "    # multiprocessing requires the fork to happen in a __main__ protected\n",
    "    # block\n",
    "\n",
    "    # find the best parameters for both the feature extraction and the\n",
    "    # classifier\n",
    "    grid_search = GridSearchCV(pipeline, parameters, n_jobs=-1, verbose=1)\n",
    "\n",
    "    print(\"Performing grid search...\")\n",
    "    print(\"pipeline:\", [name for name, _ in pipeline.steps])\n",
    "    print(\"parameters:\")\n",
    "    pprint(parameters)\n",
    "    t0 = time()\n",
    "    grid_search.fit(data.data, data.target)\n",
    "    print(\"done in %0.3fs\" % (time() - t0))\n",
    "    print()\n",
    "\n",
    "    print(\"Best score: %0.3f\" % grid_search.best_score_)\n",
    "    print(\"Best parameters set:\")\n",
    "    best_parameters = grid_search.best_estimator_.get_params()\n",
    "    for param_name in sorted(parameters.keys()):\n",
    "        print(\"\\t%s: %r\" % (param_name, best_parameters[param_name]))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "From: mangoe@cs.umd.edu (Charley Wingate)\n",
      "Subject: Benediktine Metaphysics\n",
      "Lines: 24\n",
      "\n",
      "Benedikt Rosenau writes, with great authority:\n",
      "\n",
      ">     IF IT IS CONTRADICTORY IT CANNOT EXIST.\n",
      "\n",
      "\"Contradictory\" is a property of language.  If I correct this to\n",
      "\n",
      "\n",
      "      THINGS DEFINED BY CONTRADICTORY LANGUAGE DO NOT EXIST\n",
      "\n",
      "I will object to definitions as reality.  If you then amend it to\n",
      "\n",
      "      THINGS DESCRIBED BY CONTRADICTORY LANGUAGE DO NOT EXIST\n",
      "\n",
      "then we've come to something which is plainly false.  Failures in\n",
      "description are merely failures in description.\n",
      "\n",
      "(I'm not an objectivist, remember.)\n",
      "\n",
      "\n",
      "-- \n",
      "C. Wingate        + \"The peace of God, it is no peace,\n",
      "                  +    but strife closed in the sod.\n",
      "mangoe@cs.umd.edu +  Yet, brothers, pray for but one thing:\n",
      "tove!mangoe       +    the marv'lous peace of God.\"\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(data.data[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "857"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data.filenames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['alt.atheism', 'talk.religion.misc']"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.target_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "sklearn.datasets.base.Bunch"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('/home/antonio/scikit_learn_data/20news_home/20news-bydate-train/alt.atheism/51267',\n",
       "  0),\n",
       " ('/home/antonio/scikit_learn_data/20news_home/20news-bydate-train/alt.atheism/51139',\n",
       "  0),\n",
       " ('/home/antonio/scikit_learn_data/20news_home/20news-bydate-train/alt.atheism/51245',\n",
       "  0),\n",
       " ('/home/antonio/scikit_learn_data/20news_home/20news-bydate-train/talk.religion.misc/84200',\n",
       "  1),\n",
       " ('/home/antonio/scikit_learn_data/20news_home/20news-bydate-train/talk.religion.misc/82815',\n",
       "  1),\n",
       " ('/home/antonio/scikit_learn_data/20news_home/20news-bydate-train/alt.atheism/51294',\n",
       "  0),\n",
       " ('/home/antonio/scikit_learn_data/20news_home/20news-bydate-train/alt.atheism/51253',\n",
       "  0),\n",
       " ('/home/antonio/scikit_learn_data/20news_home/20news-bydate-train/talk.religion.misc/84186',\n",
       "  1),\n",
       " ('/home/antonio/scikit_learn_data/20news_home/20news-bydate-train/alt.atheism/53459',\n",
       "  0),\n",
       " ('/home/antonio/scikit_learn_data/20news_home/20news-bydate-train/alt.atheism/51217',\n",
       "  0)]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "zip(data.filenames, data.target)[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1,\n",
       "       0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0,\n",
       "       1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1,\n",
       "       1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0,\n",
       "       1, 0, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0, 0, 0, 1, 0,\n",
       "       1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 0, 1, 1, 1, 0,\n",
       "       0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0,\n",
       "       0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0,\n",
       "       1, 0, 0, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 0,\n",
       "       0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 0, 1, 0,\n",
       "       1, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 1,\n",
       "       1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 1,\n",
       "       1, 1, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 1,\n",
       "       1, 0, 1, 1, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 0, 1,\n",
       "       0, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 1,\n",
       "       1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 0, 0, 1, 1,\n",
       "       0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 0,\n",
       "       1, 0, 0, 0, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 1, 0, 1,\n",
       "       0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 1, 1,\n",
       "       0, 0, 1, 0, 1, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1,\n",
       "       1, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n",
       "       0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0,\n",
       "       1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 1,\n",
       "       0, 0, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 1, 1, 1, 0, 1, 0,\n",
       "       1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 0, 1, 0, 0, 0, 0, 0,\n",
       "       1, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 1, 1,\n",
       "       1, 1, 0, 0, 1, 1, 1, 0, 0, 1, 0, 0, 0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0,\n",
       "       0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0,\n",
       "       0, 1, 0, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0,\n",
       "       0, 1, 1, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 1, 0, 0, 0, 0, 0,\n",
       "       1, 0, 1, 0, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 1,\n",
       "       0, 1, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1,\n",
       "       1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 0, 0, 1, 1, 1, 0, 1, 0, 0, 0,\n",
       "       0, 1, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0,\n",
       "       0, 1, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 0,\n",
       "       1, 0, 1, 0, 0, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 0, 1, 0, 0,\n",
       "       1, 0, 0, 0, 0, 0])"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading 20 newsgroups dataset for categories:\n",
      "['alt.atheism', 'talk.religion.misc', 'comp.graphics', 'sci.space']\n",
      "data loaded\n",
      "2034 documents - 3.980MB (training set)\n",
      "1353 documents - 2.867MB (test set)\n",
      "4 categories\n",
      "\n",
      "Extracting features from the training data using a sparse vectorizer\n",
      "done in 1.822811s at 2.183MB/s\n",
      "n_samples: 2034, n_features: 33810\n",
      "\n",
      "Extracting features from the test data using the same vectorizer\n",
      "done in 0.910150s at 3.151MB/s\n",
      "n_samples: 1353, n_features: 33810\n",
      "\n",
      "Extracting 10 best features by a chi-squared test\n",
      "done in 0.078601s\n",
      "\n",
      "================================================================================\n",
      "Ridge Classifier\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "RidgeClassifier(alpha=1.0, class_weight=None, copy_X=True, fit_intercept=True,\n",
      "        max_iter=None, normalize=False, solver='lsqr', tol=0.01)\n",
      "  (5, 0)\t0.109539901025\n",
      "  (6, 9)\t0.0508006478447\n",
      "  (8, 4)\t0.228862916069\n",
      "  (9, 3)\t0.0528959324173\n",
      "  (9, 7)\t0.051936762369\n",
      "  (10, 7)\t0.0936955852303\n",
      "  (13, 7)\t0.093446403874\n",
      "  (15, 9)\t0.065652060404\n",
      "  (17, 9)\t0.0352881367249\n",
      "  (18, 9)\t0.116324195232\n",
      "  (18, 4)\t0.226288675468\n",
      "  (18, 7)\t0.109019685025\n",
      "  (22, 2)\t0.0839867464462\n",
      "  (23, 2)\t0.0247831362042\n",
      "  (24, 9)\t0.0370420411367\n",
      "  (26, 7)\t0.127283310161\n",
      "  (28, 7)\t0.131328020617\n",
      "  (30, 0)\t0.0925840536084\n",
      "  (30, 2)\t0.0724051829482\n",
      "  (32, 3)\t0.101574677288\n",
      "  (33, 9)\t0.0201930576547\n",
      "  (33, 3)\t0.0915766698775\n",
      "  (34, 0)\t0.0463392287851\n",
      "  (34, 6)\t0.13375138285\n",
      "  (34, 1)\t0.0834658323948\n",
      "  :\t:\n",
      "  (2003, 9)\t0.0356647699378\n",
      "  (2003, 7)\t0.0414297004317\n",
      "  (2003, 1)\t0.104301128545\n",
      "  (2004, 2)\t0.0929260825\n",
      "  (2005, 3)\t0.0571237522049\n",
      "  (2007, 0)\t0.127324222877\n",
      "  (2007, 2)\t0.123418992721\n",
      "  (2010, 9)\t0.0840456306694\n",
      "  (2011, 2)\t0.0417182983245\n",
      "  (2012, 2)\t0.0537837053108\n",
      "  (2013, 9)\t0.0541242943606\n",
      "  (2013, 7)\t0.104405541704\n",
      "  (2015, 2)\t0.0731606620069\n",
      "  (2015, 6)\t0.191586250924\n",
      "  (2016, 2)\t0.0425110784733\n",
      "  (2018, 3)\t0.0670008777694\n",
      "  (2019, 4)\t0.199997798636\n",
      "  (2019, 7)\t0.0963534607641\n",
      "  (2020, 3)\t0.0646407215144\n",
      "  (2022, 2)\t0.118658071206\n",
      "  (2023, 9)\t0.0860318288725\n",
      "  (2027, 7)\t0.0738797930691\n",
      "  (2028, 3)\t0.115363996787\n",
      "  (2030, 9)\t0.0758672059456\n",
      "  (2032, 2)\t0.0429486940651\n",
      "[1 3 2 ..., 1 0 1]\n",
      "train time: 0.011s\n",
      "test time:  0.000s\n",
      "accuracy:   0.514\n",
      "dimensionality: 10\n",
      "density: 1.000000\n",
      "qqq [[ 4.88202355  1.33950519  4.36088177 -3.09448316 -1.26444407  3.50602791\n",
      "   2.8342832  -1.7823368   1.09878977 -2.76082182]\n",
      " [-1.5508548  -0.51269472 -3.89276672  8.80441027 -1.64010873 -1.43545506\n",
      "  -1.16675396 -1.46120663 -0.33830337 -3.29334431]\n",
      " [-1.67227166 -0.18184519 -4.24603687 -3.54856863  3.92531675 -1.52819798\n",
      "  -1.21604787  4.97236923 -0.45059597  8.54817258]\n",
      " [-1.41051809 -0.79631508  3.87591511 -2.35709283 -0.92786681 -0.52583422\n",
      "  -0.38825777 -1.54900857 -0.39326307 -2.35611645]]\n",
      "top 10 keywords per class:\n",
      ">>> 0 alt.atheism\n",
      "www [3 9 7 4 8 1 6 5 2 0]\n",
      "alt.atheism: graphics space nasa henry schneider caltech livesey keith god at...\n",
      ">>> 1 comp.graphics\n",
      "www [2 9 4 0 7 5 6 1 8 3]\n",
      "comp.graphics: god space henry atheists nasa keith livesey caltech schneider ...\n",
      ">>> 2 sci.space\n",
      "www [2 3 0 5 6 8 1 4 7 9]\n",
      "sci.space: god graphics atheists keith livesey schneider caltech henry nasa s...\n",
      ">>> 3 talk.religion.misc\n",
      "www [3 9 7 0 4 1 5 8 6 2]\n",
      "talk.religion.misc: graphics space nasa atheists henry caltech keith schneide...\n",
      "\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.62      0.56      0.59       319\n",
      "     comp.graphics       0.99      0.34      0.51       389\n",
      "         sci.space       0.41      0.97      0.58       394\n",
      "talk.religion.misc       0.00      0.00      0.00       251\n",
      "\n",
      "       avg / total       0.55      0.51      0.45      1353\n",
      "\n",
      "confusion matrix:\n",
      "[[180   0 139   0]\n",
      " [  1 133 255   0]\n",
      " [ 10   2 382   0]\n",
      " [ 99   0 152   0]]\n",
      "\n",
      "================================================================================\n",
      "L2 penalty\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "LinearSVC(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
      "     intercept_scaling=1, loss='l2', max_iter=1000, multi_class='ovr',\n",
      "     penalty='l2', random_state=None, tol=0.001, verbose=0)\n",
      "  (5, 0)\t0.109539901025\n",
      "  (6, 9)\t0.0508006478447\n",
      "  (8, 4)\t0.228862916069\n",
      "  (9, 3)\t0.0528959324173\n",
      "  (9, 7)\t0.051936762369\n",
      "  (10, 7)\t0.0936955852303\n",
      "  (13, 7)\t0.093446403874\n",
      "  (15, 9)\t0.065652060404\n",
      "  (17, 9)\t0.0352881367249\n",
      "  (18, 9)\t0.116324195232\n",
      "  (18, 4)\t0.226288675468\n",
      "  (18, 7)\t0.109019685025\n",
      "  (22, 2)\t0.0839867464462\n",
      "  (23, 2)\t0.0247831362042\n",
      "  (24, 9)\t0.0370420411367\n",
      "  (26, 7)\t0.127283310161\n",
      "  (28, 7)\t0.131328020617\n",
      "  (30, 0)\t0.0925840536084\n",
      "  (30, 2)\t0.0724051829482\n",
      "  (32, 3)\t0.101574677288\n",
      "  (33, 9)\t0.0201930576547\n",
      "  (33, 3)\t0.0915766698775\n",
      "  (34, 0)\t0.0463392287851\n",
      "  (34, 6)\t0.13375138285\n",
      "  (34, 1)\t0.0834658323948\n",
      "  :\t:\n",
      "  (2003, 9)\t0.0356647699378\n",
      "  (2003, 7)\t0.0414297004317\n",
      "  (2003, 1)\t0.104301128545\n",
      "  (2004, 2)\t0.0929260825\n",
      "  (2005, 3)\t0.0571237522049\n",
      "  (2007, 0)\t0.127324222877\n",
      "  (2007, 2)\t0.123418992721\n",
      "  (2010, 9)\t0.0840456306694\n",
      "  (2011, 2)\t0.0417182983245\n",
      "  (2012, 2)\t0.0537837053108\n",
      "  (2013, 9)\t0.0541242943606\n",
      "  (2013, 7)\t0.104405541704\n",
      "  (2015, 2)\t0.0731606620069\n",
      "  (2015, 6)\t0.191586250924\n",
      "  (2016, 2)\t0.0425110784733\n",
      "  (2018, 3)\t0.0670008777694\n",
      "  (2019, 4)\t0.199997798636\n",
      "  (2019, 7)\t0.0963534607641\n",
      "  (2020, 3)\t0.0646407215144\n",
      "  (2022, 2)\t0.118658071206\n",
      "  (2023, 9)\t0.0860318288725\n",
      "  (2027, 7)\t0.0738797930691\n",
      "  (2028, 3)\t0.115363996787\n",
      "  (2030, 9)\t0.0758672059456\n",
      "  (2032, 2)\t0.0429486940651\n",
      "[1 3 2 ..., 1 0 1]\n",
      "train time: 0.101s\n",
      "test time:  0.000s\n",
      "accuracy:   0.602\n",
      "dimensionality: 10\n",
      "density: 1.000000\n",
      "qqq [[  7.83904459   1.57551128   5.40600894  -3.12501697  -1.4056031\n",
      "    5.45142145   4.21140401  -1.99506467   1.49566489  -2.88928225]\n",
      " [ -3.00847115  -0.76600756  -5.94107802  10.5589947   -2.62035462\n",
      "   -2.7216287   -2.17276959  -2.41428555  -0.43472127  -5.12762986]\n",
      " [ -2.25194305   0.36898885  -4.50444969  -3.75303546   5.61519143\n",
      "   -1.97096728  -1.47592088   6.946436    -0.49126642  11.86270047]\n",
      " [ -2.99315188  -1.37825692   3.89918837  -3.81955666  -1.65942176\n",
      "   -1.15637825  -0.79559098  -2.84656506  -0.61794609  -4.07138192]]\n",
      "top 10 keywords per class:\n",
      ">>> 0 alt.atheism\n",
      "www [3 9 7 4 8 1 6 2 5 0]\n",
      "alt.atheism: graphics space nasa henry schneider caltech livesey god keith at...\n",
      ">>> 1 comp.graphics\n",
      "www [2 9 0 5 4 7 6 1 8 3]\n",
      "comp.graphics: god space atheists keith henry nasa livesey caltech schneider ...\n",
      ">>> 2 sci.space\n",
      "www [2 3 0 5 6 8 1 4 7 9]\n",
      "sci.space: god graphics atheists keith livesey schneider caltech henry nasa s...\n",
      ">>> 3 talk.religion.misc\n",
      "www [9 3 0 7 4 1 5 6 8 2]\n",
      "talk.religion.misc: space graphics atheists nasa henry caltech keith livesey ...\n",
      "\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.67      0.48      0.56       319\n",
      "     comp.graphics       0.48      0.96      0.64       389\n",
      "         sci.space       0.88      0.66      0.75       394\n",
      "talk.religion.misc       0.54      0.12      0.20       251\n",
      "\n",
      "       avg / total       0.65      0.60      0.57      1353\n",
      "\n",
      "confusion matrix:\n",
      "[[152 127  14  26]\n",
      " [  1 372  16   0]\n",
      " [  7 128 259   0]\n",
      " [ 67 148   5  31]]\n",
      "\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "SGDClassifier(alpha=0.0001, average=False, class_weight=None, epsilon=0.1,\n",
      "       eta0=0.0, fit_intercept=True, l1_ratio=0.15,\n",
      "       learning_rate='optimal', loss='hinge', n_iter=50, n_jobs=1,\n",
      "       penalty='l2', power_t=0.5, random_state=None, shuffle=True,\n",
      "       verbose=0, warm_start=False)\n",
      "  (5, 0)\t0.109539901025\n",
      "  (6, 9)\t0.0508006478447\n",
      "  (8, 4)\t0.228862916069\n",
      "  (9, 3)\t0.0528959324173\n",
      "  (9, 7)\t0.051936762369\n",
      "  (10, 7)\t0.0936955852303\n",
      "  (13, 7)\t0.093446403874\n",
      "  (15, 9)\t0.065652060404\n",
      "  (17, 9)\t0.0352881367249\n",
      "  (18, 9)\t0.116324195232\n",
      "  (18, 4)\t0.226288675468\n",
      "  (18, 7)\t0.109019685025\n",
      "  (22, 2)\t0.0839867464462\n",
      "  (23, 2)\t0.0247831362042\n",
      "  (24, 9)\t0.0370420411367\n",
      "  (26, 7)\t0.127283310161\n",
      "  (28, 7)\t0.131328020617\n",
      "  (30, 0)\t0.0925840536084\n",
      "  (30, 2)\t0.0724051829482\n",
      "  (32, 3)\t0.101574677288\n",
      "  (33, 9)\t0.0201930576547\n",
      "  (33, 3)\t0.0915766698775\n",
      "  (34, 0)\t0.0463392287851\n",
      "  (34, 6)\t0.13375138285\n",
      "  (34, 1)\t0.0834658323948\n",
      "  :\t:\n",
      "  (2003, 9)\t0.0356647699378\n",
      "  (2003, 7)\t0.0414297004317\n",
      "  (2003, 1)\t0.104301128545\n",
      "  (2004, 2)\t0.0929260825\n",
      "  (2005, 3)\t0.0571237522049\n",
      "  (2007, 0)\t0.127324222877\n",
      "  (2007, 2)\t0.123418992721\n",
      "  (2010, 9)\t0.0840456306694\n",
      "  (2011, 2)\t0.0417182983245\n",
      "  (2012, 2)\t0.0537837053108\n",
      "  (2013, 9)\t0.0541242943606\n",
      "  (2013, 7)\t0.104405541704\n",
      "  (2015, 2)\t0.0731606620069\n",
      "  (2015, 6)\t0.191586250924\n",
      "  (2016, 2)\t0.0425110784733\n",
      "  (2018, 3)\t0.0670008777694\n",
      "  (2019, 4)\t0.199997798636\n",
      "  (2019, 7)\t0.0963534607641\n",
      "  (2020, 3)\t0.0646407215144\n",
      "  (2022, 2)\t0.118658071206\n",
      "  (2023, 9)\t0.0860318288725\n",
      "  (2027, 7)\t0.0738797930691\n",
      "  (2028, 3)\t0.115363996787\n",
      "  (2030, 9)\t0.0758672059456\n",
      "  (2032, 2)\t0.0429486940651\n",
      "[1 3 2 ..., 1 0 1]\n",
      "train time: 0.075s"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/antonio/.virtualenvs/lexisnexis/local/lib/python2.7/site-packages/sklearn/svm/classes.py:192: DeprecationWarning: loss='l2' has been deprecated in favor of loss='squared_hinge' as of 0.16. Backward compatibility for the loss='l2' will be removed in 1.0\n",
      "  DeprecationWarning)\n",
      "/home/antonio/.virtualenvs/lexisnexis/local/lib/python2.7/site-packages/sklearn/svm/classes.py:192: DeprecationWarning: loss='l2' has been deprecated in favor of loss='squared_hinge' as of 0.16. Backward compatibility for the loss='l2' will be removed in 1.0\n",
      "  DeprecationWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "test time:  0.000s\n",
      "accuracy:   0.515\n",
      "dimensionality: 10\n",
      "density: 1.000000\n",
      "qqq [[  1.69871106e+01   3.28955507e-01   5.39285823e+00  -8.68305023e-01\n",
      "   -2.37549445e-01   1.15824831e+01   9.15344463e+00  -4.68959876e-01\n",
      "    1.94206363e+00  -4.77878413e-01]\n",
      " [ -1.67229833e-01   2.86628680e-02  -1.37961884e-01   2.55797147e+01\n",
      "   -1.77484060e-01  -1.18744888e-01  -9.41056077e-02  -2.76498470e-02\n",
      "   -6.79999401e-03  -9.22555936e-02]\n",
      " [ -1.11391383e+00   1.40101762e-02  -5.19658871e+00  -2.64361260e+00\n",
      "    1.17643826e+01  -5.66645656e-02  -4.75176623e-02   1.50904796e+01\n",
      "   -4.92954198e-02   2.35311910e+01]\n",
      " [ -3.97354533e-01  -1.60239371e-02   1.81854296e-01  -2.19500121e-01\n",
      "   -3.17700651e-02  -1.71055997e-01  -8.66916107e-02  -2.09812776e-01\n",
      "   -7.60989174e-02  -5.29278420e-01]]\n",
      "top 10 keywords per class:\n",
      ">>> 0 alt.atheism\n",
      "www [3 9 7 4 1 8 2 6 5 0]\n",
      "alt.atheism: graphics space nasa henry caltech schneider god livesey keith at...\n",
      ">>> 1 comp.graphics\n",
      "www [4 0 2 5 6 9 7 8 1 3]\n",
      "comp.graphics: henry atheists god keith livesey space nasa schneider caltech ...\n",
      ">>> 2 sci.space\n",
      "www [2 3 0 5 8 6 1 4 7 9]\n",
      "sci.space: god graphics atheists keith schneider livesey caltech henry nasa s...\n",
      ">>> 3 talk.religion.misc\n",
      "www [9 0 3 7 5 6 8 4 1 2]\n",
      "talk.religion.misc: space atheists graphics nasa keith livesey schneider henr...\n",
      "\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.33      0.95      0.49       319\n",
      "     comp.graphics       0.99      0.34      0.51       389\n",
      "         sci.space       0.88      0.66      0.75       394\n",
      "talk.religion.misc       0.00      0.00      0.00       251\n",
      "\n",
      "       avg / total       0.62      0.52      0.48      1353\n",
      "\n",
      "confusion matrix:\n",
      "[[304   0  15   0]\n",
      " [238 134  17   0]\n",
      " [133   2 259   0]\n",
      " [246   0   5   0]]\n",
      "\n",
      "================================================================================\n",
      "L1 penalty\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "LinearSVC(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
      "     intercept_scaling=1, loss='l2', max_iter=1000, multi_class='ovr',\n",
      "     penalty='l1', random_state=None, tol=0.001, verbose=0)\n",
      "  (5, 0)\t0.109539901025\n",
      "  (6, 9)\t0.0508006478447\n",
      "  (8, 4)\t0.228862916069\n",
      "  (9, 3)\t0.0528959324173\n",
      "  (9, 7)\t0.051936762369\n",
      "  (10, 7)\t0.0936955852303\n",
      "  (13, 7)\t0.093446403874\n",
      "  (15, 9)\t0.065652060404\n",
      "  (17, 9)\t0.0352881367249\n",
      "  (18, 9)\t0.116324195232\n",
      "  (18, 4)\t0.226288675468\n",
      "  (18, 7)\t0.109019685025\n",
      "  (22, 2)\t0.0839867464462\n",
      "  (23, 2)\t0.0247831362042\n",
      "  (24, 9)\t0.0370420411367\n",
      "  (26, 7)\t0.127283310161\n",
      "  (28, 7)\t0.131328020617\n",
      "  (30, 0)\t0.0925840536084\n",
      "  (30, 2)\t0.0724051829482\n",
      "  (32, 3)\t0.101574677288\n",
      "  (33, 9)\t0.0201930576547\n",
      "  (33, 3)\t0.0915766698775\n",
      "  (34, 0)\t0.0463392287851\n",
      "  (34, 6)\t0.13375138285\n",
      "  (34, 1)\t0.0834658323948\n",
      "  :\t:\n",
      "  (2003, 9)\t0.0356647699378\n",
      "  (2003, 7)\t0.0414297004317\n",
      "  (2003, 1)\t0.104301128545\n",
      "  (2004, 2)\t0.0929260825\n",
      "  (2005, 3)\t0.0571237522049\n",
      "  (2007, 0)\t0.127324222877\n",
      "  (2007, 2)\t0.123418992721\n",
      "  (2010, 9)\t0.0840456306694\n",
      "  (2011, 2)\t0.0417182983245\n",
      "  (2012, 2)\t0.0537837053108\n",
      "  (2013, 9)\t0.0541242943606\n",
      "  (2013, 7)\t0.104405541704\n",
      "  (2015, 2)\t0.0731606620069\n",
      "  (2015, 6)\t0.191586250924\n",
      "  (2016, 2)\t0.0425110784733\n",
      "  (2018, 3)\t0.0670008777694\n",
      "  (2019, 4)\t0.199997798636\n",
      "  (2019, 7)\t0.0963534607641\n",
      "  (2020, 3)\t0.0646407215144\n",
      "  (2022, 2)\t0.118658071206\n",
      "  (2023, 9)\t0.0860318288725\n",
      "  (2027, 7)\t0.0738797930691\n",
      "  (2028, 3)\t0.115363996787\n",
      "  (2030, 9)\t0.0758672059456\n",
      "  (2032, 2)\t0.0429486940651\n",
      "[1 3 2 ..., 1 0 1]\n",
      "train time: 0.008s\n",
      "test time:  0.000s\n",
      "accuracy:   0.619\n",
      "dimensionality: 10\n",
      "density: 0.850000\n",
      "qqq [[ 16.94196265   0.           5.95609294  -4.15137347  -1.30213023\n",
      "    9.28380976   6.38074476  -2.06672649   0.          -3.33584605]\n",
      " [ -5.74764772   0.         -10.39617116  18.22472982  -3.78757482\n",
      "   -5.32737597  -3.14659498  -2.96112969   0.          -7.41195845]\n",
      " [ -3.35259078   0.52087665  -7.83541047  -5.3908473    9.5871469\n",
      "   -2.62572157  -1.39472698   9.20126891   0.          18.44287668]\n",
      " [ -6.53893042  -1.64457552   4.67131922  -6.94533351  -2.1450983\n",
      "   -1.78382147  -0.70969021  -5.3272554    0.          -7.96985177]]\n",
      "top 10 keywords per class:\n",
      ">>> 0 alt.atheism\n",
      "www [3 9 7 4 1 8 2 6 5 0]\n",
      "alt.atheism: graphics space nasa henry caltech schneider god livesey keith at...\n",
      ">>> 1 comp.graphics\n",
      "www [2 9 0 5 4 6 7 1 8 3]\n",
      "comp.graphics: god space atheists keith henry livesey nasa caltech schneider ...\n",
      ">>> 2 sci.space\n",
      "www [2 3 0 5 6 8 1 7 4 9]\n",
      "sci.space: god graphics atheists keith livesey schneider caltech nasa henry s...\n",
      ">>> 3 talk.religion.misc\n",
      "www [9 3 0 7 4 5 1 6 8 2]\n",
      "talk.religion.misc: space graphics atheists nasa henry keith caltech livesey ...\n",
      "\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.79      0.43      0.56       319\n",
      "     comp.graphics       0.48      0.96      0.64       389\n",
      "         sci.space       0.89      0.65      0.75       394\n",
      "talk.religion.misc       0.60      0.27      0.37       251\n",
      "\n",
      "       avg / total       0.70      0.62      0.60      1353\n",
      "\n",
      "confusion matrix:\n",
      "[[137 126  12  44]\n",
      " [  0 374  15   0]\n",
      " [  6 129 258   1]\n",
      " [ 30 148   5  68]]\n",
      "\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "SGDClassifier(alpha=0.0001, average=False, class_weight=None, epsilon=0.1,\n",
      "       eta0=0.0, fit_intercept=True, l1_ratio=0.15,\n",
      "       learning_rate='optimal', loss='hinge', n_iter=50, n_jobs=1,\n",
      "       penalty='l1', power_t=0.5, random_state=None, shuffle=True,\n",
      "       verbose=0, warm_start=False)\n",
      "  (5, 0)\t0.109539901025\n",
      "  (6, 9)\t0.0508006478447\n",
      "  (8, 4)\t0.228862916069\n",
      "  (9, 3)\t0.0528959324173\n",
      "  (9, 7)\t0.051936762369\n",
      "  (10, 7)\t0.0936955852303\n",
      "  (13, 7)\t0.093446403874\n",
      "  (15, 9)\t0.065652060404\n",
      "  (17, 9)\t0.0352881367249\n",
      "  (18, 9)\t0.116324195232\n",
      "  (18, 4)\t0.226288675468\n",
      "  (18, 7)\t0.109019685025\n",
      "  (22, 2)\t0.0839867464462\n",
      "  (23, 2)\t0.0247831362042\n",
      "  (24, 9)\t0.0370420411367\n",
      "  (26, 7)\t0.127283310161\n",
      "  (28, 7)\t0.131328020617\n",
      "  (30, 0)\t0.0925840536084\n",
      "  (30, 2)\t0.0724051829482\n",
      "  (32, 3)\t0.101574677288\n",
      "  (33, 9)\t0.0201930576547\n",
      "  (33, 3)\t0.0915766698775\n",
      "  (34, 0)\t0.0463392287851\n",
      "  (34, 6)\t0.13375138285\n",
      "  (34, 1)\t0.0834658323948\n",
      "  :\t:\n",
      "  (2003, 9)\t0.0356647699378\n",
      "  (2003, 7)\t0.0414297004317\n",
      "  (2003, 1)\t0.104301128545\n",
      "  (2004, 2)\t0.0929260825\n",
      "  (2005, 3)\t0.0571237522049\n",
      "  (2007, 0)\t0.127324222877\n",
      "  (2007, 2)\t0.123418992721\n",
      "  (2010, 9)\t0.0840456306694\n",
      "  (2011, 2)\t0.0417182983245\n",
      "  (2012, 2)\t0.0537837053108\n",
      "  (2013, 9)\t0.0541242943606\n",
      "  (2013, 7)\t0.104405541704\n",
      "  (2015, 2)\t0.0731606620069\n",
      "  (2015, 6)\t0.191586250924\n",
      "  (2016, 2)\t0.0425110784733\n",
      "  (2018, 3)\t0.0670008777694\n",
      "  (2019, 4)\t0.199997798636\n",
      "  (2019, 7)\t0.0963534607641\n",
      "  (2020, 3)\t0.0646407215144\n",
      "  (2022, 2)\t0.118658071206\n",
      "  (2023, 9)\t0.0860318288725\n",
      "  (2027, 7)\t0.0738797930691\n",
      "  (2028, 3)\t0.115363996787\n",
      "  (2030, 9)\t0.0758672059456\n",
      "  (2032, 2)\t0.0429486940651\n",
      "[1 3 2 ..., 1 0 1]\n",
      "train time: 0.036s\n",
      "test time:  0.000s\n",
      "accuracy:   0.532\n",
      "dimensionality: 10\n",
      "density: 0.850000\n",
      "qqq [[  3.51609254e+01   0.00000000e+00   7.61332940e-02  -3.90236153e-01\n",
      "   -1.92391555e-02   2.12033149e+01   1.32795113e+01  -2.00437192e-02\n",
      "    1.38098464e+00  -1.49438590e-02]\n",
      " [ -3.47014065e-02   0.00000000e+00  -6.12130732e-02   4.22350576e+01\n",
      "   -1.26058509e-02  -2.90348035e-02  -3.93255187e-02  -2.70193880e-02\n",
      "    0.00000000e+00  -5.15775410e-02]\n",
      " [ -1.22084365e+00   0.00000000e+00  -1.57548834e+01  -9.90948003e+00\n",
      "    1.65291304e+01  -4.74016732e-02  -2.45427519e-02   2.24601170e+01\n",
      "    0.00000000e+00   3.85634184e+01]\n",
      " [ -3.59112390e-01  -1.90950141e-02   1.90448009e-01  -8.59675243e-02\n",
      "   -2.60591881e-02  -1.35674794e-01  -3.63488020e-02  -1.33137452e-01\n",
      "    0.00000000e+00  -9.91529893e-01]]\n",
      "top 10 keywords per class:\n",
      ">>> 0 alt.atheism\n",
      "www [3 7 4 9 1 2 8 6 5 0]\n",
      "alt.atheism: graphics nasa henry space caltech god schneider livesey keith at...\n",
      ">>> 1 comp.graphics\n",
      "www [2 9 6 0 5 7 4 1 8 3]\n",
      "comp.graphics: god space livesey atheists keith nasa henry caltech schneider ...\n",
      ">>> 2 sci.space\n",
      "www [2 3 0 5 6 1 8 4 7 9]\n",
      "sci.space: god graphics atheists keith livesey caltech schneider henry nasa s...\n",
      ">>> 3 talk.religion.misc\n",
      "www [9 0 5 7 3 6 4 1 8 2]\n",
      "talk.religion.misc: space atheists keith nasa graphics livesey henry caltech ...\n",
      "\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.83      0.38      0.52       319\n",
      "     comp.graphics       0.99      0.35      0.51       389\n",
      "         sci.space       0.41      0.98      0.58       394\n",
      "talk.religion.misc       0.57      0.32      0.41       251\n",
      "\n",
      "       avg / total       0.71      0.53      0.51      1353\n",
      "\n",
      "confusion matrix:\n",
      "[[120   0 139  60]\n",
      " [  0 135 254   0]\n",
      " [  6   2 385   1]\n",
      " [ 18   0 153  80]]\n",
      "\n",
      "================================================================================\n",
      "Elastic-Net penalty\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "SGDClassifier(alpha=0.0001, average=False, class_weight=None, epsilon=0.1,\n",
      "       eta0=0.0, fit_intercept=True, l1_ratio=0.15,\n",
      "       learning_rate='optimal', loss='hinge', n_iter=50, n_jobs=1,\n",
      "       penalty='elasticnet', power_t=0.5, random_state=None, shuffle=True,\n",
      "       verbose=0, warm_start=False)\n",
      "  (5, 0)\t0.109539901025\n",
      "  (6, 9)\t0.0508006478447\n",
      "  (8, 4)\t0.228862916069\n",
      "  (9, 3)\t0.0528959324173\n",
      "  (9, 7)\t0.051936762369\n",
      "  (10, 7)\t0.0936955852303\n",
      "  (13, 7)\t0.093446403874\n",
      "  (15, 9)\t0.065652060404\n",
      "  (17, 9)\t0.0352881367249\n",
      "  (18, 9)\t0.116324195232\n",
      "  (18, 4)\t0.226288675468\n",
      "  (18, 7)\t0.109019685025\n",
      "  (22, 2)\t0.0839867464462\n",
      "  (23, 2)\t0.0247831362042\n",
      "  (24, 9)\t0.0370420411367\n",
      "  (26, 7)\t0.127283310161\n",
      "  (28, 7)\t0.131328020617\n",
      "  (30, 0)\t0.0925840536084\n",
      "  (30, 2)\t0.0724051829482\n",
      "  (32, 3)\t0.101574677288\n",
      "  (33, 9)\t0.0201930576547\n",
      "  (33, 3)\t0.0915766698775\n",
      "  (34, 0)\t0.0463392287851\n",
      "  (34, 6)\t0.13375138285\n",
      "  (34, 1)\t0.0834658323948\n",
      "  :\t:\n",
      "  (2003, 9)\t0.0356647699378\n",
      "  (2003, 7)\t0.0414297004317\n",
      "  (2003, 1)\t0.104301128545\n",
      "  (2004, 2)\t0.0929260825\n",
      "  (2005, 3)\t0.0571237522049\n",
      "  (2007, 0)\t0.127324222877\n",
      "  (2007, 2)\t0.123418992721\n",
      "  (2010, 9)\t0.0840456306694\n",
      "  (2011, 2)\t0.0417182983245\n",
      "  (2012, 2)\t0.0537837053108\n",
      "  (2013, 9)\t0.0541242943606\n",
      "  (2013, 7)\t0.104405541704\n",
      "  (2015, 2)\t0.0731606620069\n",
      "  (2015, 6)\t0.191586250924\n",
      "  (2016, 2)\t0.0425110784733\n",
      "  (2018, 3)\t0.0670008777694\n",
      "  (2019, 4)\t0.199997798636\n",
      "  (2019, 7)\t0.0963534607641\n",
      "  (2020, 3)\t0.0646407215144\n",
      "  (2022, 2)\t0.118658071206\n",
      "  (2023, 9)\t0.0860318288725\n",
      "  (2027, 7)\t0.0738797930691\n",
      "  (2028, 3)\t0.115363996787\n",
      "  (2030, 9)\t0.0758672059456\n",
      "  (2032, 2)\t0.0429486940651\n",
      "[1 3 2 ..., 1 0 1]\n",
      "train time: 0.056s\n",
      "test time:  0.000s\n",
      "accuracy:   0.598\n",
      "dimensionality: 10\n",
      "density: 0.925000\n",
      "qqq [[  1.73663590e+01   1.12090908e-01   5.45190356e+00  -8.47641777e-01\n",
      "   -1.22463585e-01   1.22965473e+01   9.96622663e+00  -2.20450293e-01\n",
      "    1.77868359e+00  -4.04105729e-01]\n",
      " [ -1.11074755e-01   0.00000000e+00  -1.37772346e-01   2.68038335e+01\n",
      "   -1.15136325e-01  -9.73531438e-02  -5.34627783e-02  -6.07759708e-02\n",
      "    0.00000000e+00  -9.20905301e-02]\n",
      " [ -1.16614234e+00   1.27179064e-02  -5.63090810e+00  -2.93264065e+00\n",
      "    1.18968810e+01  -4.50833280e-02  -3.66183981e-02   1.54549953e+01\n",
      "    0.00000000e+00   2.44714404e+01]\n",
      " [ -3.40789198e-01  -3.17129511e-02   2.25806829e-01  -1.65998657e-01\n",
      "   -4.00501605e-02  -1.19997149e-01  -3.74080520e-02  -1.69712591e-01\n",
      "   -2.64795124e-02  -4.76379937e-01]]\n",
      "top 10 keywords per class:\n",
      ">>> 0 alt.atheism\n",
      "www [3 9 7 4 1 8 2 6 5 0]\n",
      "alt.atheism: graphics space nasa henry caltech schneider god livesey keith at...\n",
      ">>> 1 comp.graphics\n",
      "www [2 4 0 5 9 7 6 1 8 3]\n",
      "comp.graphics: god henry atheists keith space nasa livesey caltech schneider ...\n",
      ">>> 2 sci.space\n",
      "www [2 3 0 5 6 8 1 4 7 9]\n",
      "sci.space: god graphics atheists keith livesey schneider caltech henry nasa s...\n",
      ">>> 3 talk.religion.misc\n",
      "www [9 0 7 3 5 4 6 1 8 2]\n",
      "talk.religion.misc: space atheists nasa graphics keith henry livesey caltech ...\n",
      "\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.62      0.56      0.59       319\n",
      "     comp.graphics       0.48      0.95      0.64       389\n",
      "         sci.space       0.88      0.66      0.75       394\n",
      "talk.religion.misc       0.00      0.00      0.00       251\n",
      "\n",
      "       avg / total       0.54      0.60      0.54      1353\n",
      "\n",
      "confusion matrix:\n",
      "[[179 125  15   0]\n",
      " [  1 371  17   0]\n",
      " [ 10 125 259   0]\n",
      " [ 99 147   5   0]]\n",
      "\n",
      "================================================================================\n",
      "NearestCentroid (aka Rocchio classifier)\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "NearestCentroid(metric='euclidean', shrink_threshold=None)\n",
      "  (5, 0)\t0.109539901025\n",
      "  (6, 9)\t0.0508006478447\n",
      "  (8, 4)\t0.228862916069\n",
      "  (9, 3)\t0.0528959324173\n",
      "  (9, 7)\t0.051936762369\n",
      "  (10, 7)\t0.0936955852303\n",
      "  (13, 7)\t0.093446403874\n",
      "  (15, 9)\t0.065652060404\n",
      "  (17, 9)\t0.0352881367249\n",
      "  (18, 9)\t0.116324195232\n",
      "  (18, 4)\t0.226288675468\n",
      "  (18, 7)\t0.109019685025\n",
      "  (22, 2)\t0.0839867464462\n",
      "  (23, 2)\t0.0247831362042\n",
      "  (24, 9)\t0.0370420411367\n",
      "  (26, 7)\t0.127283310161\n",
      "  (28, 7)\t0.131328020617\n",
      "  (30, 0)\t0.0925840536084\n",
      "  (30, 2)\t0.0724051829482\n",
      "  (32, 3)\t0.101574677288\n",
      "  (33, 9)\t0.0201930576547\n",
      "  (33, 3)\t0.0915766698775\n",
      "  (34, 0)\t0.0463392287851\n",
      "  (34, 6)\t0.13375138285\n",
      "  (34, 1)\t0.0834658323948\n",
      "  :\t:\n",
      "  (2003, 9)\t0.0356647699378\n",
      "  (2003, 7)\t0.0414297004317\n",
      "  (2003, 1)\t0.104301128545\n",
      "  (2004, 2)\t0.0929260825\n",
      "  (2005, 3)\t0.0571237522049\n",
      "  (2007, 0)\t0.127324222877\n",
      "  (2007, 2)\t0.123418992721\n",
      "  (2010, 9)\t0.0840456306694\n",
      "  (2011, 2)\t0.0417182983245\n",
      "  (2012, 2)\t0.0537837053108\n",
      "  (2013, 9)\t0.0541242943606\n",
      "  (2013, 7)\t0.104405541704\n",
      "  (2015, 2)\t0.0731606620069\n",
      "  (2015, 6)\t0.191586250924\n",
      "  (2016, 2)\t0.0425110784733\n",
      "  (2018, 3)\t0.0670008777694\n",
      "  (2019, 4)\t0.199997798636\n",
      "  (2019, 7)\t0.0963534607641\n",
      "  (2020, 3)\t0.0646407215144\n",
      "  (2022, 2)\t0.118658071206\n",
      "  (2023, 9)\t0.0860318288725\n",
      "  (2027, 7)\t0.0738797930691\n",
      "  (2028, 3)\t0.115363996787\n",
      "  (2030, 9)\t0.0758672059456\n",
      "  (2032, 2)\t0.0429486940651\n",
      "[1 3 2 ..., 1 0 1]\n",
      "train time: 0.008s\n",
      "test time:  0.001s\n",
      "accuracy:   0.527\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.76      0.29      0.42       319\n",
      "     comp.graphics       0.99      0.35      0.52       389\n",
      "         sci.space       0.89      0.64      0.74       394\n",
      "talk.religion.misc       0.29      0.92      0.44       251\n",
      "\n",
      "       avg / total       0.77      0.53      0.55      1353\n",
      "\n",
      "confusion matrix:\n",
      "[[ 93   0  13 213]\n",
      " [  1 136  15 237]\n",
      " [ 13   2 252 127]\n",
      " [ 16   0   3 232]]\n",
      "\n",
      "================================================================================\n",
      "Naive Bayes\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "MultinomialNB(alpha=0.01, class_prior=None, fit_prior=True)\n",
      "  (5, 0)\t0.109539901025\n",
      "  (6, 9)\t0.0508006478447\n",
      "  (8, 4)\t0.228862916069\n",
      "  (9, 3)\t0.0528959324173\n",
      "  (9, 7)\t0.051936762369\n",
      "  (10, 7)\t0.0936955852303\n",
      "  (13, 7)\t0.093446403874\n",
      "  (15, 9)\t0.065652060404\n",
      "  (17, 9)\t0.0352881367249\n",
      "  (18, 9)\t0.116324195232\n",
      "  (18, 4)\t0.226288675468\n",
      "  (18, 7)\t0.109019685025\n",
      "  (22, 2)\t0.0839867464462\n",
      "  (23, 2)\t0.0247831362042\n",
      "  (24, 9)\t0.0370420411367\n",
      "  (26, 7)\t0.127283310161\n",
      "  (28, 7)\t0.131328020617\n",
      "  (30, 0)\t0.0925840536084\n",
      "  (30, 2)\t0.0724051829482\n",
      "  (32, 3)\t0.101574677288\n",
      "  (33, 9)\t0.0201930576547\n",
      "  (33, 3)\t0.0915766698775\n",
      "  (34, 0)\t0.0463392287851\n",
      "  (34, 6)\t0.13375138285\n",
      "  (34, 1)\t0.0834658323948\n",
      "  :\t:\n",
      "  (2003, 9)\t0.0356647699378\n",
      "  (2003, 7)\t0.0414297004317\n",
      "  (2003, 1)\t0.104301128545\n",
      "  (2004, 2)\t0.0929260825\n",
      "  (2005, 3)\t0.0571237522049\n",
      "  (2007, 0)\t0.127324222877\n",
      "  (2007, 2)\t0.123418992721\n",
      "  (2010, 9)\t0.0840456306694\n",
      "  (2011, 2)\t0.0417182983245\n",
      "  (2012, 2)\t0.0537837053108\n",
      "  (2013, 9)\t0.0541242943606\n",
      "  (2013, 7)\t0.104405541704\n",
      "  (2015, 2)\t0.0731606620069\n",
      "  (2015, 6)\t0.191586250924\n",
      "  (2016, 2)\t0.0425110784733\n",
      "  (2018, 3)\t0.0670008777694\n",
      "  (2019, 4)\t0.199997798636\n",
      "  (2019, 7)\t0.0963534607641\n",
      "  (2020, 3)\t0.0646407215144\n",
      "  (2022, 2)\t0.118658071206\n",
      "  (2023, 9)\t0.0860318288725\n",
      "  (2027, 7)\t0.0738797930691\n",
      "  (2028, 3)\t0.115363996787\n",
      "  (2030, 9)\t0.0758672059456\n",
      "  (2032, 2)\t0.0429486940651\n",
      "[1 3 2 ..., 1 0 1]\n",
      "train time: 0.004s\n",
      "test time:  0.000s\n",
      "accuracy:   0.487\n",
      "dimensionality: 10\n",
      "density: 1.000000\n",
      "qqq [[-1.87237883 -1.7345077  -1.70380446 -8.92141424 -7.51828426 -1.47300357\n",
      "  -2.01554028 -5.21530958 -2.20000304 -4.7831985 ]\n",
      " [-7.66506936 -3.64719421 -4.13862727 -0.19717179 -7.66506936 -5.71154772\n",
      "  -7.66506936 -2.50367123 -7.66506936 -2.99625033]\n",
      " [-8.58086424 -3.52751197 -5.08911354 -4.74612422 -1.58381538 -5.24964877\n",
      "  -8.58086424 -1.18865314 -8.58086424 -0.8206648 ]\n",
      " [-3.89062008 -4.34562889 -0.20712182 -7.14133525 -5.5960296  -2.5573432\n",
      "  -2.79811901 -7.14133525 -7.14133525 -4.6928273 ]]\n",
      "top 10 keywords per class:\n",
      ">>> 0 alt.atheism\n",
      "www [3 4 7 9 8 6 0 1 2 5]\n",
      "alt.atheism: graphics henry nasa space schneider livesey atheists caltech god...\n",
      ">>> 1 comp.graphics\n",
      "www [0 4 6 8 5 2 1 9 7 3]\n",
      "comp.graphics: atheists henry livesey schneider keith god caltech space nasa ...\n",
      ">>> 2 sci.space\n",
      "www [0 6 8 5 2 3 1 4 7 9]\n",
      "sci.space: atheists livesey schneider keith god graphics caltech henry nasa s...\n",
      ">>> 3 talk.religion.misc\n",
      "www [3 7 8 4 9 1 0 6 5 2]\n",
      "talk.religion.misc: graphics nasa schneider henry space caltech atheists live...\n",
      "\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.70      0.41      0.52       319\n",
      "     comp.graphics       0.60      0.35      0.44       389\n",
      "         sci.space       0.41      0.97      0.58       394\n",
      "talk.religion.misc       0.75      0.02      0.05       251\n",
      "\n",
      "       avg / total       0.60      0.49      0.43      1353\n",
      "\n",
      "confusion matrix:\n",
      "[[132  42 143   2]\n",
      " [  1 137 251   0]\n",
      " [  7   3 384   0]\n",
      " [ 48  45 152   6]]\n",
      "\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "BernoulliNB(alpha=0.01, binarize=0.0, class_prior=None, fit_prior=True)\n",
      "  (5, 0)\t0.109539901025\n",
      "  (6, 9)\t0.0508006478447\n",
      "  (8, 4)\t0.228862916069\n",
      "  (9, 3)\t0.0528959324173\n",
      "  (9, 7)\t0.051936762369\n",
      "  (10, 7)\t0.0936955852303\n",
      "  (13, 7)\t0.093446403874\n",
      "  (15, 9)\t0.065652060404\n",
      "  (17, 9)\t0.0352881367249\n",
      "  (18, 9)\t0.116324195232\n",
      "  (18, 4)\t0.226288675468\n",
      "  (18, 7)\t0.109019685025\n",
      "  (22, 2)\t0.0839867464462\n",
      "  (23, 2)\t0.0247831362042\n",
      "  (24, 9)\t0.0370420411367\n",
      "  (26, 7)\t0.127283310161\n",
      "  (28, 7)\t0.131328020617\n",
      "  (30, 0)\t0.0925840536084\n",
      "  (30, 2)\t0.0724051829482\n",
      "  (32, 3)\t0.101574677288\n",
      "  (33, 9)\t0.0201930576547\n",
      "  (33, 3)\t0.0915766698775\n",
      "  (34, 0)\t0.0463392287851\n",
      "  (34, 6)\t0.13375138285\n",
      "  (34, 1)\t0.0834658323948\n",
      "  :\t:\n",
      "  (2003, 9)\t0.0356647699378\n",
      "  (2003, 7)\t0.0414297004317\n",
      "  (2003, 1)\t0.104301128545\n",
      "  (2004, 2)\t0.0929260825\n",
      "  (2005, 3)\t0.0571237522049\n",
      "  (2007, 0)\t0.127324222877\n",
      "  (2007, 2)\t0.123418992721\n",
      "  (2010, 9)\t0.0840456306694\n",
      "  (2011, 2)\t0.0417182983245\n",
      "  (2012, 2)\t0.0537837053108\n",
      "  (2013, 9)\t0.0541242943606\n",
      "  (2013, 7)\t0.104405541704\n",
      "  (2015, 2)\t0.0731606620069\n",
      "  (2015, 6)\t0.191586250924\n",
      "  (2016, 2)\t0.0425110784733\n",
      "  (2018, 3)\t0.0670008777694\n",
      "  (2019, 4)\t0.199997798636\n",
      "  (2019, 7)\t0.0963534607641\n",
      "  (2020, 3)\t0.0646407215144\n",
      "  (2022, 2)\t0.118658071206\n",
      "  (2023, 9)\t0.0860318288725\n",
      "  (2027, 7)\t0.0738797930691\n",
      "  (2028, 3)\t0.115363996787\n",
      "  (2030, 9)\t0.0758672059456\n",
      "  (2032, 2)\t0.0429486940651\n",
      "[1 3 2 ..., 1 0 1]\n",
      "train time: 0.004s\n",
      "test time:  0.001s\n",
      "accuracy:   0.611\n",
      "dimensionality: 10\n",
      "density: 1.000000\n",
      "qqq [[ -1.19015266  -1.60937542  -1.02627516 -10.77899796  -6.16387744\n",
      "   -1.29095009  -2.01478845  -4.56239185  -1.67390699  -3.40061424]\n",
      " [-10.97510541  -4.57651048  -4.42259753  -1.05176622 -10.97510541\n",
      "   -5.67180051 -10.97510541  -3.27843833 -10.97510541  -3.42497007]\n",
      " [-10.99039831  -3.6120146   -4.18689305  -4.18689305  -2.10842313\n",
      "   -4.77379221 -10.99039831  -1.09187305 -10.99039831  -0.63262374]\n",
      " [ -4.543507    -5.92234791  -0.97640076 -10.53746842  -5.92234791\n",
      "   -3.29252688  -4.13887349 -10.53746842 -10.53746842  -4.83035816]]\n",
      "top 10 keywords per class:\n",
      ">>> 0 alt.atheism\n",
      "www [3 4 7 9 6 8 1 5 0 2]\n",
      "alt.atheism: graphics henry nasa space livesey schneider caltech keith atheis...\n",
      ">>> 1 comp.graphics\n",
      "www [0 4 6 8 5 1 2 9 7 3]\n",
      "comp.graphics: atheists henry livesey schneider keith caltech god space nasa ...\n",
      ">>> 2 sci.space\n",
      "www [0 6 8 5 2 3 1 4 7 9]\n",
      "sci.space: atheists livesey schneider keith god graphics caltech henry nasa s...\n",
      ">>> 3 talk.religion.misc\n",
      "www [3 7 8 1 4 9 0 6 5 2]\n",
      "talk.religion.misc: graphics nasa schneider caltech henry space atheists live...\n",
      "\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.81      0.38      0.51       319\n",
      "     comp.graphics       0.48      0.95      0.64       389\n",
      "         sci.space       0.84      0.65      0.74       394\n",
      "talk.religion.misc       0.61      0.31      0.41       251\n",
      "\n",
      "       avg / total       0.69      0.61      0.60      1353\n",
      "\n",
      "confusion matrix:\n",
      "[[120 125  25  49]\n",
      " [  1 370  18   0]\n",
      " [  9 125 258   2]\n",
      " [ 19 147   6  79]]\n",
      "\n",
      "================================================================================\n",
      "LinearSVC with L1-based feature selection\n",
      "________________________________________________________________________________\n",
      "Training: \n",
      "Pipeline(steps=[('feature_selection', LinearSVC(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
      "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
      "     multi_class='ovr', penalty='l1', random_state=None, tol=0.001,\n",
      "     verbose=0)), ('classification', LinearSVC(C=1.0, class_weight=None, dual=True, fit_intercept=True,\n",
      "     intercept_scaling=1, loss='squared_hinge', max_iter=1000,\n",
      "     multi_class='ovr', penalty='l2', random_state=None, tol=0.0001,\n",
      "     verbose=0))])\n",
      "  (5, 0)\t0.109539901025\n",
      "  (6, 9)\t0.0508006478447\n",
      "  (8, 4)\t0.228862916069\n",
      "  (9, 3)\t0.0528959324173\n",
      "  (9, 7)\t0.051936762369\n",
      "  (10, 7)\t0.0936955852303\n",
      "  (13, 7)\t0.093446403874\n",
      "  (15, 9)\t0.065652060404\n",
      "  (17, 9)\t0.0352881367249\n",
      "  (18, 9)\t0.116324195232\n",
      "  (18, 4)\t0.226288675468\n",
      "  (18, 7)\t0.109019685025\n",
      "  (22, 2)\t0.0839867464462\n",
      "  (23, 2)\t0.0247831362042\n",
      "  (24, 9)\t0.0370420411367\n",
      "  (26, 7)\t0.127283310161\n",
      "  (28, 7)\t0.131328020617\n",
      "  (30, 0)\t0.0925840536084\n",
      "  (30, 2)\t0.0724051829482\n",
      "  (32, 3)\t0.101574677288\n",
      "  (33, 9)\t0.0201930576547\n",
      "  (33, 3)\t0.0915766698775\n",
      "  (34, 0)\t0.0463392287851\n",
      "  (34, 6)\t0.13375138285\n",
      "  (34, 1)\t0.0834658323948\n",
      "  :\t:\n",
      "  (2003, 9)\t0.0356647699378\n",
      "  (2003, 7)\t0.0414297004317\n",
      "  (2003, 1)\t0.104301128545\n",
      "  (2004, 2)\t0.0929260825\n",
      "  (2005, 3)\t0.0571237522049\n",
      "  (2007, 0)\t0.127324222877\n",
      "  (2007, 2)\t0.123418992721\n",
      "  (2010, 9)\t0.0840456306694\n",
      "  (2011, 2)\t0.0417182983245\n",
      "  (2012, 2)\t0.0537837053108\n",
      "  (2013, 9)\t0.0541242943606\n",
      "  (2013, 7)\t0.104405541704\n",
      "  (2015, 2)\t0.0731606620069\n",
      "  (2015, 6)\t0.191586250924\n",
      "  (2016, 2)\t0.0425110784733\n",
      "  (2018, 3)\t0.0670008777694\n",
      "  (2019, 4)\t0.199997798636\n",
      "  (2019, 7)\t0.0963534607641\n",
      "  (2020, 3)\t0.0646407215144\n",
      "  (2022, 2)\t0.118658071206\n",
      "  (2023, 9)\t0.0860318288725\n",
      "  (2027, 7)\t0.0738797930691\n",
      "  (2028, 3)\t0.115363996787\n",
      "  (2030, 9)\t0.0758672059456\n",
      "  (2032, 2)\t0.0429486940651\n",
      "[1 3 2 ..., 1 0 1]\n",
      "train time: 0.029s\n",
      "test time:  0.002s\n",
      "accuracy:   0.602\n",
      "classification report:\n",
      "                    precision    recall  f1-score   support\n",
      "\n",
      "       alt.atheism       0.67      0.47      0.55       319\n",
      "     comp.graphics       0.48      0.96      0.64       389\n",
      "         sci.space       0.88      0.66      0.75       394\n",
      "talk.religion.misc       0.53      0.13      0.21       251\n",
      "\n",
      "       avg / total       0.65      0.60      0.57      1353\n",
      "\n",
      "confusion matrix:\n",
      "[[150 127  14  28]\n",
      " [  1 372  16   0]\n",
      " [  7 127 259   1]\n",
      " [ 65 148   5  33]]\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Author: Peter Prettenhofer <peter.prettenhofer@gmail.com>\n",
    "#         Olivier Grisel <olivier.grisel@ensta.org>\n",
    "#         Mathieu Blondel <mathieu@mblondel.org>\n",
    "#         Lars Buitinck <L.J.Buitinck@uva.nl>\n",
    "# License: BSD 3 clause\n",
    "\n",
    "from __future__ import print_function\n",
    "\n",
    "import logging\n",
    "import numpy as np\n",
    "from optparse import OptionParser\n",
    "import sys\n",
    "from time import time\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from sklearn.datasets import fetch_20newsgroups\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "from sklearn.feature_extraction.text import HashingVectorizer\n",
    "from sklearn.feature_selection import SelectKBest, chi2\n",
    "from sklearn.linear_model import RidgeClassifier\n",
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.svm import LinearSVC\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.linear_model import Perceptron\n",
    "from sklearn.linear_model import PassiveAggressiveClassifier\n",
    "from sklearn.naive_bayes import BernoulliNB, MultinomialNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.neighbors import NearestCentroid\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.utils.extmath import density\n",
    "from sklearn import metrics\n",
    "\n",
    "\n",
    "# Display progress logs on stdout\n",
    "logging.basicConfig(level=logging.INFO,\n",
    "                    format='%(asctime)s %(levelname)s %(message)s')\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "###############################################################################\n",
    "# Load some categories from the training set\n",
    "\n",
    "categories = [\n",
    "    'alt.atheism',\n",
    "    'talk.religion.misc',\n",
    "    'comp.graphics',\n",
    "    'sci.space',\n",
    "]\n",
    "\n",
    "remove = ()\n",
    "\n",
    "print(\"Loading 20 newsgroups dataset for categories:\")\n",
    "print(categories if categories else \"all\")\n",
    "\n",
    "data_train = fetch_20newsgroups(subset='train', categories=categories,\n",
    "                                shuffle=True, random_state=42,\n",
    "                                remove=remove)\n",
    "\n",
    "data_test = fetch_20newsgroups(subset='test', categories=categories,\n",
    "                               shuffle=True, random_state=42,\n",
    "                               remove=remove)\n",
    "print('data loaded')\n",
    "\n",
    "categories = data_train.target_names    # for case categories == None\n",
    "\n",
    "\n",
    "def size_mb(docs):\n",
    "    return sum(len(s.encode('utf-8')) for s in docs) / 1e6\n",
    "\n",
    "data_train_size_mb = size_mb(data_train.data)\n",
    "data_test_size_mb = size_mb(data_test.data)\n",
    "\n",
    "print(\"%d documents - %0.3fMB (training set)\" % (\n",
    "    len(data_train.data), data_train_size_mb))\n",
    "print(\"%d documents - %0.3fMB (test set)\" % (\n",
    "    len(data_test.data), data_test_size_mb))\n",
    "print(\"%d categories\" % len(categories))\n",
    "print()\n",
    "\n",
    "# split a training set and a test set\n",
    "y_train, y_test = data_train.target, data_test.target\n",
    "\n",
    "print(\"Extracting features from the training data using a sparse vectorizer\")\n",
    "t0 = time()\n",
    "vectorizer = TfidfVectorizer(sublinear_tf=True, max_df=0.5,\n",
    "                             stop_words='english')\n",
    "X_train = vectorizer.fit_transform(data_train.data)\n",
    "duration = time() - t0\n",
    "print(\"done in %fs at %0.3fMB/s\" % (duration, data_train_size_mb / duration))\n",
    "print(\"n_samples: %d, n_features: %d\" % X_train.shape)\n",
    "print()\n",
    "\n",
    "print(\"Extracting features from the test data using the same vectorizer\")\n",
    "t0 = time()\n",
    "X_test = vectorizer.transform(data_test.data)\n",
    "duration = time() - t0\n",
    "print(\"done in %fs at %0.3fMB/s\" % (duration, data_test_size_mb / duration))\n",
    "print(\"n_samples: %d, n_features: %d\" % X_test.shape)\n",
    "print()\n",
    "\n",
    "# mapping from integer feature name to original token string\n",
    "feature_names = vectorizer.get_feature_names()\n",
    "\n",
    "opts_select_chi2 = 10\n",
    "print(\"Extracting %d best features by a chi-squared test\" %\n",
    "      opts_select_chi2)\n",
    "t0 = time()\n",
    "ch2 = SelectKBest(chi2, k=opts_select_chi2)\n",
    "X_train = ch2.fit_transform(X_train, y_train)\n",
    "X_test = ch2.transform(X_test)\n",
    "if feature_names:\n",
    "    # keep selected feature names\n",
    "    feature_names = [feature_names[i] for i\n",
    "                     in ch2.get_support(indices=True)]\n",
    "print(\"done in %fs\" % (time() - t0))\n",
    "print()\n",
    "\n",
    "if feature_names:\n",
    "    feature_names = np.asarray(feature_names)\n",
    "\n",
    "\n",
    "def trim(s):\n",
    "    \"\"\"Trim string to fit on terminal (assuming 80-column display)\"\"\"\n",
    "    return s if len(s) <= 80 else s[:77] + \"...\"\n",
    "\n",
    "\n",
    "###############################################################################\n",
    "# Benchmark classifiers\n",
    "def benchmark(clf):\n",
    "    print('_' * 80)\n",
    "    print(\"Training: \")\n",
    "    print(clf)\n",
    "    t0 = time()\n",
    "    print(X_train)\n",
    "    print(y_train)\n",
    "    clf.fit(X_train, y_train)\n",
    "    train_time = time() - t0\n",
    "    print(\"train time: %0.3fs\" % train_time)\n",
    "\n",
    "    t0 = time()\n",
    "    pred = clf.predict(X_test)\n",
    "    test_time = time() - t0\n",
    "    print(\"test time:  %0.3fs\" % test_time)\n",
    "\n",
    "    score = metrics.accuracy_score(y_test, pred)\n",
    "    print(\"accuracy:   %0.3f\" % score)\n",
    "\n",
    "    if hasattr(clf, 'coef_'):\n",
    "        print(\"dimensionality: %d\" % clf.coef_.shape[1])\n",
    "        print(\"density: %f\" % density(clf.coef_))\n",
    "        \n",
    "        print('qqq',clf.coef_)\n",
    "        \n",
    "        if feature_names is not None:\n",
    "            print(\"top 10 keywords per class:\")\n",
    "            for i, category in enumerate(categories):\n",
    "                print('>>>',i, category)\n",
    "                print('www',np.argsort(clf.coef_[i]))\n",
    "                top10 = np.argsort(clf.coef_[i])[-10:]\n",
    "                print(trim(\"%s: %s\"\n",
    "                      % (category, \" \".join(feature_names[top10]))))\n",
    "        print()\n",
    "\n",
    "    print(\"classification report:\")\n",
    "    print(metrics.classification_report(y_test, pred,\n",
    "                                        target_names=categories))\n",
    "\n",
    "    print(\"confusion matrix:\")\n",
    "    print(metrics.confusion_matrix(y_test, pred))\n",
    "\n",
    "    print()\n",
    "    clf_descr = str(clf).split('(')[0]\n",
    "    return clf_descr, score, train_time, test_time\n",
    "\n",
    "\n",
    "results = []\n",
    "for clf, name in (\n",
    "        (RidgeClassifier(tol=1e-2, solver=\"lsqr\"), \"Ridge Classifier\"),\n",
    "        #(Perceptron(n_iter=50), \"Perceptron\"),\n",
    "        #(PassiveAggressiveClassifier(n_iter=50), \"Passive-Aggressive\"),\n",
    "        #(KNeighborsClassifier(n_neighbors=10), \"kNN\"),\n",
    "        #(RandomForestClassifier(n_estimators=100), \"Random forest\")\n",
    "    ):\n",
    "    print('=' * 80)\n",
    "    print(name)\n",
    "    results.append(benchmark(clf))\n",
    "\n",
    "for penalty in [\"l2\", \"l1\"]:\n",
    "    print('=' * 80)\n",
    "    print(\"%s penalty\" % penalty.upper())\n",
    "    # Train Liblinear model\n",
    "    results.append(benchmark(LinearSVC(loss='l2', penalty=penalty,\n",
    "                                            dual=False, tol=1e-3)))\n",
    "\n",
    "    # Train SGD model\n",
    "    results.append(benchmark(SGDClassifier(alpha=.0001, n_iter=50,\n",
    "                                           penalty=penalty)))\n",
    "\n",
    "# Train SGD with Elastic Net penalty\n",
    "print('=' * 80)\n",
    "print(\"Elastic-Net penalty\")\n",
    "results.append(benchmark(SGDClassifier(alpha=.0001, n_iter=50,\n",
    "                                       penalty=\"elasticnet\")))\n",
    "\n",
    "# Train NearestCentroid without threshold\n",
    "print('=' * 80)\n",
    "print(\"NearestCentroid (aka Rocchio classifier)\")\n",
    "results.append(benchmark(NearestCentroid()))\n",
    "\n",
    "# Train sparse Naive Bayes classifiers\n",
    "print('=' * 80)\n",
    "print(\"Naive Bayes\")\n",
    "results.append(benchmark(MultinomialNB(alpha=.01)))\n",
    "results.append(benchmark(BernoulliNB(alpha=.01)))\n",
    "\n",
    "print('=' * 80)\n",
    "print(\"LinearSVC with L1-based feature selection\")\n",
    "# The smaller C, the stronger the regularization.\n",
    "# The more regularization, the more sparsity.\n",
    "results.append(benchmark(Pipeline([\n",
    "  ('feature_selection', LinearSVC(penalty=\"l1\", dual=False, tol=1e-3)),\n",
    "  ('classification', LinearSVC())\n",
    "])))\n",
    "\n",
    "# make some plots\n",
    "\n",
    "indices = np.arange(len(results))\n",
    "\n",
    "results = [[x[i] for x in results] for i in range(4)]\n",
    "\n",
    "clf_names, score, training_time, test_time = results\n",
    "training_time = np.array(training_time) / np.max(training_time)\n",
    "test_time = np.array(test_time) / np.max(test_time)\n",
    "\n",
    "plt.figure(figsize=(12, 8))\n",
    "plt.title(\"Score\")\n",
    "plt.barh(indices, score, .2, label=\"score\", color='r')\n",
    "plt.barh(indices + .3, training_time, .2, label=\"training time\", color='g')\n",
    "plt.barh(indices + .6, test_time, .2, label=\"test time\", color='b')\n",
    "plt.yticks(())\n",
    "plt.legend(loc='best')\n",
    "plt.subplots_adjust(left=.25)\n",
    "plt.subplots_adjust(top=.95)\n",
    "plt.subplots_adjust(bottom=.05)\n",
    "\n",
    "for i, c in zip(indices, clf_names):\n",
    "    plt.text(-.3, i, c)\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 3, 2, ..., 1, 0, 1])"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train.target"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['alt.atheism', 'comp.graphics', 'sci.space', 'talk.religion.misc']"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train.target_names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u\"From: rych@festival.ed.ac.uk (R Hawkes)\\nSubject: 3DS: Where did all the texture rules go?\\nLines: 21\\n\\nHi,\\n\\nI've noticed that if you only save a model (with all your mapping planes\\npositioned carefully) to a .3DS file that when you reload it after restarting\\n3DS, they are given a default position and orientation.  But if you save\\nto a .PRJ file their positions/orientation are preserved.  Does anyone\\nknow why this information is not stored in the .3DS file?  Nothing is\\nexplicitly said in the manual about saving texture rules in the .PRJ file. \\nI'd like to be able to read the texture rule information, does anyone have \\nthe format for the .PRJ file?\\n\\nIs the .CEL file format available from somewhere?\\n\\nRych\\n\\n======================================================================\\nRycharde Hawkes\\t\\t\\t\\temail: rych@festival.ed.ac.uk\\nVirtual Environment Laboratory\\nDept. of Psychology\\t\\t\\tTel  : +44 31 650 3426\\nUniv. of Edinburgh\\t\\t\\tFax  : +44 31 667 0150\\n======================================================================\\n\""
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_train.data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}