analysis.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:51c711c78d336bd4e269643a4500bffc7b4e2d74e0fbe67c6ad3f2c08448331f"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import pandas as pd"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 1
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "df = pd.read_csv(\"raw_parallel_execution_extended.csv\")\n",
      "for k, v in df.iterrows():\n",
      "    if \"rxwiki\" in str(v.snippet).lower():\n",
      "        print str(v.snippet)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "participants = 200.0\n",
      "seconds_per_sentence = 20.0\n",
      "time = 45.0 * 60.0\n",
      "reduncancy = 3.0\n",
      "sentences_per_participant = time / seconds_per_sentence\n",
      "participants * sentences_per_participant / reduncancy"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 33,
       "text": [
        "9000.0"
       ]
      }
     ],
     "prompt_number": 33
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "d = df[df.search_term=='\"procter & gamble\" OR \"procter and gamble\"'][['abstract', 'lead_paragraph', 'snippet']].to_dict()\n",
      "for v in d.values():\n",
      "    for vv in v.values():\n",
      "        print vv\n",
      "    print \"\\n\""
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Kraft Foods and Procter & Gamble plan to share part \u2014 but not all \u2014 of their Super Bowl commercials before the game.\n",
        "Charmin is providing some toilet paper relief on behalf of one of Ohio's newest million-dollar lottery winners.\n",
        "If the latest week of earnings season has told investors anything, it is that strong banks and energy companies are getting stronger, while weaker banks and technology companies are far from conquering the challenges they have faced in the last few...\n",
        "The United States economy is supposed to approach normal growth this year. Has the recession taught consumers anything?\n",
        "The information technology sector, which once disdained paying dividends to shareholders, now pays more dividends than any other sector in the Standard & Poor\u2019s 500.\n",
        "Stock investors have already picked much of the low-hanging fruit in foreign markets, leaving some of the stocks overpriced, advisers say.\n",
        "A reader criticizes the Time Warner Cable spot that refers to \u201cShameless,\u201d the cable series on Showtime.\n",
        "Eric Ryan, a founder of the cleaning products maker Method, says that finding a suitor with similar values \u2014 in this case, the Belgian company Ecover \u2014 was important to the company.\n",
        "Being a celebrity does not make starting a makeup company easy, several have found out.\n",
        "Procter &amp; Gamble is cutting around 150 jobs in Spain and Portugal following the decision to integrate hygiene company Arbora &amp; Ausonia (A&amp;A) into its existing operations.\n",
        "\n",
        "\n",
        "Stuart Elliott Advertising column; some marketers, including Kraft Foods and Procter & Gamble, are deciding they will publicly share part--but not all--of their Super Bowl commercials online before official broadcast; decisions stand in contrast from trend of companies pre-releasing entire spots or longer versions online to capitalize on social media.\n",
        "nan\n",
        "nan\n",
        "Adam Davidson It's the Economy column examines what United States economy will look like in 2013, given that it is supposed to approach normal growth despite big changes in consumer behavior.\n",
        "nan\n",
        "Paul J Lim Fundamentally column notes that strategists say stock investors have already picked much of the low-hanging fruit in foreign markets, leaving some stocks overpriced; points out that risks are clearly on the rise.\n",
        "nan\n",
        "Eric Ryan, co-founder of cleaning products maker Method, says finding a suitor with similar values--in this case, Belgian company Ecover--was important to the company.\n",
        "Many celebrities who have gone from endorsing a company's cosmetics to selling their own makeup line are finding that starting their own company is a much more difficult and riskier proposition.\n",
        "nan\n",
        "\n",
        "\n",
        "Kraft Foods and Procter & Gamble plan to share part \u2014 but not all \u2014 of their Super Bowl commercials before the game.\n",
        "Charmin is providing some toilet paper relief on behalf of one of Ohio's newest million-dollar lottery winners.\n",
        "If the latest week of earnings season has told investors anything, it is that strong banks and energy companies are getting stronger, while weaker banks and technology companies are far from conquering the challenges they have faced in the last few years.\n",
        "The United States economy is supposed to approach normal growth this year. Has the recession taught consumers anything?\n",
        "The information technology sector, which once disdained paying dividends to shareholders, now pays more dividends than any other sector in the Standard & Poor\u2019s 500.\n",
        "Stock investors have already picked much of the low-hanging fruit in foreign markets, leaving some of the stocks overpriced, advisers say.\n",
        "A reader criticizes the Time Warner Cable spot that refers to \u201cShameless,\u201d the cable series on Showtime.\n",
        "Eric Ryan, a founder of the cleaning products maker Method, says that finding a suitor with similar values \u2014 in this case, the Belgian company Ecover \u2014 was important to the company.\n",
        "Being a celebrity does not make starting a makeup company easy, several have found out.\n",
        "Procter &amp; Gamble is cutting around 150 jobs in Spain and Portugal following the decision to integrate hygiene company Arbora &amp; Ausonia (A&amp;A) into its existing operations.\n",
        "\n",
        "\n"
       ]
      }
     ],
     "prompt_number": 14
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "!ls -lsa"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "total 1578064\r\n",
        "     4 drwxrwxr-x 5 versae versae      4096 Jan 13 16:37 .\r\n",
        "     4 drwxr-xr-x 9 versae versae      4096 Jan 13 16:37 ..\r\n",
        "  5328 -rw-r----- 1 versae versae   5449433 Jan  7 18:40 9k_dataset_cleaned.csv\r\n",
        "  5280 -rw-rw-r-- 1 versae versae   5402839 Jan 10 18:23 9k_dataset.csv\r\n",
        "  5280 -rw-rw-r-- 1 versae versae   5402656 Jan 13 14:19 9k_dataset_latin1.csv\r\n",
        "    12 -rw-rw-r-- 1 versae versae      9783 Jan  5 12:56 analysis.ipynb\r\n",
        "     4 drwxrwxr-x 2 versae versae      4096 Jan 13 16:37 bossa\r\n",
        "     4 drwxrwxr-x 3 versae versae      4096 Nov 29 10:22 data\r\n",
        "203020 -rw-rw-r-- 1 versae versae 207887267 Oct  2 15:27 data_and_csv_v1.tar.gz\r\n",
        "104944 -rw-rw-r-- 1 versae versae 107456670 Nov  6 06:26 data_and_csv_v2.tar.gz\r\n",
        " 14064 -rw-rw-r-- 1 versae versae  14399989 Oct  2 14:16 dataframe_v1.csv\r\n",
        "270380 -rw-rw-r-- 1 versae versae 276863369 Nov  6 06:09 dataframe_v2.csv\r\n",
        "269464 -rw-rw-r-- 1 versae versae 275930420 Nov  5 17:40 dataframe_v2_full.csv\r\n",
        " 98628 -rw-rw-r-- 1 versae versae 100993634 Nov  5 18:08 dataframe_v2_full.csv.tar.gz\r\n",
        "    40 -rw-rw-r-- 1 versae versae     37916 Dec  2 16:14 downloader.ipynb\r\n",
        "    24 -rw-rw-r-- 1 versae versae     22209 Jan  5 11:38 enterpreneurship.ipynb\r\n",
        "     4 -rw-rw-r-- 1 versae versae      1459 Aug 25 13:01 example_new_york_times.json\r\n",
        " 82076 -rw-rw-r-- 1 versae versae  84040980 Sep 10 09:32 financial_times_files.tar.gz\r\n",
        "     4 -rw-rw-r-- 1 versae versae       641 Nov 26 15:09 ipcontroller_config.py\r\n",
        "     4 drwxrwxr-x 2 versae versae      4096 Jan  5 21:27 .ipynb_checkpoints\r\n",
        "133848 -rw-rw-r-- 1 versae versae 137052360 Sep  8 12:06 new_york_times.tar.gz\r\n",
        "    32 -rw-rw-r-- 1 versae versae     30593 Nov 26 16:34 nytime_all_in_one.ipynb\r\n",
        "160104 -rw-rw-r-- 1 versae versae 163945213 Nov 28 19:03 raw_parallel_execution.csv\r\n",
        "  1368 -rw-rw-r-- 1 versae versae   1400783 Nov 30 17:15 raw_parallel_execution_extended.csv\r\n",
        "163324 -rw-rw-r-- 1 versae versae 167242809 Nov 29 00:38 raw_parallel_execution_extended_nonexact_search.csv\r\n",
        "     8 -rw-rw-r-- 1 versae versae      5987 Nov 28 19:14 search_terms.txt\r\n",
        " 20220 -rw-rw-r-- 1 versae versae  20702204 Nov  6 15:42 sentences_v2.csv\r\n",
        "    28 -rw-rw-r-- 1 versae versae     24592 Oct 31 13:30 startups.odt\r\n",
        "  6568 -rw-rw-r-- 1 versae versae   6725486 Nov  6 21:24 test_sentences_20k.csv\r\n",
        "  5552 -rw-r----- 1 versae versae   5679098 Jan  7 10:34 test_sentences_9k.csv\r\n",
        " 10716 -rw-rw-r-- 1 versae versae  10972075 Nov  6 21:24 test_sentences.csv\r\n",
        "  2932 -rw-rw-r-- 1 versae versae   3000574 Nov  7 11:26 test_sentences.docx\r\n",
        "  2800 -rw-rw-r-- 1 versae versae   2864895 Nov  7 11:23 test_sentences.odt\r\n",
        "  7476 -rw-rw-r-- 1 versae versae   7653975 Nov  7 11:23 test_sentences.pdf\r\n",
        "  4520 -rw-rw-r-- 1 versae versae   4628251 Nov  6 21:32 test_sentences.xlsx\r\n"
       ]
      }
     ],
     "prompt_number": 2
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "import ftfy\n",
      "sen = pd.read_csv(\"9k_dataset_latin1.csv\", encoding=\"latin1\", index_col=\"sentence_id\")\n",
      "sen.head(1)"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "html": [
        "<div style=\"max-height:1000px;max-width:1500px;overflow:auto;\">\n",
        "<table border=\"1\" class=\"dataframe\">\n",
        "  <thead>\n",
        "    <tr style=\"text-align: right;\">\n",
        "      <th></th>\n",
        "      <th>appears_in_noun_phrases</th>\n",
        "      <th>appears_in_sentence</th>\n",
        "      <th>is_company</th>\n",
        "      <th>media</th>\n",
        "      <th>noun_phrases</th>\n",
        "      <th>pub_date</th>\n",
        "      <th>search_words</th>\n",
        "      <th>sentence</th>\n",
        "      <th>text</th>\n",
        "      <th>url</th>\n",
        "    </tr>\n",
        "    <tr>\n",
        "      <th>sentence_id</th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "      <th></th>\n",
        "    </tr>\n",
        "  </thead>\n",
        "  <tbody>\n",
        "    <tr>\n",
        "      <th>0</th>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> 0</td>\n",
        "      <td> nyt</td>\n",
        "      <td> [joel towers, executive dean, parsons, chance ...</td>\n",
        "      <td> 2013-05-05T00:00:00Z</td>\n",
        "      <td> executive</td>\n",
        "      <td> Joel Towers, the executive dean of Parsons, ha...</td>\n",
        "      <td> Joel Towers, the executive dean of Parsons, ha...</td>\n",
        "      <td> http://www.nytimes.com/2013/05/05/education/in...</td>\n",
        "    </tr>\n",
        "  </tbody>\n",
        "</table>\n",
        "</div>"
       ],
       "metadata": {},
       "output_type": "pyout",
       "prompt_number": 173,
       "text": [
        "             appears_in_noun_phrases  appears_in_sentence  is_company media  \\\n",
        "sentence_id                                                                   \n",
        "0                                  0                    0           0   nyt   \n",
        "\n",
        "                                                  noun_phrases  \\\n",
        "sentence_id                                                      \n",
        "0            [joel towers, executive dean, parsons, chance ...   \n",
        "\n",
        "                         pub_date search_words  \\\n",
        "sentence_id                                      \n",
        "0            2013-05-05T00:00:00Z    executive   \n",
        "\n",
        "                                                      sentence  \\\n",
        "sentence_id                                                      \n",
        "0            Joel Towers, the executive dean of Parsons, ha...   \n",
        "\n",
        "                                                          text  \\\n",
        "sentence_id                                                      \n",
        "0            Joel Towers, the executive dean of Parsons, ha...   \n",
        "\n",
        "                                                           url  \n",
        "sentence_id                                                     \n",
        "0            http://www.nytimes.com/2013/05/05/education/in...  "
       ]
      }
     ],
     "prompt_number": 173
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "test = sen[sen.sentence.str.contains(u\"Nicola Horlick\")].sentence.to_dict().values()[0]\n",
      "print test"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Nicola Horlick, a fund manager, also raised \u00c2\u00a3150,000 in July through Seedrs in less than a day.\n"
       ]
      }
     ],
     "prompt_number": 174
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sen.sentence = sen.sentence.apply(lambda x: ftfy.fix_text(x))\n",
      "sen.text = sen.text.apply(lambda x: ftfy.fix_text(x))\n",
      "sen.noun_phrases = sen.noun_phrases.apply(lambda x: ftfy.fix_text(x))"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 175
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "test = sen[sen.sentence.str.contains(u\"Nicola Horlick\")].sentence.to_dict().values()[0]\n",
      "print test"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "Nicola Horlick, a fund manager, also raised \u00a3150,000 in July through Seedrs in less than a day.\n"
       ]
      }
     ],
     "prompt_number": 176
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "sen.to_csv(\"9k_dataset_latin1.csv\", encoding=\"utf8\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [],
     "prompt_number": 179
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "for sentence_id, sentence in sen.sentence.iteritems():\n",
      "    try:\n",
      "        ftfy.fix_text(sentence).decode(\"latin1\")\n",
      "    except Exception as e:\n",
      "        print ftfy.fix_text(sentence)\n",
      "        raise e"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "In recent months, fund manager Nicola Horlick has raised \u00a3150,000 to fund a business backing films through Seedrs and Kevin McCloud, presenter of Channel 4's Grand Designs, raised \u00a31.5m through Crowdcube to build customised homes.\n"
       ]
      },
      {
       "ename": "UnicodeEncodeError",
       "evalue": "'ascii' codec can't encode character u'\\xa3' in position 57: ordinal not in range(128)",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mUnicodeEncodeError\u001b[0m                        Traceback (most recent call last)",
        "\u001b[1;32m<ipython-input-178-f920434cd062>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0mException\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      5\u001b[0m         \u001b[1;32mprint\u001b[0m \u001b[0mftfy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfix_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msentence\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 6\u001b[1;33m         \u001b[1;32mraise\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m",
        "\u001b[1;31mUnicodeEncodeError\u001b[0m: 'ascii' codec can't encode character u'\\xa3' in position 57: ordinal not in range(128)"
       ]
      }
     ],
     "prompt_number": 178
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "print \"\u00a3\".decode(\"latin1\")"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "\u00c2\u00a3\n"
       ]
      }
     ],
     "prompt_number": 164
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "f = codecs.open(\"9k_dataset_latin1.csv\", \"r\", encoding=\"latin1\")\n",
      "o = codecs.open(\"/tmp/9k_dataset_latin1.csv\", \"w\", encoding=\"latin1\")\n",
      "for line in f.readlines():\n",
      "    print line\n",
      "    o.write(ftfy.fix_text(line))\n",
      "f.close()\n",
      "o.close()"
     ],
     "language": "python",
     "metadata": {},
     "outputs": [
      {
       "output_type": "stream",
       "stream": "stdout",
       "text": [
        "sentence_id,appears_in_noun_phrases,appears_in_sentence,is_company,media,noun_phrases,pub_date,search_words,sentence,text,url\n",
        "\n",
        "0,0,0,0,nyt,\"[joel towers, executive dean, parsons, chance 'to, negative examples.', mr. galliano]\",2013-05-05T00:00:00Z,executive,\"Joel Towers, the executive dean of Parsons, has called the class a chance 'to learn from positive and negative examples.' Mr. Galliano is volunteering his time.\",\"Joel Towers, the executive dean of Parsons, has called the class a chance 'to learn from positive and negative examples.' Mr. Galliano is volunteering his time.\",http://www.nytimes.com/2013/05/05/education/in-disgrace-yet-in-demand-as-college-teachers.html\n",
        "\n",
        "1,1,1,1,nyt,\"[early advertisers, adidas, ben, jerry's, burberry, general electric, levi's, lexus, macy's, michael kors, paypal, starwood]\",2013-10-24T23:36:56Z,general electric,\"The company said early advertisers include Adidas, Ben & Jerry's, Burberry, General Electric, Levi's, Lexus, Macy's, Michael Kors, PayPal and Starwood.\",\"The company said early advertisers include Adidas, Ben & Jerry's, Burberry, General Electric, Levi's, Lexus, Macy's, Michael Kors, PayPal and Starwood.\",http://bits.blogs.nytimes.com/2013/10/24/instagram-will-begin-showing-ads-in-the-photo-stream/\n",
        "\n",
        "2,0,0,0,nyt,\"[gap, corte madera, calif. mr. lang, copy manager, shutterstock, stock image, video website, manhattan]\",2013-11-10T00:00:00Z,manager,\"Mr. Lang, 33, is the copy manager for Shutterstock, a stock image and video website in Manhattan.\",\"Her mother was a sales associate at a Gap store in Corte Madera, Calif.\n",
        "\n",
        "\n",
        "\n",
        "Mr. Lang, 33, is the copy manager for Shutterstock, a stock image and video website in Manhattan.\",http://www.nytimes.com/2013/11/10/fashion/weddings/sydney-applegate-daryl-lang.html\n",
        "\n",
        "3,0,0,0,ft,\"[big alternative asset managers, different strategies]\",2013-11-06T00:00:00Z,manager,The varying performance of the big alternative asset managers comes as the groups adopt different strategies.,The varying performance of the big alternative asset managers comes as the groups adopt different strategies.,\"http://www.ft.com/cms/s/c44f8c52-4704-11e3-9c1b-00144feabdc0,s01=1.html\"\n",
        "\n",
        "4,1,1,1,nyt,\"[lists, microsoft, windows, windows]\",2013-03-08T07:00:48Z,microsoft,\"Those 'Jump Lists,' as Microsoft calls them in Windows 7 and Windows 8, are intended to be a convenience since they let you 'jump' right back into whatever you were recently working on without having to hunt around the computer for the files.\",\"Those 'Jump Lists,' as Microsoft calls them in Windows 7 and Windows 8, are intended to be a convenience since they let you 'jump' right back into whatever you were recently working on without having to hunt around the computer for the files.\",http://gadgetwise.blogs.nytimes.com/2013/03/08/qa-dumping-the-windows-jump-list/\n",
        "\n",
        "5,0,0,0,nyt,\"[incredible treasure trove, facebook's, chief executive, mark zuckerberg, september]\",2013-01-15T00:00:00Z,executive,\"Even a tease about search from Facebook's chief executive, Mark Zuckerberg, last September sent the shares up.\",\"'Facebook has this incredible treasure trove of unstructured data on the site.'\n",
        "\n",
        "\n",
        "\n",
        "Even a tease about search from Facebook's chief executive, Mark Zuckerberg, last September sent the shares up.\",http://www.nytimes.com/2013/01/15/technology/fortunes-of-facebook-may-hinge-on-searches.html\n",
        "\n",
        "6,1,1,1,nyt,\"[bride's father, san francisco, sales division, microsoft]\",2013-06-02T00:00:00Z,microsoft,The bride's father works in San Francisco for the sales division of Microsoft.,The bride's father works in San Francisco for the sales division of Microsoft.,http://www.nytimes.com/2013/06/02/fashion/weddings/brynn-jinnett-and-lowell-putnam-weddings.html\n",
        "\n",
        "7,1,1,1,nyt,\"[technology companies, google, apple, television networks]\",2013-08-12T00:00:00Z,apple,\"Such 'acqui-hiring,' in which a start-up is bought primarily for its talent, is normally associated with technology companies like Google and Apple, not television networks like NBC.\",\"Such 'acqui-hiring,' in which a start-up is bought primarily for its talent, is normally associated with technology companies like Google and Apple, not television networks like NBC.\",http://www.nytimes.com/2013/08/12/business/media/nbc-buying-web-service-to-stream-phone-video.html\n",
        "\n",
        "9,0,0,0,ft,\"[recent months, fund manager, nicola horlick, seedrs, kevin mccloud, grand designs, crowdcube]\",2013-09-27T00:00:00Z,manager,\"In recent months, fund manager Nicola Horlick has raised \u00c2\u00a3150,000 to fund a business backing films through Seedrs and Kevin McCloud, presenter of Channel 4's Grand Designs, raised \u00c2\u00a31.5m through Crowdcube to build customised homes.\",\"In recent months, fund manager Nicola Horlick has raised \u00c2\u00a3150,000 to fund a business backing films through Seedrs and Kevin McCloud, presenter of Channel 4's Grand Designs, raised \u00c2\u00a31.5m through Crowdcube to build customised homes.\",\"http://www.ft.com/cms/s/5589ee08-210b-11e3-a92a-00144feab7de,s01=1.html\"\n",
        "\n",
        "10,1,1,1,nyt,\"[motors, ford, chrysler, obama, trans-pacific partnership, japan, currency manipulation, american cars]\",2013-07-24T11:15:47Z,general motors,\"More than 80,000 General Motors, Ford and Chrysler employees have signed a petition opposing the Obama administration's proposed Trans-Pacific Partnership unless Japan deals with currency manipulation and moves toward importing more American cars into the Japanese market.\",\"More than 80,000 General Motors, Ford and Chrysler employees have signed a petition opposing the Obama administration's proposed Trans-Pacific Partnership unless Japan deals with currency manipulation and moves toward importing more American cars into the Japanese market.\",http://wheels.blogs.nytimes.com/2013/07/24/wheelies-the-dueling-fords-edition/\n",
        "\n",
        "11,0,0,0,nyt,\"[granovskaia, chelsea, executive board]\",2013-08-26T00:00:00Z,executive,Granovskaia has been around Chelsea for three years and joined the executive board this summer.,Granovskaia has been around Chelsea for three years and joined the executive board this summer.,http://www.nytimes.com/2013/08/26/sports/soccer/a-shuffle-of-players-then-a-reshuffle.html\n",
        "\n",
        "12,0,0,0,nyt,\"[ralph reed, faith, coalition, executive director, coalition, phone interview, republican, large block, loyal voters, same-sex marriage]\",2013-03-20T23:02:00Z,founder,\"Ralph Reed, founder of the Faith and Freedom Coalition and former executive director of the Christian Coalition, contended in a phone interview that the Republican Party risked alienating a large block of loyal voters if it moved to the left on same-sex marriage.\",\"Ralph Reed, founder of the Faith and Freedom Coalition and former executive director of the Christian Coalition, contended in a phone interview that the Republican Party risked alienating a large block of loyal voters if it moved to the left on same-sex marriage.\",http://opinionator.blogs.nytimes.com/2013/03/20/the-republican-autopsy-report/\n",
        "\n",
        "14,1,1,1,ft,\"[eu, european champions, google, apple, nsa]\",2013-10-24T00:00:00Z,apple,Some EU officials have urged European champions in the field of cloud computing to compete with Google and Apple so servers would be outside the US and could more easily avoid the NSA's snooping.,Some EU officials have urged European champions in the field of cloud computing to compete with Google and Apple so servers would be outside the US and could more easily avoid the NSA's snooping.,\"http://www.ft.com/cms/s/0bc3041a-3c92-11e3-a8c4-00144feab7de,s01=1.html\"\n",
        "\n",
        "15,1,1,1,nyt,\"[kazakhstan, china, important energy investments, kashagan, oil field, caspian, vast project, international consortium, kazmunaigaz, kazakhstan's, energy company, exxon mobil, dutch shell, italian company, agip]\",2013-09-08T00:00:00Z,exxon,\"In Kazakhstan, China has made some important energy investments, including the purchase this year of an 8.4 percent stake in the Kashagan oil field in the Caspian Sea, a vast project being developed by an international consortium including KazMunaiGaz, Kazakhstan's state-owned energy company; Exxon Mobil; Royal Dutch Shell; and the Italian company Agip.\",\"In Kazakhstan, China has made some important energy investments, including the purchase this year of an 8.4 percent stake in the Kashagan oil field in the Caspian Sea, a vast project being developed by an international consortium including KazMunaiGaz, Kazakhstan's state-owned energy company; Exxon Mobil; Royal Dutch Shell; and the Italian company Agip.\",http://www.nytimes.com/2013/09/08/world/asia/china-looks-west-as-it-strengthens-regional-ties.html\n",
        "\n",
        "16,1,1,1,ft,\"[chevron, shell, ukraine]\",2013-12-16T00:00:00Z,chevron,The Chevron and Shell deals are seen as vital for Ukraine if it is to reduce its dependency on imports.,The Chevron and Shell deals are seen as vital for Ukraine if it is to reduce its dependency on imports.,\"http://www.ft.com/cms/s/064a3926-5dc7-11e3-95bd-00144feabdc0,s01=1.html\"\n",
        "\n",
        "17,0,0,0,nyt,\"[gupta, mckinsey, bill gates, henry kravis, henry m. paulson jr., goldman's, chief executive, indonesia, komodo]\",2013-05-17T14:14:48Z,executive,\"'While Gupta departed McKinsey with a fortune, he was now mingling with a crowd that included Bill Gates, Henry Kravis and Henry M. Paulson Jr., then Goldman's chief executive, with whom he traveled to Indonesia to see the Komodo dragons.\",\"'While Gupta departed McKinsey with a fortune, he was now mingling with a crowd that included Bill Gates, Henry Kravis and Henry M. Paulson Jr., then Goldman's chief executive, with whom he traveled to Indonesia to see the Komodo dragons.\",http://dealbook.nytimes.com/2013/05/17/rajat-guptas-lust-for-zeros/\n",
        "\n",
        "18,1,1,1,nyt,\"[caterpillar, disappointing results, europe, china]\",2013-04-28T00:00:00Z,caterpillar,\"Already, some prominent companies, including I.B.M. and Caterpillar, have reported disappointing results, a reflection of waning demand not only in the United States but in recessionary Europe and in China, where growth has been below expectations. \",\"and Caterpillar, have reported disappointing results, a reflection of waning demand not only in the United States but in recessionary Europe and in China, where growth has been below expectations.\",http://www.nytimes.com/2013/04/28/opinion/sunday/the-economy-is-heading-the-wrong-way.html\n",
        "\n",
        "19,0,0,0,nyt,\"[enigmatic entrepreneur, softbank's masayoshi]\",2013-04-15T16:02:37Z,entrepreneur,\"But beating the offer from another enigmatic entrepreneur, SoftBank's Masayoshi Son, needed a stretch.\",\"But beating the offer from another enigmatic entrepreneur, SoftBank's Masayoshi Son, needed a stretch.\",http://dealbook.nytimes.com/2013/04/15/the-vague-promises-of-dishs-bid-for-sprint/\n",
        "\n",
        "20,1,1,1,nyt,\"[struggling, adjust, mexico iz\u00c3\u00bacar de matamoros, mexico, jeffrey isidoro, fifth-grade classroom, mexico, designer glasses, nike, nike]\",2013-01-01T00:00:00Z,nike,\"IZ\u00c3\u009aCAR DE MATAMOROS, Mexico \u00e2\u0080\u0094 Jeffrey Isidoro sat near the door of his fifth-grade classroom here in central Mexico, staring outside through designer glasses that, like his Nike sneakers and Nike backpack, signaled a life lived almost entirely in the United States.\",\"American Children, Now Struggling to Adjust to Life in Mexico\n",
        "\n"
       ]
      },
      {
       "ename": "UnicodeEncodeError",
       "evalue": "'latin-1' codec can't encode character u'\\u2014' in position 205: ordinal not in range(256)",
       "output_type": "pyerr",
       "traceback": [
        "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m\n\u001b[1;31mUnicodeEncodeError\u001b[0m                        Traceback (most recent call last)",
        "\u001b[1;32m<ipython-input-185-4af42f5d3312>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      3\u001b[0m \u001b[1;32mfor\u001b[0m \u001b[0mline\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mreadlines\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      4\u001b[0m     \u001b[1;32mprint\u001b[0m \u001b[0mline\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m----> 5\u001b[1;33m     \u001b[0mo\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mftfy\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mfix_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mline\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m      6\u001b[0m \u001b[0mf\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m      7\u001b[0m \u001b[0mo\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mclose\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/home/versae/.venvs/enterpreneurship/lib/python2.7/codecs.pyc\u001b[0m in \u001b[0;36mwrite\u001b[1;34m(self, data)\u001b[0m\n\u001b[0;32m    686\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    687\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 688\u001b[1;33m         \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwriter\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    689\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    690\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mwritelines\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;32m/home/versae/.venvs/enterpreneurship/lib/python2.7/codecs.pyc\u001b[0m in \u001b[0;36mwrite\u001b[1;34m(self, object)\u001b[0m\n\u001b[0;32m    349\u001b[0m         \"\"\" Writes the object's contents encoded to self.stream.\n\u001b[0;32m    350\u001b[0m         \"\"\"\n\u001b[1;32m--> 351\u001b[1;33m         \u001b[0mdata\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mconsumed\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mobject\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0merrors\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    352\u001b[0m         \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mstream\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrite\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    353\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
        "\u001b[1;31mUnicodeEncodeError\u001b[0m: 'latin-1' codec can't encode character u'\\u2014' in position 205: ordinal not in range(256)"
       ]
      }
     ],
     "prompt_number": 185
    }
   ],
   "metadata": {}
  }
 ]
}