nytime_all_in_one.ipynb

{
 "metadata": {
  "name": "",
  "signature": "sha256:901400e354a598595dcef6d83ee8763f652c0572b74e9fad4f908f9806d4804b"
 },
 "nbformat": 3,
 "nbformat_minor": 0,
 "worksheets": [
  {
   "cells": [
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#########\n",
      "#\n",
      "# This cell contains all the key words that we need to search\n",
      "#\n",
      "#########\n",
      "\n",
      "tag_lists = [['entrepreneur'], ['startup'], ['new', 'venture'], ['manager'], ['executive'], ['founder'], ['wal-mart'], ['exxon'],\n",
      "             ['chevron'], ['berkshire', 'hathaway'], ['apple'], ['samsung'], ['phillips', '66'], ['general', 'motors'], ['ford'],\n",
      "             ['valero'], ['exelon', 'corporation'], ['dow', 'chemical'], ['conocophillips'], ['morgan', 'chase'],\n",
      "             ['world', 'fuel', 'services'], ['illinois', 'tool', 'works'], ['boeing'], ['intel', 'corporation'], ['pepsico'],\n",
      "             ['tjx', 'companies'], ['costco', 'wholesale'], ['metlife'], ['blackrock'], ['at&t'], ['johnson','&','johnson'],\n",
      "             ['abbott', 'laboratories'], ['cvs', 'caremark'], ['visa'], ['cisco', 'systems'], ['walt', 'disney', 'company'],\n",
      "             ['rock', 'tenn', 'company'], ['halliburton', 'company'], ['alcoa'], ['fluor', 'corporation'],\n",
      "             ['general', 'electric'], ['cbre', 'group'], ['energy', 'transfer', 'equity'], ['express', 'scripts', 'holding'],\n",
      "             ['hca', 'holdings'], ['nrg', 'energy'], ['aramark', 'holdings', 'corporation'], ['honeywell', 'international'],\n",
      "             ['nike'], ['amazon.com'], ['procter', '&', 'gamble'], ['unitedhealth', 'group'], ['microsoft'],\n",
      "             ['Google'], ['Salesforce'], ['SAS'], ['Quicken'], ['Intuit'],\n",
      "             ['international', 'bussiness', 'machines'], ['las', 'vegas', 'sands'], ['autonation'], ['3m', 'company'],\n",
      "             ['danaher', 'corporation'], ['new', 'york', 'life', 'insurance', 'company'], ['mcdonalds', 'corporation'],\n",
      "             ['caterpillar'], ['ryder', 'systems'], ['d.r.', 'horton'], ['stanley', 'black', '&', 'decker'],\n",
      "             ['archer', 'daniels', 'midland'], ['coca-cola', 'company'], ['united', 'continental', 'holdings'],\n",
      "             ['ingram', 'micro'], ['chs'], ['international', 'paper', 'company'], ['mckesson'],\n",
      "             ['phillip', 'morris', 'international'], ['state', 'farm', 'insurance'], ['owens', 'corning'],\n",
      "             ['waste', 'management'], ['c.h.', 'robinson', 'worldwide'], ['harley-davidson'], ['manpower', 'group'],\n",
      "             ['union', 'pacific'], ['r.r', 'donnelley', '&', 'sons', 'company'], ['graham', 'holdings', 'company'],\n",
      "             ['emc', 'corporation'], ['mattel'], ['united', 'parcel', 'service'], ['omnicom', 'group'],\n",
      "             ['mckinsey', '&', 'company'], ['pricewaterhousecoopers'], ['intuitive', 'research', 'and', 'tech'],\n",
      "             ['international', 'bussiness', 'machines'], ['formerly', 'BrightTag'],\n",
      "             ['aka', 'Wright', 'Foods'], ['turospree.com'], ['nirvanix.com'], ['getpostrocket.com'], ['votertide.com'],\n",
      "             ['gamelayers.com'], ['unifyo.com'], ['lookery.com'], ['canv.as'], ['blurtt.com'], ['manilla.com'], ['admaze.ly.com'],\n",
      "             ['springpad.com'], ['ex.fm'], ['sambamobile.com'], ['qik.com'], ['inbloom.org'], ['intonow.com'], ['getfindit.com'],\n",
      "             ['inqmobile.com'], ['outboxmail.com'], ['argylesocial.com'], ['iamexec.com'], ['stippleit.com']\n",
      "             ['500px', 'Inc.'], ['Wajam'], ['Keek', 'Inc'], ['GuySpy'], ['Woozworld'], ['TinEye', 'by', 'Idee', 'Inc.'],\n",
      "             ['Jigsy'], ['Voices.com'], ['Snapsort'], ['EmpireAvenue'], ['Frank', 'and', 'Oak'], ['LoginRadius'],\n",
      "             ['Chango', 'Inc'], ['Pure', 'Energies', 'Group', 'Inc.'], ['Buytopia.ca'], ['Smoke', 'NV', 'Inc.'],\n",
      "             ['Just', 'Quality', 'International', 'Inc.'], ['Sterling', 'Cross', 'Defense', 'Systems', 'Corp.'],\n",
      "             ['Magnet', 'Forensics', 'Inc.'], ['Greenlife', 'Water', 'Corp.'], ['Secure', 'Sense', 'Solutions'],\n",
      "             ['Sandstorm', 'Gold', 'Ltd'], ['ScribbleLive', 'Technologies', 'Inc.)'],\n",
      "             ['TDot', 'Performance', 'Ltd.'], ['Redwood', 'Strategic', 'Inc.'], ['Emcision', 'International', 'Inc.'],\n",
      "             ['Henkaa', 'Inc.'], ['Whirlscape', 'Inc.'], ['Bionym'], ['Thalmic', 'Labs'], ['Recon', 'Instruments'],\n",
      "             ['Om', 'Malik', 'Technology'], ['Affino'], ['AnalyzeRE'], ['Atomic', 'Reach'], ['Cloud', 'Dynamics'],\n",
      "             ['Exo', 'U'], ['Fabric', 'Engine'], ['Finmaven'], ['Granify'], ['InfraDog'], ['Introhive'],\n",
      "             ['Invici', 'Technologies'], ['Login', 'Radius'], ['mnubo'], ['Obero', 'Solutions'], ['PeopleInsight'],\n",
      "             ['Procima', 'Experts'], ['reelyActive'], ['RtTech', 'Software'], ['Seamless', 'Mobile', 'Health'],\n",
      "             ['Slyce'], ['Smart', 'Employee', 'Benefits'], ['SweetIQ'], ['Thalmic', 'Labs'], ['Vena', 'Solutions'],\n",
      "             ['WestonExprerssions'], ['Linkett'], ['360pi'], ['AcademixDirect,', 'Inc.'], ['Adconion', 'Direct'],\n",
      "             ['Adesto', 'Technologies', 'Corporation'], ['Alfresco'], ['Aitierre', 'Corporation'], ['Altitude', 'Digital'],\n",
      "             ['Anametrix'], ['Aphios', 'Corporation'], ['AppDirect'], ['Applied', 'Predictive', 'Technologies'],\n",
      "             ['appsFreedom'], ['Apttus'], ['Argos', 'Therapeutics,', 'Inc.'], ['Aseptia/Wright', 'Foods'],\n",
      "             ['Atlantis', 'Computing'], ['Avere', 'Systems'], ['Become,', 'Inc.'], ['BeyondTrust'],\n",
      "             ['Bluebeam', 'Software,', 'Inc.'], ['Signal'], ['Cinsay,', 'Inc.'],\n",
      "             ['Cognitive', 'Networks'], ['Compass-EOS'], ['Credit', 'Sesame'], ['Curemark', 'LLC'],\n",
      "             ['Daintree', 'Networks'], ['Data', 'Dynamics', 'Inc.'], ['Defense.Net'], ['DigiCert,', 'Inc.'],\n",
      "             ['DiscGenics'], ['Duda'], ['Electric', 'Cloud'], ['Flybits'], ['Fruition', 'Partners'], ['GENBAND'],\n",
      "             ['globalVCard,', 'a', 'CSI', 'Enterprise', 'company'], ['GuideSpark'], ['HasOffers'],\n",
      "             ['Health', 'Catalyst'], ['Hortonworks'], ['iBuildApp'], ['iCIMS,', 'Inc.'], ['ImaginAb,', 'Inc.'],\n",
      "             ['iPipeline'], ['JAB', 'Broadband'], ['Kahuna,', 'Inc.'], ['Kareo'],\n",
      "             ['Klocwork', 'Inc.,', 'a', 'Rogue', 'Wave', 'company'], ['Lancope'], ['Load', 'DynamiX'],\n",
      "             ['LocalResponse'], ['Looker'], ['Malcovery', 'Security'], ['Maxta,', 'Inc.'],\n",
      "             ['MicroGREEN', 'Polymers,', 'Inc.'], ['Modernizing', 'Medicine,', 'Inc.'], ['myoscience'], ['NearWoo'],\n",
      "             ['Nomis', 'Solutions,', 'Inc.'], ['OxiCool,', 'Inc.'], ['Packsize', 'LLC'], ['Panaya'], ['Paxata'],\n",
      "             ['Perfecto', 'Mobile'], ['Perseus', 'Telecom'], ['Phunware'], ['Pluribus', 'Networks'],\n",
      "             ['PowerPlan', 'Inc.'], ['Quanterix'], ['RealMatch'], ['Red', 'Lambda', 'Inc.'], ['Rise', 'Interactive'],\n",
      "             ['RootMetrics'], ['RxWiki', 'Inc.'], ['Scaled', 'Agile,', 'Inc.'], ['Scribe', 'Software'],\n",
      "             ['Seamless', 'Medical', 'Systems', 'Inc.'], ['SecureAuth'], ['SeeChange', 'Health'], ['Shape', 'Security'],\n",
      "             ['SkyCross'], ['Skyhigh', 'Networks'], ['Skyonic', 'Corporation'], ['SmartZip', 'Analytics,', 'Inc.'],\n",
      "             ['Solace', 'Systems'], ['Solavei'], ['SpiderCloud', 'Wireless', 'Inc.'], ['Splice', 'Machine'],\n",
      "             ['SpotXchange,', 'Inc.'], ['SPR', 'Therapeutics,', 'LLC'], ['Stoke,', 'Inc.'], ['StreetLight', 'Data'],\n",
      "             ['Sureline', 'Systems,', 'Inc.'], ['T3', 'Motion,', 'Inc.'], ['TechLeads', 'Online'],\n",
      "             ['Tenable', 'Network', 'Security,', 'Inc.'], ['ThreatTrack', 'Security'], ['Trueffect'],\n",
      "             ['TRX', 'Systems,', 'Inc.'], ['Ubiquity', 'Global', 'Services', 'Inc.'], ['Welltok'],\n",
      "             ['Xactly', 'Corporation'], ['XCOR', 'Aerospace,', 'Inc.'], ['Yashi'], ['Zeta', 'Interactive'], ['Fuh'],\n",
      "             ['Quest', 'Nutrition'], ['Reliant', 'Asset', 'Management'], ['Superfish'], ['Acacia', 'Communications'],\n",
      "             ['Provider', 'Power'], ['Crescendo', 'Bioscience'], ['Plexus', 'Worldwide'], ['Vacasa'], ['Go', 'Energies'],\n",
      "             ['Minute', 'Key'], ['Sainstore'], ['The', 'HCI', 'Group'], ['Dynamic', 'Dental', 'Partners', 'Group'],\n",
      "             ['Aseptia'], ['Asentra', 'Health'], ['American', 'Solar', 'Direct'],\n",
      "             ['Prescient', 'Edge'], ['BlueSquare', 'Resolutions'], ['BES', 'Design', 'Build'], ['Simpler', 'Trading'],\n",
      "             ['sweetFrog', 'Premium', 'Frozen', 'Yogurt'], ['Base', 'Commerce'], ['CPSG', 'Partners'], ['MedHOK'],\n",
      "             ['Showroom', 'Logic'], ['WeVeel'], ['PackIt'], ['American', 'Retirement', 'Advisors'],\n",
      "             ['Highpoint', 'Global', ''], ['AdKarma'], ['Axtria'], ['Dolls', 'Kill'], ['Centric', 'Digital'],\n",
      "             ['Retail', 'Capital'], ['Restore', 'Health'], ['OpenRoad', 'Lending'], ['Pontchartrain', 'Partners'],\n",
      "             ['Team', 'Extreme', 'Marketing', 'International'], ['Choice', 'Energy'], ['ReviMedia'],\n",
      "             ['SmartZip', 'Analytics'], ['Thompson', 'Gray'], ['MCSG', 'Technologies'],\n",
      "             ['Innovative', 'Surveillance', 'Solutions'], ['Nordic'], ['Michigan', 'Realty', 'Solutions'],\n",
      "             ['Ohio', 'Investments'], ['Adore', 'Me'], ['IQ', 'Formulations'], ['Alcohol', 'by', 'Volume'],\n",
      "             ['H.Bloom'], ['Tealium'], ['Boostability'], ['Solve', 'Media'], ['Heartland', 'Energy', 'Partners'],\n",
      "             ['Bridger'], ['Jeunesse', 'Global'], ['Touchsuite'], ['Accleration', 'Partners'],\n",
      "             ['Moore', 'Heating', '&', 'Air', 'Conditioning'], ['Optimatic', 'Media'], ['EPI', 'Engineering'],\n",
      "             ['Simplicity', 'Laser'], ['Cinium', 'Financial', 'Services'], ['TNH', 'Advanced', 'Specialty', 'Pharmacy'],\n",
      "             ['Human', 'Movement', 'Management'], ['Allegheny', 'Science', '&', 'Technology'], ['Hudl'], ['GuideSpark'],\n",
      "             ['Optimus', 'Technology'], ['Luxe', 'Royale'], ['SoluComp'], ['RPM', 'Freight', 'Systems'],\n",
      "             ['Motivational', 'Press'], ['99dresses'], ['Dinnr'], ['Seesmic', 'Video'], ['Treehouse', 'Logic'],\n",
      "             ['Backchat'], ['Patient', 'Communicator'], ['Twitpic'], ['BERG'], ['Wishberg'], ['GreenGar', 'Studios'],\n",
      "             ['Rivet', '&', 'Sway'], ['Dijiwan'], ['Wantful'], ['Disruptive', 'Media'], ['Calexda'], ['TurnTable.fm'],\n",
      "             ['Tutorspree'], ['Nirvanix'], ['PostRocket'], ['VoterTide'], ['SkyRocket'], ['GameLayers'], ['Serendip'],\n",
      "             ['Unifyo'], ['Lookery'], ['Canvas', 'Networks'], ['Blurtt'], ['Manilla'], ['Pumodo'], ['Admazely'],\n",
      "             ['Springpad'], ['ExFM'], ['Samba', 'Mobile'], ['Qik'], ['inBloom'], ['IntoNow'], ['FindIt'],\n",
      "             ['Inq', 'Mobile'], ['Outbox'], ['Argyle', 'Company'], ['Exec'], ['Bloom.fm'],\n",
      "             ['Stipple'], ['Zumbox'], ['Behance'], ['Eventbrite'], ['MailChimp'], ['Twitch'], ['Moz'],\n",
      "             ['HubSpot'], ['Indeed'], ['BufferApp'], ['Airbnb'], ['ThinkGeek'], ['Quora'], ['Tesla', 'Motors'],\n",
      "             ['Grooveshark'], ['Klout'], ['Zendesk'], ['About.me'], ['Uber'], ['Lumosity']]"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#########\n",
      "#\n",
      "# NY Times - API requests automated script\n",
      "#\n",
      "# CulturePlex lab.\n",
      "#\n",
      "# @versae, @josemazo and @gabmunrio\n",
      "#\n",
      "# This cell contains unified in the same cell the downloading files and the article treatment.\n",
      "#\n",
      "#########\n",
      "\n",
      "import io\n",
      "import os\n",
      "import pandas as pd\n",
      "import urllib2\n",
      "import ujson as json\n",
      "\n",
      "from dateutil import parser\n",
      "from newspaper import Article\n",
      "from pandas.io.json import json_normalize\n",
      "from time import sleep\n",
      "\n",
      "# API Keys\n",
      "\n",
      "# api_key = '730e30f5220059551e666430644fbf87:11:69642501'\n",
      "api_key = 'd7655429355ab2df4621a10c01d04865:8:69135199'\n",
      "\n",
      "# We need to take into account the tag list used for the requests\n",
      "\n",
      "tag_lists = [['startup']]\n",
      "\n",
      "# This variable is used to create the appropiate path for the articles of the files\n",
      "\n",
      "actual_tags = ''\n",
      "\n",
      "# Dates for the requests\n",
      "\n",
      "begin_date = '20130101'\n",
      "end_date = '20131231'\n",
      "\n",
      "# Url for the requests and the paths to save the files\n",
      "\n",
      "entries_url = 'http://api.nytimes.com/svc/search/v2/articlesearch.json?q={0}&sort=oldest&begin_date={1}&end_date={2}&api-key={3}&page={4}'\n",
      "data_path = 'data/files_new_york_times/json_original/'\n",
      "\n",
      "# Urls and variables needed for the Article extraction\n",
      "\n",
      "json_original_data_path = 'data/files_new_york_times/json_original/'\n",
      "csv_url_data_path = 'data/files_new_york_times/csv_url/'\n",
      "txt_article_data_path = 'data/files_new_york_times/txt_article/'\n",
      "txt_article_temp_data_path = ''\n",
      "\n",
      "data_total = pd.DataFrame()\n",
      "\n",
      "wanted_columns = ['_id', 'web_url']\n",
      "new_column = 'url_works'\n",
      "\n",
      "change_columns_name = {'_id': 'id', 'web_url': 'url'}\n",
      "index_column = 'id'\n",
      "\n",
      "ordered_columns = ['url_works', 'url']\n",
      "\n",
      "##\n",
      "# get_url_works(row, tags)\n",
      "#\n",
      "# Function that checks if the article url works and, in that case, download and treat\n",
      "# the article text.\n",
      "##\n",
      "\n",
      "def get_url_works(row):\n",
      "    works = 0\n",
      "    url = row.url\n",
      "    \n",
      "    txt_path = txt_article_data_path + '_'.join(actual_tags)\n",
      "    if not os.path.exists(txt_path):\n",
      "        os.makedirs(txt_path)\n",
      "    \n",
      "    if url.startswith('http://www.nytimes.com/') or url.startswith('https://www.nytimes.com/'):\n",
      "        url += '?pagewanted=all'\n",
      "    \n",
      "    attempts = 3\n",
      "    while attempts > 0:\n",
      "        try:\n",
      "            a = Article(url, fetch_images=False, memorize_articles=False)\n",
      "            a.download()\n",
      "            a.parse()\n",
      "            \n",
      "            if a.is_valid_body():\n",
      "                works = 1\n",
      "                \n",
      "                with io.open(txt_path + '/' + row.id + '.txt', 'w', encoding='utf8') as outfile:\n",
      "                    outfile.write(a.url)\n",
      "                    outfile.write(u'\\n\\n')\n",
      "                    outfile.write(a.title)\n",
      "                    outfile.write(u'\\n\\n')\n",
      "                    outfile.write(a.text)\n",
      "            \n",
      "            attempts = 0\n",
      "        \n",
      "        except:\n",
      "            attempts -= 1\n",
      "    \n",
      "    return pd.Series({new_column: works})\n",
      "\n",
      "# We create the csv path and txt article path if they don't exist.\n",
      "\n",
      "if not os.path.exists(csv_url_data_path):\n",
      "    os.makedirs(csv_url_data_path)\n",
      "    \n",
      "if not os.path.exists(txt_article_data_path):\n",
      "    os.makedirs(txt_article_data_path)\n",
      "\n",
      "# This is the main loop to request to the NY Times API for every tags in the tags list\n",
      "\n",
      "for tags in tag_lists:\n",
      "    \n",
      "    # Initialization variables\n",
      "    \n",
      "    actual_tags = tags\n",
      "    perform_requests = True\n",
      "    first_request = True\n",
      "    \n",
      "    iteration = 0\n",
      "    page = 0\n",
      "    offset = 0\n",
      "    hits = 0\n",
      "    begin_date_aux = begin_date\n",
      "    \n",
      "    # We prepare the necessary paths\n",
      "    \n",
      "    full_data_path = data_path + '_'.join(tags)\n",
      "    print full_data_path\n",
      "    if not os.path.exists(full_data_path):\n",
      "        os.makedirs(full_data_path)\n",
      "    \n",
      "    # And now, we start with the requests\n",
      "    \n",
      "    while perform_requests:\n",
      "        request_url = entries_url.format('+'.join(tags), begin_date_aux, end_date, api_key, page)\n",
      "        print request_url\n",
      "        response = urllib2.urlopen(request_url)\n",
      "        sleep(0.1)  # Max. 10 request per second\n",
      "        \n",
      "        # Right now, we load the reponse in the data variable. This variable contains the JSON result.\n",
      "        \n",
      "        data = json.load(response)\n",
      "        \n",
      "        if (len(data['response']['docs']) == 0):\n",
      "            perform_requests = False\n",
      "            print '\\tFinishing, tags:', '_'.join(tags)\n",
      "            break\n",
      "        elif (data['status'] != 'OK'):\n",
      "            perform_requests = False\n",
      "            print '\\tFinishing with errors, tags:', '_'.join(tags)\n",
      "            print '\\tPrinting the response'\n",
      "            print '\\t\\t' + response\n",
      "            break\n",
      "        \n",
      "        if first_request:\n",
      "            hits = data['response']['meta']['hits']\n",
      "            first_request = False\n",
      "            print 'Total hits for ' + '_'.join(tags) + ': ' +  str(hits)\n",
      "        \n",
      "        # We need to control the page, because the pagination ends when reach to 100.\n",
      "        # So, we need to change the begin date to start a new pagination from 0.\n",
      "        \n",
      "        page += 1\n",
      "        if page > 100:\n",
      "            last_index = len(data['response']['docs']) - 1\n",
      "            \n",
      "            last_date = parser.parse(data['response']['docs'][last_index]['pub_date'])\n",
      "            begin_date_aux = str(last_date.year) + '%02d' % last_date.month + '%02d' % last_date.day\n",
      "\n",
      "            page = 0\n",
      "            iteration += 1\n",
      "        \n",
      "        # We save the content in the JSON file.\n",
      "        \n",
      "        filename = '_'.join(tags) + '_' + str((iteration * 100) + page + iteration) + '.json'\n",
      "        print '\\tSaving file', filename\n",
      "        \n",
      "        with open(full_data_path + '/' + filename, 'w') as outfile:            \n",
      "            json.dump(data, outfile)\n",
      "        \n",
      "        # Once the file is saved, we focus on the article extraction.\n",
      "        \n",
      "        data_aux = json_normalize(data['response'], 'docs')[wanted_columns]\n",
      "            \n",
      "        data_aux.rename(columns=change_columns_name, inplace=True)\n",
      "        data_aux[new_column] = 0\n",
      "\n",
      "        url_works = data_aux.apply(get_url_works, axis=1)\n",
      "        data_aux.update(url_works)\n",
      "\n",
      "        if first_request:\n",
      "            data_total = data_aux.copy()\n",
      "            first_request = False\n",
      "        else:\n",
      "            data_total = data_total.append(data_aux)\n",
      "\n",
      "        data_aux = None\n",
      "\n",
      "    # And we finalize saving the csv files that links for every folder.    \n",
      "    \n",
      "    print '\\tTreating the big dataFrame'\n",
      "    \n",
      "    print '\\tShape before remove bad rows:', data_total.shape\n",
      "    filter_empty_url = (data_total.url != '')  # If doing this after saving to CSV, instead of that you need this: data.text.notnull()\n",
      "    data_total = data_total[(filter_empty_url)].copy()\n",
      "    print '\\tShape after remove bad rows:', data_total.shape\n",
      "        \n",
      "    print '\\tShape before remove duplicated rows:', data_total.shape\n",
      "    data_total.drop_duplicates(['id'], inplace=True)\n",
      "    print '\\tShape after remove duplicated rows:', data_total.shape\n",
      "    \n",
      "    data_total.set_index('id', inplace=True)\n",
      "    \n",
      "    print '\\tSaving to CSV'\n",
      "    \n",
      "    data_total = data_total.reindex_axis(ordered_columns, axis=1)\n",
      "    folder = '_'.join(tags)\n",
      "    data_total.to_csv(csv_url_data_path + '/' + folder + '.csv', encoding='utf-8')\n",
      "    \n",
      "    data_total = None\n",
      "\n",
      "print 'Finished!'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "#########\n",
      "#\n",
      "# NY Times - API requests automated script\n",
      "#\n",
      "# CulturePlex lab.\n",
      "#\n",
      "# @versae, @josemazo and @gabmunrio\n",
      "#\n",
      "# This cell contains all the logic to get the readability, diversity and sentiment from the texts.\n",
      "#\n",
      "#########\n",
      "\n",
      "import datetime\n",
      "import os\n",
      "\n",
      "import numpy as np\n",
      "import pandas as pd\n",
      "import ujson as json\n",
      "\n",
      "from pandas.io.json import json_normalize\n",
      "from pattern.metrics import readability\n",
      "from pattern.metrics import ttr as diversity\n",
      "from pattern.en import sentiment\n",
      "\n",
      "# Necessary paths to save and access to the files\n",
      "\n",
      "json_data_path = 'data/files_new_york_times/json_original/'\n",
      "article_data_path = 'data/files_new_york_times/txt_article/'\n",
      "\n",
      "csv_data_path = 'data/files_new_york_times/'\n",
      "text_data_path = 'data/files_new_york_times/data/'\n",
      "\n",
      "# Some variables for a correct behaviour\n",
      "\n",
      "exist_article_column = 'exist_article'\n",
      "article_text_column = 'text'\n",
      "text_file_path_column = 'file'\n",
      "\n",
      "no_company_topics = ['entrepreneur', 'startup', 'new_venture', 'manager', 'executive', 'founder']\n",
      "media = 'nyt'\n",
      "is_company = 0\n",
      "search_words = []\n",
      "dir_name = ''\n",
      "\n",
      "first = True\n",
      "data = None\n",
      "\n",
      "# Columns to reorder the different csv files\n",
      "\n",
      "sentiment_columns = ['readability', 'diversity', 'polarity', 'subjetivity']\n",
      "\n",
      "change_columns_name = {'web_url': 'url'}\n",
      "\n",
      "ordered_columns = ['media', 'search_words', 'is_company', 'pub_date',\n",
      "                   'readability', 'diversity', 'polarity', 'subjetivity',\n",
      "                   'text', 'url', 'file']\n",
      "\n",
      "ordered_no_text_columns = ['media', 'search_words', 'is_company', 'pub_date',\n",
      "                           'readability', 'diversity', 'polarity', 'subjetivity',\n",
      "                           'url', 'file']\n",
      "\n",
      "##\n",
      "# check_article_and_proceed(row)\n",
      "#\n",
      "# Function that checks if the article exists and proceed to treat it it is. In the treatment,\n",
      "# we calculate the values for readability, diversity and sentiment.\n",
      "#\n",
      "##\n",
      "\n",
      "def check_article_and_proceed(row):\n",
      "    file_name = article_data_path + dir_name + '/' + row._id + '.txt' \n",
      "    exist = os.path.isfile(file_name)\n",
      "    \n",
      "    save_file_name = ''\n",
      "    text = ''\n",
      "    \n",
      "    read = 0\n",
      "    diver = 0\n",
      "    pol = 0\n",
      "    sub = 0\n",
      "    \n",
      "    if exist:\n",
      "        if is_company:\n",
      "            company_topic = 'company'\n",
      "        else:\n",
      "            company_topic = 'topic'\n",
      "            \n",
      "        dt = datetime.datetime.strptime(row.pub_date, '%Y-%m-%dT%H:%M:%SZ')\n",
      "        dt = dt.strftime('%Y%m%dT%H%M%S')\n",
      "        \n",
      "        save_file_name = '_'.join([media, company_topic, search_words[0], dt, row._id]) + '.txt'\n",
      "        \n",
      "        with open(file_name, 'r') as file_in:\n",
      "            file_lines = file_in.read().splitlines(True)\n",
      "        \n",
      "        with open(text_data_path + save_file_name, 'w') as file_out:\n",
      "            file_out.writelines(file_lines[4:])\n",
      "        \n",
      "        with open(text_data_path + save_file_name, 'r') as file_in:\n",
      "            text = file_in.read()\n",
      "        \n",
      "        try:\n",
      "            read = readability(text)\n",
      "        except:\n",
      "            read = 0\n",
      "        \n",
      "        try:\n",
      "            diver = diversity(text)\n",
      "        except:\n",
      "            diver = 0\n",
      "            \n",
      "        try:\n",
      "            sent = sentiment(text)\n",
      "        except:\n",
      "            sent = (0, 0)\n",
      "        \n",
      "        pol = sent[0]\n",
      "        sub = sent[1]\n",
      "        \n",
      "    return pd.Series({\n",
      "        exist_article_column: exist,\n",
      "        article_text_column: text,\n",
      "        text_file_path_column: 'data/' + save_file_name,\n",
      "        sentiment_columns[0]: read,\n",
      "        sentiment_columns[1]: diver,\n",
      "        sentiment_columns[2]: pol,\n",
      "        sentiment_columns[3]: sub\n",
      "    })\n",
      "\n",
      "# We create the text_data_path if it does not exist.\n",
      "\n",
      "if not os.path.exists(text_data_path):\n",
      "    os.makedirs(text_data_path)\n",
      "\n",
      "# Main loop to create the final csv file. We are going to have two csv files, one with the\n",
      "# associated text of every row, and another without that text.\n",
      "    \n",
      "for dir_name in os.walk(json_data_path).next()[1]:\n",
      "    json_topic_path = json_data_path + dir_name + '/'\n",
      "    \n",
      "    if dir_name in no_company_topics:\n",
      "        is_company = 0\n",
      "    else:\n",
      "        is_company = 1\n",
      "    \n",
      "    search_words = dir_name.split('_')\n",
      "    \n",
      "    for file_name in os.walk(json_topic_path).next()[2]:\n",
      "        json_topic_file = json_topic_path + file_name\n",
      "        \n",
      "        json_file = json.load(open(json_topic_file))\n",
      "        data_aux = json_normalize(json_file['response'], 'docs')\n",
      "        \n",
      "        data_aux.rename(columns=change_columns_name, inplace=True)\n",
      "        \n",
      "        data_aux[exist_article_column] = False\n",
      "        data_aux[article_text_column] = ''\n",
      "        data_aux[text_file_path_column] = ''\n",
      "        data_aux[sentiment_columns[0]] = 0\n",
      "        data_aux[sentiment_columns[1]] = 0\n",
      "        data_aux[sentiment_columns[2]] = 0\n",
      "        data_aux[sentiment_columns[3]] = 0\n",
      "        \n",
      "        ordered_columns = ['media', 'search_words', 'is_company', 'pub_date',\n",
      "                   'readability', 'diversity', 'polarity', 'subjetivity',\n",
      "                   'text', 'url', 'file']\n",
      "        \n",
      "        data_aux[ordered_columns[0]] = media\n",
      "        data_aux[ordered_columns[1]] = ' '.join(search_words)\n",
      "        data_aux[ordered_columns[2]] = is_company\n",
      "        \n",
      "        data_exist = data_aux[['_id', 'pub_date']].apply(check_article_and_proceed, axis=1)\n",
      "        data_aux.update(data_exist)\n",
      "        \n",
      "        exist_filter = (data_aux[exist_article_column] == True)\n",
      "        data_aux = data_aux[exist_filter]\n",
      "        \n",
      "        empty_text_filter = (data_aux[article_text_column] != '')\n",
      "        data_aux = data_aux[empty_text_filter]\n",
      "        \n",
      "        data_aux = data_aux[ordered_columns]\n",
      "        data_aux = data_aux.reindex_axis(ordered_columns, axis=1)\n",
      "        \n",
      "        if first:\n",
      "            data = data_aux.copy()\n",
      "            first = False\n",
      "        else:\n",
      "            data = data.append(data_aux, ignore_index=True)\n",
      "\n",
      "        data_aux = None\n",
      "    \n",
      "    print 'Finished with ' + dir_name\n",
      "    \n",
      "data.to_csv(csv_data_path + 'new_york_times' + '.csv', index=False, encoding='utf-8')\n",
      "\n",
      "data[ordered_no_text_columns].to_csv(csv_data_path + 'new_york_times_no_text' + '.csv', index=False, encoding='utf-8')\n",
      "\n",
      "print 'FINISHED!'"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "pd.read_csv('data/files_new_york_times/new_york_times_no_text.csv', encoding='utf-8')"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    },
    {
     "cell_type": "code",
     "collapsed": false,
     "input": [
      "dataFrame = pd.read_csv('data/files_new_york_times/csv_url/startup.csv', encoding='utf-8')\n",
      "count = 0\n",
      "for result in dataFrame['url_works']:\n",
      "    if result == 1:\n",
      "        count = count + 1\n",
      "print count"
     ],
     "language": "python",
     "metadata": {},
     "outputs": []
    }
   ],
   "metadata": {}
  }
 ]
}