Selenium_Gov_Census_Oct18.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Web Scrpe Using Selenium "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Yi Yin "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Last Updated: Oct 18 2:30 AM"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import  selenium"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "import time\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Start a browser and Visiting the website"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "driver = webdriver.Chrome()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# your url\n",
    "url = 'https://www.census.gov/data/tables.html'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "# visit the url\n",
    "driver.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "survey_menu = driver.find_element_by_xpath('//*[@id=\"programsFacetGroup\"]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'x': 120, 'y': 0}"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "survey_menu.location_once_scrolled_into_view"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scrape and Print data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1990 Census (72)\n",
      "2000 Census (64)\n",
      "2010 Census (4)\n",
      "Advance Monthly Retail Trade Survey (MARTS) (1)\n",
      "American Community Survey (ACS) (55)\n",
      "Annual Capital Expenditures Survey (ACES) (29)\n",
      "Annual Retail Trade Survey (ARTS) (75)\n",
      "Annual Survey of Entrepreneurs (ASE) (10)\n",
      "Annual Survey of Manufactures (ASM) (12)\n",
      "Annual Survey of Public Employment & Payroll (APES) (96)\n",
      "Annual Survey of Public Pensions (ASPP) (26)\n",
      "Annual Survey of School System Finances (33)\n",
      "Annual Survey of State Government Tax Collections (STC) (68)\n",
      "Annual Wholesale Trade Survey (AWTS) (15)\n",
      "Building Permits Survey (BPS) (51)\n",
      "Business Expenses Supplement (BES) (9)\n",
      "Census of Governments (COG) (63)\n",
      "Commodity Flow Survey (CFS) (79)\n",
      "Consolidated Federal Fund Report (CFFR) (79)\n",
      "County Business Patterns (CBP) (149)\n",
      "Current Industrial Reports (CIR) (2)\n",
      "Current Population Survey (CPS) (382)\n",
      "Decennial Census (96)\n",
      "Economic Census (EC) (60)\n",
      "Economic Census of Island Areas (IA) (4)\n",
      "Enterprise Statistics Program (ESP) (6)\n",
      "Export Statistics (1)\n",
      "Governments Units Survey (GUS) (14)\n",
      "Housing Vacancy Survey (HVS) (119)\n",
      "Information and Communication Technology (ICT) Survey (10)\n",
      "International Programs (4)\n",
      "Management and Organizational Practices Survey (MOPS) (1)\n",
      "Manufactured Homes Survey (MHS) (43)\n",
      "Manufacturers' Shipments, Inventories, and Orders Survey (M3) (1)\n",
      "Metropolitan & Micropolitan Program (13)\n",
      "Monthly Retail Trade Survey (MRTS) (14)\n",
      "Monthly Wholesale Trade Survey (MWTS) (22)\n",
      "New Residential Construction (NRC) (3)\n",
      "New Residential Sales (NRS) (2)\n",
      "Nonemployer Statistics (28)\n",
      "Population Estimates (58)\n",
      "Population Projections (11)\n",
      "Property Owners and Managers Survey (POMS) (1)\n",
      "Quarterly Services Survey (QSS) (13)\n",
      "Quarterly Summary of State and Local Government Tax Revenue (QTAX) (225)\n",
      "Quarterly Survey of Plant Capacity Utilization (QPC) (26)\n",
      "Quarterly Survey of Public Pensions (QSPP) (2)\n",
      "Redistricting Data Office (RDO) (2)\n",
      "Residential Finance Survey (RFS) (1)\n",
      "Service Annual Survey (SAS) (27)\n",
      "Small Area Income and Poverty Estimates (SAIPE) (14)\n",
      "Statistics of U.S. Businesses (SUSB) (49)\n",
      "Survey of Business Owners (SBO) (33)\n",
      "Survey of Construction (SOC) (2)\n",
      "Survey of Income and Program Participation (SIPP) (92)\n",
      "Survey of Market Absorption of New Multifamily Units (SOMA) (90)\n",
      "Survey of Program Dynamics (SPD) (1)\n",
      "Tribal Affairs (AIAN) Program (1)\n",
      "Value of Construction Put in Place Survey (4)\n",
      "Voting Rights (2)\n"
     ]
    }
   ],
   "source": [
    "for p in survey_menu.find_elements_by_xpath('//li[@ng-show=\"allSurveysAndProgramFacet.count > 0\"]'):\n",
    "    print(p.text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Search Automation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "survey_search = driver.find_element_by_xpath('//*[@id=\"searchSurveysPrograms\"]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'x': 171, 'y': 0}"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "survey_search.location_once_scrolled_into_view"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "survey_search.clear()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "survey_search.send_keys(\"economic\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Economic Census (EC) (60)\n",
      "Economic Census of Island Areas (IA) (4)\n"
     ]
    }
   ],
   "source": [
    "for p in survey_menu.find_elements_by_xpath('//li[@ng-show=\"allSurveysAndProgramFacet.count > 0\"]'):\n",
    "    print(p.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "econ_survey =  survey_menu.find_elements_by_xpath('//li[@ng-show=\"allSurveysAndProgramFacet.count > 0\"]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "econ_survey [0].click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "article_list = driver.find_element_by_xpath('//*[@id=\"facetedListArticles\"]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'x': 359, 'y': 0}"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "article_list.location_once_scrolled_into_view"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "articles_1 = article_list.find_elements_by_xpath('//*[@ng-repeat=\"article in listArticles\"]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "20"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(articles_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SEPTEMBER 2019\n",
      "Quarterly Summary of State & Local Tax Revenue: Quarter Two\n",
      "The First Quarter 2019 data were released on June 18, 2019.\n",
      "SEPTEMBER 2019\n",
      "Income and Poverty in the United States: 2018\n",
      "These tables present data on income, earnings, income inequality & poverty in the United States based on information collected in the 2019 and earlier CPS ASEC.\n",
      "SEPTEMBER 10, 2019\n",
      "Health Insurance in the United States: 2018- Tables\n",
      "These tables present data on health insurance coverage in the United States during 2018.\n",
      "2019 CPS Poverty Table: POV-30. School Enrollment by Poverty Status, Sex, and Age\n",
      "School Enrollment by Poverty Status, Sex, and Age\n",
      "SEPTEMBER 10, 2019\n",
      "The Supplemental Poverty Measure: 2018\n",
      "View and download tables on the Supplemental Poverty Measure: 2018.\n",
      "Poverty Thresholds: 2018\n",
      "View and download tables on Poverty Thresholds.\n",
      "SEPTEMBER 2019\n",
      "MHS Annual Data\n",
      "2014 – 2018 Average Sales Price by State, Selected Characteristics by Region, Average and Median Square Feet of Floor Area.\n",
      "County-to-County Migration Flows: 2013-2017 ACS\n",
      "These tables are based on the 2013-2017 ACS data\n",
      "Metro Area-to-Metro Area Migration Flows: 2013-2017 American Community Survey\n",
      "These tables are based on the 2013-2017 ACS.\n",
      "AUGUST 2019\n",
      "CPS Historical Migration/Geographic Mobility Tables\n",
      "This table package shows historical geographical mobility estimates and rates going back to 1948.\n",
      "2019\n",
      "2017 CBP First Look\n",
      "JULY 2019\n",
      "MHS Latest Data\n",
      "Manufactured Housing statistics at a glance.\n",
      "County Population by Characteristics: 2010-2018\n",
      "This page features Vintage 2018 population estimates by age, sex, race, and Hispanic origin.\n",
      "Puerto Rico Municipios Population by Characteristics: 2010-2018\n",
      "This page features Vintage 2018 population estimates by age and sex.\n",
      "2019\n",
      "2019 Quarterly Survey of Plant Capacity Utilization\n",
      "View the 2019 quarterly tables from the Quarterly Survey of Plant Capacity Utilization.\n",
      "JUNE 2019\n",
      "Quarterly Summary of State & Local Tax Revenue: Quarter One\n",
      "The First Quarter 2019 data were released on June 18, 2019.\n",
      "Characteristics of Driver/Sales Workers and Truck Drivers\n",
      "View and download tables on Drive/sales and Truck drivers by selected characteristics.\n",
      "MAY 29, 2019\n",
      "2019 Capital Spending Report Tables\n",
      "Tables in this report are from the U.S. Census Bureau's 2008 - 2017 Annual Capital Expenditures Survey (ACES).\n",
      "City and Town Population Totals: 2010-2018\n",
      "This page features Vintage 2018 population estimates totals.\n"
     ]
    }
   ],
   "source": [
    "for article in articles_1:\n",
    "    print(article.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "pub_dates = []\n",
    "titles = []\n",
    "notes = []\n",
    "links = []\n",
    "for article in articles_1:\n",
    "    try: \n",
    "        pub_dates.append(article.find_element_by_class_name('uscb-uppercase').text)\n",
    "    except:\n",
    "        pub_dates.append(None)\n",
    "        \n",
    "    try: \n",
    "        titles.append(article.find_element_by_class_name('uscb-title-3').text)\n",
    "    except:\n",
    "        titles.append(None)\n",
    "        \n",
    "    try: \n",
    "        notes.append(article.find_element_by_class_name('uscb-sub-heading-2').text)\n",
    "    except:\n",
    "        notes.append(None)\n",
    "    \n",
    "    try:\n",
    "        links.append(article.find_element_by_class_name('uscb-title-3').\\\n",
    "                     find_element_by_xpath('..').\\\n",
    "                     get_attribute(\"href\"))\n",
    "    \n",
    "    except:\n",
    "        links.append(None)\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Across Page"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "page_menu = driver.find_element_by_xpath('//*[@id=\"listpagination\"]/ul')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "page_menu.find_element_by_class_name('nextButton').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "articles_2 = driver.find_element_by_xpath('//*[@id=\"facetedListArticles\"]').\\\n",
    "find_elements_by_xpath('//*[@ng-repeat=\"article in listArticles\"]')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "for article in articles_2:\n",
    "    try: \n",
    "        pub_dates.append(article.find_element_by_class_name('uscb-uppercase').text)\n",
    "    except:\n",
    "        pub_dates.append(None)\n",
    "        \n",
    "    try: \n",
    "        titles.append(article.find_element_by_class_name('uscb-title-3').text)\n",
    "    except:\n",
    "        titles.append(None)\n",
    "        \n",
    "    try: \n",
    "        notes.append(article.find_element_by_class_name('uscb-sub-heading-2').text)\n",
    "    except:\n",
    "        notes.append(None)\n",
    "    \n",
    "    try:\n",
    "        links.append(article.find_element_by_class_name('uscb-title-3').\\\n",
    "                     find_element_by_xpath('..').\\\n",
    "                     get_attribute(\"href\"))\n",
    "    \n",
    "    except:\n",
    "        links.append(None)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Next Page Automation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "tp=int(driver.find_element_by_xpath('//*[@id=\"listpagination\"]').text[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "def collect_data():\n",
    "    article_list = driver.find_element_by_xpath('//*[@id=\"facetedListArticles\"]').\\\n",
    "    find_elements_by_xpath('//*[@ng-repeat=\"article in listArticles\"]')\n",
    "    time.sleep(1)\n",
    "    for article in article_list:\n",
    "        try: \n",
    "            pub_dates.append(article.find_element_by_class_name('uscb-uppercase').text)\n",
    "        except:\n",
    "            pub_dates.append(None)\n",
    "            \n",
    "        try: \n",
    "            titles.append(article.find_element_by_class_name('uscb-title-3').text)\n",
    "        except:\n",
    "            titles.append(None)\n",
    "            \n",
    "        try: \n",
    "            notes.append(article.find_element_by_class_name('uscb-sub-heading-2').text)\n",
    "        except:\n",
    "            notes.append(None)\n",
    "        \n",
    "        try:\n",
    "            links.append(article.find_element_by_class_name('uscb-title-3').\\\n",
    "                         find_element_by_xpath('..').\\\n",
    "                         get_attribute(\"href\"))\n",
    "        \n",
    "        except:\n",
    "            links.append(None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.find_element_by_xpath('//*[@id=\"listpagination\"]/ul').find_element_by_class_name('prevButton').click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "i=0\n",
    "pub_dates = []\n",
    "titles = []\n",
    "notes = []\n",
    "links = []\n",
    "\n",
    "while i<=tp:\n",
    "    i = i+1\n",
    "    collect_data()\n",
    "    if i<tp:\n",
    "        # next page\n",
    "        driver.find_element_by_xpath('//*[@id=\"listpagination\"]/ul').\\\n",
    "        find_element_by_class_name('nextButton').\\\n",
    "        click()\n",
    "    else: \n",
    "        i = i+1    \n",
    "    time.sleep(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save to Dataframe"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "econ_file_summary = pd.DataFrame({'title': titles,\n",
    "              'note': notes,\n",
    "    'pub_date': pub_dates,\n",
    "              'link': links})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>note</th>\n",
       "      <th>pub_date</th>\n",
       "      <th>link</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>55</th>\n",
       "      <td>2002 Management of Companies and Enterprises (...</td>\n",
       "      <td>These data tables are for the Management of Co...</td>\n",
       "      <td>2002</td>\n",
       "      <td>https://www.census.gov/content/census/en/data/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>56</th>\n",
       "      <td>2002 Educational Services (NAICS Sector 61)</td>\n",
       "      <td>These data tables are for the Educational Serv...</td>\n",
       "      <td>2002</td>\n",
       "      <td>https://www.census.gov/content/census/en/data/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>57</th>\n",
       "      <td>2002 Arts, Entertainment, and Recreation (NAIC...</td>\n",
       "      <td>These data tables are for the Arts, Entertainm...</td>\n",
       "      <td>2002</td>\n",
       "      <td>https://www.census.gov/content/census/en/data/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58</th>\n",
       "      <td>1992 Economic Census of Island Areas - Puerto ...</td>\n",
       "      <td>View tables for the 1992 Economic Census of Is...</td>\n",
       "      <td>1992</td>\n",
       "      <td>https://www.census.gov/content/census/en/data/...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>59</th>\n",
       "      <td>1947 - 1992 Economic Census Manufacturing Conc...</td>\n",
       "      <td>View and download the table for 1947 - 1992 Ec...</td>\n",
       "      <td>1992</td>\n",
       "      <td>https://www.census.gov/content/census/en/data/...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                title  \\\n",
       "55  2002 Management of Companies and Enterprises (...   \n",
       "56        2002 Educational Services (NAICS Sector 61)   \n",
       "57  2002 Arts, Entertainment, and Recreation (NAIC...   \n",
       "58  1992 Economic Census of Island Areas - Puerto ...   \n",
       "59  1947 - 1992 Economic Census Manufacturing Conc...   \n",
       "\n",
       "                                                 note pub_date  \\\n",
       "55  These data tables are for the Management of Co...     2002   \n",
       "56  These data tables are for the Educational Serv...     2002   \n",
       "57  These data tables are for the Arts, Entertainm...     2002   \n",
       "58  View tables for the 1992 Economic Census of Is...     1992   \n",
       "59  View and download the table for 1947 - 1992 Ec...     1992   \n",
       "\n",
       "                                                 link  \n",
       "55  https://www.census.gov/content/census/en/data/...  \n",
       "56  https://www.census.gov/content/census/en/data/...  \n",
       "57  https://www.census.gov/content/census/en/data/...  \n",
       "58  https://www.census.gov/content/census/en/data/...  \n",
       "59  https://www.census.gov/content/census/en/data/...  "
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "econ_file_summary.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "econ_file_summary.to_csv('econ_census.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "econ_survey[0].click()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Bonus: Download Automation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "d_links = []\n",
    "for article in driver.find_element_by_xpath('//*[@id=\"facetedListArticles\"]').\\\n",
    "find_elements_by_xpath('//*[@ng-repeat=\"article in listArticles\"]'):\n",
    "\n",
    "    try:\n",
    "        d_links.append(article.find_element_by_class_name('uscb-title-3').\\\n",
    "                     find_element_by_xpath('..').\\\n",
    "                     get_attribute(\"href\"))\n",
    "    \n",
    "    except:\n",
    "        d_links.append(None)\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['https://www.census.gov/content/census/en/data/tables/2019/econ/qtax/historical/q2.html',\n",
       " 'https://www.census.gov/content/census/en/data/tables/2019/demo/income-poverty/p60-266.html',\n",
       " 'https://www.census.gov/content/census/en/data/tables/2019/demo/health-insurance/p60-267.html',\n",
       " 'https://www.census.gov/content/census/en/data/tables/2019/demo/cps/pov-30.html',\n",
       " 'https://www.census.gov/content/census/en/data/tables/2019/demo/income-poverty/p60-268.html',\n",
       " 'https://www.census.gov/content/census/en/data/tables/2018/demo/supplemental-poverty-measure/poverty-thresholds.html',\n",
       " 'https://www.census.gov/content/census/en/data/tables/time-series/econ/mhs/annual-data.html',\n",
       " 'https://www.census.gov/content/census/en/data/tables/2017/demo/geographic-mobility/county-to-county-migration-2013-2017.html',\n",
       " 'https://www.census.gov/content/census/en/data/tables/2017/demo/geographic-mobility/metro-to-metro-migration.html',\n",
       " 'https://www.census.gov/content/census/en/data/tables/time-series/demo/geographic-mobility/historic.html',\n",
       " 'https://www.census.gov/content/census/en/data/tables/2017/econ/cbp/2017-first-look.html',\n",
       " 'https://www.census.gov/content/census/en/data/tables/time-series/econ/mhs/latest-data.html',\n",
       " 'https://www.census.gov/content/census/en/data/tables/time-series/demo/popest/2010s-counties-detail.html',\n",
       " 'https://www.census.gov/content/census/en/data/tables/time-series/demo/popest/2010s-detail-puerto-rico-municipios.html',\n",
       " 'https://www.census.gov/content/census/en/data/tables/2019/econ/qpc/qpc-quarterly-tables.html',\n",
       " 'https://www.census.gov/content/census/en/data/tables/2019/econ/qtax/historical/q1.html',\n",
       " 'https://www.census.gov/content/census/en/data/tables/2017/demo/industry-occupation/truckers-acs17.html',\n",
       " 'https://www.census.gov/content/census/en/data/tables/time-series/econ/aces/2019-csr.html',\n",
       " 'https://www.census.gov/content/census/en/data/tables/time-series/demo/popest/2010s-total-cities-and-towns.html']"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "d_links "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "for page in d_links[0:2]:\n",
    "    try:\n",
    "        driver.get(page)\n",
    "        time.sleep(1)\n",
    "        table_list = driver.find_element_by_id('detailContent')\n",
    "        files = table_list.find_elements_by_class_name(\"uscb-text-link\")\n",
    "        for f in files:\n",
    "            f.click()\n",
    "            time.sleep(1)\n",
    "    except:\n",
    "        pass\n",
    "        \n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Close your driver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}