mars_scraping.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "from splinter import Browser\n",
    "from bs4 import BeautifulSoup as bs\n",
    "import time\n",
    "import pandas as pd"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# @NOTE: Replace the path with your actual path to the chromedriver\n",
    "executable_path = {\"executable_path\": \"/usr/local/bin/chromedriver\"}\n",
    "browser=Browser(\"chrome\", **executable_path, headless=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Curiosity Tastes First Sample in 'Clay-Bearing Unit'\n",
      "This new region on Mars might reveal more about the role of water on Mount Sharp.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# browser = init_browser()\n",
    "url = 'https://mars.nasa.gov/news/'\n",
    "browser.visit(url)\n",
    "\n",
    "# Scrape page into Soup\n",
    "html = browser.html\n",
    "soup = bs(html, 'html.parser')\n",
    "# latest article title \n",
    "news_title = soup.find('div', class_='content_title').find('a').text\n",
    "# latest article paragraph \n",
    "news_p = soup.find('div', class_='article_teaser_body').text\n",
    "browser.quit()\n",
    "\n",
    "print(news_title)\n",
    "print(news_p)\n",
    "    \n",
    "\n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [],
   "source": [
    "#scrape freatured image on nasa.gov\n",
    "img_url = \"https://www.jpl.nasa.gov/spaceimages/?search=&category=Mars\"\n",
    "browser.visit(img_url)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "html = browser.html\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.jpl.nasa.gov//spaceimages/images/wallpaper/PIA17793-1920x1200.jpg\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Scrape page into Soup\n",
    "soup = bs(html, \"html.parser\")\n",
    "# Featured image on mars \n",
    "# Return results\n",
    "articles=soup.find('div',class_=\"carousel_items\").find('article')\n",
    "imgtag=articles['style']\n",
    "\n",
    "base_path='https://www.jpl.nasa.gov/'\n",
    "#extract url from style\n",
    "#get image URL through a string split\n",
    "rel_path=imgtag.split(\"('\", 1)[1].split(\"')\")[0]\n",
    "featured_image = base_path+ rel_path\n",
    "print(featured_image)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<p class=\"TweetTextSize TweetTextSize--normal js-tweet-text tweet-text\" data-aria-label-part=\"0\" lang=\"en\">InSight sol 138 (2019-04-17) low -97.7ºC (-143.9ºF) high -17.3ºC (0.9ºF)\n",
      "winds from the W at 4.3 m/s (9.5 mph) gusting to 12.6 m/s (28.1 mph)\n",
      "pressure at 7.30 hPa<a class=\"twitter-timeline-link u-hidden\" data-pre-embedded=\"true\" dir=\"ltr\" href=\"https://t.co/ofMyPZM2vS\">pic.twitter.com/ofMyPZM2vS</a></p>\n"
     ]
    }
   ],
   "source": [
    "# scrape Mars weather\n",
    "weather_url = \"https://twitter.com/marswxreport?lang=en\"\n",
    "browser.visit(weather_url)\n",
    "html = browser.html\n",
    "soup = bs(html, \"html.parser\")\n",
    "weather_rel_path = soup.find(\"div\", class_= \"js-tweet-text-container\")\n",
    "mars_weather=weather_rel_path.find(class_=\"tweet-text\")\n",
    "print(mars_weather)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'bs4.element.Tag'>\n",
      "InSight sol 138 (2019-04-17) low -97.7ºC (-143.9ºF) high -17.3ºC (0.9ºF)\n",
      "winds from the W at 4.3 m/s (9.5 mph) gusting to 12.6 m/s (28.1 mph)\n",
      "pressure at 7.30 hPa\n",
      "pic.twitter.com/ofMyPZM2vS\n"
     ]
    }
   ],
   "source": [
    "\n",
    "print(type(mars_weather))   \n",
    "myweather=''\n",
    "for tag in mars_weather:\n",
    "    myweather=tag.string\n",
    "\n",
    "    print(myweather)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 216,
   "metadata": {},
   "outputs": [],
   "source": [
    "# scrape mars facts table\n",
    "marsFacts_url = \"https://space-facts.com/mars/\"\n",
    "browser.visit(marsFacts_url)\n",
    "html = browser.html\n",
    "soup = bs(html, \"html.parser\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 232,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Description</th>\n",
       "      <th>Measurement</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Equatorial Diameter:</td>\n",
       "      <td>6,792 km</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Polar Diameter:</td>\n",
       "      <td>6,752 km</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Mass:</td>\n",
       "      <td>6.42 x 10^23 kg (10.7% Earth)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Moons:</td>\n",
       "      <td>2 (Phobos &amp; Deimos)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Orbit Distance:</td>\n",
       "      <td>227,943,824 km (1.52 AU)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Orbit Period:</td>\n",
       "      <td>687 days (1.9 years)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Surface Temperature:</td>\n",
       "      <td>-153 to 20 °C</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>First Record:</td>\n",
       "      <td>2nd millennium BC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Recorded By:</td>\n",
       "      <td>Egyptian astronomers</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            Description                    Measurement\n",
       "0  Equatorial Diameter:                       6,792 km\n",
       "1       Polar Diameter:                       6,752 km\n",
       "2                 Mass:  6.42 x 10^23 kg (10.7% Earth)\n",
       "3                Moons:            2 (Phobos & Deimos)\n",
       "4       Orbit Distance:       227,943,824 km (1.52 AU)\n",
       "5         Orbit Period:           687 days (1.9 years)\n",
       "6  Surface Temperature:                  -153 to 20 °C\n",
       "7         First Record:              2nd millennium BC\n",
       "8          Recorded By:           Egyptian astronomers"
      ]
     },
     "execution_count": 232,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Find Table and turn into data frame\n",
    "marsFacts_table = soup.find(\"table\",class_=\"tablepress-id-mars\")\n",
    "table_rows = marsFacts_table.find_all('tr')\n",
    "\n",
    "res = []\n",
    "for tr in table_rows:\n",
    "    td = tr.find_all('td')\n",
    "    row = [tr.text.strip() for tr in td if tr.text.strip()]\n",
    "    if row:\n",
    "        res.append(row)\n",
    "\n",
    "\n",
    "#Initialize the pandas data frame\n",
    "df = pd.DataFrame(res,columns=['Description','Measurement'])\n",
    "df.head(30)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "marsurl='https://astrogeology.usgs.gov/search/results?q=hemisphere+enhanced&k1=target&v1=Mars'\n",
    "browser.visit(marsurl)\n",
    "html=browser.html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Find all the items\n",
    "soup = bs(html, \"html.parser\")\n",
    "#Find all links('a') on the page\n",
    "items= soup.find_all(\"a\",class_=\"itemLink product-item\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://astrogeology.usgs.gov/search/map/Mars/Viking/cerberus_enhanced\n",
      "Cerberus Hemisphere Enhanced\n",
      "Title is Cerberus Hemisphere Enhanced\n",
      "----<>-----\n",
      "https://astrogeology.usgs.gov/search/map/Mars/Viking/schiaparelli_enhanced\n",
      "Schiaparelli Hemisphere Enhanced\n",
      "Title is Schiaparelli Hemisphere Enhanced\n",
      "----<>-----\n",
      "https://astrogeology.usgs.gov/search/map/Mars/Viking/syrtis_major_enhanced\n",
      "Syrtis Major Hemisphere Enhanced\n",
      "Title is Syrtis Major Hemisphere Enhanced\n",
      "----<>-----\n",
      "https://astrogeology.usgs.gov/search/map/Mars/Viking/valles_marineris_enhanced\n",
      "Valles Marineris Hemisphere Enhanced\n",
      "Title is Valles Marineris Hemisphere Enhanced\n",
      "----<>-----\n"
     ]
    }
   ],
   "source": [
    "marsroot='https://astrogeology.usgs.gov'\n",
    "\n",
    "def get_link_image(browser,link):\n",
    "    browser.visit(link)\n",
    "    html=browser.html\n",
    "    sp=bs(html,\"html.parser\")\n",
    "    tags=sp.find(\"div\",class_=\"downloads\").find('ul').find('li').find('a')\n",
    "    imglink=tags['href']\n",
    "    if 'jpg' in imglink:\n",
    "        return(imglink)\n",
    "    else:\n",
    "        return(None)\n",
    "\n",
    "i=0\n",
    "hemisphere_image_urls=[]\n",
    "for tag in items:\n",
    "    imageURL=marsroot+tag['href']\n",
    "    title=tag.find('h3')\n",
    "    if title is not None:\n",
    "        i=i+1\n",
    "        print(imageURL)\n",
    "        name=title.string\n",
    "        print(name)\n",
    "        print(\"Title is \"+title.string)\n",
    "        #Call the function to follow the links\n",
    "        fullimglink=get_link_image(browser,imageURL)\n",
    "        if fullimglink is not None:\n",
    "            hemisphere_image_urls.append({\"title\" : name, 'img_url':fullimglink })\n",
    "\n",
    "        print(\"----<>-----\")\n",
    "        "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[{'title': 'Cerberus Hemisphere Enhanced', 'img_url': 'http://astropedia.astrogeology.usgs.gov/download/Mars/Viking/cerberus_enhanced.tif/full.jpg'}, {'title': 'Schiaparelli Hemisphere Enhanced', 'img_url': 'http://astropedia.astrogeology.usgs.gov/download/Mars/Viking/schiaparelli_enhanced.tif/full.jpg'}, {'title': 'Syrtis Major Hemisphere Enhanced', 'img_url': 'http://astropedia.astrogeology.usgs.gov/download/Mars/Viking/syrtis_major_enhanced.tif/full.jpg'}, {'title': 'Valles Marineris Hemisphere Enhanced', 'img_url': 'http://astropedia.astrogeology.usgs.gov/download/Mars/Viking/valles_marineris_enhanced.tif/full.jpg'}]\n"
     ]
    }
   ],
   "source": [
    "#Print result dictionary\n",
    "print(hemisphere_image_urls)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}