PostMetaDataScraper.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from itertools import chain\n",
    "\n",
    "class PostMetaDataScraper:\n",
    "        \n",
    "    # example link: 'http://www.1point3acres.com/bbs/forum-80-1.html'\n",
    "    def getLinksForMainPage(self):\n",
    "        main_page_links = []\n",
    "        prefix = 'http://www.1point3acres.com/bbs/forum.php?mod=forumdisplay&fid=80&sortid=195&filter=sortid&sortid=195&page='\n",
    "        for i in range(9):\n",
    "            link = prefix + str(i)\n",
    "            main_page_links.append(link)\n",
    "        return main_page_links\n",
    "    \n",
    "    # soup: <td> elements of a post\n",
    "    def getPostsSoupForMainPage(self, main_page_links):\n",
    "        soups = []\n",
    "        for i in range(len(main_page_links)):\n",
    "            page = requests.get(main_page_links[i])\n",
    "            soup = BeautifulSoup(page.text, 'html.parser')\n",
    "            soup = soup.findAll(\"tbody\", id=lambda x: x and x.startswith('normalthread_'))\n",
    "            soups.append(soup)\n",
    "        soups = list(chain(*soups))\n",
    "        return soups\n",
    "        \n",
    "    # parse the soup <td> elements into dictionary meta data\n",
    "    # also do some filtering (return None if the post is not a dating post)\n",
    "    def getPostMetaDataFromSoup(self, soup):\n",
    "        type_special_href = 'http://www.1point3acres.com/bbs/forum.php?mod=forumdisplay&fid=80&filter=sortid&sortid=195'\n",
    "        is_dating_post = soup.find('a', {'href': type_special_href})\n",
    "        if is_dating_post is None:\n",
    "            return None\n",
    "        \n",
    "        post_creation_date = soup.findAll('td')[1].find('span').text\n",
    "        post_link = soup.find('a')['href']\n",
    "        post_comment_num = int(soup.find('td', {'class': 'num'}).find('a').text)\n",
    "        post_title = soup.find('a', {'class': 's xst'}).text\n",
    "        \n",
    "        return {\n",
    "            'post_creation_date': post_creation_date,\n",
    "            'post_link': post_link,\n",
    "            'post_comment_num': post_comment_num,\n",
    "            'post_title': post_title,\n",
    "        }\n",
    "    \n",
    "    # main function, get all filtered posts link with meta data\n",
    "    def run(self):\n",
    "        main_page_links = self.getLinksForMainPage()\n",
    "        posts_soup = self.getPostsSoupForMainPage(main_page_links)\n",
    "        posts_metadata = []\n",
    "        for post_soup in posts_soup:\n",
    "            post_metadata = self.getPostMetaDataFromSoup(post_soup)\n",
    "            if post_metadata is None:\n",
    "                continue\n",
    "            posts_metadata.append(post_metadata)   \n",
    "        return posts_metadata\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "post_metadata_scraper = PostMetaDataScraper()\n",
    "posts_metadata = post_metadata_scraper.run()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "450"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(posts_metadata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "%run FileManager.ipynb\n",
    "file_manager = FileManager()\n",
    "file_manager.saveToFile(posts_metadata, 'posts_metadata')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}