web-scrapping-2.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.012539,
     "end_time": "2022-05-22T10:10:43.935248",
     "exception": false,
     "start_time": "2022-05-22T10:10:43.922709",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Import Libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-22T10:10:43.964962Z",
     "iopub.status.busy": "2022-05-22T10:10:43.964237Z",
     "iopub.status.idle": "2022-05-22T10:10:44.184503Z",
     "shell.execute_reply": "2022-05-22T10:10:44.183885Z"
    },
    "papermill": {
     "duration": 0.237757,
     "end_time": "2022-05-22T10:10:44.184623",
     "exception": false,
     "start_time": "2022-05-22T10:10:43.946866",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import numpy as np\n",
    "import bs4\n",
    "import time\n",
    "import requests\n",
    "import lxml\n",
    "import urllib\n",
    "import urllib.request\n",
    "\n",
    "from pprint import pprint\n",
    "from bs4 import BeautifulSoup\n",
    "from tqdm import tqdm\n",
    "from getpass import getpass\n",
    "from urllib.parse import urljoin, urlparse"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.011193,
     "end_time": "2022-05-22T10:10:44.209370",
     "exception": false,
     "start_time": "2022-05-22T10:10:44.198177",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Parse all websites of the Manga"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-22T10:10:44.236220Z",
     "iopub.status.busy": "2022-05-22T10:10:44.235482Z",
     "iopub.status.idle": "2022-05-22T10:10:44.238182Z",
     "shell.execute_reply": "2022-05-22T10:10:44.238674Z"
    },
    "papermill": {
     "duration": 0.017863,
     "end_time": "2022-05-22T10:10:44.238788",
     "exception": false,
     "start_time": "2022-05-22T10:10:44.220925",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# one pirce: https://ww9.readonepiece.com/manga/one-piece-digital-colored-comics/\n",
    "# my hero academia: https://ww7.readmha.com/manga/boku-no-hero-academia-colored/"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.011979,
     "end_time": "2022-05-22T10:10:44.262116",
     "exception": false,
     "start_time": "2022-05-22T10:10:44.250137",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## colored"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-22T10:10:44.292208Z",
     "iopub.status.busy": "2022-05-22T10:10:44.291393Z",
     "iopub.status.idle": "2022-05-22T10:10:45.675709Z",
     "shell.execute_reply": "2022-05-22T10:10:45.674677Z"
    },
    "papermill": {
     "duration": 1.402154,
     "end_time": "2022-05-22T10:10:45.675858",
     "exception": false,
     "start_time": "2022-05-22T10:10:44.273704",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64\n",
      "'https://ww7.readmha.com/chapter/boku-no-hero-academia-colored-chapter-347'\n",
      "'https://ww7.readmha.com/chapter/boku-no-hero-academia-colored-chapter-125'\n"
     ]
    }
   ],
   "source": [
    "html_text = requests.get('https://ww7.readmha.com/manga/boku-no-hero-academia-colored/', 'r').text\n",
    "soup = BeautifulSoup(html_text, 'lxml')\n",
    "\n",
    "cl_chapters = soup.find_all('div', class_ = 'bg-bg-secondary p-3 rounded mb-3 shadow')\n",
    "cl_chapter_site = []\n",
    "for chapter in cl_chapters:\n",
    "    site = chapter.find('a', class_ = 'uppercase text-xs bg-text-success text-white font-bold py-1 px-2 rounded').attrs['href']\n",
    "    cl_chapter_site.append(site)\n",
    "\n",
    "# cl_chapter_site.sort()\n",
    "pprint(len(cl_chapter_site))\n",
    "pprint(cl_chapter_site[0])\n",
    "pprint(cl_chapter_site[-1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.012117,
     "end_time": "2022-05-22T10:10:45.700782",
     "exception": false,
     "start_time": "2022-05-22T10:10:45.688665",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## grayscale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-22T10:10:45.732586Z",
     "iopub.status.busy": "2022-05-22T10:10:45.732060Z",
     "iopub.status.idle": "2022-05-22T10:10:47.395457Z",
     "shell.execute_reply": "2022-05-22T10:10:47.394262Z"
    },
    "papermill": {
     "duration": 1.682763,
     "end_time": "2022-05-22T10:10:47.395591",
     "exception": false,
     "start_time": "2022-05-22T10:10:45.712828",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "372\n",
      "'https://ww7.readmha.com/chapter/boku-no-hero-academia-chapter-354'\n",
      "'https://ww7.readmha.com/chapter/boku-no-hero-academia-chapter-0'\n"
     ]
    }
   ],
   "source": [
    "html_text = requests.get('https://ww7.readmha.com/manga/boku-no-hero-academia/', 'r').text\n",
    "soup = BeautifulSoup(html_text, 'lxml')\n",
    "\n",
    "bw_chapters = soup.find_all('div', class_ = 'bg-bg-secondary p-3 rounded mb-3 shadow')\n",
    "bw_chapter_site = []\n",
    "for chapter in bw_chapters:\n",
    "    site = chapter.find('a', class_ = 'uppercase text-xs bg-text-success text-white font-bold py-1 px-2 rounded').attrs['href']\n",
    "    bw_chapter_site.append(site)\n",
    "\n",
    "# bw_chapter_site.sort()\n",
    "pprint(len(bw_chapter_site))\n",
    "pprint(bw_chapter_site[0])\n",
    "pprint(bw_chapter_site[-1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-22T10:10:47.426818Z",
     "iopub.status.busy": "2022-05-22T10:10:47.426130Z",
     "iopub.status.idle": "2022-05-22T10:10:47.446406Z",
     "shell.execute_reply": "2022-05-22T10:10:47.445840Z"
    },
    "papermill": {
     "duration": 0.03795,
     "end_time": "2022-05-22T10:10:47.446494",
     "exception": false,
     "start_time": "2022-05-22T10:10:47.408544",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "colored, chapter 900 and above: 100%|██████████| 64/64 [00:00<00:00, 47730.34it/s]\n",
      "grayscale, chapter 900 and above: 100%|██████████| 72/72 [00:00<00:00, 345131.30it/s]\n"
     ]
    }
   ],
   "source": [
    "cl_chapters = []\n",
    "bw_chapters = []\n",
    "\n",
    "for i in tqdm(range(len(cl_chapter_site)), 'colored, chapter 900 and above'):\n",
    "    cl_chapters.append(cl_chapter_site[i])\n",
    "        \n",
    "for j in tqdm(range(len(bw_chapter_site) - 300), 'grayscale, chapter 900 and above'):\n",
    "    bw_chapters.append(bw_chapter_site[j])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-22T10:10:47.480199Z",
     "iopub.status.busy": "2022-05-22T10:10:47.479378Z",
     "iopub.status.idle": "2022-05-22T10:10:47.484781Z",
     "shell.execute_reply": "2022-05-22T10:10:47.483948Z"
    },
    "papermill": {
     "duration": 0.024342,
     "end_time": "2022-05-22T10:10:47.484896",
     "exception": false,
     "start_time": "2022-05-22T10:10:47.460554",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "64\n",
      "72\n",
      "['https://ww7.readmha.com/chapter/boku-no-hero-academia-colored-chapter-127',\n",
      " 'https://ww7.readmha.com/chapter/boku-no-hero-academia-colored-chapter-126',\n",
      " 'https://ww7.readmha.com/chapter/boku-no-hero-academia-colored-chapter-125']\n",
      "['https://ww7.readmha.com/chapter/boku-no-hero-academia-chapter-286',\n",
      " 'https://ww7.readmha.com/chapter/boku-no-hero-academia-chapter-285',\n",
      " 'https://ww7.readmha.com/chapter/boku-no-hero-academia-chapter-284']\n"
     ]
    }
   ],
   "source": [
    "pprint(len(cl_chapters))\n",
    "pprint(len(bw_chapters))\n",
    "\n",
    "pprint(cl_chapters[-3:None])\n",
    "pprint(bw_chapters[-3:None])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.014705,
     "end_time": "2022-05-22T10:10:47.514674",
     "exception": false,
     "start_time": "2022-05-22T10:10:47.499969",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Parsing all image urls from each chapter"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.014376,
     "end_time": "2022-05-22T10:10:47.543490",
     "exception": false,
     "start_time": "2022-05-22T10:10:47.529114",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## colored"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-22T10:10:47.579792Z",
     "iopub.status.busy": "2022-05-22T10:10:47.578994Z",
     "iopub.status.idle": "2022-05-22T10:11:49.845699Z",
     "shell.execute_reply": "2022-05-22T10:11:49.845260Z"
    },
    "papermill": {
     "duration": 62.28772,
     "end_time": "2022-05-22T10:11:49.845815",
     "exception": false,
     "start_time": "2022-05-22T10:10:47.558095",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "extracting images: 100%|██████████| 64/64 [01:02<00:00,  1.03it/s]\n"
     ]
    }
   ],
   "source": [
    "cl = []\n",
    "for i in tqdm(range(len(cl_chapters)), \"extracting images\"):\n",
    "# for i in tqdm(range(1), \"extracting images\"):\n",
    "    cl_imgs = []\n",
    "    \n",
    "    html_text = requests.get(cl_chapters[i], 'r').text\n",
    "    soup = BeautifulSoup(html_text, 'lxml')\n",
    "\n",
    "    all_imgs = soup.find_all('div', class_ = 'text-center')\n",
    "    for img in all_imgs:\n",
    "        site = img.find(\"img\", class_ = 'mb-3 mx-auto js-page')\n",
    "        if site is not None:\n",
    "            cl_imgs.append(site['src'])\n",
    "            \n",
    "    cl = cl + cl_imgs[1:None]\n",
    "            \n",
    "#     pprint(cl_imgs)\n",
    "#     pprint(cl)\n",
    "#     raise ValueError(\"done\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-22T10:11:49.916560Z",
     "iopub.status.busy": "2022-05-22T10:11:49.915875Z",
     "iopub.status.idle": "2022-05-22T10:11:49.920609Z",
     "shell.execute_reply": "2022-05-22T10:11:49.919877Z"
    },
    "papermill": {
     "duration": 0.041323,
     "end_time": "2022-05-22T10:11:49.920726",
     "exception": false,
     "start_time": "2022-05-22T10:11:49.879403",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1050\n",
      "https://i.imgur.com/x4BaC0S.jpeg\r\n"
     ]
    }
   ],
   "source": [
    "print(len(cl))\n",
    "print(cl[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.033146,
     "end_time": "2022-05-22T10:11:49.987627",
     "exception": false,
     "start_time": "2022-05-22T10:11:49.954481",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## grayscale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-22T10:11:50.063082Z",
     "iopub.status.busy": "2022-05-22T10:11:50.062531Z",
     "iopub.status.idle": "2022-05-22T10:13:03.749209Z",
     "shell.execute_reply": "2022-05-22T10:13:03.749614Z"
    },
    "papermill": {
     "duration": 73.728218,
     "end_time": "2022-05-22T10:13:03.749744",
     "exception": false,
     "start_time": "2022-05-22T10:11:50.021526",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "extracting images: 100%|██████████| 72/72 [01:13<00:00,  1.02s/it]\n"
     ]
    }
   ],
   "source": [
    "bw = []\n",
    "for i in tqdm(range(len(bw_chapters)), \"extracting images\"):\n",
    "# for i in tqdm(range(1), \"extracting images\"):\n",
    "    bw_imgs = []\n",
    "    \n",
    "    html_text = requests.get(bw_chapters[i], 'r').text\n",
    "    soup = BeautifulSoup(html_text, 'lxml')\n",
    "\n",
    "    all_imgs = soup.find_all('div', class_ = 'text-center')\n",
    "    for img in all_imgs:\n",
    "        site = img.find(\"img\", class_ = 'mb-3 mx-auto js-page')\n",
    "        if site is not None:\n",
    "            bw_imgs.append(site['src'])    \n",
    "            \n",
    "    bw = bw + bw_imgs[1:None]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-22T10:13:03.863195Z",
     "iopub.status.busy": "2022-05-22T10:13:03.862494Z",
     "iopub.status.idle": "2022-05-22T10:13:03.865867Z",
     "shell.execute_reply": "2022-05-22T10:13:03.866480Z"
    },
    "papermill": {
     "duration": 0.062394,
     "end_time": "2022-05-22T10:13:03.866600",
     "exception": false,
     "start_time": "2022-05-22T10:13:03.804206",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1094\n",
      "https://cdn.readmha.com/file/CDN-M-A-N/bnha_353_01.png\n"
     ]
    }
   ],
   "source": [
    "print(len(bw))\n",
    "print(bw[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.054556,
     "end_time": "2022-05-22T10:13:03.975733",
     "exception": false,
     "start_time": "2022-05-22T10:13:03.921177",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "# Download the images"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.055012,
     "end_time": "2022-05-22T10:13:04.085678",
     "exception": false,
     "start_time": "2022-05-22T10:13:04.030666",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## colored"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-22T10:13:04.202093Z",
     "iopub.status.busy": "2022-05-22T10:13:04.201138Z",
     "iopub.status.idle": "2022-05-22T10:48:59.782298Z",
     "shell.execute_reply": "2022-05-22T10:48:59.782792Z"
    },
    "papermill": {
     "duration": 2155.642859,
     "end_time": "2022-05-22T10:48:59.783001",
     "exception": false,
     "start_time": "2022-05-22T10:13:04.140142",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "downloading images: 100%|██████████| 1050/1050 [35:55<00:00,  2.05s/it]\n"
     ]
    }
   ],
   "source": [
    "pathname = './colored'\n",
    "# if path doesn't exist, make that path dir\n",
    "if not os.path.isdir(pathname):\n",
    "    os.makedirs(pathname)\n",
    "    \n",
    "for i in tqdm(range(len(cl)), \"downloading images\"):\n",
    "    url = cl[i].replace('\\r', '')  \n",
    "#     url = 'https://cdn.readonepiece.com/file/mangap/3258/10900000/1.jpeg'\n",
    "    filename = os.path.join(pathname, str(i) + str('.png'))\n",
    "\n",
    "    response = requests.get(url)\n",
    "    if response.ok:    \n",
    "        with open(filename, 'wb') as f:\n",
    "            f.write(requests.get(url).content)\n",
    "    else:\n",
    "        print(response)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 0.345697,
     "end_time": "2022-05-22T10:49:00.478253",
     "exception": false,
     "start_time": "2022-05-22T10:49:00.132556",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "## grayscale"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-05-22T10:49:01.183342Z",
     "iopub.status.busy": "2022-05-22T10:49:01.182317Z",
     "iopub.status.idle": "2022-05-22T10:56:58.529459Z",
     "shell.execute_reply": "2022-05-22T10:56:58.529849Z"
    },
    "papermill": {
     "duration": 477.70401,
     "end_time": "2022-05-22T10:56:58.530068",
     "exception": false,
     "start_time": "2022-05-22T10:49:00.826058",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "downloading images: 100%|██████████| 1094/1094 [07:57<00:00,  2.29it/s]\n"
     ]
    }
   ],
   "source": [
    "pathname = './grayscale'\n",
    "# if path doesn't exist, make that path dir\n",
    "if not os.path.isdir(pathname):\n",
    "    os.makedirs(pathname)\n",
    "    \n",
    "for i in tqdm(range(len(bw)), \"downloading images\"):\n",
    "    url = bw[i].replace('\\r', '')\n",
    "    filename = os.path.join(pathname, str(i) + str('.png'))\n",
    "\n",
    "    response = requests.get(url)\n",
    "    if response.ok:    \n",
    "        with open(filename, 'wb') as f:\n",
    "            f.write(requests.get(url).content)\n",
    "    else:\n",
    "        print(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 0.656139,
     "end_time": "2022-05-22T10:56:59.843920",
     "exception": false,
     "start_time": "2022-05-22T10:56:59.187781",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "papermill": {
   "duration": 2780.217482,
   "end_time": "2022-05-22T10:57:00.607213",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-05-22T10:10:40.389731",
   "version": "2.1.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}