parse_and_build_dataset.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "source": [
    "from bs4 import BeautifulSoup\n",
    "import os\n",
    "import re\n",
    "import shutil"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "source": [
    "def get_category_soup(cat):\n",
    "    root_dir = f'./data/{cat}/www.diablo2.io'\n",
    "    soup = BeautifulSoup(open(f'{root_dir}/{cat}/index.html'), 'html.parser')\n",
    "    return root_dir, soup"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 73,
   "source": [
    "def parse(cat):\n",
    "    root_dir, soup = get_category_soup(cat)\n",
    "    items = soup.find_all('article', class_=\"element-item\")\n",
    "    items = [{\n",
    "        'caption': re.sub(r'(discord\\nMissing v0.3 Data)', '', \\\n",
    "            \"\\n\".join([line.strip() for line in items[0].text.split(\"\\n\") if line.strip() != \"\"][:-7])).strip(),\n",
    "        'image': root_dir+item.find('div', {'data-background-image': True})['data-background-image'],\n",
    "    } for item in items]\n",
    "    return items\n",
    "\n",
    "def write_to_dir(items):\n",
    "    # Build a new folder with all images in the same folder and .txt file pair containing their metadata\n",
    "    new_dir = f'./data/clean'\n",
    "    try:\n",
    "        os.rmdir(new_dir)\n",
    "    except:\n",
    "        pass\n",
    "    os.makedirs(new_dir, exist_ok=True)\n",
    "    for e in items:\n",
    "        new_path = f'{new_dir}/{e[\"image\"].split(\"/\")[-1]}'\n",
    "        shutil.copyfile(e['image'], f'{new_path}')\n",
    "        image_name = e[\"image\"].split(\"/\")[-1]\n",
    "        caption_path = f'{new_dir}/{image_name.split(\".\")[0]}.txt'\n",
    "        with open(caption_path, 'w') as f:\n",
    "            f.write(e['caption'])\n",
    "\n",
    "try: \n",
    "    os.rmdir('./data/clean')\n",
    "except:\n",
    "    pass\n",
    "for e in os.listdir('./data'):\n",
    "    if 'clean' in e: continue\n",
    "    write_to_dir(parse(e))\n"
   ],
   "outputs": [],
   "metadata": {}
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "source": [
    "!zip -r -q data.zip data/clean"
   ],
   "outputs": [],
   "metadata": {}
  }
 ],
 "metadata": {
  "orig_nbformat": 4,
  "language_info": {
   "name": "python",
   "version": "3.9.6",
   "mimetype": "text/x-python",
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "pygments_lexer": "ipython3",
   "nbconvert_exporter": "python",
   "file_extension": ".py"
  },
  "kernelspec": {
   "name": "python3",
   "display_name": "Python 3.9.6 64-bit ('env': venv)"
  },
  "interpreter": {
   "hash": "0ec106e6ffc2f491b439fa307e957472bffcd720914339563c8a31bbd4f1d547"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}