merge_txt_files.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Take all files from the /datasets folder and merge them into one file\n",
    "\n",
    "import os\n",
    "import glob\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "goethe_works = []\n",
    "for fname in glob.glob('source_datasets/*.txt'):\n",
    "    with open(fname, 'r', encoding='utf8') as infile:\n",
    "        for line in infile:\n",
    "            goethe_works.append(line)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Prometheus. Dramatisches Fragment. 1773.\\n\\n\\nErster Act.\\nPrometheus. Merkur.\\nPrometheus.\\nIch will nicht, sag' es ihnen!\\nUnd kurz und gut, ich will nicht!\\nIhr Wille gegen meinen!\\nEins gegen eins,\\n\""
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "''.join(goethe_works[0:10])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Clean the text by removing unnecessary characters and altering the format of words.\n",
    "goethe_string = ''.join(goethe_works)\n",
    "# goethe_string_clean = goethe_string.replace('\\n\\n', '\\n').replace('\\n \\n', '')\n",
    "# Remove all \\n that come before '.' or '!' or '?'\n",
    "goethe_string_clean = goethe_string.replace('\\n.', '.').replace('\\n!', '!').replace('\\n?', '?')\n",
    "# Trim all leading and trailing spaces\n",
    "goethe_string_clean = goethe_string_clean.strip()\n",
    "# Remove all linebreaks that are within a sentence\n",
    "goethe_string_clean = re.sub(r\"(?<=[^.!?])[\\r\\n]+(?=[^.!?])\", \" \", goethe_string_clean)\n",
    "# Remove all leading spaces after a line break\n",
    "goethe_string_clean = re.sub(r\"(?<=[\\r\\n]) +\", \"\", goethe_string_clean)\n",
    "# Remove all duplicate spaces\n",
    "goethe_string_clean = re.sub(r\" +\", \" \", goethe_string_clean)\n",
    "# Remove all commas and periods before \")\"\n",
    "goethe_string_clean = re.sub(r\",\\)\", \")\", goethe_string_clean)\n",
    "goethe_string_clean = re.sub(r\"\\.\\)\", \")\", goethe_string_clean)\n",
    "# Remove all characters in the following list\n",
    "chars_to_remove = [\"¬\", \"½\", \"¾\", \" \"]\n",
    "goethe_string_clean = ''.join([i for i in goethe_string_clean if not i in chars_to_remove])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Prometheus. Dramatisches Fragment. 1773.\\nErster Act.\\nPrometheus. Merkur.\\nPrometheus.\\nIch will nicht, sag' es ihnen!\\nUnd kurz und gut, ich will nicht!\\nIhr Wille gegen meinen!\\nEins gegen eins, Mich duen\""
      ]
     },
     "execution_count": 61,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "goethe_string_clean[:200]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Save the cleaned text to a file\n",
    "with open('tiny-goethe.txt', 'w', encoding='utf8') as outfile:\n",
    "    outfile.write(goethe_string_clean)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "CS330_project",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "d1431fde99e0ebc468d53b7b8f653cc219b31c336519b7c4e31df4c6588d9b00"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}