0Data-Analysis.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "text = \"the prior reviewer need a little schooling in widescreen. the 1st . \\ntrue widescreen film be the robe in Cinemascope 1953 and follow by how to marry a Millionaire both by 20th Century Fox who own the process . not the king and I. \\nnot either use cinemascope or use one of they own . quite possibly disney could give this film a matting since the industry . also , disney do produce 2 film in widescreen Lady & The Tramp and sleep Beauty Cinemascope and Techniram . \\nboth beautifully do on dvd and vh . so please get you fact straight when make string stetement about other comment .\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "docs = text.split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['the prior reviewer need a little schooling in widescreen. the 1st . ', 'true widescreen film be the robe in Cinemascope 1953 and follow by how to marry a Millionaire both by 20th Century Fox who own the process . not the king and I. ', 'not either use cinemascope or use one of they own . quite possibly disney could give this film a matting since the industry . also , disney do produce 2 film in widescreen Lady & The Tramp and sleep Beauty Cinemascope and Techniram . ', 'both beautifully do on dvd and vh . so please get you fact straight when make string stetement about other comment .']\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print(docs)\n",
    "len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# finding total number of sentences in a doc\n",
    "\n",
    "count_sentences = 0\n",
    "for i in docs:\n",
    "    for char in i:\n",
    "        if char == '.':\n",
    "            count_sentences += 1\n",
    "            \n",
    "\n",
    "count_sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "9"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# finding total number of sentences in a doc wtih find()\n",
    "\n",
    "count_sentences = 0\n",
    "for i in docs:\n",
    "    index = 0\n",
    "    while index != -1:\n",
    "        index = i.find('.', index+1, len(i))\n",
    "        count_sentences += 1\n",
    "    count_sentences -= 1\n",
    "        \n",
    "count_sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "113"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#  finding number of words in docs\n",
    "\n",
    "count_words = 0\n",
    "for i in docs:\n",
    "    words = i.split(' ')\n",
    "    count_words += len(words)\n",
    "    \n",
    "count_words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "# building dictionary/vocabulary --- unique words\n",
    "\n",
    "vocab = []\n",
    "for i in docs:\n",
    "    words = i.split(' ')\n",
    "    for word in words:\n",
    "        if word not in vocab:\n",
    "            vocab.append(word)\n",
    "            \n",
    "# vocab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "26.318181818181817\n"
     ]
    }
   ],
   "source": [
    "# ratio of characters per word\n",
    "\n",
    "count_chars = 0\n",
    "for i in docs:\n",
    "    count_chars += len(i)\n",
    "    \n",
    "count_chars\n",
    "\n",
    "print(count_chars/len(words))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2.25"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#average sentences per document\n",
    "\n",
    "count_sentences / len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12.555555555555555"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#average words per sentences\n",
    "\n",
    "count_words / count_sentences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "28.25"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# average words per document\n",
    "\n",
    "count_words / len(docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.3950617283950617"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ratio of words per unique words(vocab)\n",
    "\n",
    "count_words / len(vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "5.123893805309734"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# average of chars per word\n",
    "\n",
    "count_chars / count_words"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.3 Most Frequent and Most Infrequent Tokens/Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "wordFreq = {}  #word: key, freq: value\n",
    "\n",
    "for i in docs:\n",
    "    for word in i.split(' '):\n",
    "        if word in wordFreq.keys():\n",
    "            wordFreq[word] += 1\n",
    "        else:\n",
    "            wordFreq[word] = 1\n",
    "            \n",
    "# print(wordFreq)\n",
    "            \n",
    "minFreq = min(wordFreq.values())\n",
    "maxFreq = max(wordFreq.values())\n",
    "\n",
    "minFreqWords = []\n",
    "maxFreqWords = []\n",
    "\n",
    "for key in wordFreq.keys():\n",
    "    if wordFreq[key] == minFreq:\n",
    "        minFreqWords.append(key)\n",
    "    elif wordFreq[key] == maxFreq:\n",
    "        maxFreqWords.append(key)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1 ['prior', 'reviewer', 'need', 'little', 'schooling', 'widescreen.', '1st', 'true', 'be', 'robe', '1953', 'follow', 'how', 'to', 'marry', 'Millionaire', '20th', 'Century', 'Fox', 'who', 'process', 'king', 'I.', 'either', 'cinemascope', 'or', 'one', 'of', 'they', 'quite', 'possibly', 'could', 'give', 'this', 'matting', 'since', 'industry', 'also', ',', 'produce', '2', 'Lady', '&', 'The', 'Tramp', 'sleep', 'Beauty', 'Techniram', 'beautifully', 'on', 'dvd', 'vh', 'so', 'please', 'get', 'you', 'fact', 'straight', 'when', 'make', 'string', 'stetement', 'about', 'other', 'comment']\n",
      "7 ['.']\n"
     ]
    }
   ],
   "source": [
    "print(minFreq, minFreqWords)\n",
    "print(maxFreq, maxFreqWords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# wordFreq.keys()\n",
    "# wordFreq.values()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2.4 Longest and Shortest Tokens/Words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['widescreen.', 'Cinemascope', 'Millionaire', 'cinemascope', 'beautifully']\n",
      "['be', 'I.', 'or', 'of', 'do', 'on', 'vh', 'so']\n"
     ]
    }
   ],
   "source": [
    "longest_word_length = len('')\n",
    "shortest_word_length = len(vocab[0])\n",
    "\n",
    "longest_word = []\n",
    "shortest_word = []\n",
    "\n",
    "stopWords = ['the','a','in','.','','and','by','how','to','&',',','2']\n",
    "\n",
    "for i in vocab:\n",
    "    if len(i) > longest_word_length and i not in stopWords:\n",
    "        longest_word_length = len(i)\n",
    "    if len(i) < shortest_word_length and i not in stopWords:\n",
    "        shortest_word_length = len(i)\n",
    "        \n",
    "for i in vocab:\n",
    "    if len(i) == shortest_word_length and i not in stopWords:\n",
    "        shortest_word.append(i)\n",
    "    if len(i) == longest_word_length and i not in stopWords:\n",
    "        longest_word.append(i)\n",
    "        \n",
    "print(longest_word)\n",
    "print(shortest_word)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Tasks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### second most frequent token and second most infrequent token"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'the': 6, 'prior': 1, 'reviewer': 1, 'need': 1, 'a': 3, 'little': 1, 'schooling': 1, 'in': 3, 'widescreen.': 1, '1st': 1, '.': 7, '': 3, 'true': 1, 'widescreen': 2, 'film': 3, 'be': 1, 'robe': 1, 'Cinemascope': 2, '1953': 1, 'and': 5, 'follow': 1, 'by': 2, 'how': 1, 'to': 1, 'marry': 1, 'Millionaire': 1, 'both': 2, '20th': 1, 'Century': 1, 'Fox': 1, 'who': 1, 'own': 2, 'process': 1, 'not': 2, 'king': 1, 'I.': 1, 'either': 1, 'use': 2, 'cinemascope': 1, 'or': 1, 'one': 1, 'of': 1, 'they': 1, 'quite': 1, 'possibly': 1, 'disney': 2, 'could': 1, 'give': 1, 'this': 1, 'matting': 1, 'since': 1, 'industry': 1, 'also': 1, ',': 1, 'do': 2, 'produce': 1, '2': 1, 'Lady': 1, '&': 1, 'The': 1, 'Tramp': 1, 'sleep': 1, 'Beauty': 1, 'Techniram': 1, 'beautifully': 1, 'on': 1, 'dvd': 1, 'vh': 1, 'so': 1, 'please': 1, 'get': 1, 'you': 1, 'fact': 1, 'straight': 1, 'when': 1, 'make': 1, 'string': 1, 'stetement': 1, 'about': 1, 'other': 1, 'comment': 1}\n",
      "-------------------------------------------\n",
      "['the'] 6\n",
      "['widescreen', 'Cinemascope', 'by', 'both', 'own', 'not', 'use', 'disney', 'do'] 2\n"
     ]
    }
   ],
   "source": [
    "wordFreq = {}  #word: key, freq: value\n",
    "\n",
    "for i in docs:\n",
    "    for word in i.split(' '):\n",
    "        if word in wordFreq.keys():\n",
    "            wordFreq[word] += 1\n",
    "        else:\n",
    "            wordFreq[word] = 1\n",
    "                        \n",
    "minFreq = min(wordFreq.values())  #1\n",
    "maxFreq = max(wordFreq.values())  #7\n",
    "\n",
    "print(wordFreq)\n",
    "print('-------------------------------------------')\n",
    "\n",
    "#################\n",
    "\n",
    "values = []\n",
    "\n",
    "for i in wordFreq.values():\n",
    "    if i in values:\n",
    "        continue\n",
    "    else:\n",
    "        values.append(i)\n",
    "\n",
    "#################\n",
    "\n",
    "minFreqWords = []\n",
    "maxFreqWords = []\n",
    "\n",
    "maxFreq2 = 0\n",
    "\n",
    "for i in vocab:\n",
    "    if wordFreq[i] > maxFreq2 and wordFreq[i] < maxFreq:\n",
    "        maxFreq2 = wordFreq[i]\n",
    "        maxFreqWords.append(i)\n",
    "        \n",
    "for i in vocab:\n",
    "    values.sort(reverse=True)\n",
    "    if wordFreq[i] == values[len(values)-2]:\n",
    "        minFreqWords.append(i)\n",
    "        \n",
    "print(maxFreqWords, maxFreq2)\n",
    "print(minFreqWords, values[len(values)-2])\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Shortest and Longest Document by number of Tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'the prior reviewer need a little schooling in widescreen. the 1st . ': 12, 'true widescreen film be the robe in Cinemascope 1953 and follow by how to marry a Millionaire both by 20th Century Fox who own the process . not the king and I. ': 32, 'not either use cinemascope or use one of they own . quite possibly disney could give this film a matting since the industry . also , disney do produce 2 film in widescreen Lady & The Tramp and sleep Beauty Cinemascope and Techniram . ': 44, 'both beautifully do on dvd and vh . so please get you fact straight when make string stetement about other comment .': 22}\n",
      "------------------------------------\n",
      "['the prior reviewer need a little schooling in widescreen. the 1st . '] 12\n",
      "['not either use cinemascope or use one of they own . quite possibly disney could give this film a matting since the industry . also , disney do produce 2 film in widescreen Lady & The Tramp and sleep Beauty Cinemascope and Techniram . '] 44\n"
     ]
    }
   ],
   "source": [
    "tokens = []\n",
    "dic = {}\n",
    "val = float('inf')\n",
    "\n",
    "for i in docs:\n",
    "    dic[i] = 0\n",
    "\n",
    "for j in docs:\n",
    "    dic[j] = len(j.strip().split(' '))\n",
    "    \n",
    "print(dic)\n",
    "print('------------------------------------')\n",
    "###############\n",
    "\n",
    "values = []\n",
    "for v in dic.values():\n",
    "    if v in values:\n",
    "        continue\n",
    "    else:\n",
    "        values.append(v)\n",
    "        \n",
    "# print(values)\n",
    "\n",
    "###############\n",
    "shortest_doc = []\n",
    "longest_doc = []\n",
    "\n",
    "for k in dic.keys():\n",
    "    \n",
    "    if dic[k] == min(values):\n",
    "        shortest_doc.append(k)\n",
    "        \n",
    "    elif dic[k] == max(values):\n",
    "        longest_doc.append(k)\n",
    "        \n",
    "print(shortest_doc, min(values))\n",
    "print(longest_doc, max(values))\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Shortest and Longest Document by number of characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'the prior reviewer need a little schooling in widescreen. the 1st . ': 68, 'true widescreen film be the robe in Cinemascope 1953 and follow by how to marry a Millionaire both by 20th Century Fox who own the process . not the king and I. ': 161, 'not either use cinemascope or use one of they own . quite possibly disney could give this film a matting since the industry . also , disney do produce 2 film in widescreen Lady & The Tramp and sleep Beauty Cinemascope and Techniram . ': 234, 'both beautifully do on dvd and vh . so please get you fact straight when make string stetement about other comment .': 116}\n",
      "------------------------------------\n",
      "['the prior reviewer need a little schooling in widescreen. the 1st . '] 68\n",
      "['not either use cinemascope or use one of they own . quite possibly disney could give this film a matting since the industry . also , disney do produce 2 film in widescreen Lady & The Tramp and sleep Beauty Cinemascope and Techniram . '] 234\n"
     ]
    }
   ],
   "source": [
    "chars = []\n",
    "count = 0\n",
    "dic = {}\n",
    "val = float('inf')\n",
    "\n",
    "for i in docs:\n",
    "    for j in i:\n",
    "        count += 1\n",
    "    dic[i] = count\n",
    "    count = 0\n",
    "\n",
    "print(dic)\n",
    "print('------------------------------------')\n",
    "###############\n",
    "\n",
    "values = []\n",
    "for v in dic.values():\n",
    "    if v in values:\n",
    "        continue\n",
    "    else:\n",
    "        values.append(v)\n",
    "        \n",
    "# print(values)\n",
    "\n",
    "###############\n",
    "shortest_doc = []\n",
    "longest_doc = []\n",
    "\n",
    "for k in dic.keys():\n",
    "    \n",
    "    if dic[k] == min(values):\n",
    "        shortest_doc.append(k)\n",
    "        \n",
    "    elif dic[k] == max(values):\n",
    "        longest_doc.append(k)\n",
    "        \n",
    "print(shortest_doc, min(values))\n",
    "print(longest_doc, max(values))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}