2024_Notebook_4_answers.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyNDPBQ9ck5sTaxlITdRaMdh",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/TurkuNLP/ATP_kurssi/blob/master/2024_Notebook_4_answers.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "ENGAzyUzNtyf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4ce75df7-0098-4851-bc65-e590c5604d43"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Cloning into 'CORE-corpus'...\n",
            "remote: Enumerating objects: 31, done.\u001b[K\n",
            "remote: Counting objects: 100% (23/23), done.\u001b[K\n",
            "remote: Compressing objects: 100% (21/21), done.\u001b[K\n",
            "remote: Total 31 (delta 12), reused 5 (delta 2), pack-reused 8 (from 1)\u001b[K\n",
            "Receiving objects: 100% (31/31), 124.72 MiB | 15.68 MiB/s, done.\n",
            "Resolving deltas: 100% (13/13), done.\n",
            "/content/CORE-corpus\n"
          ]
        }
      ],
      "source": [
        "# Prepare this notebook to correspond to the main one so that this notebook can be run smoothly\n",
        "! git clone https://github.com/TurkuNLP/CORE-corpus.git\n",
        "%cd CORE-corpus/\n",
        "! zcat train.tsv.gz | cut -f 3 | sort | uniq | gzip > cleaned.txt.gz\n",
        "! zcat cleaned.txt.gz  | tr ' ' '\\n' | egrep -v \"^$\" > one-per-line.txt"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BOVdYRF6KCjV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c617ada6-1944-43a3-ba74-67d15ec56e76"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "$17.99\n",
            "A\n",
            "high-end\n",
            "Vic\n",
            "Trafford\n"
          ]
        }
      ],
      "source": [
        "#Check the file contents\n",
        "\n",
        "! cat one-per-line.txt | head -5"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Exercises\n",
        "\n",
        "In the file one-per-line.txt grep lines\n",
        "\n",
        "22. with one capital letter\n",
        "23. with one or more capital letters\n",
        "24. starting with the letter a followed by any character 0 or more times, and ending with ing\n",
        "25. with proper nouns on A in the possessive form\n",
        "26. with words in the form as in wOrd, wORD, woRD, worD"
      ],
      "metadata": {
        "id": "-R2McTewxIDV"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HYFJ6nuxPQkN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ae8a1e30-6380-430f-b42d-17950f0f0df4"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "A\n",
            "A\n",
            "A\n",
            "I\n",
            "A\n"
          ]
        }
      ],
      "source": [
        "#22. One-letter lines in caps\n",
        "\n",
        "! cat one-per-line.txt | egrep \"^[A-Z]$\" | head -5"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "si3xIP_SWLob",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "af579dd8-ac3c-43d3-9e03-8119b4f418e9"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "ADA\n",
            "A\n",
            "ADA\n",
            "QUICK\n",
            "LINKS\n"
          ]
        }
      ],
      "source": [
        "# 23. One or more capital letters per line\n",
        "\n",
        "! cat one-per-line.txt | egrep \"^[A-Z]+$\" | tail -5"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "RBpETUCuWUgc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ecb86245-a4ea-4311-dd23-3e53e77a893c"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "awaiting\n",
            "associating\n",
            "award-winning\n",
            "asking\n",
            "according\n",
            "anything\n",
            "anything\n",
            "anything\n",
            "appealing\n",
            "avoiding\n"
          ]
        }
      ],
      "source": [
        "#24. Lines starting with the letter a, then any character 0 or more times, and ends with ing\n",
        "\n",
        "! cat one-per-line.txt | egrep \"^a.*ing$\" | head"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "V3UkI3A5acXj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "b56543f3-aaaf-4316-8517-9861166b8299"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "America's\n",
            "Abakhan's\n",
            "Adriana's\n",
            "Apple's\n",
            "Allah's\n",
            "Arthur's\n",
            "Airport's\n",
            "Arsene's\n",
            "Arshavin's\n",
            "Arsene's\n"
          ]
        }
      ],
      "source": [
        "#25. Lines with proper nouns starting with A in the possessive form\n",
        "\n",
        "# (i.e. words starting with the capital A followed by any character 0 or more times followed by 's and line end)\n",
        "\n",
        "! cat one-per-line.txt | egrep \"^A.*'s$\" | uniq | head"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_agNhUFga6Ww",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "64ae7982-09a4-4569-c023-fdd37c37e0ca"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "iPad,\n",
            "toiPOds\n",
            "iTunes\n",
            "iOS\n",
            "iPhones\n"
          ]
        }
      ],
      "source": [
        "#26. Lines with words in the form wOrd, wORD, woRD, worD\n",
        "\n",
        "#Lines that start with one or more lower case letter(s) followed by one or more capital letter(s)\n",
        "\n",
        "! cat one-per-line.txt  | egrep \"^[[:lower:]]+[[:upper:]]+\" | head -5"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "## Now try these yourselves\n",
        "\n",
        "**N.B.**\n",
        "\n",
        "`.` stands for any **character**; i.e. not only letters but also e.g. numbers\n",
        "\n",
        "`[a-z]` stands for any lower case **letter**\n",
        "\n",
        "`[A-Z]` stands for any upper case **letter**\n"
      ],
      "metadata": {
        "id": "Ku_woq_85bHX"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# 28. Lines with the word \"cat\" ending with just one punctuation mark\n",
        "! cat one-per-line.txt  | egrep \"^cat[[:punct:]]$\" | head -10"
      ],
      "metadata": {
        "id": "n8qTo_wq7Czg",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2b2c5456-3944-496a-acb6-a96d730f8557"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cat\"\n",
            "cat!\n",
            "cat.\n",
            "cat.\n",
            "cat.\n",
            "cat.\n",
            "cat.\n",
            "cat.\n",
            "cat,\n",
            "cat.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 29. Lines with compound words with a hyphen, e.g. \"co-operate\"\n",
        "! cat one-per-line.txt  | egrep \"^.+-.+$\" | head -10  # this will match also e.g. 123-123\n",
        "! echo '----------------------------------------'\n",
        "! cat one-per-line.txt  | egrep \"^[a-zA-Z]+-[a-zA-Z]+$\" | head -10  # this will only match words that consists of letters, but it even omits words ending in commas"
      ],
      "metadata": {
        "id": "a1_GPHrL7KS4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c4d7e356-e7c8-4318-a28a-ce4995d9ef0e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "high-end\n",
            "Action-Adventure.\n",
            "sword-wielding\n",
            "killer-squads\n",
            "tomb-raider\n",
            "mind-blowing\n",
            "nausea-inducing\n",
            "3-year-old\n",
            "African-Americans\n",
            "African-Americans,\n",
            "----------------------------------------\n",
            "high-end\n",
            "sword-wielding\n",
            "killer-squads\n",
            "tomb-raider\n",
            "mind-blowing\n",
            "nausea-inducing\n",
            "African-Americans\n",
            "African-Americans\n",
            "African-Americans\n",
            "re-development\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 30. Lines that end in full stop\n",
        "! cat one-per-line.txt  | egrep \"^.+\\.$\" | head -10"
      ],
      "metadata": {
        "id": "KOXHMkVD7ns4",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e1da4625-f687-4275-d1ac-53e5d9ae4e5c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Action-Adventure.\n",
            "Australia.\n",
            "notice.\n",
            "Pacific.\n",
            "destruction.\n",
            "park.\n",
            "Antietam.\n",
            "events.\n",
            "activity.\n",
            "Family.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 31. Lines written fully in capitals\n",
        "# (The output might make more sense if you remove 'I')\n",
        "! cat one-per-line.txt  | egrep \"^[[:upper:]]+$\" | egrep -v \"^I$\" | head -10"
      ],
      "metadata": {
        "id": "aWsSmSnp8OPs",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "ce83dfb1-9537-41ab-dcc0-9d8cd659d9da"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "A\n",
            "A\n",
            "PM\n",
            "TVPG\n",
            "A\n",
            "MA\n",
            "NUL\n",
            "A\n",
            "NAACP\n",
            "WWI\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 32. Lines starting with punctuation\n",
        "! cat one-per-line.txt | egrep \"^[[:punct:]]+\" | head -10"
      ],
      "metadata": {
        "id": "ru8NyOZ09gl5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "02d75d4d-a494-4e7b-914f-619d1ba27e10"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "$17.99\n",
            "\"Riveting,\n",
            "_\"\n",
            "#1122\n",
            "$19.99\n",
            "#50\n",
            "\"the\n",
            "\"the\n",
            "\"the\n",
            "(NAACP),\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 33. Can you tell what the difference is between these two lines:\n",
        "! cat one-per-line.txt | egrep \"^[[:punct:]]$\" | tail -10  # finds lines with just one punctuation\n",
        "! echo '----------------------------------'\n",
        "! cat one-per-line.txt | egrep \"^[[:punct:]]+$\" | tail -10  # finds lines with one or more punctuations"
      ],
      "metadata": {
        "id": "vBmblmQ5-BfT",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "18ba6437-e9b7-4d4e-8e4f-45d5b84706bd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            ".\n",
            ".\n",
            "?\n",
            "-\n",
            "-\n",
            "-\n",
            ".\n",
            "?\n",
            "?\n",
            ",\n",
            "----------------------------------\n",
            ".\n",
            "?\n",
            "-\n",
            "-\n",
            "-\n",
            ".\n",
            "?\"\n",
            "?\n",
            "?\n",
            ",\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 34. Can you tell what the difference is between these two lines:\n",
        "! cat one-per-line.txt | egrep \"^.[[:punct:]].[[:punct:]].\" | head -10  # one character + one punct + one character + one punct + one character\n",
        "! echo '-----------------------------------'\n",
        "! cat one-per-line.txt | egrep \"^[a-z][[:punct:]][a-z][[:punct:]][a-z]\" | head -10  # one lower case letter + one punct + one lower case letter + one punct + one ower case letter\n"
      ],
      "metadata": {
        "id": "BNj8DzfBIL2B",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "5e61f5a7-d41d-45c0-ac70-7af708421555"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\"...there\n",
            "!...\"\n",
            "?݃?݃?݃?..Which\n",
            "______________\n",
            "($8),\n",
            "e.t.c.\n",
            "________\n",
            "E.g.,\n",
            "E.g.,\n",
            "8.2%.\n",
            "-----------------------------------\n",
            "e.t.c.\n",
            "a.k.a\n",
            "l.l.a.m.a.s.\n",
            "p.o.q.u.i.t.o.\n",
            "s.o.b.r.e.\n",
            "u.s.t.e.d.e.s.\n",
            "x,y,z'\n",
            "u.s.a\n",
            "w-o-r-s-h\n",
            "a.s.o.\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# 37. Lines with words only (i.e. lines with only letters)\n",
        "! cat one-per-line.txt | egrep '^[a-zA-Z]+$' | head -10"
      ],
      "metadata": {
        "id": "8v4KuVJ-DkgC",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "80532d2c-f2f1-4f7c-ffce-32dc4ee46b24"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "A\n",
            "Vic\n",
            "Trafford\n",
            "And\n",
            "a\n",
            "true\n",
            "is\n",
            "a\n",
            "turbulent\n",
            "time\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "! ls\n"
      ],
      "metadata": {
        "id": "JgDI0kTI0DWl",
        "outputId": "fa94279b-7e28-4ea2-910e-7ee1d0950d7e",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "cleaned.txt.gz\tLICENSE.txt\t  README.md\t\t\t    test.tsv.gz\n",
            "dev.tsv.gz\tone-per-line.txt  register_label_abbreviations.txt  train.tsv.gz\n"
          ]
        }
      ]
    }
  ]
}