lexicon_validation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import csv\n",
    "import validate\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "df = pd.read_csv('ted2020.tsv.gz', sep='\\t', keep_default_na=False, encoding='utf8', quoting=csv.QUOTE_NONE)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "outputs": [],
   "source": [
    "en = df[['talkid', 'en']]\n",
    "ja = df[['talkid', 'ja']]\n",
    "de = df[['talkid', 'de']]\n",
    "ar = df[['talkid', 'ar']]\n",
    "es = df[['talkid', 'es']]\n",
    "pt = df[['talkid', 'pt']]\n",
    "ru = df[['talkid', 'ru']]\n",
    "id = df[['talkid', 'id']]\n",
    "zh = df[['talkid', 'zh-cn']]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "def read_list(filename):\n",
    "    with open(filename, 'rb') as f:\n",
    "        n_list = pickle.load(f, encoding='utf8')\n",
    "        return n_list"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "outputs": [],
   "source": [
    "male_words_en = read_list('wordlist/male_word_file_en')\n",
    "female_words_en = read_list('wordlist/female_word_file_en')\n",
    "male_words_de = read_list('wordlist/male_word_file_de')\n",
    "female_words_de = read_list('wordlist/female_word_file_de')\n",
    "male_words_es = read_list('wordlist/male_word_file_es')\n",
    "female_words_es = read_list('wordlist/female_word_file_es')\n",
    "male_name_de = read_list('wordlist/male_name_file_de')\n",
    "female_name_de = read_list('wordlist/female_name_file_de')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "outputs": [],
   "source": [
    "# source tokens lists needed for validation\n",
    "src_tokens_list = validate.tokenize(en, 'en')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "outputs": [],
   "source": [
    "zh = zh.rename(columns={'zh-cn': 'zh'})"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [
    {
     "data": {
      "text/plain": "(997, 1325, 0.7524528301886793)"
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "male_words_zh = read_list('eval_words/only_male_words_cn')\n",
    "female_words_zh = read_list('eval_words/only_female_words_cn')\n",
    "dest_tokens_list_zh = validate.tokenize(zh, 'zh')\n",
    "validate.validate(src_tokens_list, dest_tokens_list_zh, male_words_en, female_words_en, male_words_zh, female_words_zh)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "outputs": [],
   "source": [
    "# extra words added to German lexicon\n",
    "\n",
    "# male_words_de.extend(male_name_de)\n",
    "# female_words_de.extend(female_name_de)\n",
    "# male_words_de = [word.lower() for word in male_words_de]\n",
    "# female_words_de = [word.lower() for word in female_words_de]\n",
    "male_words_de.append('ihm')\n",
    "male_words_de.append('sich')\n",
    "male_words_de.append('den')\n",
    "female_words_de.append('ihr')\n",
    "female_words_de.append('die')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "outputs": [],
   "source": [
    "dest_tokens_list = validate.tokenize(de, 'de')\n",
    "validate.validate(src_tokens_list, dest_tokens_list, male_words_en, female_words_en, male_words_de, female_words_de)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "outputs": [],
   "source": [
    "dest_tokens_list = validate.tokenize(es, 'es')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "outputs": [],
   "source": [
    "male_words_es.append('él')\n",
    "male_words_es.append('sí')\n",
    "male_words_es.append('lo')\n",
    "male_words_es.append('le')\n",
    "female_words_es.append('la')\n",
    "female_words_es.append('le')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "outputs": [
    {
     "data": {
      "text/plain": "(1125, 1380, 0.8152173913043478)"
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validate.validate(src_tokens_list, dest_tokens_list, male_words_en, female_words_en, male_words_es, female_words_es)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [],
   "source": [
    "male_words_pt = read_list('wordlist/male_word_file_pt')\n",
    "female_words_pt = read_list('wordlist/female_word_file_pt')\n",
    "male_words_pt.append('dele')\n",
    "male_words_pt.append('se')\n",
    "male_words_pt.append('seu')\n",
    "female_words_pt.append('seu')\n",
    "male_words_pt.append('ihe')\n",
    "female_words_pt.append('ihe')\n",
    "female_words_es.append('elas')\n",
    "male_words_pt.append('eles')\n",
    "male_words_pt.append('seus')\n",
    "female_words_pt.append('suas')\n",
    "dest_tokens_list_pt = validate.tokenize(pt, 'pt')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "data": {
      "text/plain": "(928, 1206, 0.7694859038142621)"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "validate.validate(src_tokens_list, dest_tokens_list_pt, male_words_en, female_words_en, male_words_pt, female_words_pt)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "data": {
      "text/plain": "(466, 1288, 0.36180124223602483)"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "male_words_ja = read_list('wordlist/male_word_file_ja')\n",
    "female_words_ja = read_list('wordlist/female_word_file_ja')\n",
    "dest_tokens_list_ja = validate.tokenize(ja, 'ja')\n",
    "validate.validate(src_tokens_list, dest_tokens_list_ja, male_words_en, female_words_en, male_words_ja, female_words_ja)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "outputs": [
    {
     "data": {
      "text/plain": "(583, 1289, 0.45228859581070596)"
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "male_words_ru = read_list('wordlist/male_word_file_ru')\n",
    "female_words_ru = read_list('wordlist/female_word_file_ru')\n",
    "dest_tokens_list_ru = validate.tokenize(ru, 'ru')\n",
    "validate.validate(src_tokens_list, dest_tokens_list_ru, male_words_en, female_words_en, male_words_ru, female_words_ru)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "outputs": [
    {
     "data": {
      "text/plain": "(312, 671, 0.46497764530551416)"
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "male_words_id = read_list('wordlist/male_word_file_id')\n",
    "female_words_id = read_list('wordlist/female_word_file_id')\n",
    "dest_tokens_list_id = validate.tokenize(id, 'id')\n",
    "validate.validate(src_tokens_list, dest_tokens_list_id, male_words_en, female_words_en, male_words_id, female_words_id)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "outputs": [
    {
     "data": {
      "text/plain": "(252, 1327, 0.18990203466465713)"
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "male_words_ar = read_list('wordlist/male_word_file_ar')\n",
    "female_words_ar = read_list('wordlist/female_word_file_ar')\n",
    "dest_tokens_list_ar = validate.tokenize(ar, 'ar')\n",
    "validate.validate(src_tokens_list, dest_tokens_list_ar, male_words_en, female_words_en, male_words_ar, female_words_ar)"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "outputs": [],
   "source": [
    "male = read_list('non_gender_de')"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "outputs": [
    {
     "data": {
      "text/plain": "['Um ehrlich zu sein, verzerrte  ich mich nach Interesse und Hilfe, und fing an, ihm von meinem seltsamen Kommentator zu erzählen.',\n 'Jetzt steht in meiner Patientenakte,  dass Eleanor die Wahnvorstellung hat, eine  Nachrichtensprecherin zu sein.',\n 'Zehn Jahre nachdem die Stimme erstmals auftrat,  machte ich schließlich meinen Abschluss, dieses Mal mit dem höchsten  Bachelorabschluss in Psychologie, den die Universität je vergeben hatte,  und ein Jahr später den höchsten Masterabschluss,  was für eine Verrückte nicht schlecht ist.',\n '(Musik) [\"Oedipus Rex\"] [\"The Lion King\"] [\"Titus\"] [\"Frida\"] [\"The Magic Flute\"] [\"Across The Universe\"] (Applaus) Julie Taymor: Danke.',\n 'Ich könnte also -- Zum Beispiel werde ich Ihnen später  einige \"Spinnenmenschen\" zeigen, unglaubliche Maschinen,  die Menschen fortbewegen.',\n 'Ich habe es immer mit einem  männlichen Prospero inszeniert.',\n 'Ariel: \"Ich kam an Bord des Königlichen Schiffes,  und sezte [...] jede Cajüte in Schreken.\"',\n 'Trinculo: \"Die Noth kan einen Menschen  mit seltsamen Bettgesellen bekannt machen.\"',\n 'Ein einziger Streich soll dich [...]  zum Liebling eines Königs machen.\"',\n 'Jetzt zeige ich Ihnen \"Spider-Man\".',\n 'Spider-Man ist nicht  an die Schwerkraft gebunden.',\n '\"Komm, komm, kümmer dich um die!',\n 'Arabisch ist die Muttersprache.',\n 'Forscher sagen, dass  die Beherrschung anderer Sprachen die Beherrschung der eigenen Muttersprache voraussetzt.',\n 'Die eigene Muttersprache zu beherrschen ist Voraussetzung für den kreativen Ausdruck in anderen Sprachen.',\n 'Kreativität in Ihrer Muttersprache ist der Weg.',\n 'Ich habe also mit diesen Leuten gearbeitet und wir haben eine Maschine gebaut.',\n 'Nennen wir es \"das rhetorische Modell\", bei dem man die Debatte an  das nahe Publikum anpassen muss.',\n 'Shü-mom ist ein Schreibsystem.',\n 'Doch nach einer Weile dachte ich,  dass es nur die Möglichkeit gab, dich hochzunehmen, Derek,  auf die andere Seite des Zimmers zu schubsen und in den zehn Sekunden, die ich hatte,  bevor Derek zurückkam, konnte ich ihm schnell etwas zum Lernen vorspielen.',\n 'Ich habe mich sehr gefreut Goldie Hawn zu sehen, weil ich mich noch bei ihr entschuldigen musste.',\n 'Ich meine, Virgin Brides – was ist denn da passiert?',\n 'Sechs Monate später, oder neun Monate, nachdem das Problem aufgetaucht war, bekam ich diesen entzückenden Brief mit einem Bild des Babys mit der Frage, ob ich Taufpate sein würde, was ich dann auch wurde.',\n 'CA: So, wir werden jetzt noch ein paar Bilder zeigen.',\n 'RB: Niemand nennt mich jemals Sir Richard.',\n 'Manchmal in Amerika. Da höre ich Leute Sir Richard sagen und denke, da wird gerade ein Shakespeare-Stück aufgeführt.']"
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "male"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "outputs": [
    {
     "data": {
      "text/plain": "True"
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'Richard' in male_words_de"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "outputs": [
    {
     "data": {
      "text/plain": "['landsmann',\n 'brüderlich',\n 'zauberer',\n 'mannknecht',\n 'vater',\n 'divo',\n 'schauspieler',\n 'junggeselle',\n 'papa',\n 'herzöge',\n 'barkeeper',\n 'landsmänner',\n 'brautpreis',\n 'gastgeber',\n 'flieger',\n 'andropause',\n 'penis',\n 'prinz',\n 'gouverneure',\n 'abt',\n 'männer',\n 'witwer',\n 'herren',\n 'zauberer',\n 'herr',\n 'bräutigame',\n 'baron',\n 'hausmänner',\n 'götter',\n 'neffe',\n 'witwer',\n 'herr',\n 'bruder',\n 'bräutigame',\n 'priester',\n 'erwachsener',\n 'andrologie',\n 'seine',\n 'marquis',\n 'prinzen',\n 'kaiser',\n 'hengst',\n 'vorsitzender',\n 'kloster',\n 'priester',\n 'jungenschaft',\n 'burschen',\n 'könig',\n 'kerle',\n 'väter',\n 'diener',\n 'sperma',\n 'pressesprecher',\n 'schneider',\n 'cowboys',\n 'kumpel',\n 'junggesellen',\n 'kaiser',\n 'papa',\n 'maskulismus',\n 'jungs',\n 'verzauberer',\n 'kerl',\n 'vaterschaft',\n 'androgene',\n 'kameraleute',\n 'pate',\n 'strongman',\n 'gott',\n 'patriarch',\n 'onkel',\n 'vorsitzender',\n 'herr',\n 'bruderschaft',\n 'gastgeber',\n 'testosteron',\n 'ehemann',\n 'vater',\n 'verwalter',\n 'männchen',\n 'cialis',\n 'sprecher',\n 'papa',\n 'beau',\n 'gestüt',\n 'junggeselle',\n 'zauberer',\n 'herr',\n 'neffen',\n 'gezeugt',\n 'stier',\n 'beaus',\n 'ratsherren',\n 'wirte',\n 'enkelsohn',\n 'verlobte',\n 'stiefväter',\n 'reiter',\n 'großväter',\n 'erwachsener',\n 'schuljunge',\n 'hahn',\n 'enkel',\n 'junggeselle',\n 'kameramann',\n 'väter',\n 'ihn',\n 'meister',\n 'junge',\n 'polizist',\n 'mönch',\n 'schauspieler',\n 'verkäufer',\n 'freund',\n 'stadtrat',\n 'kerl',\n 'staatsmann',\n 'väterlicherseits',\n 'kerl',\n 'vermieter',\n 'geschwister',\n 'gebieter',\n 'burschen',\n 'burschenschaft',\n 'herzog',\n 'balletttänzer',\n 'kerle',\n 'verlobte',\n 'colts',\n 'ehemänner',\n 'freier',\n 'vaterschaft',\n 'er',\n 'geschäftsmann',\n 'masseure',\n 'held',\n 'hirsche',\n 'busboys',\n 'freunde',\n 'könige',\n 'brüder',\n 'meister',\n 'stiefvater',\n 'bräutigame',\n 'sohn',\n 'hengste',\n 'cowboy',\n 'söhne',\n 'bariton',\n 'verkäufer',\n 'paramour',\n 'wirt',\n 'mönche',\n 'dienerschaft',\n 'herr',\n 'schulleiter',\n 'burschen',\n 'abgeordneter',\n 'flieger',\n 'hausmann',\n 'pfarrer',\n 'barkeeper',\n 'barone',\n 'äbte',\n 'handwerker',\n 'bart',\n 'bruderschaften',\n 'stewards',\n 'hengstfohlen ',\n 'zar',\n 'stiefsöhne',\n 'jungen',\n 'löwen',\n 'herr',\n 'penis',\n 'seine',\n 'masseur',\n 'stiere',\n 'onkel',\n 'kerl',\n 'bärte',\n 'ehemann',\n 'löwe',\n 'zauberer',\n 'macho',\n 'vater',\n 'schwule',\n 'männlich',\n 'kellner',\n 'sperma',\n 'prostata',\n 'stiefsohn',\n 'prostata ultrikel',\n 'geschäftsleute',\n 'erbe',\n 'kellner',\n 'schuldirektor',\n 'mann',\n 'gouverneur',\n 'gott',\n 'bräutigam',\n 'großvater',\n 'bräutigam',\n 'kumpel',\n 'schwul',\n 'herren',\n 'junge',\n 'großvater',\n 'wallach',\n 'vaterschaft',\n 'hähne',\n 'prostata ultrikel',\n 'priester',\n 'dienerschaft',\n 'landstreicher',\n 'busjunge',\n 'helden',\n 'pagen',\n 'friseur',\n 'hotelpage',\n 'selbst',\n 'bill',\n 'andrew',\n 'jerome',\n 'markus',\n 'bob',\n 'joe',\n 'brayden',\n 'jerry',\n 'john',\n 'sidney',\n 'adam',\n 'ron',\n 'robert',\n 'eric',\n 'tommy',\n 'ben',\n 'jeremy',\n 'mohammad',\n 'carl',\n 'chang',\n 'tom',\n 'gary',\n 'treyvone',\n 'greuze',\n 'jose',\n 'denny',\n 'michael',\n 'paul',\n 'rocco',\n 'mike',\n 'simon',\n 'silas',\n 'kurt',\n 'nick',\n 'matt',\n 'jorge',\n 'chan',\n 'jack',\n 'reagan',\n 'carter',\n 'aaron',\n 'tim',\n 'josh',\n 'jafari',\n 'cal',\n 'steve',\n 'jimmy',\n 'david',\n 'craig',\n 'wright',\n 'danny',\n 'prescott',\n 'chen',\n 'patrick',\n 'alfred',\n 'deshawn',\n 'james',\n 'joxer',\n 'biff',\n 'dave',\n 'charlie',\n 'terry',\n 'jamel',\n 'thomas',\n 'kyle',\n 'jones',\n 'peter',\n 'raymond',\n 'ed',\n 'dan',\n 'juan',\n 'george',\n 'lebron',\n 'johnny',\n 'jeff',\n 'andy',\n 'jim',\n 'benjamin',\n 'edward',\n 'benny',\n 'shawn',\n 'charles',\n 'richard',\n 'ryan',\n 'lee',\n 'gabriel',\n 'brad',\n \"ca'daan\",\n 'jon',\n 'bill',\n 'andrew',\n 'jerome',\n 'markus',\n 'bob',\n 'joe',\n 'brayden',\n 'jerry',\n 'john',\n 'sidney',\n 'adam',\n 'ron',\n 'robert',\n 'eric',\n 'tommy',\n 'ben',\n 'jeremy',\n 'mohammad',\n 'carl',\n 'chang',\n 'tom',\n 'gary',\n 'treyvone',\n 'greuze',\n 'jose',\n 'denny',\n 'michael',\n 'paul',\n 'rocco',\n 'mike',\n 'simon',\n 'silas',\n 'kurt',\n 'nick',\n 'matt',\n 'jorge',\n 'chan',\n 'jack',\n 'reagan',\n 'carter',\n 'aaron',\n 'tim',\n 'josh',\n 'jafari',\n 'cal',\n 'steve',\n 'jimmy',\n 'david',\n 'craig',\n 'wright',\n 'danny',\n 'prescott',\n 'chen',\n 'patrick',\n 'alfred',\n 'deshawn',\n 'james',\n 'joxer',\n 'biff',\n 'dave',\n 'charlie',\n 'terry',\n 'jamel',\n 'thomas',\n 'kyle',\n 'jones',\n 'peter',\n 'raymond',\n 'ed',\n 'dan',\n 'juan',\n 'george',\n 'lebron',\n 'johnny',\n 'jeff',\n 'andy',\n 'jim',\n 'benjamin',\n 'edward',\n 'benny',\n 'shawn',\n 'charles',\n 'richard',\n 'ryan',\n 'lee',\n 'gabriel',\n 'brad',\n \"ca'daan\",\n 'jon']"
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "male_words_de"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "outputs": [
    {
     "data": {
      "text/plain": "False"
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'den' in male_words_de"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "outputs": [
    {
     "data": {
      "text/plain": "False"
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'die' in female_words_de"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "outputs": [
    {
     "data": {
      "text/plain": "False"
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'seus' in male_words_pt"
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}