assn3_s2s.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Neural Machine Translation\n",
    "\n",
    "**Reference:** Sutskever, Ilya, Oriol Vinyals, and Quoc V. Le. \"Sequence to sequence learning with neural networks.\" In Advances in neural information processing systems, pp. 3104-3112. 2014. ([Paper](https://papers.nips.cc/paper/5346-sequence-to-sequence-learning-with-neural-networks), [Sample code](https://github.com/tensorflow/nmt))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m registry at `~/.julia/registries/General`\n",
      "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m git-repo `https://github.com/JuliaRegistries/General.git`\n",
      "\u001b[?25l\u001b[2K\u001b[?25h\u001b[32m\u001b[1m Resolving\u001b[22m\u001b[39m package versions...\n",
      "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `~/.julia/environments/v1.2/Project.toml`\n",
      "\u001b[90m [no changes]\u001b[39m\n",
      "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `~/.julia/environments/v1.2/Manifest.toml`\n",
      "\u001b[90m [no changes]\u001b[39m\n"
     ]
    }
   ],
   "source": [
    "import Pkg; using Pkg; Pkg.add(\"Knet\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[32m\u001b[1m Resolving\u001b[22m\u001b[39m package versions...\n",
      "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `~/.julia/environments/v1.2/Project.toml`\n",
      "\u001b[90m [no changes]\u001b[39m\n",
      "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `~/.julia/environments/v1.2/Manifest.toml`\n",
      "\u001b[90m [no changes]\u001b[39m\n",
      "\u001b[32m\u001b[1m Resolving\u001b[22m\u001b[39m package versions...\n",
      "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `~/.julia/environments/v1.2/Project.toml`\n",
      "\u001b[90m [no changes]\u001b[39m\n",
      "\u001b[32m\u001b[1m  Updating\u001b[22m\u001b[39m `~/.julia/environments/v1.2/Manifest.toml`\n",
      "\u001b[90m [no changes]\u001b[39m\n"
     ]
    }
   ],
   "source": [
    "Pkg.add(\"IterTools\"); Pkg.add(\"AutoGrad\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "@size (macro with 1 method)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "using Knet, Test, Base.Iterators, IterTools, Random # , LinearAlgebra, StatsBase\n",
    "using AutoGrad: @gcheck  # to check gradients, use with Float64\n",
    "#Knet.atype() = KnetArray{Float32}  # determines what Knet.param() uses.\n",
    "macro size(z, s); esc(:(@assert (size($z) == $s) string(summary($z),!=,$s))); end # for debugging"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part -1. Types from the last project\n",
    "\n",
    "Please copy the following types and related functions from the last project: `Vocab`,\n",
    "`TextReader`, `Embed`, `Linear`, `mask!`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "mask! (generic function with 1 method)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "struct Vocab\n",
    "    w2i::Dict{String,Int}\n",
    "    i2w::Vector{String}\n",
    "    unk::Int\n",
    "    eos::Int\n",
    "    tokenizer\n",
    "end\n",
    "\n",
    "function Vocab(file::String; tokenizer=split, vocabsize=Inf, mincount=1, unk=\"<unk>\", eos=\"<s>\")\n",
    "    M = 100000\n",
    "    wdict = Dict()\n",
    "    wcount = Dict()\n",
    "    w2i(x) = get!(wdict, x, 1+length(wdict))\n",
    "    w2c(key) = haskey(wcount, key) ? wcount[key] = wcount[key] + 1 : get!(wcount, key, 1)\n",
    "    wcount[unk] = M; wcount[eos] = M\n",
    "    i2w = []; \n",
    "\n",
    "    \n",
    "    for line in eachline(file)\n",
    "        words = tokenizer(line)\n",
    "        w2c.(words)\n",
    "    end\n",
    "    \n",
    "    sortedcount = sort(collect(wcount), by=x->x[2])\n",
    "    words = sortedcount[findfirst(x-> x[2]>=mincount, sortedcount):length(sortedcount)]\n",
    "    \n",
    "    #vocabsize excludes unk & eos\n",
    "    if(length(words) > vocabsize)\n",
    "        words = words[length(words) - vocabsize + 1 : length(words)]\n",
    "    end\n",
    "\n",
    "    map(x-> w2i(x[1]) , words)\n",
    "    map(x-> push!(i2w, x[1]), words)\n",
    "    \n",
    "    Vocab(wdict, i2w, wdict[unk], wdict[eos], tokenizer)\n",
    "end\n",
    "#=\n",
    "function Vocab(file::String; tokenizer=split, vocabsize=Inf, mincount=1, unk=\"<unk>\", eos=\"<s>\")\n",
    "    M = 1000000000000000\n",
    "    wdict = Dict()\n",
    "    wcount = Dict()\n",
    "    w2i(x) = get!(wdict, x, 1+length(wdict))\n",
    "    w2c(key) = haskey(wcount, key) ? wcount[key] = wcount[key] + 1 : get!(wcount, key, 1)\n",
    "    wcount[eos] = M; wcount[unk] = M - 1\n",
    "    i2w = []; \n",
    "\n",
    "    \n",
    "    for line in eachline(file)\n",
    "        words = tokenizer(line)\n",
    "        w2c.(words)\n",
    "    end\n",
    "    \n",
    "    sortedcount = sort(collect(wcount), by=x->x[2])\n",
    "    words = sortedcount[findfirst(x-> x[2]>=mincount, sortedcount):length(sortedcount)]\n",
    "    \n",
    "    #vocabsize excludes unk & eos\n",
    "    if(length(words) > vocabsize)\n",
    "        words = words[length(words) - vocabsize + 1 : length(words)]\n",
    "    end\n",
    "    \n",
    "    words = reverse(words)\n",
    "\n",
    "    map(x-> w2i(x[1]) , words)\n",
    "    map(x-> push!(i2w, x[1]), words)\n",
    "    \n",
    "    Vocab(wdict, i2w, wdict[unk], wdict[eos], tokenizer)\n",
    "end\n",
    "=#\n",
    "\n",
    "struct TextReader\n",
    "    file::String\n",
    "    vocab::Vocab\n",
    "end\n",
    "\n",
    "function Base.iterate(r::TextReader, s=nothing)\n",
    "    w2i(x) = get(r.vocab.w2i, x, r.vocab.unk)\n",
    "    if (s === nothing) \n",
    "        s = open(r.file, \"r\")\n",
    "    end\n",
    "\n",
    "    if eof(s) \n",
    "        close(s)\n",
    "        return nothing\n",
    "    \n",
    "    else\n",
    "        tmp = readline(s)\n",
    "        line = r.vocab.tokenizer(tmp)\n",
    "        words = w2i.(line) \n",
    "        return words, s\n",
    "    end    \n",
    "end\n",
    "\n",
    "Base.IteratorSize(::Type{TextReader}) = Base.SizeUnknown()\n",
    "Base.IteratorEltype(::Type{TextReader}) = Base.HasEltype()\n",
    "Base.eltype(::Type{TextReader}) = Vector{Int}\n",
    "\n",
    "struct Embed; w; end\n",
    "\n",
    "function Embed(vocabsize::Int, embedsize::Int)\n",
    "    Embed(param(embedsize, vocabsize))\n",
    "end\n",
    "\n",
    "function (l::Embed)(x)\n",
    "    embedsz, vocabsz = size(l.w)\n",
    "    tmparr = [embedsz]\n",
    "    for dim in size(x)\n",
    "        push!(tmparr, dim)\n",
    "    end\n",
    "    reshape(l.w[:,collect(flatten(x))], tuple(tmparr...))\n",
    "end\n",
    "\n",
    "struct Linear; w; b; end\n",
    "\n",
    "function Linear(inputsize::Int, outputsize::Int)\n",
    "    w = param(outputsize, inputsize)\n",
    "    b = param0(outputsize)\n",
    "    Linear(w,b)\n",
    "end\n",
    "\n",
    "function (l::Linear)(x)\n",
    "    l.w * x .+ l.b\n",
    "end\n",
    "\n",
    "function mask!(a,pad)\n",
    "    x,y = size(a)\n",
    "    \n",
    "    for i = 1:x\n",
    "        tmp_mem = []\n",
    "        isfirst = true\n",
    "        for j = 1:y\n",
    "            if a[i, j] == pad\n",
    "                \n",
    "                if isfirst\n",
    "                    isfirst = false\n",
    "                else\n",
    "                    push!(tmp_mem, j)\n",
    "                end\n",
    "            else\n",
    "                isfirst = true\n",
    "                tmp_mem = []\n",
    "            end\n",
    "        end\n",
    "        tmp_mem = convert(Array{Int,1}, tmp_mem)\n",
    "        a[i, tmp_mem] .= 0\n",
    "    end\n",
    "    return a\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 0. Load data\n",
    "\n",
    "We will use the Turkish-English pair from the [TED Talks Dataset](https://github.com/neulab/word-embeddings-for-nmt) for our experiments."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: Testing data\n",
      "└ @ Main In[5]:18\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m\u001b[1mTest Passed\u001b[22m\u001b[39m"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "datadir = \"datasets/tr_to_en\"\n",
    "\n",
    "if !isdir(datadir)\n",
    "    download(\"http://www.phontron.com/data/qi18naacl-dataset.tar.gz\", \"qi18naacl-dataset.tar.gz\")\n",
    "    run(`tar xzf qi18naacl-dataset.tar.gz`)\n",
    "end\n",
    "\n",
    "\n",
    "if !isdefined(Main, :tr_vocab)\n",
    "    tr_vocab = Vocab(\"$datadir/tr.train\", mincount=5)\n",
    "    en_vocab = Vocab(\"$datadir/en.train\", mincount=5)\n",
    "    tr_train = TextReader(\"$datadir/tr.train\", tr_vocab)\n",
    "    en_train = TextReader(\"$datadir/en.train\", en_vocab)\n",
    "    tr_dev = TextReader(\"$datadir/tr.dev\", tr_vocab)\n",
    "    en_dev = TextReader(\"$datadir/en.dev\", en_vocab)\n",
    "    tr_test = TextReader(\"$datadir/tr.test\", tr_vocab)\n",
    "    en_test = TextReader(\"$datadir/en.test\", en_vocab)\n",
    "    @info \"Testing data\"\n",
    "    @test length(tr_vocab.i2w) == 38126\n",
    "    @test length(first(tr_test)) == 16\n",
    "    @test length(collect(tr_test)) == 5029\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 1. Minibatching\n",
    "\n",
    "For minibatching we are going to design a new iterator: `MTData`. This iterator is built\n",
    "on top of two TextReaders `src` and `tgt` that produce parallel sentences for source and\n",
    "target languages."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "struct MTData\n",
    "    src::TextReader        # reader for source language data\n",
    "    tgt::TextReader        # reader for target language data\n",
    "    batchsize::Int         # desired batch size\n",
    "    maxlength::Int         # skip if source sentence above maxlength\n",
    "    batchmajor::Bool       # batch dims (B,T) if batchmajor=false (default) or (T,B) if true.\n",
    "    bucketwidth::Int       # batch sentences with length within bucketwidth of each other\n",
    "    buckets::Vector        # sentences collected in separate arrays called buckets for each length range\n",
    "    batchmaker::Function   # function that turns a bucket into a batch.\n",
    "end\n",
    "\n",
    "#batchsize 128\n",
    "function MTData(src::TextReader, tgt::TextReader; batchmaker = arraybatch, batchsize = 128, maxlength = typemax(Int),\n",
    "                batchmajor = false, bucketwidth = 10, numbuckets = min(128, maxlength ÷ bucketwidth))\n",
    "    buckets = [ [] for i in 1:numbuckets ] # buckets[i] is an array of sentence pairs with similar length\n",
    "    MTData(src, tgt, batchsize, maxlength, batchmajor, bucketwidth, buckets, batchmaker)\n",
    "end\n",
    "\n",
    "Base.IteratorSize(::Type{MTData}) = Base.SizeUnknown()\n",
    "Base.IteratorEltype(::Type{MTData}) = Base.HasEltype()\n",
    "Base.eltype(::Type{MTData}) = NTuple{2}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### iterate(::MTData)\n",
    "\n",
    "Define the `iterate` function for the `MTData` iterator. `iterate` should return a\n",
    "`(batch, state)` pair or `nothing` if there are no more batches.  The `batch` is a\n",
    "`(x::Matrix{Int},y::Matrix{Int})` pair where `x` is a `(batchsize,srclength)` batch of\n",
    "source language sentences and `y` is a `(batchsize,tgtlength)` batch of parallel target\n",
    "language translations. The `state` is a pair of `(src_state,tgt_state)` which can be used\n",
    "to iterate `d.src` and `d.tgt` to get more sentences.  `iterate(d)` without a second\n",
    "argument should initialize `d` by emptying its buckets and calling `iterate` on the inner\n",
    "iterators `d.src` and `d.tgt` without a state. Please review the documentation on\n",
    "iterators from the last project.\n",
    "\n",
    "To keep similar length sentences together `MTData` uses arrays of similar length sentence\n",
    "pairs called buckets.  Specifically, the `(src_sentence,tgt_sentence)` pairs coming from\n",
    "`src` and `tgt` are pushed into `d.buckets[i]` when the length of the source sentence is\n",
    "in the range `((i-1)*d.bucketwidth+1):(i*d.bucketwidth)`. When one of the buckets reaches\n",
    "`d.batchsize` `d.batchmaker` is called with the full bucket producing a 2-D batch, the\n",
    "bucket is emptied and the batch is returned. If `src` and `tgt` are exhausted the\n",
    "remaining partially full buckets are turned into batches and returned in any order. If the\n",
    "source sentence length is larger than `length(d.buckets)*d.bucketwidth`, the last bucket\n",
    "is used.\n",
    "\n",
    "Sentences above a certain length can be skipped using the `d.maxlength` field, and\n",
    "transposed `x,y` arrays can be produced using the `d.batchmajor` field."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "function Base.iterate(d::MTData, state=nothing)\n",
    "    if (state === nothing) \n",
    "        \n",
    "        for i = 1:length(d.buckets)\n",
    "            d.buckets[i] = []\n",
    "        end\n",
    "        src = d.src\n",
    "        tgt = d.tgt\n",
    "        src = Iterators.Stateful(src)\n",
    "        tgt = Iterators.Stateful(tgt)\n",
    "    else\n",
    "        src = state[1]\n",
    "        tgt = state[2]\n",
    "    end\n",
    "    \n",
    "    \n",
    "    if(isempty(src)&&isempty(tgt))\n",
    "        for i = 1:length(d.buckets)\n",
    "            if(length(d.buckets[i]) > 0)\n",
    "                tmp_batch = d.batchmaker(d, d.buckets[i])\n",
    "                 if(d.batchmajor == true)\n",
    "                    tmp_batch = (transpose(tmp_batch[1]), transpose(tmp_batch[2]))\n",
    "                end\n",
    "                d.buckets[i] = []\n",
    "                return (tmp_batch, (src, tgt))\n",
    "            end\n",
    "        end\n",
    "    end    \n",
    "        \n",
    "    while(!isempty(src) && !isempty(tgt))\n",
    "        sentences = (popfirst!(src), popfirst!(tgt))\n",
    "        src_sentence = sentences[1]\n",
    "        tgt_sentence = sentences[2]\n",
    "        src_length = length(src_sentence)\n",
    "        \n",
    "        if(src_length > d.maxlength)\n",
    "            continue\n",
    "        elseif(length(d.buckets)*d.bucketwidth < src_length)\n",
    "            index_in_buckets = length(d.buckets)\n",
    "        else\n",
    "            index_in_buckets = ceil(src_length/d.bucketwidth)\n",
    "        end\n",
    "        \n",
    "        index_in_buckets = convert(Int64, index_in_buckets)\n",
    "        push!(d.buckets[index_in_buckets], (src_sentence, tgt_sentence))\n",
    "        \n",
    "        if(isempty(src) && isempty(tgt))\n",
    "                tmp_batch = d.batchmaker(d, d.buckets[index_in_buckets])\n",
    "                if(d.batchmajor == true)\n",
    "                    tmp_batch = (transpose(tmp_batch[1]), transpose(tmp_batch[2]))\n",
    "                end\n",
    "                d.buckets[index_in_buckets] = []\n",
    "                return (tmp_batch, (src, tgt))\n",
    "        end  \n",
    "        \n",
    "        if(length(d.buckets[index_in_buckets]) == d.batchsize)\n",
    "            tmp_batch = d.batchmaker(d, d.buckets[index_in_buckets])\n",
    "            if(d.batchmajor == true)\n",
    "                tmp_batch = (transpose(tmp_batch[1]), transpose(tmp_batch[2]))\n",
    "            end\n",
    "            d.buckets[index_in_buckets] = []\n",
    "            return (tmp_batch, (src, tgt))\n",
    "        end \n",
    "    end   \n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### arraybatch\n",
    "\n",
    "Define `arraybatch(d, bucket)` to be used as the default `d.batchmaker`. `arraybatch`\n",
    "takes an `MTData` object and an array of sentence pairs `bucket` and returns a\n",
    "`(x::Matrix{Int},y::Matrix{Int})` pair where `x` is a `(batchsize,srclength)` batch of\n",
    "source language sentences and `y` is a `(batchsize,tgtlength)` batch of parallel target\n",
    "language translations. Note that the sentences in the bucket do not have any `eos` tokens\n",
    "and they may have different lengths. `arraybatch` should copy the source sentences into\n",
    "`x` padding shorter ones on the left with `eos` tokens. It should copy the target\n",
    "sentences into `y` with an `eos` token in the beginning and end of each sentence and\n",
    "shorter sentences padded on the right with extra `eos` tokens."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "arraybatch (generic function with 1 method)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function arraybatch(d::MTData, bucket)\n",
    "    # Your code here\n",
    "    x = []\n",
    "    y = []\n",
    "    \n",
    "    padded_x = Array{Int64,1}[]\n",
    "    padded_y = Array{Int64,1}[]\n",
    "    \n",
    "    max_length_x = 0\n",
    "    max_length_y = 0\n",
    "    \n",
    "    for sent_pair in bucket\n",
    "        push!(x, sent_pair[1])\n",
    "        push!(sent_pair[2], d.tgt.vocab.eos)\n",
    "        pushfirst!(sent_pair[2], d.tgt.vocab.eos)\n",
    "        push!(y, sent_pair[2])\n",
    "        \n",
    "        if(length(sent_pair[1]) > max_length_x)\n",
    "            max_length_x = length(sent_pair[1])\n",
    "        end\n",
    "        \n",
    "        if(length(sent_pair[2]) > max_length_y)\n",
    "            max_length_y = length(sent_pair[2])\n",
    "        end\n",
    "    end\n",
    "    for sent_pair in zip(x,y)\n",
    "        x_pad_length = max_length_x - length(sent_pair[1])\n",
    "        y_pad_length = max_length_y - length(sent_pair[2])\n",
    "        x_pad_seq = repeat([d.src.vocab.eos], x_pad_length)\n",
    "        y_pad_seq = repeat([d.tgt.vocab.eos], y_pad_length)\n",
    "        push!(padded_x, append!(x_pad_seq, sent_pair[1]))\n",
    "        push!(padded_y, append!(sent_pair[2], y_pad_seq))\n",
    "    end\n",
    "    \n",
    "    no_of_sentences = length(padded_x)\n",
    "\n",
    "    \n",
    "    padded_x = permutedims(hcat(padded_x...), (2,1))\n",
    "    padded_y = permutedims(hcat(padded_y...), (2,1))\n",
    "    \n",
    "    return (padded_x,padded_y)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: Testing MTData\n",
      "└ @ Main In[9]:1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m\u001b[1mTest Passed\u001b[22m\u001b[39m"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@info \"Testing MTData\"\n",
    "dtrn = MTData(tr_train, en_train)\n",
    "ddev = MTData(tr_dev, en_dev)\n",
    "dtst = MTData(tr_test, en_test)\n",
    "x,y = first(dtst)\n",
    "\n",
    "@test length(collect(dtst)) == 48\n",
    "@test size.((x,y)) == ((128,10),(128,24))\n",
    "@test x[1,1] == tr_vocab.eos\n",
    "@test x[1,end] != tr_vocab.eos\n",
    "@test y[1,1] == en_vocab.eos;\n",
    "@test y[1,2] != en_vocab.eos\n",
    "@test y[1,end] == en_vocab.eos"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 2. Sequence to sequence model without attention\n",
    "\n",
    "In this part we will define a simple sequence to sequence encoder-decoder model for\n",
    "machine translation."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "mutable struct S2S_v1\n",
    "    srcembed::Embed     # source language embedding\n",
    "    encoder::RNN        # encoder RNN (can be bidirectional)\n",
    "    tgtembed::Embed     # target language embedding\n",
    "    decoder::RNN        # decoder RNN\n",
    "    projection::Linear  # converts decoder output to vocab scores\n",
    "    dropout::Real       # dropout probability to prevent overfitting\n",
    "    srcvocab::Vocab     # source language vocabulary\n",
    "    tgtvocab::Vocab     # target language vocabulary\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### S2S_v1 constructor\n",
    "\n",
    "Define the S2S_v1 constructor using your predefined layer types (Embed, Linear), and the\n",
    "Knet RNN type. Please review the RNN documentation using `@doc RNN`, paying attention to\n",
    "the following options in particular: `numLayers`, `bidirectional`, `dropout`, `dataType`,\n",
    "`usegpu`. The last two are important if you experiment with array types other than the\n",
    "default `KnetArray{Float32}`: make sure the RNNs use the same array type as the other\n",
    "layers. Note that if the encoder is bidirectional, its `numLayers` should be half of the\n",
    "decoder so that their hidden states match in size."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "S2S_v1"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function S2S_v1(hidden::Int,         # hidden size for both the encoder and decoder RNN\n",
    "                srcembsz::Int,       # embedding size for source language\n",
    "                tgtembsz::Int,       # embedding size for target language\n",
    "                srcvocab::Vocab,     # vocabulary for source language\n",
    "                tgtvocab::Vocab;     # vocabulary for target language\n",
    "                layers=1,            # number of layers\n",
    "                bidirectional=false, # whether encoder RNN is bidirectional\n",
    "                dropout=0)           # dropout probability\n",
    "    \n",
    "    srcembed = Embed(length(srcvocab.i2w), srcembsz)\n",
    "    tgtembed = Embed(length(tgtvocab.i2w), tgtembsz)\n",
    "    decoder_layers = layers\n",
    "    if(bidirectional == true)\n",
    "        decoder_layers = 2 * layers\n",
    "    end\n",
    "    \n",
    "    encoder = RNN(srcembsz, hidden, rnnType = :lstm, bidirectional = bidirectional, dropout = dropout, numLayers = layers, h = 0)\n",
    "    decoder = RNN(tgtembsz, hidden, rnnType = :lstm, dropout = dropout, numLayers = decoder_layers, h = 0)\n",
    "    projection = Linear(hidden, length(tgtvocab.i2w))\n",
    "    \n",
    "    S2S_v1(srcembed, encoder, tgtembed, decoder, projection, dropout, srcvocab, tgtvocab)\n",
    "    \n",
    "end "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### S2S_v1 loss function\n",
    "\n",
    "Define the S2S_v1 loss function that takes `src`, a source language minibatch, and `tgt`,\n",
    "a target language minibatch and returns either a `(total_loss, num_words)` pair if\n",
    "`average=false`, or `(total_loss/num_words)` average if `average=true`.\n",
    "\n",
    "Assume that `src` and `tgt` are integer arrays of size `(B,Tx)` and `(B,Ty)` respectively,\n",
    "where `B` is the batch size, `Tx` is the length of the longest source sequence, `Ty` is\n",
    "the length of the longest target sequence. The `src` sequences only contain words, the\n",
    "`tgt` sequences surround the words with `eos` tokens at the start and end. This allows\n",
    "columns `tgt[:,1:end-1]` to be used as the decoder input and `tgt[:,2:end]` as the desired\n",
    "decoder output.\n",
    "\n",
    "Assume any shorter sentences in the batches have been padded with extra `eos` tokens on\n",
    "the left for `src` and on the right for `tgt`. Don't worry about masking `src` for the\n",
    "encoder, it doesn't have a significant effect on the loss. However do mask `tgt` before\n",
    "`nll`: you do not want the padding tokens to be counted in the loss calculation.\n",
    "\n",
    "Please review `@doc RNN`: in particular the `r.c` and `r.h` fields can be used to get/set\n",
    "the cell and hidden arrays of an RNN (note that `0` and `nothing` act as special values).\n",
    "\n",
    "RNNs take a dropout value at construction and apply dropout to the input of every layer if\n",
    "it is non-zero. You need to handle dropout for other layers in the loss function or in\n",
    "layer definitions as necessary."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "function (s::S2S_v1)(src, tgt; average=true)\n",
    "    src_embed_tensor = s.srcembed(src)\n",
    "    s.encoder.h = 0\n",
    "    s.encoder.c = 0\n",
    "    y_enc = s.encoder(src_embed_tensor)\n",
    "    tgt_embed_tensor = s.tgtembed(tgt[:,1:end-1])\n",
    "    s.decoder.h = copy(s.encoder.h)\n",
    "    s.decoder.c = copy(s.encoder.c)\n",
    "    y_dec = s.decoder(tgt_embed_tensor)\n",
    "    hy, b ,ty = size(y_dec)\n",
    "    y_dec = reshape(y_dec, (hy, b*ty))\n",
    "    scores = s.projection(y_dec)\n",
    "    #check dropout\n",
    "    y_gold = mask!(tgt[:,2:end], s.tgtvocab.eos)\n",
    "    nll(scores, y_gold; average = average)\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: Testing S2S_v1\n",
      "└ @ Main In[13]:1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m\u001b[1mTest Passed\u001b[22m\u001b[39m"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@info \"Testing S2S_v1\"\n",
    "Knet.seed!(1)\n",
    "\n",
    "model = S2S_v1(512, 512, 512, tr_vocab, en_vocab; layers=2, bidirectional=true, dropout=0.2)\n",
    "(x,y) = first(dtst)\n",
    "# Your loss can be slightly different due to different ordering of words in the vocabulary.\n",
    "# The reference vocabulary starts with eos, unk, followed by words in decreasing frequency.\n",
    "@test model(x,y; average=false)[2] == (14097.471f0, 1432)[2] #our version\n",
    "@test model(x,y; average=false)[1] ≈ (14097.471f0, 1432)[1]\n",
    "#@test model(x,y; average=false) == (14097.471f0, 1432) ,original"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Loss for a whole dataset\n",
    "\n",
    "Define a `loss(model, data)` which returns a `(Σloss, Nloss)` pair if `average=false` and\n",
    "a `Σloss/Nloss` average if `average=true` for a whole dataset. Assume that `data` is an\n",
    "iterator of `(x,y)` pairs such as `MTData` and `model(x,y;average)` is a model like\n",
    "`S2S_v1` that computes loss on a single `(x,y)` pair."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "loss (generic function with 1 method)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function loss(model, data; average=true)\n",
    "    instances = 0\n",
    "    cumulative_loss = 0\n",
    "    for batch in data\n",
    "        x, y = batch\n",
    "        batch_loss, batch_instances = model(x,y; average=false)\n",
    "        cumulative_loss += batch_loss\n",
    "        instances += batch_instances\n",
    "    end\n",
    "    if (average)\n",
    "        cumulative_loss / instances\n",
    "    else\n",
    "        cumulative_loss, instances\n",
    "    end\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: Testing loss\n",
      "└ @ Main In[15]:1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "\u001b[32m\u001b[1mTest Passed\u001b[22m\u001b[39m"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@info \"Testing loss\"\n",
    "#@test loss(model, dtst, average=false) == (1.0429117f6, 105937) ,true\n",
    "@test loss(model, dtst, average=false)[1] ≈ (1.0429117f6, 105937)[1] #our version\n",
    "@test loss(model, dtst, average=false)[2] == (1.0429117f6, 105937)[2] #our version\n",
    "# Your loss can be slightly different due to different ordering of words in the vocabulary.\n",
    "# The reference vocabulary starts with eos, unk, followed by words in decreasing frequency.\n",
    "# Also, because we do not mask src, different batch sizes may lead to slightly different\n",
    "# losses. The test above gives (1.0429178f6, 105937) with batchsize==1."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Training SGD_v1\n",
    "\n",
    "The following function can be used to train our model. `trn` is the training data, `dev`\n",
    "is used to determine the best model, `tst...` can be zero or more small test datasets for\n",
    "loss reporting. It returns the model that does best on `dev`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "train! (generic function with 1 method)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function train!(model, trn, dev, tst...)\n",
    "\n",
    "    bestmodel, bestloss = deepcopy(model), loss(model, dev)\n",
    "    progress!(adam(model, trn), steps=100) do y\n",
    "        losses = [ loss(model, d) for d in (dev,tst...) ]\n",
    "        if losses[1] < bestloss\n",
    "            bestmodel, bestloss = deepcopy(model), losses[1]\n",
    "        end\n",
    "        return (losses...,)\n",
    "    end\n",
    "    return bestmodel\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "You should be able to get under 3.40 dev loss with the following settings in 10\n",
    "epochs. The training speed on a V100 is about 3 mins/epoch or 40K words/sec, K80 is about\n",
    "6 times slower. Using settings closer to the Luong paper (per-sentence loss rather than\n",
    "per-word loss, SGD with lr=1, gclip=1 instead of Adam), you can get to 3.17 dev loss in\n",
    "about 25 epochs. Using dropout and shuffling batches before each epoch significantly\n",
    "improve the dev loss. You can play around with hyperparameters but I doubt results will\n",
    "get much better without attention. To verify your training, here is the dev loss I\n",
    "observed at the beginning of each epoch in one training session:\n",
    "`[9.83, 4.60, 3.98, 3.69, 3.52, 3.41, 3.35, 3.32, 3.30, 3.31, 3.33]`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "┌ Info: Training S2S_v1\n",
      "└ @ Main In[17]:1\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "38-element Array{Tuple{T,T} where T,1}:\n",
       " ([38124 38124 … 38123 38126; 38124 38124 … 38123 38126; … ; 38124 38124 … 38123 38126; 38124 38124 … 36514 38126], [18853 18815 … 18853 18853; 18853 18821 … 18853 18853; … ; 18853 18829 … 18853 18853; 18853 18847 … 18853 18853])\n",
       " ([37131 36447 … 37057 38126; 38124 38124 … 38102 38126; … ; 38124 38124 … 35612 38126; 38124 38124 … 38106 38126], [18853 18798 … 18856 18853; 18853 18849 … 18853 18853; … ; 18853 18837 … 18853 18853; 18853 18837 … 18853 18853])\n",
       " ([38124 38124 … 29405 38126; 38124 38124 … 38123 38126; … ; 38124 38124 … 38123 38126; 38124 38124 … 37789 38126], [18853 18837 … 18853 18853; 18853 18845 … 18853 18853; … ; 18853 18854 … 18853 18853; 18853 18774 … 18853 18853])\n",
       " ([38124 38073 … 37495 38126; 38124 38124 … 38043 38126; … ; 38124 38124 … 38123 38126; 38124 38124 … 37559 38126], [18853 18847 … 18853 18853; 18853 18821 … 18853 18853; … ; 18853 18844 … 18853 18853; 18853 18845 … 18853 18853])\n",
       " ([38124 38124 … 10541 38126; 38124 38124 … 37789 38126; … ; 38124 38124 … 37748 38126; 38124 38124 … 13915 38126], [18853 18847 … 18853 18853; 18853 18847 … 18853 18853; … ; 18853 18838 … 18853 18853; 18853 18854 … 18853 18853])\n",
       " ([38124 38124 … 37300 38126; 38124 38124 … 38123 38126; … ; 33621 36898 … 38123 38126; 38124 38124 … 38092 38126], [18853 18838 … 18853 18853; 18853 18849 … 18853 18853; … ; 18853 18837 … 18853 18853; 18853 18846 … 18853 18853])\n",
       " ([38124 38124 … 38123 35678; 38124 38121 … 36319 38126; … ; 38121 19071 … 11214 38126; 38124 38124 … 35410 38126], [18853 18847 … 18853 18853; 18853 18854 … 18853 18853; … ; 18853 18854 … 18853 18853; 18853 18846 … 18853 18853])\n",
       " ([38124 38124 … 37340 31556; 38124 38124 … 35295 38126; … ; 38124 38124 … 37357 38126; 38124 38124 … 38123 38126], [18853 18641 … 18853 18853; 18853 18838 … 18853 18853; … ; 18853 18837 … 18853 18853; 18853 18830 … 18853 18853])\n",
       " ([38124 38124 … 37962 38126; 38124 38124 … 38123 38126; … ; 38124 38124 … 26716 38126; 38124 38124 … 38123 33609], [18853 18855 … 18853 18853; 18853 18854 … 18853 18853; … ; 18853 18807 … 18853 18853; 18853 18847 … 18853 18853])\n",
       " ([38124 38124 … 22093 38126; 38124 38124 … 35363 38113; … ; 38124 38124 … 38123 38126; 38124 38124 … 38123 38126], [18853 18521 … 18853 18853; 18853 18838 … 18853 18853; … ; 18853 18845 … 18853 18853; 18853 18854 … 18853 18853])\n",
       " ([38124 38124 … 35905 38126; 38124 38004 … 37540 38126; … ; 38124 38124 … 31808 38126; 38124 38124 … 38123 38126], [18853 18268 … 18853 18853; 18853 18750 … 18853 18853; … ; 18853 18847 … 18853 18853; 18853 18847 … 18853 18853])\n",
       " ([38124 38124 … 2962 38126; 38124 38124 … 8420 38126; … ; 38124 38124 … 36814 38126; 38124 38124 … 37252 38126], [18853 18843 … 18853 18853; 18853 18807 … 18853 18853; … ; 18853 18692 … 18853 18853; 18853 17797 … 18853 18853])  \n",
       " ([38120 37112 … 38123 38126; 38123 38122 … 26130 38126; … ; 38124 38124 … 38092 38126; 38124 38084 … 36827 38126], [18853 18839 … 18853 18853; 18853 18854 … 18853 18853; … ; 18853 18855 … 18853 18853; 18853 18847 … 18853 18853])\n",
       " ⋮                                                                                                                                                                                                                                   \n",
       " ([38124 38124 … 36597 38126; 38124 38124 … 37998 38126; … ; 38124 35729 … 36009 38126; 38124 38124 … 26780 38123], [18853 18831 … 18853 18853; 18853 18847 … 18853 18853; … ; 18853 18847 … 18853 18853; 18853 18829 … 18853 18853])\n",
       " ([38124 38124 … 38123 38126; 38124 38124 … 37987 38126; … ; 38124 38124 … 38123 38126; 38124 38124 … 34200 38113], [18853 6537 … 18853 18853; 18853 18854 … 18853 18853; … ; 18853 18847 … 18853 18853; 18853 18835 … 18853 18853]) \n",
       " ([38124 38124 … 14168 38126; 38121 35817 … 8193 38126; … ; 38124 38124 … 37955 38126; 38124 38124 … 37979 38110], [18853 18563 … 18853 18853; 18853 18854 … 18853 18853; … ; 18853 18695 … 18853 18853; 18853 18801 … 18853 18853]) \n",
       " ([38124 38124 … 23995 38126; 38124 38124 … 34546 38126; … ; 38124 38124 … 17896 38126; 38124 38124 … 35953 38126], [18853 18845 … 18853 18853; 18853 18688 … 18853 18853; … ; 18853 18834 … 18853 18853; 18853 18855 … 18853 18853])\n",
       " ([38124 38124 … 38123 38126; 38124 38121 … 35972 38126; … ; 38124 38124 … 33610 38126; 38124 38124 … 37976 38126], [18853 18831 … 18853 18853; 18853 18854 … 18856 18853; … ; 18853 17215 … 18853 18853; 18853 18854 … 18853 18853])\n",
       " ([38124 38124 … 34884 38126; 38124 38124 … 38123 38126; … ; 38124 38124 … 36955 38126; 38118 37278 … 38126 38119], [18853 18854 … 18853 18853; 18853 18852 … 18853 18853; … ; 18853 18852 … 18853 18853; 18853 18835 … 18853 18853])\n",
       " ([38124 38124 … 38126 38119; 38124 38124 … 38064 38119; … ; 38124 38124 … 38123 38126; 38124 38124 … 37945 38126], [18853 18835 … 18853 18853; 18853 18835 … 18853 18853; … ; 18853 18846 … 18853 18853; 18853 18746 … 18853 18853])\n",
       " ([38118 38118 … 38126 38119; 38124 38124 … 38126 38119; … ; 38124 38124 … 35581 38126; 38124 38124 … 38126 38119], [18853 18835 … 18853 18853; 18853 18835 … 18853 18853; … ; 18853 18854 … 18853 18853; 18853 18835 … 18853 18853])\n",
       " ([38124 38124 … 6430 38126; 38124 38124 … 36043 38103; … ; 38124 38124 … 34891 38126; 38124 38124 … 38041 38126], [18853 18817 … 18853 18853; 18853 18847 … 18853 18853; … ; 18853 18831 … 18853 18853; 18853 18843 … 18853 18853]) \n",
       " ([38124 38124 … 38126 38119; 38121 37575 … 38123 34571; … ; 38124 38124 … 38110 38119; 38118 38102 … 38126 38119], [18853 18835 … 18853 18853; 18853 16206 … 18853 18853; … ; 18853 18835 … 18853 18853; 18853 18835 … 18841 18853])\n",
       " ([38124 38124 … 38126 38119; 38124 38118 … 38126 38119; 32062 37061 … 37369 38126], [18853 18835 … 18841 18853; 18853 18835 … 18853 18853; 18853 18847 … 18853 18853])                                                              \n",
       " ([38118 37852 … 38123 38119], [18853 18835 … 18841 18853])                                                                                                                                                                          "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "@info \"Training S2S_v1\"\n",
    "\n",
    "model = Knet.load(\"s2s_v1.jld2\",\"model\")\n",
    "epochs = 1\n",
    "ctrn = collect(dtrn)\n",
    "trnx10 = collect(flatten(shuffle!(ctrn) for i in 1:epochs))\n",
    "trn20 = ctrn[1:20]\n",
    "dev38 = collect(ddev)\n",
    "# Uncomment this to train the model (This takes about 30 mins on a V100):\n",
    "#model = train!(model, trnx10, dev38, trn20)\n",
    "# Uncomment this to save the model:\n",
    "#Knet.save(\"s2s_v2.jld2\",\"model\",model)\n",
    "# Uncomment this to load the model:\n",
    "#model = Knet.load(\"s2s_vDY.jld2\",\"model\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Generating translations\n",
    "\n",
    "With a single argument, a `S2S_v1` object should take it as a batch of source sentences\n",
    "and generate translations for them. After passing `src` through the encoder and copying\n",
    "its hidden states to the decoder, the decoder is run starting with an initial input of all\n",
    "`eos` tokens. Highest scoring tokens are appended to the output and used as input for the\n",
    "subsequent decoder steps.  The decoder should stop generating when all sequences in the\n",
    "batch have generated `eos` or when `stopfactor * size(src,2)` decoder steps are reached. A\n",
    "correctly shaped target language batch should be returned."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "function (s::S2S_v1)(src::Matrix{Int}; stopfactor = 3)\n",
    "\n",
    "    isDone = false\n",
    "    batch_size = size(src,1)\n",
    "    first_input = repeat([s.tgtvocab.eos], batch_size)\n",
    "    is_all_finished = zeros(batch_size)\n",
    "    translated_sentences = copy(first_input)\n",
    "    max_length_output = 0\n",
    "    s.encoder.h = 0\n",
    "    s.encoder.c = 0\n",
    "    src_embed_tensor = s.srcembed(src)\n",
    "    y_enc = s.encoder(src_embed_tensor)\n",
    "    s.decoder.h = copy(s.encoder.h)\n",
    "    s.decoder.c = copy(s.encoder.c)\n",
    "    input = first_input\n",
    "    \n",
    "    while (!isDone && max_length_output < stopfactor*size(src,2))\n",
    "        \n",
    "        \n",
    "        tgt_embed_tensor = s.tgtembed(input)\n",
    "        y = s.decoder(tgt_embed_tensor)\n",
    "    \n",
    "        scores = s.projection(y)\n",
    "        \n",
    "        \n",
    "        output_words = reshape(map(x->x[1], argmax(scores, dims = 1)), batch_size)\n",
    "        translated_sentences = hcat(translated_sentences, output_words')\n",
    "        max_length_output = size(translated_sentences, 2)\n",
    "        input = output_words\n",
    "\n",
    "        \n",
    "        tmp_output_words = copy(output_words)\n",
    "        tmp_output_words = tmp_output_words .== s.tgtvocab.eos\n",
    "        is_all_finished += tmp_output_words\n",
    "        if(sum(is_all_finished.==0)==0)\n",
    "            isDone = true\n",
    "        end\n",
    "    end\n",
    "    \n",
    "    return translated_sentences\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "int2str (generic function with 1 method)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Utility to convert int arrays to sentence strings\n",
    "function int2str(y,vocab)\n",
    "    y = vec(y)\n",
    "    ysos = findnext(w->!isequal(w,vocab.eos), y, 1)\n",
    "    ysos == nothing && return \"\"\n",
    "    yeos = something(findnext(isequal(vocab.eos), y, ysos), 1+length(y))\n",
    "    join(vocab.i2w[y[ysos:yeos-1]], \" \")\n",
    "end"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@info \"Generating some translations\"\n",
    "d = MTData(tr_dev, en_dev, batchsize=1) |> collect\n",
    "(src,tgt) = rand(d)\n",
    "out = model(src)\n",
    "println(\"SRC: \", int2str(src,model.srcvocab))\n",
    "println(\"REF: \", int2str(tgt,model.tgtvocab))\n",
    "println(\"OUT: \", int2str(out,model.tgtvocab))\n",
    "# Here is a sample output:\n",
    "# SRC: çin'e 15 şubat 2006'da ulaştım .\n",
    "# REF: i made it to china on february 15 , 2006 .\n",
    "# OUT: i got to china , china , at the last 15 years ."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Calculating BLEU\n",
    "\n",
    "BLEU is the most commonly used metric to measure translation quality. The following should\n",
    "take a model and some data, generate translations and calculate BLEU."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "bleu (generic function with 1 method)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "function bleu(s2s,d::MTData)\n",
    "    d = MTData(d.src,d.tgt,batchsize=1)\n",
    "    reffile = d.tgt.file\n",
    "    hypfile,hyp = mktemp()\n",
    "    for (x,y) in progress(collect(d))\n",
    "        g = s2s(x)\n",
    "        for i in 1:size(y,1)\n",
    "            println(hyp, int2str(g[i,:], d.tgt.vocab))\n",
    "        end\n",
    "    end\n",
    "    close(hyp)\n",
    "    isfile(\"multi-bleu.perl\") || download(\"https://github.com/moses-smt/mosesdecoder/raw/master/scripts/generic/multi-bleu.perl\", \"multi-bleu.perl\")\n",
    "    run(pipeline(`cat $hypfile`,`perl multi-bleu.perl $reffile`))\n",
    "    return hypfile\n",
    "end"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Calculating dev BLEU takes about 45 secs on a V100. We get about 8.0 BLEU which is pretty\n",
    "low. As can be seen from the sample translations a loss of ~3+ (perplexity ~20+) or a BLEU\n",
    "of ~8 is not sufficient to generate meaningful translations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "@info \"Calculating BLEU\"\n",
    "bleu(model, ddev)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "To improve the quality of translations we can use more training data, different training\n",
    "and model parameters, or preprocess the input/output: e.g. splitting Turkish words to make\n",
    "suffixes look more like English function words may help. Other architectures,\n",
    "e.g. attention and transformer, perform significantly better than this simple S2S model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "*This notebook was generated using [Literate.jl](https://github.com/fredrikekre/Literate.jl).*"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Julia 1.2.0",
   "language": "julia",
   "name": "julia-1.2"
  },
  "language_info": {
   "file_extension": ".jl",
   "mimetype": "application/julia",
   "name": "julia",
   "version": "1.2.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}