initial download and unzipping done

.gitignore

@@ -2,3 +2,4 @@ logs
 /download
 /list
 /brick
+/raw

stages/01_download.sh

@@ -13,7 +13,7 @@ mkdir -p $listpath
 cd $listpath;
 
 # Define the FTP base address
-export ftpbase=""
+export ftpbase="https://ftp.ncbi.nlm.nih.gov/pub/lu/PubTator3/"
 
 # Retrieve the list of files to download from FTP base address
 wget --no-remove-listing $ftpbase

stages/02_unzip.sh

@@ -24,5 +24,8 @@ cat $listpath/files.txt | tail -n +2 | xargs -P14 -n1 bash -c '
   filename="${0%.*}"
   echo $downloadpath/$0
   echo $rawpath/$filename
-  unzip $downloadpath/$0 -d $rawpath/$filename
-'
+  if [[ "$downloadpath/$0" == *.gz && "$downloadpath/$0" != *.tar.gz ]]; then
+    gunzip -d "$downloadpath/$0" -c > "$rawpath/$filename"
+  elif [[ "$downloadpath/$0" == *.tar.gz ]]; then
+    tar -xzvf "$downloadpath/$0" -C "$rawpath"
+  fi'