From b9eb75fa8b8389ef0cbb9b879dc56a6150174251 Mon Sep 17 00:00:00 2001
From: Cihan Emre <cihan@vngrs.com>
Date: Mon, 6 Apr 2015 10:50:12 +0300
Subject: [PATCH 1/2] SPARK-4888 mount and format all preformatted disks

---
 setup-slave.sh | 74 ++++++++++++++++++++++++++++----------------------
 1 file changed, 42 insertions(+), 32 deletions(-)

diff --git a/setup-slave.sh b/setup-slave.sh
index 606c602..c07377b 100755
--- a/setup-slave.sh
+++ b/setup-slave.sh
@@ -27,38 +27,49 @@ instance_type=$(curl http://169.254.169.254/latest/meta-data/instance-type 2> /d
 
 echo "Setting up slave on `hostname`... of type $instance_type"
 
+# Format & mount using ext4, which has the best performance among ext3, ext4, and xfs based
+# on our shuffle heavy benchmark
+function to_ext4 {
+  device=$1
+  mount_point=$2
+  trim_on=$3
+  EXT4_MOUNT_OPTS="defaults,noatime,nodiratime"
+
+  mkdir -p $mount_point
+  
+  if [[ trim_on == true ]]; then
+    echo $device $mount_point ' ext4 defaults,noatime,nodiratime,discard 0 0' >> /etc/fstab
+  fi
+
+  mkfs.ext4 -E lazy_itable_init=0,lazy_journal_init=0 $device
+  mount -o $EXT4_MOUNT_OPTS $device $mount_point
+}
+
 if [[ $instance_type == r3* || $instance_type == i2* || $instance_type == hi1* ]]; then
   # Format & mount using ext4, which has the best performance among ext3, ext4, and xfs based
   # on our shuffle heavy benchmark
-  EXT4_MOUNT_OPTS="defaults,noatime,nodiratime"
   rm -rf /mnt*
-  mkdir /mnt
-  # To turn TRIM support on, uncomment the following line.
-  #echo '/dev/sdb /mnt  ext4  defaults,noatime,nodiratime,discard 0 0' >> /etc/fstab
-  mkfs.ext4 -E lazy_itable_init=0,lazy_journal_init=0 /dev/sdb
-  mount -o $EXT4_MOUNT_OPTS /dev/sdb /mnt
-
-  if [[ $instance_type == "r3.8xlarge" || $instance_type == "hi1.4xlarge" ]]; then
-    mkdir /mnt2
-    # To turn TRIM support on, uncomment the following line.
-    #echo '/dev/sdc /mnt2  ext4  defaults,noatime,nodiratime,discard 0 0' >> /etc/fstab
-    if [[ $instance_type == "r3.8xlarge" ]]; then
-      mkfs.ext4 -E lazy_itable_init=0,lazy_journal_init=0 /dev/sdc      
-      mount -o $EXT4_MOUNT_OPTS /dev/sdc /mnt2
-    fi
-    # To turn TRIM support on, uncomment the following line.
-    #echo '/dev/sdf /mnt2  ext4  defaults,noatime,nodiratime,discard 0 0' >> /etc/fstab
-    if [[ $instance_type == "hi1.4xlarge" ]]; then
-      mkfs.ext4 -E lazy_itable_init=0,lazy_journal_init=0 /dev/sdf      
-      mount -o $EXT4_MOUNT_OPTS /dev/sdf /mnt2
-    fi    
-  fi
+  device_array=(/dev/sdb /dev/sdc /dev/sdd /dev/sde /dev/sdf /dev/sdg /dev/sdh /dev/sdi)
+
+  total=${#device_array[*]}
+ 
+  # To turn TRIM support on, change the third argument to true
+  for (( i=0; i<=$(( $total - 1 )); i++ ))
+  do
+      case "$i" in
+        0) mount_index="" ;;
+        *) mount_index=$((i + 1)) ;;
+      esac
+      to_ext4 ${device_array[$i]} /mnt$mount_index false
+  done
 fi
 
 # Mount options to use for ext3 and xfs disks (the ephemeral disks
 # are ext3, but we use xfs for EBS volumes to format them faster)
 XFS_MOUNT_OPTS="defaults,noatime,nodiratime,allocsize=8m"
 
+yum install -q -y xfsprogs
+
 function setup_ebs_volume {
   device=$1
   mount_point=$2
@@ -66,7 +77,6 @@ function setup_ebs_volume {
     # Check if device is already formatted
     if ! blkid $device; then
       mkdir $mount_point
-      yum install -q -y xfsprogs
       if mkfs.xfs -q $device; then
         mount -o $XFS_MOUNT_OPTS $device $mount_point
         chmod -R a+w $mount_point
@@ -87,15 +97,15 @@ function setup_ebs_volume {
   fi
 }
 
-# Format and mount EBS volume (/dev/sd[s, t, u, v, w, x, y, z]) as /vol[x] if the device exists
-setup_ebs_volume /dev/sds /vol0
-setup_ebs_volume /dev/sdt /vol1
-setup_ebs_volume /dev/sdu /vol2
-setup_ebs_volume /dev/sdv /vol3
-setup_ebs_volume /dev/sdw /vol4
-setup_ebs_volume /dev/sdx /vol5
-setup_ebs_volume /dev/sdy /vol6
-setup_ebs_volume /dev/sdz /vol7
+devices=(/dev/sds /dev/sdt /dev/sdu /dev/sdv /dev/sdw /dev/sdx /dev/sdy /dev/sdz)
+
+# Format and mount EBS volume (/dev/sd[s, t, u, v, w, x, y, z] as /vol[x] if the device exists
+total=${#devices[*]}
+ 
+for (( i=0; i<=$(( $total - 1 )); i++ ))
+do
+    setup_ebs_volume ${devices[$i]} /vol$i
+done
 
 # Alias vol to vol3 for backward compatibility: the old spark-ec2 script supports only attaching
 # one EBS volume at /dev/sdv.

From efd7af6f756378753a3bf9cbe1fca05816b25f81 Mon Sep 17 00:00:00 2001
From: Cihan Emre <cihan@vngrs.com>
Date: Tue, 7 Apr 2015 11:24:06 +0300
Subject: [PATCH 2/2] SPARK-4888 check if device exists & mounted before
 formatting

---
 setup-slave.sh | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/setup-slave.sh b/setup-slave.sh
index c07377b..32f3523 100755
--- a/setup-slave.sh
+++ b/setup-slave.sh
@@ -29,20 +29,28 @@ echo "Setting up slave on `hostname`... of type $instance_type"
 
 # Format & mount using ext4, which has the best performance among ext3, ext4, and xfs based
 # on our shuffle heavy benchmark
-function to_ext4 {
+function format_ext4 {
   device=$1
   mount_point=$2
   trim_on=$3
   EXT4_MOUNT_OPTS="defaults,noatime,nodiratime"
 
-  mkdir -p $mount_point
+  #check if device exists
+  if [[ -e "$device" ]]; then
+    #unmount the drive if it's mounted already
+    if grep -qs "$device" /proc/mounts; then
+      umount $mount_point
+    fi
+
+    mkdir -p $mount_point
   
-  if [[ trim_on == true ]]; then
-    echo $device $mount_point ' ext4 defaults,noatime,nodiratime,discard 0 0' >> /etc/fstab
-  fi
+    if [[ trim_on == true ]]; then
+      echo $device $mount_point ' ext4 defaults,noatime,nodiratime,discard 0 0' >> /etc/fstab
+    fi
 
-  mkfs.ext4 -E lazy_itable_init=0,lazy_journal_init=0 $device
-  mount -o $EXT4_MOUNT_OPTS $device $mount_point
+    mkfs.ext4 -E lazy_itable_init=0,lazy_journal_init=0 $device
+    mount -o $EXT4_MOUNT_OPTS $device $mount_point
+  fi
 }
 
 if [[ $instance_type == r3* || $instance_type == i2* || $instance_type == hi1* ]]; then