Merge pull request #106 from luke-heberling/master

Use --copy-dest, enabling the rsync algorithm when copying from remote to staging

README.md

@@ -129,7 +129,8 @@ optional arguments:
 usage: carbon-sync [-h] [-c CONFIG_FILE] [-C CLUSTER] [-f METRICS_FILE] -s
                    SOURCE_NODE [-d STORAGE_DIR] [-b BATCH_SIZE]
                    [--source-storage-dir SOURCE_STORAGE_DIR]
-                   [--rsync-options RSYNC_OPTIONS] [--dirty] [-l] [-o]
+                   [--rsync-options RSYNC_OPTIONS] [--rsync-disable-copy-dest]
+                   [--dirty] [-l] [-o]
 
 Sync local metrics using remote nodes in the cluster
 
@@ -155,10 +156,13 @@ optional arguments:
   --rsync-options RSYNC_OPTIONS
                         Pass option(s) to rsync. Make sure to use "--rsync-
                         options=" if option starts with '-' (default: -azpS)
+  --rsync-disable-copy-dest
+                        Avoid --copy-dest, transfer all whisper data between
+                        nodes. (default: False)
   --dirty               If set, don't clean temporary rsync directory
                         (default: False)
   -l, --lock            Lock whisper files during filling (default: False)
-  -o, --overwrite        Write all non nullpoints from src to dst (default:
+  -o, --overwrite       Write all non nullpoints from src to dst (default:
                         False)
 ```
 

carbonate/cli.py

@@ -172,6 +172,12 @@ def carbon_sync():
         help='Pass option(s) to rsync. Make sure to use ' +
         '"--rsync-options=" if option starts with \'-\'')
 
+    parser.add_argument(
+        '--rsync-disable-copy-dest',
+        default=False,
+        action='store_true',
+        help='Avoid --copy-dest, transfer all whisper data between nodes.')
+
     parser.add_argument(
         '--dirty',
         action='store_true',
@@ -210,6 +216,10 @@ def carbon_sync():
     total_metrics = 0
     batch_size = int(args.batch_size)
 
+    rsync_options = args.rsync_options
+    if not args.rsync_disable_copy_dest:
+        rsync_options += ' --copy-dest="%s"' % args.storage_dir
+
     for metric in metrics:
         total_metrics += 1
         metric = metric.strip()
@@ -221,7 +231,7 @@ def carbon_sync():
             print("* Running batch %s-%s"
                   % (total_metrics-batch_size+1, total_metrics))
             run_batch(metrics_to_sync, remote,
-                      args.storage_dir, args.rsync_options,
+                      args.storage_dir, rsync_options,
                       remote_ip, args.dirty, lock_writes=whisper_lock_writes,
                       overwrite=args.overwrite)
             metrics_to_sync = []
@@ -230,7 +240,7 @@ def carbon_sync():
         print("* Running batch %s-%s"
               % (total_metrics-len(metrics_to_sync)+1, total_metrics))
         run_batch(metrics_to_sync, remote,
-                  args.storage_dir, args.rsync_options,
+                  args.storage_dir, rsync_options,
                   remote_ip, args.dirty, lock_writes=whisper_lock_writes)
 
     elapsed = (time() - start)