From 00f8a0b984d45adf0e64347b671e18beef7c9a26 Mon Sep 17 00:00:00 2001 From: gpratt Date: Wed, 20 Aug 2014 23:07:01 -0700 Subject: [PATCH 1/4] Fixed file parsing issue (.'s weren't allowed) --- trackhub/track.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/trackhub/track.py b/trackhub/track.py index 1007dac2..ff794979 100644 --- a/trackhub/track.py +++ b/trackhub/track.py @@ -19,7 +19,7 @@ def _check_name(name): - regex = re.compile('[^a-zA-Z0-9-_]') + regex = re.compile('[^a-zA-Z0-9-_.]') if regex.search(name): raise ValueError('Non-alphanumeric character in name "%s"' % name) From c463175ee83aead447b1109b158813c81bc3415c Mon Sep 17 00:00:00 2001 From: gpratt Date: Wed, 20 Aug 2014 23:07:31 -0700 Subject: [PATCH 2/4] Added basic s3 integration, only works for yeo-lab bucket at the moment. --- trackhub/upload.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/trackhub/upload.py b/trackhub/upload.py index a37a5a11..29e29aa3 100644 --- a/trackhub/upload.py +++ b/trackhub/upload.py @@ -3,6 +3,18 @@ from fabric.api import local, settings, run, abort, cd, env, hide, put from fabric.contrib.console import confirm import track +import boto +from boto.s3.key import Key + + +def upload_s3(local_fn, remote_fn, **kwargs): + s3 = boto.connect_s3() + bucket = s3.get_bucket("sauron-yeo") + k = Key(bucket) + k.key = remote_fn + k.set_contents_from_filename(local_fn) + k.make_public() + return ["done"] def upload_file(host, user, local_fn, remote_fn, port=22, rsync_options='-azvr --progress', run_local=False): @@ -33,12 +45,13 @@ def upload_file(host, user, local_fn, remote_fn, port=22, def upload_hub(host, user, hub, port=22, rsync_options='-azvr --progress', - run_local=False): + run_local=False, run_s3=False): kwargs = dict(host=host, user=user, port=port, rsync_options=rsync_options, run_local=run_local) print kwargs results = [] - + if run_s3: + upload_file = upload_s3 # First the hub file: results.extend( upload_file( @@ -79,7 +92,11 @@ def upload_hub(host, user, hub, port=22, rsync_options='-azvr --progress', def upload_track(host, user, track, port=22, rsync_options='-azvr --progress', - run_local=False): + run_local=False, run_s3=False): + + if run_s3: + upload_file = upload_s3 + kwargs = dict(host=host, user=user, local_fn=track.local_fn, remote_fn=track.remote_fn, rsync_options=rsync_options, run_local=run_local) From a1ad5f35d01383ad85d06b6996f3b618bbc5e238 Mon Sep 17 00:00:00 2001 From: gpratt Date: Mon, 6 Oct 2014 14:33:23 -0700 Subject: [PATCH 3/4] fixed bug where I couldn't disable s3 --- trackhub/upload.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/trackhub/upload.py b/trackhub/upload.py index 29e29aa3..77aa7a16 100644 --- a/trackhub/upload.py +++ b/trackhub/upload.py @@ -16,6 +16,7 @@ def upload_s3(local_fn, remote_fn, **kwargs): k.make_public() return ["done"] + def upload_file(host, user, local_fn, remote_fn, port=22, rsync_options='-azvr --progress', run_local=False): results = [] @@ -95,12 +96,13 @@ def upload_track(host, user, track, port=22, rsync_options='-azvr --progress', run_local=False, run_s3=False): if run_s3: - upload_file = upload_s3 - + cur_upload_file = upload_s3 + else: + cur_upload_file = upload_file kwargs = dict(host=host, user=user, local_fn=track.local_fn, remote_fn=track.remote_fn, rsync_options=rsync_options, run_local=run_local) - results = upload_file(**kwargs) + results = cur_upload_file(**kwargs) if track.tracktype == 'bam': kwargs['local_fn'] += '.bai' kwargs['remote_fn'] += '.bai' From cdbc2f82400ed6a91307d8f624675dd555a9c12a Mon Sep 17 00:00:00 2001 From: Katannya Kapeli Date: Tue, 2 Dec 2014 13:03:33 -0800 Subject: [PATCH 4/4] fixed bug in uploading tracks to local server instead of s3 --- trackhub/upload.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/trackhub/upload.py b/trackhub/upload.py index 77aa7a16..5fc8a6b1 100644 --- a/trackhub/upload.py +++ b/trackhub/upload.py @@ -52,10 +52,12 @@ def upload_hub(host, user, hub, port=22, rsync_options='-azvr --progress', print kwargs results = [] if run_s3: - upload_file = upload_s3 + cur_upload_file = upload_s3 + else: + cur_upload_file = upload_file # First the hub file: results.extend( - upload_file( + cur_upload_file( local_fn=hub.local_fn, remote_fn=hub.remote_fn, **kwargs) @@ -64,7 +66,7 @@ def upload_hub(host, user, hub, port=22, rsync_options='-azvr --progress', # Then the genomes file: print hub.genomes_file.local_fn results.extend( - upload_file( + cur_upload_file( local_fn=hub.genomes_file.local_fn, remote_fn=hub.genomes_file.remote_fn, **kwargs) @@ -73,7 +75,7 @@ def upload_hub(host, user, hub, port=22, rsync_options='-azvr --progress', # then the trackDB file: for g in hub.genomes_file.genomes: results.extend( - upload_file( + cur_upload_file( local_fn=g.trackdb.local_fn, remote_fn=g.trackdb.remote_fn, **kwargs @@ -84,7 +86,7 @@ def upload_hub(host, user, hub, port=22, rsync_options='-azvr --progress', print repr(t) if t._html: results.extend( - upload_file( + cur_upload_file( local_fn=t._html.local_fn, remote_fn=t._html.remote_fn, **kwargs)