From 7ba14fa44a53443f58fa28856773f25404fee318 Mon Sep 17 00:00:00 2001 From: Piotr Kaleta Date: Fri, 1 Mar 2013 11:47:55 +0100 Subject: [PATCH 01/76] .gitignore added. --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..0d20b64 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +*.pyc From 68af3c15756339c3e8445d7c33764193af3d29ca Mon Sep 17 00:00:00 2001 From: Piotr Kaleta Date: Fri, 1 Mar 2013 11:48:20 +0100 Subject: [PATCH 02/76] Multipart upload added. --- README.md | 1 + glacier.py | 40 ++++++++++++++++++++++++++++++++++------ 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index fb7b5f4..d8e3035 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,7 @@ Commands * glacier vault sync [--wait] [--fix] [--max-age hours] vault-name * glacier archive list vault-name * glacier archive upload [--name archive-name] vault-name filename +* glacier archive multipart_upload [--name archive-name] [--part-size part-size] vault-name filename * glacier archive retrieve [--wait] [-o filename] [--multipart-size bytes] vault-name archive-name * glacier archive retrieve [--wait] [--multipart-size bytes] vault-name archive-name [archive-name...] * glacier archive delete vault-name archive-name diff --git a/glacier.py b/glacier.py index c28a637..3a36035 100755 --- a/glacier.py +++ b/glacier.py @@ -49,6 +49,8 @@ INVENTORY_LAG = 24 * 60 * 60 * 3 PROGRAM_NAME = 'glacier' +DEFAULT_PART_SIZE = 4194304 + class ConsoleError(RuntimeError): def __init__(self, m): @@ -463,7 +465,7 @@ def archive_list(self, args): if archive_list: print(*archive_list, sep="\n") - def archive_upload(self, args): + def archive_upload(self, args, multipart=False): # XXX: "Leading whitespace in archive descriptions is removed." # XXX: "The description must be less than or equal to 1024 bytes. The # allowable characters are 7 bit ASCII without control codes, @@ -473,15 +475,25 @@ def archive_upload(self, args): name = args.name else: try: - full_name = args.file.name + full_name = args.file except: - raise RuntimeError('Archive name not specified. Use --name') + raise RuntimeError("Archive name not specified. Use --name.") name = os.path.basename(full_name) vault = self.connection.get_vault(args.vault) - archive_id = vault.create_archive_from_file(file_obj=args.file, description=name) + + if not multipart: + archive_id = vault.create_archive_from_file( + filename=args.file, description=name) + else: + archive_id = vault.concurrent_create_archive_from_file( + filename=args.file, description=name) + self.cache.add_archive(args.vault, name, archive_id) + def multipart_archive_upload(self, args): + return self.archive_upload(args, multipart=True) + @staticmethod def _write_archive_retrieval_job(f, job, multipart_size): if job.archive_size > multipart_size: @@ -639,12 +651,28 @@ def main(self): archive_list_subparser = archive_subparser.add_parser('list') archive_list_subparser.set_defaults(func=self.archive_list) archive_list_subparser.add_argument('vault') + + # Upload command archive_upload_subparser = archive_subparser.add_parser('upload') archive_upload_subparser.set_defaults(func=self.archive_upload) archive_upload_subparser.add_argument('vault') - archive_upload_subparser.add_argument('file', - type=argparse.FileType('rb')) + archive_upload_subparser.add_argument('file') archive_upload_subparser.add_argument('--name') + + # Multipart upload command + archive_multipart_upload_subparser = archive_subparser.add_parser( + 'multipart_upload') + archive_multipart_upload_subparser.set_defaults( + func=self.multipart_archive_upload) + archive_multipart_upload_subparser.add_argument('vault') + archive_multipart_upload_subparser.add_argument('file') + archive_multipart_upload_subparser.add_argument('--name') + archive_multipart_upload_subparser.add_argument( + '--part-size', + default=DEFAULT_PART_SIZE, + dest="part_size" + ) + archive_retrieve_subparser = archive_subparser.add_parser('retrieve') archive_retrieve_subparser.set_defaults(func=self.archive_retrieve) archive_retrieve_subparser.add_argument('vault') From 6a90f7f73e5828965c0f88a8bc50463c3ca7f018 Mon Sep 17 00:00:00 2001 From: Piotr Kaleta Date: Fri, 1 Mar 2013 14:28:55 +0100 Subject: [PATCH 03/76] Importing default part size from boto. --- glacier.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/glacier.py b/glacier.py index 3a36035..b212507 100755 --- a/glacier.py +++ b/glacier.py @@ -33,6 +33,7 @@ import sys import time +from boto.glacier.utils import DEFAULT_PART_SIZE import boto.glacier import iso8601 import sqlalchemy @@ -49,7 +50,6 @@ INVENTORY_LAG = 24 * 60 * 60 * 3 PROGRAM_NAME = 'glacier' -DEFAULT_PART_SIZE = 4194304 class ConsoleError(RuntimeError): @@ -57,7 +57,8 @@ def __init__(self, m): self.message = m -class RetryConsoleError(ConsoleError): pass +class RetryConsoleError(ConsoleError): + pass def info(message): From 4be17d7a73a2b61df892ae5406f308a00c2c6002 Mon Sep 17 00:00:00 2001 From: Piotr Kaleta Date: Fri, 1 Mar 2013 14:31:58 +0100 Subject: [PATCH 04/76] Added option to specify number of threads in multipart upload. --- glacier.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/glacier.py b/glacier.py index b212507..aa64863 100755 --- a/glacier.py +++ b/glacier.py @@ -33,6 +33,7 @@ import sys import time +from boto.glacier.utils import DEFAULT_NUM_THREADS from boto.glacier.utils import DEFAULT_PART_SIZE import boto.glacier import iso8601 @@ -466,7 +467,9 @@ def archive_list(self, args): if archive_list: print(*archive_list, sep="\n") - def archive_upload(self, args, multipart=False): + def archive_upload(self, args, multipart=False, + part_size=DEFAULT_PART_SIZE, + num_threads=DEFAULT_NUM_THREADS): # XXX: "Leading whitespace in archive descriptions is removed." # XXX: "The description must be less than or equal to 1024 bytes. The # allowable characters are 7 bit ASCII without control codes, @@ -488,7 +491,8 @@ def archive_upload(self, args, multipart=False): filename=args.file, description=name) else: archive_id = vault.concurrent_create_archive_from_file( - filename=args.file, description=name) + filename=args.file, description=name, + part_size=part_size, num_threads=num_threads) self.cache.add_archive(args.vault, name, archive_id) @@ -673,6 +677,11 @@ def main(self): default=DEFAULT_PART_SIZE, dest="part_size" ) + archive_multipart_upload_subparser.add_argument( + '--num-threads', + default=DEFAULT_NUM_THREADS, + dest="num_threads" + ) archive_retrieve_subparser = archive_subparser.add_parser('retrieve') archive_retrieve_subparser.set_defaults(func=self.archive_retrieve) From ec13d9313fa853d9409a0b06f8eb8b8c43c54f04 Mon Sep 17 00:00:00 2001 From: Piotr Kaleta Date: Tue, 12 Mar 2013 18:03:31 +0100 Subject: [PATCH 05/76] Using ConcurrentUploader directly to upload file to glacier. --- glacier.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/glacier.py b/glacier.py index aa64863..68ce7de 100755 --- a/glacier.py +++ b/glacier.py @@ -34,7 +34,7 @@ import time from boto.glacier.utils import DEFAULT_NUM_THREADS -from boto.glacier.utils import DEFAULT_PART_SIZE +from boto.glacier.concurrent import ConcurrentUploader import boto.glacier import iso8601 import sqlalchemy @@ -52,6 +52,8 @@ PROGRAM_NAME = 'glacier' +DEFAULT_PART_SIZE = 4194304 + class ConsoleError(RuntimeError): def __init__(self, m): @@ -490,9 +492,11 @@ def archive_upload(self, args, multipart=False, archive_id = vault.create_archive_from_file( filename=args.file, description=name) else: - archive_id = vault.concurrent_create_archive_from_file( - filename=args.file, description=name, - part_size=part_size, num_threads=num_threads) + uploader = ConcurrentUploader(self.connection.layer1, + vault.name, + part_size=part_size, + num_threads=num_threads) + archive_id = uploader.upload(args.file, description=name) self.cache.add_archive(args.vault, name, archive_id) From 6f7174c9145013eadac7d8a9113f4046e3154a22 Mon Sep 17 00:00:00 2001 From: Piotr Kaleta Date: Wed, 13 Mar 2013 01:13:18 +0100 Subject: [PATCH 06/76] First shot at GPG encryption/decryption of transferred messages. --- glacier.py | 102 ++++++++++++++++++++++++++++++++++++++--------------- gpg.py | 40 +++++++++++++++++++++ 2 files changed, 113 insertions(+), 29 deletions(-) create mode 100644 gpg.py diff --git a/glacier.py b/glacier.py index 68ce7de..44c2314 100755 --- a/glacier.py +++ b/glacier.py @@ -24,6 +24,7 @@ from __future__ import print_function from __future__ import unicode_literals +from functools import partial import argparse import calendar import errno @@ -31,16 +32,19 @@ import os import os.path import sys +import tempfile import time -from boto.glacier.utils import DEFAULT_NUM_THREADS from boto.glacier.concurrent import ConcurrentUploader +from boto.glacier.utils import DEFAULT_NUM_THREADS import boto.glacier import iso8601 import sqlalchemy import sqlalchemy.ext.declarative import sqlalchemy.orm +from gpg import Encryptor + # There is a lag between an archive being created and the archive # appearing on an inventory. Even if the inventory has an InventoryDate @@ -54,6 +58,8 @@ DEFAULT_PART_SIZE = 4194304 +DEFAULT_REGION = 'us-east-1' + class ConsoleError(RuntimeError): def __init__(self, m): @@ -469,9 +475,13 @@ def archive_list(self, args): if archive_list: print(*archive_list, sep="\n") - def archive_upload(self, args, multipart=False, + def archive_upload(self, + args, + multipart=False, + encryptor=None, part_size=DEFAULT_PART_SIZE, num_threads=DEFAULT_NUM_THREADS): + # XXX: "Leading whitespace in archive descriptions is removed." # XXX: "The description must be less than or equal to 1024 bytes. The # allowable characters are 7 bit ASCII without control codes, @@ -486,39 +496,53 @@ def archive_upload(self, args, multipart=False, raise RuntimeError("Archive name not specified. Use --name.") name = os.path.basename(full_name) + if encryptor: + tmpfile = tempfile.NamedTemporaryFile() + encryptor.encrypt_file(args.file, tmpfile.name) + filename = tmpfile.name + else: + filename = args.file + vault = self.connection.get_vault(args.vault) if not multipart: archive_id = vault.create_archive_from_file( - filename=args.file, description=name) + filename=filename, description=name) else: uploader = ConcurrentUploader(self.connection.layer1, vault.name, part_size=part_size, num_threads=num_threads) - archive_id = uploader.upload(args.file, description=name) + archive_id = uploader.upload(filename, description=name) self.cache.add_archive(args.vault, name, archive_id) - def multipart_archive_upload(self, args): - return self.archive_upload(args, multipart=True) + if encryptor: + tmpfile.close() + + def multipart_archive_upload(self, args, encryptor=None): + return self.archive_upload(args, multipart=True, encryptor=encryptor) @staticmethod - def _write_archive_retrieval_job(f, job, multipart_size): + def _write_archive_retrieval_job(f, job, multipart_size, + encryptor=None): if job.archive_size > multipart_size: def fetch(start, end): - byte_range = start, end-1 + byte_range = start, end - 1 f.write(job.get_output(byte_range).read()) whole_parts = job.archive_size // multipart_size for first_byte in xrange(0, whole_parts * multipart_size, - multipart_size): + multipart_size): fetch(first_byte, first_byte + multipart_size) remainder = job.archive_size % multipart_size if remainder: fetch(job.archive_size - remainder, job.archive_size) else: - f.write(job.get_output().read()) + data = job.get_output().read() + if encryptor: + data = encryptor.decrypt(data) + f.write(data) # Make sure that the file now exactly matches the downloaded archive, # even if the file existed before and was longer. @@ -531,19 +555,21 @@ def fetch(start, end): raise @classmethod - def _archive_retrieve_completed(cls, args, job, name): + def _archive_retrieve_completed(cls, args, job, name, encryptor=None): if args.output_filename == '-': cls._write_archive_retrieval_job( - sys.stdout, job, args.multipart_size) + sys.stdout, job, args.multipart_size, + encryptor=encryptor) else: if args.output_filename: filename = args.output_filename else: filename = os.path.basename(name) with open(filename, 'wb') as f: - cls._write_archive_retrieval_job(f, job, args.multipart_size) + cls._write_archive_retrieval_job(f, job, args.multipart_size, + encryptor=encryptor) - def archive_retrieve_one(self, args, name): + def archive_retrieve_one(self, args, name, encryptor=None): try: archive_id = self.cache.get_archive_id(args.vault, name) except KeyError: @@ -554,13 +580,16 @@ def archive_retrieve_one(self, args, name): complete_job = find_complete_job(retrieval_jobs) if complete_job: - self._archive_retrieve_completed(args, complete_job, name) + self._archive_retrieve_completed(args, complete_job, name, + encryptor=encryptor) elif has_pending_job(retrieval_jobs): if args.wait: complete_job = wait_until_job_completed(retrieval_jobs) - self._archive_retrieve_completed(args, complete_job, name) + self._archive_retrieve_completed(args, complete_job, name, + encryptor=encryptor) else: - raise RetryConsoleError('job still pending for archive %r' % name) + raise RetryConsoleError('job still pending for archive %r' + % name) else: # create an archive retrieval job job = vault.retrieve_archive(archive_id) @@ -568,16 +597,18 @@ def archive_retrieve_one(self, args, name): wait_until_job_completed([job]) self._archive_retrieve_completed(args, job, name) else: - raise RetryConsoleError('queued retrieval job for archive %r' % name) + raise RetryConsoleError('queued retrieval job for archive %r' + % name) - def archive_retrieve(self, args): + def archive_retrieve(self, args, encryptor=None): if len(args.names) > 1 and args.output_filename: - raise ConsoleError('cannot specify output filename with multi-archive retrieval') + raise ConsoleError("cannot specify output filename with " + "multi-archive retrieval") success_list = [] retry_list = [] for name in args.names: try: - self.archive_retrieve_one(args, name) + self.archive_retrieve_one(args, name, encryptor=encryptor) except RetryConsoleError, e: retry_list.append(e.message) else: @@ -642,7 +673,7 @@ def too_old(last_seen): def main(self): parser = argparse.ArgumentParser() - parser.add_argument('--region', default='us-east-1') + parser.add_argument('--region', default=DEFAULT_REGION) subparsers = parser.add_subparsers() vault_subparser = subparsers.add_parser('vault').add_subparsers() vault_subparser.add_parser('list').set_defaults(func=self.vault_list) @@ -661,18 +692,23 @@ def main(self): archive_list_subparser.set_defaults(func=self.archive_list) archive_list_subparser.add_argument('vault') + encryptor = Encryptor() + # Upload command + archive_upload_func = partial(self.archive_upload, encryptor=encryptor) archive_upload_subparser = archive_subparser.add_parser('upload') - archive_upload_subparser.set_defaults(func=self.archive_upload) + archive_upload_subparser.set_defaults(func=archive_upload_func) archive_upload_subparser.add_argument('vault') archive_upload_subparser.add_argument('file') archive_upload_subparser.add_argument('--name') # Multipart upload command + multipart_archive_upload_func = partial( + self.archive_upload, encryptor=encryptor) archive_multipart_upload_subparser = archive_subparser.add_parser( 'multipart_upload') archive_multipart_upload_subparser.set_defaults( - func=self.multipart_archive_upload) + func=multipart_archive_upload_func) archive_multipart_upload_subparser.add_argument('vault') archive_multipart_upload_subparser.add_argument('file') archive_multipart_upload_subparser.add_argument('--name') @@ -687,24 +723,31 @@ def main(self): dest="num_threads" ) + # Retrieve command + archive_retrieve_func = partial( + self.archive_retrieve, encryptor=encryptor) archive_retrieve_subparser = archive_subparser.add_parser('retrieve') - archive_retrieve_subparser.set_defaults(func=self.archive_retrieve) + archive_retrieve_subparser.set_defaults(func=archive_retrieve_func) archive_retrieve_subparser.add_argument('vault') archive_retrieve_subparser.add_argument('names', nargs='+', metavar='name') archive_retrieve_subparser.add_argument('--multipart-size', type=int, - default=(8*1024*1024)) + default=(8 * 1024 * 1024)) archive_retrieve_subparser.add_argument('-o', dest='output_filename', metavar='OUTPUT_FILENAME') archive_retrieve_subparser.add_argument('--wait', action='store_true') + + # Delete command archive_delete_subparser = archive_subparser.add_parser('delete') archive_delete_subparser.set_defaults(func=self.archive_delete) archive_delete_subparser.add_argument('vault') archive_delete_subparser.add_argument('name') + + # Checkpresent command archive_checkpresent_subparser = archive_subparser.add_parser( - 'checkpresent') + 'checkpresent') archive_checkpresent_subparser.set_defaults( - func=self.archive_checkpresent) + func=self.archive_checkpresent) archive_checkpresent_subparser.add_argument('vault') archive_checkpresent_subparser.add_argument('name') archive_checkpresent_subparser.add_argument('--wait', @@ -712,7 +755,8 @@ def main(self): archive_checkpresent_subparser.add_argument('--quiet', action='store_true') archive_checkpresent_subparser.add_argument( - '--max-age', type=int, default=80, dest='max_age_hours') + '--max-age', type=int, default=80, dest='max_age_hours') + job_subparser = subparsers.add_parser('job').add_subparsers() job_subparser.add_parser('list').set_defaults(func=self.job_list) args = parser.parse_args() diff --git a/gpg.py b/gpg.py new file mode 100644 index 0000000..7a3b2ef --- /dev/null +++ b/gpg.py @@ -0,0 +1,40 @@ +import gnupg +import os + + +GNUPG_DIRECTORY = os.path.join(os.path.abspath(os.path.dirname(__file__)), + "gnupg") +GNUPG_KEY_TYPE = "RSA" +GNUPG_KEY_LENGTH = 2048 + + +class Encryptor(object): + + def __init__(self): + self.gpg = gnupg.GPG(gnupghome=GNUPG_DIRECTORY) + self.gpg.encoding = "utf-8" + if self.gpg.list_keys() == []: + self.generate_keypair() + + def generate_keypair(self): + input_data = self.gpg.gen_key_input(key_type=GNUPG_KEY_TYPE, + key_length=GNUPG_KEY_LENGTH) + key = self.gpg.gen_key(input_data) + + return key + + def export_keypair(self, key): + public_key = self.gpg.export_keys(key) + private_key = self.gpg.export_keys(key, True) + + return public_key, private_key + + def encrypt_file(self, input_filename, output_filename): + fingerprint = self.gpg.list_keys()[0]["fingerprint"] + + with open(input_filename) as input_file: + self.gpg.encrypt_file(input_file, fingerprint, + output=output_filename) + + def decrypt(self, data): + return self.gpg.decrypt(data) From 389a1b8d1695fbf818a07c38a7f8fa258e29c887 Mon Sep 17 00:00:00 2001 From: Piotr Kaleta Date: Wed, 13 Mar 2013 13:38:28 +0100 Subject: [PATCH 07/76] Fixed decryption when retrieving file. --- glacier.py | 31 ++++++++++++++++++++++++------- gpg.py | 11 +++++++++-- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/glacier.py b/glacier.py index 44c2314..609e178 100755 --- a/glacier.py +++ b/glacier.py @@ -352,7 +352,17 @@ def recent_enough(job): def find_complete_job(jobs): - for job in sorted(filter(lambda job: job.completed, jobs), key=lambda job: iso8601.parse_date(job.completion_date), reverse=True): + complete_jobs = filter(lambda job: job.completed, jobs) + + def most_recent_job(job): + return iso8601.parse_date(job.completion_date) + + sorted_completed_jobs = sorted( + complete_jobs, + key=most_recent_job, reverse=True + ) + + for job in sorted_completed_jobs: return job @@ -526,10 +536,15 @@ def multipart_archive_upload(self, args, encryptor=None): @staticmethod def _write_archive_retrieval_job(f, job, multipart_size, encryptor=None): + if encryptor: + destfile = tempfile.NamedTemporaryFile() + else: + destfile = f + if job.archive_size > multipart_size: def fetch(start, end): byte_range = start, end - 1 - f.write(job.get_output(byte_range).read()) + destfile.write(job.get_output(byte_range).read()) whole_parts = job.archive_size // multipart_size for first_byte in xrange(0, whole_parts * multipart_size, @@ -539,21 +554,23 @@ def fetch(start, end): if remainder: fetch(job.archive_size - remainder, job.archive_size) else: - data = job.get_output().read() - if encryptor: - data = encryptor.decrypt(data) - f.write(data) + destfile.write(job.get_output().read()) # Make sure that the file now exactly matches the downloaded archive, # even if the file existed before and was longer. try: - f.truncate(job.archive_size) + destfile.truncate(job.archive_size) except IOError, e: # Allow ESPIPE, since the "file" couldn't have existed before in # this case. if e.errno != errno.ESPIPE: raise + # Decrypt file if encryptor is given + if encryptor: + encryptor.decrypt_file(destfile.name, f.name) + destfile.close() + @classmethod def _archive_retrieve_completed(cls, args, job, name, encryptor=None): if args.output_filename == '-': diff --git a/gpg.py b/gpg.py index 7a3b2ef..f7bdeaf 100644 --- a/gpg.py +++ b/gpg.py @@ -29,12 +29,19 @@ def export_keypair(self, key): return public_key, private_key + def _get_fingerprint(self): + return self.gpg.list_keys()[0]["fingerprint"] + def encrypt_file(self, input_filename, output_filename): - fingerprint = self.gpg.list_keys()[0]["fingerprint"] + fingerprint = self._get_fingerprint() - with open(input_filename) as input_file: + with open(input_filename, "r") as input_file: self.gpg.encrypt_file(input_file, fingerprint, output=output_filename) + def decrypt_file(self, input_filename, output_filename): + with open(input_filename, "r") as input_file: + self.gpg.decrypt_file(input_file, output=output_filename) + def decrypt(self, data): return self.gpg.decrypt(data) From 85cc6b4c48af59a7c2b9f6f570ce7c5431f945a6 Mon Sep 17 00:00:00 2001 From: Piotr Kaleta Date: Wed, 13 Mar 2013 18:29:51 +0100 Subject: [PATCH 08/76] Fixed missing encryptor. --- glacier.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/glacier.py b/glacier.py index 609e178..99a4840 100755 --- a/glacier.py +++ b/glacier.py @@ -612,7 +612,8 @@ def archive_retrieve_one(self, args, name, encryptor=None): job = vault.retrieve_archive(archive_id) if args.wait: wait_until_job_completed([job]) - self._archive_retrieve_completed(args, job, name) + self._archive_retrieve_completed(args, job, name, + encryptor=encryptor) else: raise RetryConsoleError('queued retrieval job for archive %r' % name) From 88e499fd99b6ba778e4566a2308873822ef7d536 Mon Sep 17 00:00:00 2001 From: Piotr Kaleta Date: Fri, 15 Mar 2013 13:19:37 +0100 Subject: [PATCH 09/76] Add some docstrings. --- glacier.py | 4 ++-- gpg.py | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/glacier.py b/glacier.py index 99a4840..28280e6 100755 --- a/glacier.py +++ b/glacier.py @@ -561,8 +561,8 @@ def fetch(start, end): try: destfile.truncate(job.archive_size) except IOError, e: - # Allow ESPIPE, since the "file" couldn't have existed before in - # this case. + # Allow ESPIPE, since the "file" couldn't have existed + # before in this case. if e.errno != errno.ESPIPE: raise diff --git a/gpg.py b/gpg.py index f7bdeaf..e7996f7 100644 --- a/gpg.py +++ b/gpg.py @@ -17,6 +17,7 @@ def __init__(self): self.generate_keypair() def generate_keypair(self): + """Generate the RSA keypair.""" input_data = self.gpg.gen_key_input(key_type=GNUPG_KEY_TYPE, key_length=GNUPG_KEY_LENGTH) key = self.gpg.gen_key(input_data) @@ -30,18 +31,26 @@ def export_keypair(self, key): return public_key, private_key def _get_fingerprint(self): + """Return the fingerprint of the default key.""" return self.gpg.list_keys()[0]["fingerprint"] def encrypt_file(self, input_filename, output_filename): + """Encrypt `input_filename` and save results as `output_filename`.""" fingerprint = self._get_fingerprint() with open(input_filename, "r") as input_file: self.gpg.encrypt_file(input_file, fingerprint, output=output_filename) + def encrypt(self, data): + """Return the encrypted version of `data`.""" + return self.gpg.encrypt(data) + def decrypt_file(self, input_filename, output_filename): + """Decrypt `input_filename` and save results as `output_filename`.""" with open(input_filename, "r") as input_file: self.gpg.decrypt_file(input_file, output=output_filename) def decrypt(self, data): + """Return decrypted version of `data`.""" return self.gpg.decrypt(data) From 724f65f396cb7e58db7aa49bce48e00cd067bd29 Mon Sep 17 00:00:00 2001 From: Piotr Kaleta Date: Fri, 15 Mar 2013 14:10:57 +0100 Subject: [PATCH 10/76] Added option to disable encryption/decryption. --- glacier.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/glacier.py b/glacier.py index 28280e6..b7aa2bc 100755 --- a/glacier.py +++ b/glacier.py @@ -692,6 +692,7 @@ def too_old(last_seen): def main(self): parser = argparse.ArgumentParser() parser.add_argument('--region', default=DEFAULT_REGION) + subparsers = parser.add_subparsers() vault_subparser = subparsers.add_parser('vault').add_subparsers() vault_subparser.add_parser('list').set_defaults(func=self.vault_list) @@ -719,6 +720,7 @@ def main(self): archive_upload_subparser.add_argument('vault') archive_upload_subparser.add_argument('file') archive_upload_subparser.add_argument('--name') + archive_upload_subparser.add_argument('--encrypt', default=True) # Multipart upload command multipart_archive_upload_func = partial( @@ -730,6 +732,8 @@ def main(self): archive_multipart_upload_subparser.add_argument('vault') archive_multipart_upload_subparser.add_argument('file') archive_multipart_upload_subparser.add_argument('--name') + archive_multipart_upload_subparser.add_argument( + '--encrypt', default=True) archive_multipart_upload_subparser.add_argument( '--part-size', default=DEFAULT_PART_SIZE, @@ -754,6 +758,8 @@ def main(self): archive_retrieve_subparser.add_argument('-o', dest='output_filename', metavar='OUTPUT_FILENAME') archive_retrieve_subparser.add_argument('--wait', action='store_true') + archive_multipart_upload_subparser.add_argument( + '--decrypt', default=True) # Delete command archive_delete_subparser = archive_subparser.add_parser('delete') @@ -777,6 +783,7 @@ def main(self): job_subparser = subparsers.add_parser('job').add_subparsers() job_subparser.add_parser('list').set_defaults(func=self.job_list) + args = parser.parse_args() self.connection = boto.glacier.connect_to_region(args.region) self.cache = Cache(get_connection_account(self.connection)) From 806c31e38e578101e4ec77034a4116f804f2fd15 Mon Sep 17 00:00:00 2001 From: Piotr Kaleta Date: Fri, 15 Mar 2013 14:35:36 +0100 Subject: [PATCH 11/76] Using command line option to decide whether to encrypt/decrypt or not. --- glacier.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/glacier.py b/glacier.py index b7aa2bc..1c185a2 100755 --- a/glacier.py +++ b/glacier.py @@ -506,7 +506,7 @@ def archive_upload(self, raise RuntimeError("Archive name not specified. Use --name.") name = os.path.basename(full_name) - if encryptor: + if args.encrypt: tmpfile = tempfile.NamedTemporaryFile() encryptor.encrypt_file(args.file, tmpfile.name) filename = tmpfile.name @@ -527,7 +527,7 @@ def archive_upload(self, self.cache.add_archive(args.vault, name, archive_id) - if encryptor: + if args.encrypt: tmpfile.close() def multipart_archive_upload(self, args, encryptor=None): @@ -624,6 +624,12 @@ def archive_retrieve(self, args, encryptor=None): "multi-archive retrieval") success_list = [] retry_list = [] + + # Let called functions know that encryption was disabled from + # command-line. + if not args.decrypt: + encryptor = None + for name in args.names: try: self.archive_retrieve_one(args, name, encryptor=encryptor) @@ -688,7 +694,6 @@ def too_old(last_seen): print(args.name) - def main(self): parser = argparse.ArgumentParser() parser.add_argument('--region', default=DEFAULT_REGION) @@ -720,7 +725,8 @@ def main(self): archive_upload_subparser.add_argument('vault') archive_upload_subparser.add_argument('file') archive_upload_subparser.add_argument('--name') - archive_upload_subparser.add_argument('--encrypt', default=True) + archive_upload_subparser.add_argument( + '--encrypt', default=False, action="store_true") # Multipart upload command multipart_archive_upload_func = partial( @@ -733,7 +739,7 @@ def main(self): archive_multipart_upload_subparser.add_argument('file') archive_multipart_upload_subparser.add_argument('--name') archive_multipart_upload_subparser.add_argument( - '--encrypt', default=True) + '--encrypt', default=False, action="store_true") archive_multipart_upload_subparser.add_argument( '--part-size', default=DEFAULT_PART_SIZE, @@ -758,8 +764,8 @@ def main(self): archive_retrieve_subparser.add_argument('-o', dest='output_filename', metavar='OUTPUT_FILENAME') archive_retrieve_subparser.add_argument('--wait', action='store_true') - archive_multipart_upload_subparser.add_argument( - '--decrypt', default=True) + archive_retrieve_subparser.add_argument( + '--decrypt', default=False, action="store_true") # Delete command archive_delete_subparser = archive_subparser.add_parser('delete') From 7f306548c8bcbd6e4b585886f44a219f67d7224a Mon Sep 17 00:00:00 2001 From: Piotr Kaleta Date: Fri, 15 Mar 2013 15:00:28 +0100 Subject: [PATCH 12/76] Updated readme. --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d8e3035..f289216 100644 --- a/README.md +++ b/README.md @@ -211,10 +211,9 @@ Commands * glacier vault create vault-name * glacier vault sync [--wait] [--fix] [--max-age hours] vault-name * glacier archive list vault-name -* glacier archive upload [--name archive-name] vault-name filename -* glacier archive multipart_upload [--name archive-name] [--part-size part-size] vault-name filename -* glacier archive retrieve [--wait] [-o filename] [--multipart-size bytes] vault-name archive-name -* glacier archive retrieve [--wait] [--multipart-size bytes] vault-name archive-name [archive-name...] +* glacier archive upload [--encrypt] [--name archive-name] vault-name filename +* glacier archive multipart_upload [--encrypt] [--name archive-name] [--part-size part-size] vault-name filename +* glacier archive retrieve [--wait] [--decrypt] [-o filename] [--multipart-size bytes] vault-name archive-name [archive-name...] * glacier archive delete vault-name archive-name * glacier job list From d5af2ed14aa3b70cfcbb06d1966b4b15fbf46334 Mon Sep 17 00:00:00 2001 From: Piotr Kaleta Date: Fri, 15 Mar 2013 15:38:13 +0100 Subject: [PATCH 13/76] Added setup.py --- setup.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 setup.py diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..b3f1f43 --- /dev/null +++ b/setup.py @@ -0,0 +1,9 @@ +from setuptools import setup + +setup(name='glacier-cli', + version='0.1', + packages=['glacier-cli'], + license='BSD', + description='Command-line interface to Amazon Glacier', + classifiers=[] +) From 720bdbd0c1c07e84b32301d88e02c42600d0ca57 Mon Sep 17 00:00:00 2001 From: Piotr Kaleta Date: Fri, 15 Mar 2013 15:39:29 +0100 Subject: [PATCH 14/76] Fix package names in setup file. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b3f1f43..f7be857 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ setup(name='glacier-cli', version='0.1', - packages=['glacier-cli'], + packages=['glacier', 'gpg'], license='BSD', description='Command-line interface to Amazon Glacier', classifiers=[] From e8a25bb830e3bfe33d397721088c06144bc9f901 Mon Sep 17 00:00:00 2001 From: Piotr Kaleta Date: Fri, 15 Mar 2013 15:43:16 +0100 Subject: [PATCH 15/76] Fixed setup.py one more time. --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index f7be857..2c068b5 100644 --- a/setup.py +++ b/setup.py @@ -2,8 +2,7 @@ setup(name='glacier-cli', version='0.1', - packages=['glacier', 'gpg'], + py_modules=['glacier', 'gpg'], license='BSD', description='Command-line interface to Amazon Glacier', - classifiers=[] ) From 2af4c6f604f5590552e8d2cf3453db531a4f8262 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Thu, 21 Nov 2013 19:29:18 -0800 Subject: [PATCH 16/76] remove comments from merge --- glacier.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/glacier.py b/glacier.py index bb879f9..a1417a4 100755 --- a/glacier.py +++ b/glacier.py @@ -38,6 +38,7 @@ from boto.glacier.concurrent import ConcurrentUploader #from boto.glacier.utils import DEFAULT_NUM_THREADS DEFAULT_NUM_THREADS = 1 + import boto.glacier import iso8601 import sqlalchemy @@ -361,7 +362,6 @@ def recent_enough(job): def find_complete_job(jobs): - #for job in sorted(filter(lambda job: job.completed, jobs), key=lambda job: iso8601.parse_date(job.completion_date), reverse=True): complete_jobs = filter(lambda job: job.completed, jobs) def most_recent_job(job): @@ -681,8 +681,6 @@ def archive_checkpresent(self): return def too_old(last_seen): - #return not last_seen or not args.max_age_hours or ( - # last_seen < time.time() - args.max_age_hours * 60 * 60) return (not last_seen or not self.args.max_age_hours or (last_seen < From 724d692fc78a88481b7f3c71efa017fbcf70ac1f Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Thu, 21 Nov 2013 19:37:49 -0800 Subject: [PATCH 17/76] fix handling of file name vs obj --- glacier.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/glacier.py b/glacier.py index a1417a4..ab07232 100755 --- a/glacier.py +++ b/glacier.py @@ -523,16 +523,16 @@ def archive_upload(self, if self.args.encrypt: tmpfile = tempfile.NamedTemporaryFile() - encryptor.encrypt_file(self.args.file.name, tmpfile.name) - file_obj = tmpfile + encryptor.encrypt_file(self.args.file, tmpfile.name) + filename = tmpfile else: - file_obj = self.args.file + filename = self.args.file vault = self.connection.get_vault(self.args.vault) if not multipart: archive_id = vault.create_archive_from_file( - file_obj=file_obj, description=name) + file_obj=filename, description=name) else: uploader = ConcurrentUploader(self.connection.layer1, vault.name, From b51724a6760118bbf672f2264cf75e433993623d Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Thu, 21 Nov 2013 19:45:29 -0800 Subject: [PATCH 18/76] fix handling of file name vs obj --- glacier.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/glacier.py b/glacier.py index ab07232..b6141f6 100755 --- a/glacier.py +++ b/glacier.py @@ -524,15 +524,16 @@ def archive_upload(self, if self.args.encrypt: tmpfile = tempfile.NamedTemporaryFile() encryptor.encrypt_file(self.args.file, tmpfile.name) - filename = tmpfile + filename = tmpfile.name else: filename = self.args.file vault = self.connection.get_vault(self.args.vault) if not multipart: + file_obj=file(filename) archive_id = vault.create_archive_from_file( - file_obj=filename, description=name) + file_obj=file_obj, description=name) else: uploader = ConcurrentUploader(self.connection.layer1, vault.name, From bfcdaa475362aea5ea4730db39c641f6f0e45fc5 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Sat, 23 Nov 2013 00:55:54 -0800 Subject: [PATCH 19/76] set DEFAULT_NUM_THREADS to 10 as boto does internally --- glacier.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/glacier.py b/glacier.py index b6141f6..0d2eae2 100755 --- a/glacier.py +++ b/glacier.py @@ -36,8 +36,7 @@ import time from boto.glacier.concurrent import ConcurrentUploader -#from boto.glacier.utils import DEFAULT_NUM_THREADS -DEFAULT_NUM_THREADS = 1 +DEFAULT_NUM_THREADS = 10 import boto.glacier import iso8601 From d4088e60e15578c5482095c127c4c3ac98424ca0 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Sat, 23 Nov 2013 14:03:01 -0800 Subject: [PATCH 20/76] Ensure archive multipart_upload passes the multipart flag. --- glacier.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/glacier.py b/glacier.py index 0d2eae2..e21a082 100755 --- a/glacier.py +++ b/glacier.py @@ -545,9 +545,6 @@ def archive_upload(self, if self.args.encrypt: tmpfile.close() - def multipart_archive_upload(self, args, encryptor=None): - return self.archive_upload(args, multipart=True, encryptor=encryptor) - @staticmethod def _write_archive_retrieval_job(f, job, multipart_size, encryptor=None): @@ -753,7 +750,7 @@ def parse_args(self, args=None): # Multipart upload command multipart_archive_upload_func = partial( - self.archive_upload, encryptor=encryptor) + self.archive_upload, encryptor=encryptor, multipart=True) archive_multipart_upload_subparser = archive_subparser.add_parser( 'multipart_upload') archive_multipart_upload_subparser.set_defaults( From 635fbb4ffc2e5fdc641915fd695f7e2b88828a1c Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Sat, 23 Nov 2013 14:43:11 -0800 Subject: [PATCH 21/76] Cache encrypted file for retries until upload completes. Also add logging. Before merging reduce from info to debug. --- glacier.py | 51 +++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 43 insertions(+), 8 deletions(-) diff --git a/glacier.py b/glacier.py index e21a082..e8ac8dd 100755 --- a/glacier.py +++ b/glacier.py @@ -29,6 +29,7 @@ import calendar import errno import itertools +import logging import os import os.path import sys @@ -60,6 +61,8 @@ DEFAULT_REGION = 'us-east-1' +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger('glacier-cli') class ConsoleError(RuntimeError): def __init__(self, m): @@ -521,29 +524,61 @@ def archive_upload(self, name = os.path.basename(full_name) if self.args.encrypt: - tmpfile = tempfile.NamedTemporaryFile() - encryptor.encrypt_file(self.args.file, tmpfile.name) - filename = tmpfile.name + tmpdir = os.path.join(get_user_cache_dir(), + 'glacier-cli', + 'encryption-tmp', + self.args.vault) + if not os.path.exists(tmpdir): + os.makedirs(tmpdir) + filename = os.path.join(tmpdir, "%s.encrypted" % self.args.name) + if os.path.exists(filename): + logger.info("Found cached encryption file %s. " + "Skipping re-encryption of " + % filename) + else: + filename_working = "%s.working.%d" % (filename, os.getpid()) + if os.path.exists(filename_working): + logger.info("cleaning up %s" % filename_working) + os.remove(filename_working) + logger.info("encrypting %s as %s(.working)" % (self.args.file, filename)) + try: + encryptor.encrypt_file(self.args.file, filename_working) + os.rename(filename_working, filename) + logger.info("encryption complete: %s" % filename) + finally: + if os.path.exists(filename_working): + os.remove(filename_working) else: filename = self.args.file vault = self.connection.get_vault(self.args.vault) + logger.info("upload initiating") + if not multipart: + logger.info("uploading in single-part %s to %s" + % (filename, vault)) file_obj=file(filename) archive_id = vault.create_archive_from_file( file_obj=file_obj, description=name) else: + logger.info("confirguring multi-part upload " + "with part_size %d " + "and num_threads %d" % (part_size, num_threads)) uploader = ConcurrentUploader(self.connection.layer1, - vault.name, - part_size=part_size, - num_threads=num_threads) + vault.name, + part_size=part_size, + num_threads=num_threads) + logger.info("uploading multi-part %s to %s with %s" + % (filename, vault, uploader)) archive_id = uploader.upload(filename, description=name) + + logger.info("upload complete") self.cache.add_archive(self.args.vault, name, archive_id) - + if self.args.encrypt: - tmpfile.close() + os.remove(filename) @staticmethod def _write_archive_retrieval_job(f, job, multipart_size, From 30a86c068d3d9ce5f319c1a3672943ccd62e7117 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Sat, 23 Nov 2013 22:24:10 -0800 Subject: [PATCH 22/76] cleanup of new code around encryption temp files --- glacier.py | 99 +++++++++++++++++++++++++++++++----------------------- 1 file changed, 57 insertions(+), 42 deletions(-) diff --git a/glacier.py b/glacier.py index e8ac8dd..cc58e43 100755 --- a/glacier.py +++ b/glacier.py @@ -61,7 +61,10 @@ DEFAULT_REGION = 'us-east-1' -logging.basicConfig(level=logging.INFO) +logging.basicConfig( + level=logging.INFO, + format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') + logger = logging.getLogger('glacier-cli') class ConsoleError(RuntimeError): @@ -523,60 +526,57 @@ def archive_upload(self, raise RuntimeError("Archive name not specified. Use --name.") name = os.path.basename(full_name) - if self.args.encrypt: - tmpdir = os.path.join(get_user_cache_dir(), - 'glacier-cli', - 'encryption-tmp', - self.args.vault) - if not os.path.exists(tmpdir): - os.makedirs(tmpdir) - filename = os.path.join(tmpdir, "%s.encrypted" % self.args.name) - if os.path.exists(filename): - logger.info("Found cached encryption file %s. " - "Skipping re-encryption of " - % filename) - else: - filename_working = "%s.working.%d" % (filename, os.getpid()) - if os.path.exists(filename_working): - logger.info("cleaning up %s" % filename_working) - os.remove(filename_working) - logger.info("encrypting %s as %s(.working)" % (self.args.file, filename)) - try: - encryptor.encrypt_file(self.args.file, filename_working) - os.rename(filename_working, filename) - logger.info("encryption complete: %s" % filename) - finally: - if os.path.exists(filename_working): - os.remove(filename_working) - else: + if self.args.encrypt is None: filename = self.args.file + else: + if self.args.encryption_dir is None: + filename = tempfile.NamedTemporaryFile().name + logger.info("Encrypting %s to %s." + % (self.args.file, filename)) + encryptor.encrypt_file(self.args.file, filename) + else: + filename = os.path.join(self.args.encryption_dir, + "%s.encrypted" % self.args.name) + if os.path.exists(filename): + logger.info("Found cached encryption file %s. " + "Skipping re-encryption." + % filename) + else: + filename_working = "%s.working.%d" % (filename, os.getpid()) + try: + logger.info("Encrypting %s to %s." + % (self.args.file, filename_working)) + encryptor.encrypt_file(self.args.file, + filename_working) + os.rename(filename_working, filename) + finally: + if os.path.exists(filename_working): + os.remove(filename_working) + + logger.info("Encryption complete: %s." % filename) vault = self.connection.get_vault(self.args.vault) - logger.info("upload initiating") - if not multipart: - logger.info("uploading in single-part %s to %s" + logger.info("Uploading in a single part: %s to %s." % (filename, vault)) file_obj=file(filename) archive_id = vault.create_archive_from_file( file_obj=file_obj, description=name) else: - logger.info("confirguring multi-part upload " - "with part_size %d " - "and num_threads %d" % (part_size, num_threads)) + logger.info("Uploading multi-part: %s to %s" + % (filename, vault)) uploader = ConcurrentUploader(self.connection.layer1, - vault.name, - part_size=part_size, - num_threads=num_threads) - logger.info("uploading multi-part %s to %s with %s" - % (filename, vault, uploader)) + vault.name, + part_size=part_size, + num_threads=num_threads) archive_id = uploader.upload(filename, description=name) - - logger.info("upload complete") + + logger.info("Upload complete.") + logger.info("New Archive ID: %s" % archive_id) self.cache.add_archive(self.args.vault, name, archive_id) - + if self.args.encrypt: os.remove(filename) @@ -782,6 +782,10 @@ def parse_args(self, args=None): archive_upload_subparser.add_argument('--name') archive_upload_subparser.add_argument( '--encrypt', default=False, action="store_true") + archive_upload_subparser.add_argument( + '--encryption-dir', + dest='encryption_dir', + ) # Multipart upload command multipart_archive_upload_func = partial( @@ -795,6 +799,10 @@ def parse_args(self, args=None): archive_multipart_upload_subparser.add_argument('--name') archive_multipart_upload_subparser.add_argument( '--encrypt', default=False, action="store_true") + archive_multipart_upload_subparser.add_argument( + '--encryption-dir', + dest='encryption_dir', + ) archive_multipart_upload_subparser.add_argument( '--part-size', default=DEFAULT_PART_SIZE, @@ -844,7 +852,14 @@ def parse_args(self, args=None): job_subparser = subparsers.add_parser('job').add_subparsers() job_subparser.add_parser('list').set_defaults(func=self.job_list) - return parser.parse_args(args) + parsed = parser.parse_args(args) + + # Additional constraints + if (1 == None and parsed.encrypt == False): + raise argparse.ArgumentError("--encryption-dir is invalid " + "when --encrypt is not specified.") + + return parsed def __init__(self, args=None, connection=None, cache=None): args = self.parse_args(args) From a72a504ecb8516bfcfeb66e6322c9503a388e153 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Sat, 23 Nov 2013 22:27:14 -0800 Subject: [PATCH 23/76] fix temporarily disabled check around --encryption-dir --- glacier.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/glacier.py b/glacier.py index cc58e43..aba245b 100755 --- a/glacier.py +++ b/glacier.py @@ -852,14 +852,15 @@ def parse_args(self, args=None): job_subparser = subparsers.add_parser('job').add_subparsers() job_subparser.add_parser('list').set_defaults(func=self.job_list) - parsed = parser.parse_args(args) + parsed_args = parser.parse_args(args) # Additional constraints - if (1 == None and parsed.encrypt == False): + if (parsed_args.encrypt == False + and parsed_args.encryption_dir is not None): raise argparse.ArgumentError("--encryption-dir is invalid " "when --encrypt is not specified.") - return parsed + return parsed_args def __init__(self, args=None, connection=None, cache=None): args = self.parse_args(args) From 20dd737b94f61183eb92b8805795517c2e7ed925 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Sun, 24 Nov 2013 14:56:55 -0800 Subject: [PATCH 24/76] remove extra argument checking: not operating cleanly --- glacier.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/glacier.py b/glacier.py index aba245b..f1e5c2a 100755 --- a/glacier.py +++ b/glacier.py @@ -854,12 +854,6 @@ def parse_args(self, args=None): job_subparser.add_parser('list').set_defaults(func=self.job_list) parsed_args = parser.parse_args(args) - # Additional constraints - if (parsed_args.encrypt == False - and parsed_args.encryption_dir is not None): - raise argparse.ArgumentError("--encryption-dir is invalid " - "when --encrypt is not specified.") - return parsed_args def __init__(self, args=None, connection=None, cache=None): From f11cf786686b523c4449f7559e7dad6d47bf4465 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Mon, 25 Nov 2013 14:20:54 -0800 Subject: [PATCH 25/76] return to default gnupg key handling (use homedir, override with env) --- gpg.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/gpg.py b/gpg.py index e7996f7..5bfebd4 100644 --- a/gpg.py +++ b/gpg.py @@ -1,9 +1,6 @@ import gnupg import os - -GNUPG_DIRECTORY = os.path.join(os.path.abspath(os.path.dirname(__file__)), - "gnupg") GNUPG_KEY_TYPE = "RSA" GNUPG_KEY_LENGTH = 2048 @@ -11,7 +8,7 @@ class Encryptor(object): def __init__(self): - self.gpg = gnupg.GPG(gnupghome=GNUPG_DIRECTORY) + self.gpg = gnupg.GPG() self.gpg.encoding = "utf-8" if self.gpg.list_keys() == []: self.generate_keypair() From d3bb1179aa237c3f4cfeb24e131bf8970d3cc835 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Mon, 2 Dec 2013 01:36:34 -0800 Subject: [PATCH 26/76] Remove --encryption-directory. --- glacier.py | 40 +++++----------------------------------- 1 file changed, 5 insertions(+), 35 deletions(-) diff --git a/glacier.py b/glacier.py index f1e5c2a..1d82343 100755 --- a/glacier.py +++ b/glacier.py @@ -529,30 +529,10 @@ def archive_upload(self, if self.args.encrypt is None: filename = self.args.file else: - if self.args.encryption_dir is None: - filename = tempfile.NamedTemporaryFile().name - logger.info("Encrypting %s to %s." - % (self.args.file, filename)) - encryptor.encrypt_file(self.args.file, filename) - else: - filename = os.path.join(self.args.encryption_dir, - "%s.encrypted" % self.args.name) - if os.path.exists(filename): - logger.info("Found cached encryption file %s. " - "Skipping re-encryption." - % filename) - else: - filename_working = "%s.working.%d" % (filename, os.getpid()) - try: - logger.info("Encrypting %s to %s." - % (self.args.file, filename_working)) - encryptor.encrypt_file(self.args.file, - filename_working) - os.rename(filename_working, filename) - finally: - if os.path.exists(filename_working): - os.remove(filename_working) - + filename = tempfile.NamedTemporaryFile().name + logger.info("Encrypting %s to %s." + % (self.args.file, filename)) + encryptor.encrypt_file(self.args.file, filename) logger.info("Encryption complete: %s." % filename) vault = self.connection.get_vault(self.args.vault) @@ -782,10 +762,6 @@ def parse_args(self, args=None): archive_upload_subparser.add_argument('--name') archive_upload_subparser.add_argument( '--encrypt', default=False, action="store_true") - archive_upload_subparser.add_argument( - '--encryption-dir', - dest='encryption_dir', - ) # Multipart upload command multipart_archive_upload_func = partial( @@ -799,10 +775,6 @@ def parse_args(self, args=None): archive_multipart_upload_subparser.add_argument('--name') archive_multipart_upload_subparser.add_argument( '--encrypt', default=False, action="store_true") - archive_multipart_upload_subparser.add_argument( - '--encryption-dir', - dest='encryption_dir', - ) archive_multipart_upload_subparser.add_argument( '--part-size', default=DEFAULT_PART_SIZE, @@ -852,9 +824,7 @@ def parse_args(self, args=None): job_subparser = subparsers.add_parser('job').add_subparsers() job_subparser.add_parser('list').set_defaults(func=self.job_list) - parsed_args = parser.parse_args(args) - - return parsed_args + return parser.parse_args(args) def __init__(self, args=None, connection=None, cache=None): args = self.parse_args(args) From b578431c80aff61563d0e505c2bb90a417ebbb99 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Mon, 2 Dec 2013 01:40:34 -0800 Subject: [PATCH 27/76] add whitespace around operator --- glacier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glacier.py b/glacier.py index 1d82343..3490a8b 100755 --- a/glacier.py +++ b/glacier.py @@ -542,7 +542,7 @@ def archive_upload(self, % (filename, vault)) file_obj=file(filename) archive_id = vault.create_archive_from_file( - file_obj=file_obj, description=name) + file_obj = file_obj, description=name) else: logger.info("Uploading multi-part: %s to %s" % (filename, vault)) From e5db4d81242665006a089b57d5fdb1c50c2ae062 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Mon, 2 Dec 2013 10:39:45 -0800 Subject: [PATCH 28/76] whitespace around operator --- glacier.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glacier.py b/glacier.py index 3490a8b..5f1ba14 100755 --- a/glacier.py +++ b/glacier.py @@ -540,7 +540,7 @@ def archive_upload(self, if not multipart: logger.info("Uploading in a single part: %s to %s." % (filename, vault)) - file_obj=file(filename) + file_obj = file(filename) archive_id = vault.create_archive_from_file( file_obj = file_obj, description=name) else: From 0643e9a550f9c952ffb899d6c71b0419c53f45b0 Mon Sep 17 00:00:00 2001 From: Chris Crutchfield Date: Mon, 9 Dec 2013 22:13:46 -0800 Subject: [PATCH 29/76] Add koality --- koality.yml | 17 +++++++++++++++++ requirements-dev.txt | 5 +++++ requirements.txt | 5 +++++ 3 files changed, 27 insertions(+) create mode 100644 koality.yml create mode 100644 requirements-dev.txt create mode 100644 requirements.txt diff --git a/koality.yml b/koality.yml new file mode 100644 index 0000000..c78ff12 --- /dev/null +++ b/koality.yml @@ -0,0 +1,17 @@ +languages: + python: 2.7 +setup: +- packages: + - pip: + - install requirements: requirements.txt + - install requirements: requirements-dev.txt +test: + machines: 1 + scripts: + - codequality: + script: codequality + timeout: 60 + - unit tests: + script: nosetests --with-xunit + xunit: nosetests.xml + timeout: 300 diff --git a/requirements-dev.txt b/requirements-dev.txt new file mode 100644 index 0000000..fa6b641 --- /dev/null +++ b/requirements-dev.txt @@ -0,0 +1,5 @@ +--index-url http://pypi.counsyl.com/counsyl/prod/+simple/ +codequality==0.2-dev +nose==1.2.1 +pep8==1.4.4 +pyflakes==0.5.0 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..3e4c5b3 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,5 @@ +--index-url http://pypi.counsyl.com/counsyl/prod/+simple/ +boto==2.19.0 +iso8601==0.1.8 +sqlalchemy==0.8.4 +python-gnupg==0.3.5 From a6c3aed5c25824121f5250f2989a5f428c2d047b Mon Sep 17 00:00:00 2001 From: Chris Crutchfield Date: Mon, 9 Dec 2013 22:16:13 -0800 Subject: [PATCH 30/76] add mock --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index fa6b641..0faf58e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,6 @@ --index-url http://pypi.counsyl.com/counsyl/prod/+simple/ codequality==0.2-dev +mock==1.0.1 nose==1.2.1 pep8==1.4.4 pyflakes==0.5.0 From 0deb23184f9a67ec0a17058122730183a275ef90 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Tue, 10 Dec 2013 16:37:01 -0800 Subject: [PATCH 31/76] add MIT license --- gpg.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/gpg.py b/gpg.py index 5bfebd4..0706021 100644 --- a/gpg.py +++ b/gpg.py @@ -1,3 +1,28 @@ +#!/usr/bin/env python + +# The MIT License (MIT) +# +# Copyright (c) 2013 Piotr Kaleta and Counsyl +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, dis- +# tribute, sublicense, and/or sell copies of the Software, and to permit +# persons to whom the Software is furnished to do so, subject to the fol- +# lowing conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS +# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL- +# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS +# IN THE SOFTWARE. + import gnupg import os From 8bd36ad5d7db4cc4d7fcf6b8d7cc7ddcf4a2c6e3 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Tue, 10 Dec 2013 17:28:05 -0800 Subject: [PATCH 32/76] make mutipart_upload a parameter to upload instead of a separate command --- glacier.py | 61 +++++++++++++++++++++++++++--------------------------- 1 file changed, 30 insertions(+), 31 deletions(-) diff --git a/glacier.py b/glacier.py index 5f1ba14..9a62500 100755 --- a/glacier.py +++ b/glacier.py @@ -506,12 +506,7 @@ def archive_list(self): if archive_list: print(*archive_list, sep="\n") - def archive_upload(self, - multipart=False, - encryptor=None, - part_size=DEFAULT_PART_SIZE, - num_threads=DEFAULT_NUM_THREADS): - + def archive_upload(self, encryptor=None): # XXX: "Leading whitespace in archive descriptions is removed." # XXX: "The description must be less than or equal to 1024 bytes. The # allowable characters are 7 bit ASCII without control codes, @@ -526,18 +521,18 @@ def archive_upload(self, raise RuntimeError("Archive name not specified. Use --name.") name = os.path.basename(full_name) - if self.args.encrypt is None: - filename = self.args.file - else: + if self.args.encrypt: filename = tempfile.NamedTemporaryFile().name logger.info("Encrypting %s to %s." % (self.args.file, filename)) encryptor.encrypt_file(self.args.file, filename) logger.info("Encryption complete: %s." % filename) + else: + filename = self.args.file vault = self.connection.get_vault(self.args.vault) - if not multipart: + if not self.args.multipart: logger.info("Uploading in a single part: %s to %s." % (filename, vault)) file_obj = file(filename) @@ -548,8 +543,8 @@ def archive_upload(self, % (filename, vault)) uploader = ConcurrentUploader(self.connection.layer1, vault.name, - part_size=part_size, - num_threads=num_threads) + part_size=self.args.part_size, + num_threads=self.args.num_threads) archive_id = uploader.upload(filename, description=name) logger.info("Upload complete.") @@ -759,31 +754,35 @@ def parse_args(self, args=None): archive_upload_subparser.set_defaults(func=archive_upload_func) archive_upload_subparser.add_argument('vault') archive_upload_subparser.add_argument('file') - archive_upload_subparser.add_argument('--name') archive_upload_subparser.add_argument( - '--encrypt', default=False, action="store_true") - - # Multipart upload command - multipart_archive_upload_func = partial( - self.archive_upload, encryptor=encryptor, multipart=True) - archive_multipart_upload_subparser = archive_subparser.add_parser( - 'multipart_upload') - archive_multipart_upload_subparser.set_defaults( - func=multipart_archive_upload_func) - archive_multipart_upload_subparser.add_argument('vault') - archive_multipart_upload_subparser.add_argument('file') - archive_multipart_upload_subparser.add_argument('--name') - archive_multipart_upload_subparser.add_argument( - '--encrypt', default=False, action="store_true") - archive_multipart_upload_subparser.add_argument( + '--name', + help='The description of the archive.' + ) + archive_upload_subparser.add_argument( + '--encrypt', default=False, action="store_true", + help="Encrypt before uploading using default GNUPG encryption." + ) + archive_upload_subparser.add_argument( + '--multi-part', default=False, action="store_true", + dest="multipart", + help='Break the upload into multiple pieces ' + '(required for large uploads).' + ) + archive_upload_subparser.add_argument( '--part-size', default=DEFAULT_PART_SIZE, - dest="part_size" + dest="part_size", + help=("For --multi-part uploads, change the " + "part size from the default of %d." + % DEFAULT_PART_SIZE) ) - archive_multipart_upload_subparser.add_argument( + archive_upload_subparser.add_argument( '--num-threads', default=DEFAULT_NUM_THREADS, - dest="num_threads" + dest="num_threads", + help=("For --multi-part uploads, change the " + "num threads from the default of %d." + % DEFAULT_NUM_THREADS) ) # Retrieve command From 3319d208645217c692cceb0e81aeb37e59ed57e4 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Wed, 11 Dec 2013 14:52:03 -0800 Subject: [PATCH 33/76] set type for archive upload file parameter --- glacier.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/glacier.py b/glacier.py index 9a62500..c1dd18a 100755 --- a/glacier.py +++ b/glacier.py @@ -753,7 +753,8 @@ def parse_args(self, args=None): archive_upload_subparser = archive_subparser.add_parser('upload') archive_upload_subparser.set_defaults(func=archive_upload_func) archive_upload_subparser.add_argument('vault') - archive_upload_subparser.add_argument('file') + archive_upload_subparser.add_argument('file', + type=argparse.FileType('rb')) archive_upload_subparser.add_argument( '--name', help='The description of the archive.' From 857d450eb9576e4b9ddbf1fa51029402364b8dd6 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Wed, 11 Dec 2013 15:46:48 -0800 Subject: [PATCH 34/76] restore support for stdin streaming for "-" --- glacier.py | 25 ++++++++++++++----------- gpg.py | 7 +++---- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/glacier.py b/glacier.py index c1dd18a..7993545 100755 --- a/glacier.py +++ b/glacier.py @@ -522,30 +522,33 @@ def archive_upload(self, encryptor=None): name = os.path.basename(full_name) if self.args.encrypt: - filename = tempfile.NamedTemporaryFile().name + temp_file_name = tempfile.NamedTemporaryFile().name logger.info("Encrypting %s to %s." - % (self.args.file, filename)) - encryptor.encrypt_file(self.args.file, filename) - logger.info("Encryption complete: %s." % filename) + % (self.args.file.name, temp_file_name)) + encryptor.encrypt_file(self.args.file, temp_file_name) + logger.info("Encryption complete: %s." % temp_file_name) + file_obj = file(temp_file_name, 'rb') + file_name = temp_file_name else: - filename = self.args.file + file_obj = self.args.file + file_name = file_obj.name vault = self.connection.get_vault(self.args.vault) if not self.args.multipart: logger.info("Uploading in a single part: %s to %s." - % (filename, vault)) - file_obj = file(filename) + % (file_name, vault)) archive_id = vault.create_archive_from_file( - file_obj = file_obj, description=name) + file_obj = file_obj, description=name + ) else: logger.info("Uploading multi-part: %s to %s" - % (filename, vault)) + % (file_name, vault)) uploader = ConcurrentUploader(self.connection.layer1, vault.name, part_size=self.args.part_size, num_threads=self.args.num_threads) - archive_id = uploader.upload(filename, description=name) + archive_id = uploader.upload(file_name, description=name) logger.info("Upload complete.") logger.info("New Archive ID: %s" % archive_id) @@ -553,7 +556,7 @@ def archive_upload(self, encryptor=None): self.cache.add_archive(self.args.vault, name, archive_id) if self.args.encrypt: - os.remove(filename) + os.remove(temp_file_name) @staticmethod def _write_archive_retrieval_job(f, job, multipart_size, diff --git a/gpg.py b/gpg.py index 0706021..969300e 100644 --- a/gpg.py +++ b/gpg.py @@ -56,13 +56,12 @@ def _get_fingerprint(self): """Return the fingerprint of the default key.""" return self.gpg.list_keys()[0]["fingerprint"] - def encrypt_file(self, input_filename, output_filename): + def encrypt_file(self, input_file, output_filename): """Encrypt `input_filename` and save results as `output_filename`.""" fingerprint = self._get_fingerprint() - with open(input_filename, "r") as input_file: - self.gpg.encrypt_file(input_file, fingerprint, - output=output_filename) + self.gpg.encrypt_file(input_file, fingerprint, + output=output_filename) def encrypt(self, data): """Return the encrypted version of `data`.""" From 8f248c2b2e5ae2ea7b1928acd0b078ce15014ba4 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Wed, 11 Dec 2013 16:17:17 -0800 Subject: [PATCH 35/76] renamed callback to "sortable_date" --- glacier.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/glacier.py b/glacier.py index 7993545..96952e1 100755 --- a/glacier.py +++ b/glacier.py @@ -369,12 +369,12 @@ def recent_enough(job): def find_complete_job(jobs): complete_jobs = filter(lambda job: job.completed, jobs) - def most_recent_job(job): + def sortable_date(job): return iso8601.parse_date(job.completion_date) sorted_completed_jobs = sorted( complete_jobs, - key=most_recent_job, reverse=True + key=sortable_date, reverse=True ) for job in sorted_completed_jobs: From d14d4da20cd709ae8023c63610c028b75867fd74 Mon Sep 17 00:00:00 2001 From: Chris Crutchfield Date: Wed, 11 Dec 2013 16:30:15 -0800 Subject: [PATCH 36/76] Remove codequality from koality --- koality.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/koality.yml b/koality.yml index c78ff12..b706abb 100644 --- a/koality.yml +++ b/koality.yml @@ -8,9 +8,6 @@ setup: test: machines: 1 scripts: - - codequality: - script: codequality - timeout: 60 - unit tests: script: nosetests --with-xunit xunit: nosetests.xml From e0cc3ad136d71083917a5d3b15c4c831822b1738 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Wed, 11 Dec 2013 16:47:59 -0800 Subject: [PATCH 37/76] test archive upload w/ encrypt, multi-part --- glacier_test.py | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/glacier_test.py b/glacier_test.py index bc93f9c..33b2e2e 100644 --- a/glacier_test.py +++ b/glacier_test.py @@ -106,22 +106,37 @@ def test_archive_list_force_ids(self): ) def test_archive_upload(self): - file_obj = Mock() - file_obj.name = 'filename' - open_mock = Mock(return_value=file_obj) - with patch('__builtin__.open', open_mock): - self.run_app(['archive', 'upload', 'vault_name', 'filename']) - self.connection.get_vault.assert_called_with('vault_name') - mock_vault = self.connection.get_vault.return_value - mock_vault.create_archive_from_file.assert_called_once_with( - file_obj=file_obj, description='filename') + + for multipart in (False, True): + for encrypt in (False, True): + args = ['archive', 'upload', 'vault_name', 'filename'] + if multipart: + args.append('--multi-part') + if encrypt: + args.append('--encrypt') + file_obj = Mock() + file_obj.name = 'filename' + open_mock = Mock(return_value=file_obj) + with patch('__builtin__.open', open_mock): + self.run_app(args) + self.connection.get_vault.assert_called_with('vault_name') + mock_vault = self.connection.get_vault.return_value + mock_vault.create_archive_from_file.assert_called_once_with( + file_obj=file_obj, description='filename') def test_archive_stdin_upload(self): - self.run_app(['archive', 'upload', 'vault_name', '-']) - self.connection.get_vault.assert_called_once_with('vault_name') - vault = self.connection.get_vault.return_value - vault.create_archive_from_file.assert_called_once_with( - file_obj=sys.stdin, description='') + for multipart in (False, True): + for encrypt in (False, True): + args = ['archive', 'upload', 'vault_name', 'filename'] + if multipart: + args.append('--multi-part') + if encrypt: + args.append('--encrypt') + self.run_app() + self.connection.get_vault.assert_called_once_with('vault_name') + vault = self.connection.get_vault.return_value + vault.create_archive_from_file.assert_called_once_with( + file_obj=sys.stdin, description='') def test_archive_retrieve_no_job(self): self.init_app(['archive', 'retrieve', 'vault_name', 'archive_name']) From dc688a2fa1bc3049f1cfc915efe0983c3e4fe315 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Wed, 11 Dec 2013 17:45:53 -0800 Subject: [PATCH 38/76] Get glacier_test.py working. (Failing on master.) TODO: extend to cover encryption and multipart upload. --- glacier_test.py | 51 +++++++++++++++++++++---------------------------- 1 file changed, 22 insertions(+), 29 deletions(-) diff --git a/glacier_test.py b/glacier_test.py index 33b2e2e..ca8cfce 100644 --- a/glacier_test.py +++ b/glacier_test.py @@ -106,37 +106,30 @@ def test_archive_list_force_ids(self): ) def test_archive_upload(self): - - for multipart in (False, True): - for encrypt in (False, True): - args = ['archive', 'upload', 'vault_name', 'filename'] - if multipart: - args.append('--multi-part') - if encrypt: - args.append('--encrypt') - file_obj = Mock() - file_obj.name = 'filename' - open_mock = Mock(return_value=file_obj) - with patch('__builtin__.open', open_mock): - self.run_app(args) - self.connection.get_vault.assert_called_with('vault_name') - mock_vault = self.connection.get_vault.return_value - mock_vault.create_archive_from_file.assert_called_once_with( - file_obj=file_obj, description='filename') + for encrypt in (False, ): + args = ['archive', 'upload', 'vault_name', 'filename', '--name', 'filename'] + if encrypt: + args.append('--encrypt') + file_obj = Mock() + file_obj.name = 'filename' + open_mock = Mock(return_value=file_obj) + with patch('__builtin__.open', open_mock): + self.run_app(args) + self.connection.get_vault.assert_called_with('vault_name') + mock_vault = self.connection.get_vault.return_value + mock_vault.create_archive_from_file.assert_called_once_with( + file_obj=file_obj, description='filename') def test_archive_stdin_upload(self): - for multipart in (False, True): - for encrypt in (False, True): - args = ['archive', 'upload', 'vault_name', 'filename'] - if multipart: - args.append('--multi-part') - if encrypt: - args.append('--encrypt') - self.run_app() - self.connection.get_vault.assert_called_once_with('vault_name') - vault = self.connection.get_vault.return_value - vault.create_archive_from_file.assert_called_once_with( - file_obj=sys.stdin, description='') + for encrypt in (False, ): + args = ['archive', 'upload', 'vault_name', '-','--name', ''] + if encrypt: + args.append('--encrypt') + self.run_app(args) + self.connection.get_vault.assert_called_once_with('vault_name') + vault = self.connection.get_vault.return_value + vault.create_archive_from_file.assert_called_once_with( + file_obj=sys.stdin, description='') def test_archive_retrieve_no_job(self): self.init_app(['archive', 'retrieve', 'vault_name', 'archive_name']) From e9f173687c3e4f666ad058a9ec8800d954044138 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Wed, 11 Dec 2013 19:25:06 -0800 Subject: [PATCH 39/76] add multipart upload unit test --- glacier_test.py | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/glacier_test.py b/glacier_test.py index ca8cfce..9c59edc 100644 --- a/glacier_test.py +++ b/glacier_test.py @@ -37,19 +37,19 @@ class TestCase(unittest.TestCase): - def init_app(self, args, memory_cache=False): + def init_app(self, xargs, memory_cache=False): self.connection = Mock() if memory_cache: self.cache = glacier.Cache(0, db_path=':memory:') else: self.cache = Mock() self.app = glacier.App( - args=args, + args=xargs, connection=self.connection, cache=self.cache) - def run_app(self, args): - self.init_app(args) + def run_app(self, xargs): + self.init_app(xargs) self.app.main() def test_vault_list(self): @@ -120,16 +120,36 @@ def test_archive_upload(self): mock_vault.create_archive_from_file.assert_called_once_with( file_obj=file_obj, description='filename') - def test_archive_stdin_upload(self): + def test_archive_upload_multipart(self): for encrypt in (False, ): - args = ['archive', 'upload', 'vault_name', '-','--name', ''] + args = ['archive', 'upload', 'vault_name', 'filename', '--name', 'filename'] + args.append('--multi-part') if encrypt: args.append('--encrypt') - self.run_app(args) - self.connection.get_vault.assert_called_once_with('vault_name') - vault = self.connection.get_vault.return_value - vault.create_archive_from_file.assert_called_once_with( - file_obj=sys.stdin, description='') + file_obj = Mock() + file_obj.name = 'filename' + open_mock = Mock(return_value=file_obj) + import glacier + with patch('__builtin__.open', open_mock), \ + patch('glacier.ConcurrentUploader'), \ + patch('glacier.ConcurrentUploader.upload', return_value=123): + self.run_app(args) + glacier.ConcurrentUploader.upload.assert_called_once() + self.connection.get_vault.assert_called_with('vault_name') + + def test_archive_stdin_upload(self): + for encrypt in (False, ): + for multipart in (False, ): + args = ['archive', 'upload', 'vault_name', '-','--name', ''] + if encrypt: + args.append('--encrypt') + if multipart: + args.append('--multi-part') + self.run_app(args) + self.connection.get_vault.assert_called_once_with('vault_name') + vault = self.connection.get_vault.return_value + vault.create_archive_from_file.assert_called_once_with( + file_obj=sys.stdin, description='') def test_archive_retrieve_no_job(self): self.init_app(['archive', 'retrieve', 'vault_name', 'archive_name']) From 2627cc1c501e349c9e91c99faec8364ae13a442c Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Wed, 11 Dec 2013 20:10:38 -0800 Subject: [PATCH 40/76] test cleanup --- glacier_test.py | 75 ++++++++++++++++++++++++------------------------- 1 file changed, 37 insertions(+), 38 deletions(-) diff --git a/glacier_test.py b/glacier_test.py index 9c59edc..e95905d 100644 --- a/glacier_test.py +++ b/glacier_test.py @@ -23,6 +23,7 @@ from __future__ import print_function +from os import environ import sys import unittest @@ -37,19 +38,28 @@ class TestCase(unittest.TestCase): - def init_app(self, xargs, memory_cache=False): + def setUp(self): + self.orig_gnupghome = environ.get('gnupg_test_home') + environ['GNUPGHOME'] = "gnupg_test_home" + patch('glacier.Encryptor', autospec=True) + + def tearDown(self): + if self.orig_gnupghome is not None: + environ['GNUPGHOME'] = self.orig_gnupghome + + def init_app(self, args, memory_cache=False): self.connection = Mock() if memory_cache: self.cache = glacier.Cache(0, db_path=':memory:') else: self.cache = Mock() self.app = glacier.App( - args=xargs, + args=args, connection=self.connection, cache=self.cache) - def run_app(self, xargs): - self.init_app(xargs) + def run_app(self, args): + self.init_app(args) self.app.main() def test_vault_list(self): @@ -106,51 +116,40 @@ def test_archive_list_force_ids(self): ) def test_archive_upload(self): - for encrypt in (False, ): - args = ['archive', 'upload', 'vault_name', 'filename', '--name', 'filename'] - if encrypt: - args.append('--encrypt') - file_obj = Mock() - file_obj.name = 'filename' - open_mock = Mock(return_value=file_obj) - with patch('__builtin__.open', open_mock): - self.run_app(args) - self.connection.get_vault.assert_called_with('vault_name') - mock_vault = self.connection.get_vault.return_value - mock_vault.create_archive_from_file.assert_called_once_with( - file_obj=file_obj, description='filename') + args = ['archive', 'upload', 'vault_name', 'filename', '--name', 'filename'] + file_obj = Mock() + file_obj.name = 'filename' + open_mock = Mock(return_value=file_obj) + with patch('__builtin__.open', open_mock): + self.run_app(args) + self.connection.get_vault.assert_called_with('vault_name') + mock_vault = self.connection.get_vault.return_value + mock_vault.create_archive_from_file.assert_called_once_with( + file_obj=file_obj, description='filename') + + def test_archive_stdin_upload(self): + args = ['archive', 'upload', 'vault_name', '-','--name', ''] + self.run_app(args) + self.connection.get_vault.assert_called_once_with('vault_name') + vault = self.connection.get_vault.return_value + vault.create_archive_from_file.assert_called_once_with( + file_obj=sys.stdin, description='') def test_archive_upload_multipart(self): - for encrypt in (False, ): - args = ['archive', 'upload', 'vault_name', 'filename', '--name', 'filename'] + for filename in ('filename', '-'): + name = '' if filename == '-' else filename + args = ['archive', 'upload', 'vault_name', filename, '--name', name] args.append('--multi-part') - if encrypt: - args.append('--encrypt') file_obj = Mock() file_obj.name = 'filename' open_mock = Mock(return_value=file_obj) - import glacier with patch('__builtin__.open', open_mock), \ - patch('glacier.ConcurrentUploader'), \ - patch('glacier.ConcurrentUploader.upload', return_value=123): + patch('glacier.ConcurrentUploader'), \ + patch('glacier.ConcurrentUploader.upload', return_value=123): self.run_app(args) glacier.ConcurrentUploader.upload.assert_called_once() self.connection.get_vault.assert_called_with('vault_name') - def test_archive_stdin_upload(self): - for encrypt in (False, ): - for multipart in (False, ): - args = ['archive', 'upload', 'vault_name', '-','--name', ''] - if encrypt: - args.append('--encrypt') - if multipart: - args.append('--multi-part') - self.run_app(args) - self.connection.get_vault.assert_called_once_with('vault_name') - vault = self.connection.get_vault.return_value - vault.create_archive_from_file.assert_called_once_with( - file_obj=sys.stdin, description='') - def test_archive_retrieve_no_job(self): self.init_app(['archive', 'retrieve', 'vault_name', 'archive_name']) mock_vault = Mock() From 3839287f0e5c4e2ae7839131520dd874e01037bd Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Wed, 11 Dec 2013 20:13:24 -0800 Subject: [PATCH 41/76] remove koality.yml --- koality.yml | 14 -------------- 1 file changed, 14 deletions(-) delete mode 100644 koality.yml diff --git a/koality.yml b/koality.yml deleted file mode 100644 index b706abb..0000000 --- a/koality.yml +++ /dev/null @@ -1,14 +0,0 @@ -languages: - python: 2.7 -setup: -- packages: - - pip: - - install requirements: requirements.txt - - install requirements: requirements-dev.txt -test: - machines: 1 - scripts: - - unit tests: - script: nosetests --with-xunit - xunit: nosetests.xml - timeout: 300 From 863a74b7a3acd15f298eb909061578171a1b5369 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Wed, 11 Dec 2013 20:33:12 -0800 Subject: [PATCH 42/76] warn the user that multipart uploads do not support streaming from stdin --- glacier.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/glacier.py b/glacier.py index 96952e1..e39742c 100755 --- a/glacier.py +++ b/glacier.py @@ -767,7 +767,7 @@ def parse_args(self, args=None): help="Encrypt before uploading using default GNUPG encryption." ) archive_upload_subparser.add_argument( - '--multi-part', default=False, action="store_true", + '--multipart', default=False, action="store_true", dest="multipart", help='Break the upload into multiple pieces ' '(required for large uploads).' @@ -776,7 +776,7 @@ def parse_args(self, args=None): '--part-size', default=DEFAULT_PART_SIZE, dest="part_size", - help=("For --multi-part uploads, change the " + help=("For --multipart uploads, change the " "part size from the default of %d." % DEFAULT_PART_SIZE) ) @@ -784,7 +784,7 @@ def parse_args(self, args=None): '--num-threads', default=DEFAULT_NUM_THREADS, dest="num_threads", - help=("For --multi-part uploads, change the " + help=("For --multipart uploads, change the " "num threads from the default of %d." % DEFAULT_NUM_THREADS) ) @@ -827,7 +827,16 @@ def parse_args(self, args=None): job_subparser = subparsers.add_parser('job').add_subparsers() job_subparser.add_parser('list').set_defaults(func=self.job_list) - return parser.parse_args(args) + parsed = parser.parse_args(args) + + if (parsed.func == archive_upload_func + and parsed.multipart + and parsed.file is sys.stdin): + raise ConsoleError( + "multipart uploads do not support streaming from stdin" + ) + + return parsed def __init__(self, args=None, connection=None, cache=None): args = self.parse_args(args) From 1f29e411289ac0737a165efbdc9218f6aadee9d1 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Wed, 11 Dec 2013 20:34:20 -0800 Subject: [PATCH 43/76] gnupg test keys --- gnupg_test_home/pubring.gpg | Bin 0 -> 664 bytes gnupg_test_home/pubring.gpg~ | Bin 0 -> 664 bytes gnupg_test_home/random_seed | Bin 0 -> 600 bytes gnupg_test_home/secring.gpg | Bin 0 -> 1315 bytes gnupg_test_home/trustdb.gpg | Bin 0 -> 1280 bytes 5 files changed, 0 insertions(+), 0 deletions(-) create mode 100644 gnupg_test_home/pubring.gpg create mode 100644 gnupg_test_home/pubring.gpg~ create mode 100644 gnupg_test_home/random_seed create mode 100644 gnupg_test_home/secring.gpg create mode 100644 gnupg_test_home/trustdb.gpg diff --git a/gnupg_test_home/pubring.gpg b/gnupg_test_home/pubring.gpg new file mode 100644 index 0000000000000000000000000000000000000000..91278d548f7b7a356b401f07d3a1952f5db8e613 GIT binary patch literal 664 zcmV;J0%!f10SyFFsWC$V2mq)yrO4j^m=W`Lq**_A1#X34Dm%ZwTM^#01YG0GK|%>7 z9V()4R&4P7$U8>x+#yXAOr51zeY+gQTy7nZZ}Lt@YTNpr^(aa?CG3zc;QJ#`avo}Y z0d8_7t6hV74b%r1yTefDU5NIZMMCz3?st9E_5s8Sw8dcyuH4jmjXDRAH}H92Y5tP* zL_?J{rVISk4+R$#S&zH;`Sb?(jQ}^3sRda%nD$0IoXhpfFSdAjeYO+3#aBoKqU*TU zrum_8B^$o3po0MOJDY#cV7^rWqBYxb8~HJbZ9^|FfB4L zF)cAUGA%JUEpv2XbZKKQXl7$BV{dI^VRLk1b#rNMWpi^bZe?^ni2*nS69EDMA_W3c zsWC$W8!rY62?z%R0tOWb0tpHW1Qr4V0RkQY0vCV)3JDNr`m^e;d`aZDoCp3jR${bR zdaoieir@$FS%CHFmylFFl=T{ePsFRp(Hi*5)z?RFP+6*oB#Z3muX=G3(#ObTu<4() zT%pQg2OL*OCS>xh!<69rzn@x&Yto8z$D3;+e|KsNc4!8)CD8sk0v9_*iqGCBs4S$> zo=c38KuaOJ9I?Y&r`>YLS+;boe@sdBTZiR!IgfpSUd-q83(d2c6$J!I(tJ>Y4{!v8 zARjy1h@>V2sEEL{xFMsXQ!f8zBw$&&7#qC}TYb})jk^EGY@A&^lNJXKMAkf-7P}Ba y-qRSG9^$rZ{ZDY4lx4n{S#UB4$p8)AaQ}WRE1FkEN^CzUD4yi#!>2Z|0ssRkEFvHP literal 0 HcmV?d00001 diff --git a/gnupg_test_home/pubring.gpg~ b/gnupg_test_home/pubring.gpg~ new file mode 100644 index 0000000000000000000000000000000000000000..d22072bf0a8a2b81aff247c225dd50859a7c9380 GIT binary patch literal 664 zcmV;J0%!f10SyFFsWC$V2mq)yrO4j^m=W`Lq**_A1#X34Dm%ZwTM^#01YG0GK|%>7 z9V()4R&4P7$U8>x+#yXAOr51zeY+gQTy7nZZ}Lt@YTNpr^(aa?CG3zc;QJ#`avo}Y z0d8_7t6hV74b%r1yTefDU5NIZMMCz3?st9E_5s8Sw8dcyuH4jmjXDRAH}H92Y5tP* zL_?J{rVISk4+R$#S&zH;`Sb?(jQ}^3sRda%nD$0IoXhpfFSdAjeYO+3#aBoKqU*TU zrum_8B^$o3po0MOJDY#cV7^rWqBYxb8~HJbZ9^|FfB4L zF)cAUGA%JUEpv2XbZKKQXl7$BV{dI^VRLk1b#rNMWpi^bZe?^ni2*nS69EDMA_W3c zsWC$W8!rY62?z%R0tOWb0tpHW1Qr4V0RkQY0vCV)3JDNr`m^e;d`aZDoCp3jR${bR zdaoieir@$FS%CHFmylFFl=T{ePsFRp(Hi*5)z?RFP+6*oB#Z3muX=G3(#ObTu<4() zT%pQg2OL*OCS>xh!<69rzn@x&Yto8z$D3;+e|KsNc4!8)CD8sk0v9_*iqGCBs4S$> zo=c38KuaOJ9I?Y&r`>YLS+;boe@sdBTZiR!IgfpSUd-q83(d2c6$J!I(tJ>Y4{!v8 zARjy1h@>V2sEEL{xFMsXQ!f8zBw$&&7#qC}TYb})jk^EGY@A&^lNJXKMAkf-7P}Ba y-qRSG9^$rZ{ZDY4lx4n{S#UB4$p8)AaQ}WRE1FkEN^CzUD4yi#!>2Z|0ssIhDIy&J literal 0 HcmV?d00001 diff --git a/gnupg_test_home/random_seed b/gnupg_test_home/random_seed new file mode 100644 index 0000000000000000000000000000000000000000..e8672bcbb1facfb944eadbbc9d5637f40ca1f57a GIT binary patch literal 600 zcmV-e0;l~7P#pQOY3faIq7O$BCzDNrhFN-VU@CY&9)GIsBcT=WCxuLs{%|(|$XlOA zTjC&c5pSdnOd6ytc%CP6#;HhMg9!kp%odc=%{7g$>Rtnb4MPYQ@BbWhbJS##P)N*) z*wf*!`_j#1pX1C(+qFCI31t}Knxr32Q|7#&`i?*E}U>?FH z&=ddnPr$Z?R?)y#w8xrm*QuMGp@KMv`7{6*Qz2>O=)-ssG#rQ|F#Ujkv|N&MQR4}c zO`LBrDq5~C9gLkRmD?tJLh)ZBE1ZKg zc~&EfS4Hd)ha^=;x@!dHwpWFMdDJEBfA>Rktv=GBqwZb&gFh@^1rpmr5-gaA44oyK{$1|H_z)wKCZOSGPTl^T^Ll;4`DmmrQB5KE??CW4k zA@-D?dp~BnG}55w=fGl)T;t&3e=P`-D(8Z67O1q%>#|(WdKyGQUO!967b2Id5s8qG z5{{g#@5mf>A}2jXQwffe7s=i;pZ4wj4Ri(KJ?-pScn#eSf_%h5+~-v&>E#b6fewiG zt+#Io1T);>C?pl7ciOeI_)~mW&5uGF2b^CFv|r5F9*XW+x`SIHm5Lt52IC-}g6zB% m0b5RkFY1qXH@KC|+dlImNY*IGpjd>H31qSfIodtHZwQiJLmE|T1DFI-sWC$V2mq)yrO4j^m=W`Lq**_A1#X34Dm%ZwTM^#01YG0GK|%>7 z9V()4R&4P7$U8>x+#yXAOr51zeY+gQTy7nZZ}Lt@YTNpr^(aa?CG3zc;QJ#`avo}Y z0d8_7t6hV74b%r1yTefDU5NIZMMCz3?st9E_5s8Sw8dcyuH4jmjXDRAH}H92Y5tP* zL_?J{rVISk4+R$#S&zH;`Sb?(jQ}^3sRda%nD$0IoXhpfFSdAjeYO+3#aBoKqU*TU zrum_8B^$o3po0MOJDY#cV78Xovk-_io^GJWty9c=-^H_`*jbOW0!!$qx&|so83N%Yv0kDF{mcfkYz!iJa^O$t=H@X113QO)lj3Z~3Q*R&mx%)wz?W5QzL z&A%wY4FH8me7B1XkNrkE-10m~$0{Qo`~l7XtzC<-ZLQ^7FzV^0v0;7rN7OA2j(R{I zgu~^wQ-%Q21LGI$un&1XMP)1OVM%S~F5s zEl@9fLAHx3&w#W&f9nI@=@{ngdc6Q&kBhG;80U?|2`)M1YOixn-5KHd3?st5bk84G zZJh;X^+E-{FvB?|Oo=fKVwGRANq>isLr>l8Z7`lr^D_PWxhUzGhcI#KmbG=Ie#A}2 zyA2mGIZSqg28&#}n4#0rb29_~a%x+E%ENEUj2bxHbmh$6+C}<<>tbX?IAMs%Rh0mR zEi(XzVQNleoG@{f=844X0IfxrQUq$s66pT1vv5paT!zG3$i2sf3uU!9S8KGyR7Uub z|D`i)RY1DYBlssy9^Hnl0V(8!O4Nv0&))1-?RslOPF-aLVa6+kOE5c+Hy*S{L3MO* zXJu|>a$$63WFSjrc_2JLb7^j6b8{|kWpqA?0XPH`0RjLb1p-p3F+&0yF9r(<2nPcK1{DYb2?`4Y76JnS z0v-VZ7k~f?2@q%cv+AyVN#wVj2mUoyVzgL#uOcyu;0N(pfc5E@kW@XC^%{du#H+~B z8u-f9*GF$qS*nO6i|pvHdT|oc$H-)`>7TS*p~_(g99KvtWb&=Ul;HcnpIV4((u#D) zn` Date: Wed, 11 Dec 2013 20:36:41 -0800 Subject: [PATCH 44/76] rename --multi-part to --multipart --- glacier_test.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/glacier_test.py b/glacier_test.py index e95905d..3d6de80 100644 --- a/glacier_test.py +++ b/glacier_test.py @@ -136,19 +136,17 @@ def test_archive_stdin_upload(self): file_obj=sys.stdin, description='') def test_archive_upload_multipart(self): - for filename in ('filename', '-'): - name = '' if filename == '-' else filename - args = ['archive', 'upload', 'vault_name', filename, '--name', name] - args.append('--multi-part') - file_obj = Mock() - file_obj.name = 'filename' - open_mock = Mock(return_value=file_obj) - with patch('__builtin__.open', open_mock), \ - patch('glacier.ConcurrentUploader'), \ - patch('glacier.ConcurrentUploader.upload', return_value=123): - self.run_app(args) - glacier.ConcurrentUploader.upload.assert_called_once() - self.connection.get_vault.assert_called_with('vault_name') + args = ['archive', 'upload', 'vault_name', 'filename', '--name', 'filename'] + args.append('--multipart') + file_obj = Mock() + file_obj.name = 'filename' + open_mock = Mock(return_value=file_obj) + with patch('__builtin__.open', open_mock), \ + patch('glacier.ConcurrentUploader'), \ + patch('glacier.ConcurrentUploader.upload', return_value=123): + self.run_app(args) + glacier.ConcurrentUploader.upload.assert_called_once() + self.connection.get_vault.assert_called_with('vault_name') def test_archive_retrieve_no_job(self): self.init_app(['archive', 'retrieve', 'vault_name', 'archive_name']) From 07a6348291cb678a7941d2cf53f4c6f8d88395f9 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Wed, 11 Dec 2013 21:52:59 -0800 Subject: [PATCH 45/76] restore koality.yml --- koality.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 koality.yml diff --git a/koality.yml b/koality.yml new file mode 100644 index 0000000..b706abb --- /dev/null +++ b/koality.yml @@ -0,0 +1,14 @@ +languages: + python: 2.7 +setup: +- packages: + - pip: + - install requirements: requirements.txt + - install requirements: requirements-dev.txt +test: + machines: 1 + scripts: + - unit tests: + script: nosetests --with-xunit + xunit: nosetests.xml + timeout: 300 From c3b1564117b9e0ad414b31889ce7bebbe43709ff Mon Sep 17 00:00:00 2001 From: Chris Gilmer Date: Fri, 13 Dec 2013 10:30:38 -0800 Subject: [PATCH 46/76] Updated setup.py --- MANIFEST.in | 4 ++++ Makefile | 49 ++++++++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 2 +- setup.py | 33 ++++++++++++++++++++++++-------- 4 files changed, 79 insertions(+), 9 deletions(-) create mode 100644 MANIFEST.in create mode 100644 Makefile diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..0f3d172 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,4 @@ +include COPYING +include MANIFEST.in +include README.md +include *.py diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..749adc1 --- /dev/null +++ b/Makefile @@ -0,0 +1,49 @@ +#! /usr/bin/make + +default: + python setup.py check build + +.PHONY: register dist inspect upload clean docs test + +register: + if [ ! -f ~/.pypirc ]; then \ + echo "Missing ~/.pypirc file"; \ + exit 1; \ + fi; \ + python setup.py register + +dist: + python setup.py sdist + +inspect: + python setup.py clean + rm -rf build/ + rm -rf dist/ + rm -rf *.egg/ + rm -rf *.egg-info/ + rm -f MANIFEST + python setup.py sdist + cd dist/ && tar xzvf *.tar.gz + +upload: + if [ ! -f ~/.pypirc ]; then \ + echo "Missing ~/.pypirc file"; \ + exit 1; \ + fi; \ + python setup.py sdist upload + +clean: + python setup.py clean + rm -rf build/ + rm -rf dist/ + rm -rf *.egg/ + rm -rf *.egg-info/ + rm -rf __pycache__/ + rm -f MANIFEST + rm -rf docs/_*/ + +docs: + cd docs && make html + +test: + nosetests -c tests/nose.cfg diff --git a/requirements.txt b/requirements.txt index 3e4c5b3..adafe9f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ --index-url http://pypi.counsyl.com/counsyl/prod/+simple/ +SQLAlchemy==0.8.4 boto==2.19.0 iso8601==0.1.8 -sqlalchemy==0.8.4 python-gnupg==0.3.5 diff --git a/setup.py b/setup.py index b2c3112..fc871ca 100644 --- a/setup.py +++ b/setup.py @@ -1,14 +1,31 @@ from setuptools import setup -setup(name='glacier-cli', - version='0.1', - py_modules=['glacier', 'gpg'], - license='BSD', - description='Command-line interface to Amazon Glacier', - install_requires=[ + +setup( + name='counsyl-glacier-cli', + version='0.1.0', + author='Counsyl', + author_email='root@counsyl.com', + maintainer='Counsyl', + maintainer_email='root@counsyl.com', + description='Command-line interface to Amazon Glacier', + long_description=open('README.md', 'rt').read(), + url='https://github.counsyl.com/dev/glacier-cli', + + py_modules=['glacier', 'gpg'], + zip_safe=False, + install_requires=[ 'boto', 'iso8601', 'sqlalchemy', - 'python-gnupg' - ] + 'python-gnupg', + ] + classifiers=( + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "License :: OSI Approved :: BSD License", + "Operating System :: OS Independent", + "Programming Language :: Python :: 2.7", + "Topic :: System :: Logging", + ), ) From c8c58fb20141af1a66524c128ab7139c13bece34 Mon Sep 17 00:00:00 2001 From: Chris Gilmer Date: Fri, 13 Dec 2013 10:40:15 -0800 Subject: [PATCH 47/76] Fix missing comma --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index fc871ca..d716edd 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ 'iso8601', 'sqlalchemy', 'python-gnupg', - ] + ], classifiers=( "Development Status :: 3 - Alpha", "Intended Audience :: Developers", From f10aa31328c8d0456b450fd98194f95cc1e3a2e9 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Fri, 20 Dec 2013 21:57:25 -0800 Subject: [PATCH 48/76] leave stubs for additional tests --- glacier_test.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/glacier_test.py b/glacier_test.py index 33b2e2e..1030393 100644 --- a/glacier_test.py +++ b/glacier_test.py @@ -106,8 +106,7 @@ def test_archive_list_force_ids(self): ) def test_archive_upload(self): - - for multipart in (False, True): + for multipart in (False, ): for encrypt in (False, True): args = ['archive', 'upload', 'vault_name', 'filename'] if multipart: @@ -121,18 +120,21 @@ def test_archive_upload(self): self.run_app(args) self.connection.get_vault.assert_called_with('vault_name') mock_vault = self.connection.get_vault.return_value - mock_vault.create_archive_from_file.assert_called_once_with( - file_obj=file_obj, description='filename') + if multipart: + pass + else: + mock_vault.create_archive_from_file.assert_called_once_with( + file_obj=file_obj, description='filename') def test_archive_stdin_upload(self): - for multipart in (False, True): - for encrypt in (False, True): - args = ['archive', 'upload', 'vault_name', 'filename'] + for multipart in (False, ): + for encrypt in (False, ): + args = ['archive', 'upload', 'vault_name', '-'] if multipart: args.append('--multi-part') if encrypt: args.append('--encrypt') - self.run_app() + self.run_app(args) self.connection.get_vault.assert_called_once_with('vault_name') vault = self.connection.get_vault.return_value vault.create_archive_from_file.assert_called_once_with( From d7bc03694c2d4cde933fb9679fe146f07b030520 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Fri, 20 Dec 2013 23:50:27 -0800 Subject: [PATCH 49/76] Add ConcurrentDownloader, rename --multipart to --concurrent. The regular uploader and downloader automatically do multipart upload/download as needed, but do not do it concurrently. I renamed the --multipart flag to --concurrent to be clearer about the feature it adds. --- glacier.py | 88 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 60 insertions(+), 28 deletions(-) diff --git a/glacier.py b/glacier.py index 96952e1..43c284a 100755 --- a/glacier.py +++ b/glacier.py @@ -36,6 +36,7 @@ import tempfile import time +from boto.glacier.concurrent import ConcurrentDownloader from boto.glacier.concurrent import ConcurrentUploader DEFAULT_NUM_THREADS = 10 @@ -535,14 +536,14 @@ def archive_upload(self, encryptor=None): vault = self.connection.get_vault(self.args.vault) - if not self.args.multipart: + if not self.args.concurrent: logger.info("Uploading in a single part: %s to %s." % (file_name, vault)) archive_id = vault.create_archive_from_file( file_obj = file_obj, description=name ) else: - logger.info("Uploading multi-part: %s to %s" + logger.info("Uploading concurrent: %s to %s" % (file_name, vault)) uploader = ConcurrentUploader(self.connection.layer1, vault.name, @@ -561,35 +562,44 @@ def archive_upload(self, encryptor=None): @staticmethod def _write_archive_retrieval_job(f, job, multipart_size, encryptor=None): + if encryptor: destfile = tempfile.NamedTemporaryFile() else: destfile = f - if job.archive_size > multipart_size: - def fetch(start, end): - byte_range = start, end - 1 - destfile.write(job.get_output(byte_range).read()) - - whole_parts = job.archive_size // multipart_size - for first_byte in xrange(0, whole_parts * multipart_size, - multipart_size): - fetch(first_byte, first_byte + multipart_size) - remainder = job.archive_size % multipart_size - if remainder: - fetch(job.archive_size - remainder, job.archive_size) + if self.args.concurrent: + downloader = ConcurrentDownloader( + job=job, + part_size=self.args.part_size, + num_threads=self.args.num_threads + ) + downloader.download(destfile.name) else: - destfile.write(job.get_output().read()) + if job.archive_size > multipart_size: + def fetch(start, end): + byte_range = start, end - 1 + destfile.write(job.get_output(byte_range).read()) + + whole_parts = job.archive_size // multipart_size + for first_byte in xrange(0, whole_parts * multipart_size, + multipart_size): + fetch(first_byte, first_byte + multipart_size) + remainder = job.archive_size % multipart_size + if remainder: + fetch(job.archive_size - remainder, job.archive_size) + else: + destfile.write(job.get_output().read()) - # Make sure that the file now exactly matches the downloaded archive, - # even if the file existed before and was longer. - try: - destfile.truncate(job.archive_size) - except IOError, e: - # Allow ESPIPE, since the "file" couldn't have existed - # before in this case. - if e.errno != errno.ESPIPE: - raise + # Make sure that the file now exactly matches the downloaded archive, + # even if the file existed before and was longer. + try: + destfile.truncate(job.archive_size) + except IOError, e: + # Allow ESPIPE, since the "file" couldn't have existed + # before in this case. + if e.errno != errno.ESPIPE: + raise # Decrypt file if encryptor is given if encryptor: @@ -767,8 +777,8 @@ def parse_args(self, args=None): help="Encrypt before uploading using default GNUPG encryption." ) archive_upload_subparser.add_argument( - '--multi-part', default=False, action="store_true", - dest="multipart", + '--concurrent', default=False, action="store_true", + dest="concurrent", help='Break the upload into multiple pieces ' '(required for large uploads).' ) @@ -776,7 +786,7 @@ def parse_args(self, args=None): '--part-size', default=DEFAULT_PART_SIZE, dest="part_size", - help=("For --multi-part uploads, change the " + help=("For --concurrent uploads, change the " "part size from the default of %d." % DEFAULT_PART_SIZE) ) @@ -784,7 +794,7 @@ def parse_args(self, args=None): '--num-threads', default=DEFAULT_NUM_THREADS, dest="num_threads", - help=("For --multi-part uploads, change the " + help=("For --concurrent uploads, change the " "num threads from the default of %d." % DEFAULT_NUM_THREADS) ) @@ -804,6 +814,28 @@ def parse_args(self, args=None): archive_retrieve_subparser.add_argument('--wait', action='store_true') archive_retrieve_subparser.add_argument( '--decrypt', default=False, action="store_true") + archive_retrieve_subparser.add_argument( + '--concurrent', default=False, action="store_true", + dest="concurrent", + help='Break the download into multiple parallel threads' + ) + archive_retrieve_subparser.add_argument( + '--part-size', + default=DEFAULT_PART_SIZE, + dest="part_size", + help=("For --concurrent downloads, change the " + "part size from the default of %d." + % DEFAULT_PART_SIZE) + ) + archive_retrieve_subparser.add_argument( + '--num-threads', + default=DEFAULT_NUM_THREADS, + dest="num_threads", + help=("For --concurrent downloads, change the " + "num threads from the default of %d." + % DEFAULT_NUM_THREADS) + ) + # Delete command archive_delete_subparser = archive_subparser.add_parser('delete') From 81b6d8996cbd526fa985d5e2032c90985b0016da Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Sat, 21 Dec 2013 00:02:49 -0800 Subject: [PATCH 50/76] fix docstring --- gpg.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpg.py b/gpg.py index 969300e..56e55bb 100644 --- a/gpg.py +++ b/gpg.py @@ -57,7 +57,7 @@ def _get_fingerprint(self): return self.gpg.list_keys()[0]["fingerprint"] def encrypt_file(self, input_file, output_filename): - """Encrypt `input_filename` and save results as `output_filename`.""" + """Encrypt `input_file` (handle) and save results as `output_filename`.""" fingerprint = self._get_fingerprint() self.gpg.encrypt_file(input_file, fingerprint, From 232e5ec89db9fb241b1fca6d5769ed354c728761 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Sat, 21 Dec 2013 00:04:50 -0800 Subject: [PATCH 51/76] add additional spaces for pep8 --- glacier_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/glacier_test.py b/glacier_test.py index 70b3dca..433d6b2 100644 --- a/glacier_test.py +++ b/glacier_test.py @@ -142,10 +142,10 @@ def test_archive_upload_concurrent(self): file_obj.name = 'filename' open_mock = Mock(return_value=file_obj) with patch('__builtin__.open', open_mock), \ - patch('glacier.ConcurrentUploader'), \ - patch('glacier.ConcurrentUploader.upload', return_value=123): - self.run_app(args) - glacier.ConcurrentUploader.upload.assert_called_once() + patch('glacier.ConcurrentUploader'), \ + patch('glacier.ConcurrentUploader.upload', return_value=123): + self.run_app(args) + glacier.ConcurrentUploader.upload.assert_called_once() self.connection.get_vault.assert_called_with('vault_name') def test_archive_retrieve_no_job(self): From e68a102c5300c1bdffe3fa3450915bd5618880a0 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Sat, 21 Dec 2013 00:19:03 -0800 Subject: [PATCH 52/76] clarify code around temp file naming --- glacier.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/glacier.py b/glacier.py index bbc7a3f..ab1a96a 100755 --- a/glacier.py +++ b/glacier.py @@ -523,13 +523,11 @@ def archive_upload(self, encryptor=None): name = os.path.basename(full_name) if self.args.encrypt: - temp_file_name = tempfile.NamedTemporaryFile().name - logger.info("Encrypting %s to %s." - % (self.args.file.name, temp_file_name)) - encryptor.encrypt_file(self.args.file, temp_file_name) - logger.info("Encryption complete: %s." % temp_file_name) - file_obj = file(temp_file_name, 'rb') - file_name = temp_file_name + file_obj = tempfile.NamedTemporaryFile() + file_name = file_obj.name + logger.info("Encrypting %s to %s." % (self.args.file, file_name)) + encryptor.encrypt_file(self.args.file, file_name) + logger.info("Encryption complete: %s." % file_name) else: file_obj = self.args.file file_name = file_obj.name @@ -557,7 +555,7 @@ def archive_upload(self, encryptor=None): self.cache.add_archive(self.args.vault, name, archive_id) if self.args.encrypt: - os.remove(temp_file_name) + os.remove(file_name) @staticmethod def _write_archive_retrieval_job(f, job, multipart_size, From c765a16f207ece2395f8af1a74b46906a01c8268 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Sat, 21 Dec 2013 00:31:16 -0800 Subject: [PATCH 53/76] make multipart downloads use the concurrent downloader by default --- glacier.py | 42 ++++++++---------------------- glacier_test.py | 68 ++++++++++++++++++++++--------------------------- 2 files changed, 41 insertions(+), 69 deletions(-) diff --git a/glacier.py b/glacier.py index ab1a96a..0de9abc 100755 --- a/glacier.py +++ b/glacier.py @@ -566,15 +566,15 @@ def _write_archive_retrieval_job(f, job, multipart_size, else: destfile = f - if self.args.concurrent: - downloader = ConcurrentDownloader( - job=job, - part_size=self.args.part_size, - num_threads=self.args.num_threads - ) - downloader.download(destfile.name) - else: if job.archive_size > multipart_size: + downloader = ConcurrentDownloader( + job=job, + part_size=DEFAULT_PART_SIZE, + num_threads=DEFAULT_NUM_THREADS + ) + downloader.download(destfile.name) + + """ def fetch(start, end): byte_range = start, end - 1 destfile.write(job.get_output(byte_range).read()) @@ -586,6 +586,7 @@ def fetch(start, end): remainder = job.archive_size % multipart_size if remainder: fetch(job.archive_size - remainder, job.archive_size) + """ else: destfile.write(job.get_output().read()) @@ -812,27 +813,6 @@ def parse_args(self, args=None): archive_retrieve_subparser.add_argument('--wait', action='store_true') archive_retrieve_subparser.add_argument( '--decrypt', default=False, action="store_true") - archive_retrieve_subparser.add_argument( - '--concurrent', default=False, action="store_true", - dest="concurrent", - help='Break the download into multiple parallel threads' - ) - archive_retrieve_subparser.add_argument( - '--part-size', - default=DEFAULT_PART_SIZE, - dest="part_size", - help=("For --concurrent downloads, change the " - "part size from the default of %d." - % DEFAULT_PART_SIZE) - ) - archive_retrieve_subparser.add_argument( - '--num-threads', - default=DEFAULT_NUM_THREADS, - dest="num_threads", - help=("For --concurrent downloads, change the " - "num threads from the default of %d." - % DEFAULT_NUM_THREADS) - ) # Delete command @@ -860,10 +840,10 @@ def parse_args(self, args=None): parsed = parser.parse_args(args) if (parsed.func == archive_upload_func - and parsed.multipart + and parsed.concurrent and parsed.file is sys.stdin): raise ConsoleError( - "multipart uploads do not support streaming from stdin" + "concurrent uploads do not support streaming from stdin" ) return parsed diff --git a/glacier_test.py b/glacier_test.py index 433d6b2..20c2c1a 100644 --- a/glacier_test.py +++ b/glacier_test.py @@ -23,7 +23,6 @@ from __future__ import print_function -from os import environ import sys import unittest @@ -38,15 +37,6 @@ class TestCase(unittest.TestCase): - def setUp(self): - self.orig_gnupghome = environ.get('gnupg_test_home') - environ['GNUPGHOME'] = "gnupg_test_home" - patch('glacier.Encryptor', autospec=True) - - def tearDown(self): - if self.orig_gnupghome is not None: - environ['GNUPGHOME'] = self.orig_gnupghome - def init_app(self, args, memory_cache=False): self.connection = Mock() if memory_cache: @@ -116,37 +106,39 @@ def test_archive_list_force_ids(self): ) def test_archive_upload(self): - args = ['archive', 'upload', 'vault_name', 'filename', '--name', 'filename'] - file_obj = Mock() - file_obj.name = 'filename' - open_mock = Mock(return_value=file_obj) - with patch('__builtin__.open', open_mock): - self.run_app(args) - self.connection.get_vault.assert_called_with('vault_name') - mock_vault = self.connection.get_vault.return_value - mock_vault.create_archive_from_file.assert_called_once_with( - file_obj=file_obj, description='filename') + for multipart in (False, ): + for encrypt in (False, ): + args = ['archive', 'upload', 'vault_name', 'filename'] + if multipart: + args.append('--multi-part') + if encrypt: + args.append('--encrypt') + file_obj = Mock() + file_obj.name = 'filename' + open_mock = Mock(return_value=file_obj) + with patch('__builtin__.open', open_mock): + self.run_app(args) + self.connection.get_vault.assert_called_with('vault_name') + mock_vault = self.connection.get_vault.return_value + if multipart: + pass + else: + mock_vault.create_archive_from_file.assert_called_once_with( + file_obj=file_obj, description='filename') def test_archive_stdin_upload(self): - args = ['archive', 'upload', 'vault_name', '-','--name', ''] - self.run_app(args) - self.connection.get_vault.assert_called_once_with('vault_name') - vault = self.connection.get_vault.return_value - vault.create_archive_from_file.assert_called_once_with( - file_obj=sys.stdin, description='') - - def test_archive_upload_concurrent(self): - args = ['archive', 'upload', 'vault_name', 'filename', '--name', 'filename'] - args.append('--concurrent') - file_obj = Mock() - file_obj.name = 'filename' - open_mock = Mock(return_value=file_obj) - with patch('__builtin__.open', open_mock), \ - patch('glacier.ConcurrentUploader'), \ - patch('glacier.ConcurrentUploader.upload', return_value=123): + for concurrent in (False, ): + for encrypt in (False, ): + args = ['archive', 'upload', 'vault_name', '-'] + if concurrent: + args.append('--concurrent') + if encrypt: + args.append('--encrypt') self.run_app(args) - glacier.ConcurrentUploader.upload.assert_called_once() - self.connection.get_vault.assert_called_with('vault_name') + self.connection.get_vault.assert_called_once_with('vault_name') + vault = self.connection.get_vault.return_value + vault.create_archive_from_file.assert_called_once_with( + file_obj=sys.stdin, description='') def test_archive_retrieve_no_job(self): self.init_app(['archive', 'retrieve', 'vault_name', 'archive_name']) From f866664ea7f8f941ffc93a8d91e43264a4f32c6f Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Tue, 31 Dec 2013 14:11:22 -0800 Subject: [PATCH 54/76] rename --multi-part to --concurrent in test case --- glacier_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/glacier_test.py b/glacier_test.py index 20c2c1a..a6148b0 100644 --- a/glacier_test.py +++ b/glacier_test.py @@ -110,7 +110,7 @@ def test_archive_upload(self): for encrypt in (False, ): args = ['archive', 'upload', 'vault_name', 'filename'] if multipart: - args.append('--multi-part') + args.append('--concurrent') if encrypt: args.append('--encrypt') file_obj = Mock() From ce133df54202a23909b94550c21a376f80a0dd48 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Thu, 2 Jan 2014 15:13:20 -0800 Subject: [PATCH 55/76] remove code to explictly delete tmp dir --- glacier.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/glacier.py b/glacier.py index 0de9abc..c19fb4e 100755 --- a/glacier.py +++ b/glacier.py @@ -554,9 +554,6 @@ def archive_upload(self, encryptor=None): self.cache.add_archive(self.args.vault, name, archive_id) - if self.args.encrypt: - os.remove(file_name) - @staticmethod def _write_archive_retrieval_job(f, job, multipart_size, encryptor=None): From e1a91145713cf908a2c884614e1d95b0a90f6cec Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Thu, 2 Jan 2014 15:14:52 -0800 Subject: [PATCH 56/76] test combinations of encryption and concurrency with uploading --- glacier_test.input | 1 + 1 file changed, 1 insertion(+) create mode 100644 glacier_test.input diff --git a/glacier_test.input b/glacier_test.input new file mode 100644 index 0000000..190a180 --- /dev/null +++ b/glacier_test.input @@ -0,0 +1 @@ +123 From 8232f81b2e88a3b16132677606667f27c1abe6c8 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Thu, 2 Jan 2014 15:22:03 -0800 Subject: [PATCH 57/76] test combinations of --encrypt and --concurrent when uploading --- glacier_test.py | 67 +++++++++++++++++++++++++++++++++++-------------- 1 file changed, 48 insertions(+), 19 deletions(-) mode change 100644 => 100755 glacier_test.py diff --git a/glacier_test.py b/glacier_test.py old mode 100644 new mode 100755 index a6148b0..13bf7e3 --- a/glacier_test.py +++ b/glacier_test.py @@ -106,27 +106,56 @@ def test_archive_list_force_ids(self): ) def test_archive_upload(self): - for multipart in (False, ): - for encrypt in (False, ): - args = ['archive', 'upload', 'vault_name', 'filename'] - if multipart: - args.append('--concurrent') - if encrypt: - args.append('--encrypt') - file_obj = Mock() - file_obj.name = 'filename' - open_mock = Mock(return_value=file_obj) - with patch('__builtin__.open', open_mock): - self.run_app(args) - self.connection.get_vault.assert_called_with('vault_name') - mock_vault = self.connection.get_vault.return_value - if multipart: - pass - else: - mock_vault.create_archive_from_file.assert_called_once_with( - file_obj=file_obj, description='filename') + for encrypt in (False, True): + args = ['archive', 'upload', 'vault_name', 'glacier_test.input','--name','glacier_test.input'] + if encrypt: + args.append('--encrypt') + + open_mock = Mock(wraps=open) + with patch('__builtin__.open', open_mock): + self.run_app(args) + + self.connection.get_vault.assert_called_with('vault_name') + + mock_vault = self.connection.get_vault.return_value + + call_args = mock_vault.create_archive_from_file.call_args + uploaded_name = call_args[1]['file_obj'].name + if encrypt: + nose.tools.assert_equals(uploaded_name.find('/tmp'), 0) + else: + nose.tools.assert_equals(uploaded_name, 'glacier_test.input') + + nose.tools.assert_equals(call_args[1]['description'], 'glacier_test.input') + + def test_archive_upload_concurrent(self): + for encrypt in (False, True): + args = ['archive', 'upload', 'vault_name', 'glacier_test.input', + '--name', 'glacier_test.input', + '--concurrent'] + if encrypt: + args.append('--encrypt') + + open_mock = Mock(wraps=open) + upload_mock = Mock(return_value='fake_archive_id') + with patch('__builtin__.open', open_mock), \ + patch('boto.glacier.concurrent.ConcurrentUploader.upload', + upload_mock): + self.run_app(args) + + self.connection.get_vault.assert_called_with('vault_name') + mock_vault = self.connection.get_vault.return_value + + call_args = upload_mock.call_args + if encrypt: + nose.tools.assert_equals(call_args[0][0].find('/tmp'), 0) + else: + nose.tools.assert_equals(call_args[0][0], 'glacier_test.input') + nose.tools.assert_equals(call_args[1]['description'], 'glacier_test.input') def test_archive_stdin_upload(self): + # NOTE: concurrency and encryption are not + # currently supported when streaming fron stdin. for concurrent in (False, ): for encrypt in (False, ): args = ['archive', 'upload', 'vault_name', '-'] From 06f5f38ff78b475af0cd501c04d6ee0b4b89e189 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Thu, 2 Jan 2014 15:23:06 -0800 Subject: [PATCH 58/76] update setup.py to wrap duplicate requirements{,-dev}.txt --- setup.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/setup.py b/setup.py index d716edd..cc46beb 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ from setuptools import setup - +from pip.req import parse_requirements setup( name='counsyl-glacier-cli', @@ -15,10 +15,10 @@ py_modules=['glacier', 'gpg'], zip_safe=False, install_requires=[ - 'boto', - 'iso8601', - 'sqlalchemy', - 'python-gnupg', + str(req.req) for req in parse_requirements('requirements.txt') + ], + tests_require=[ + str(req.req) for req in parse_requirements('requirements-dev.txt') ], classifiers=( "Development Status :: 3 - Alpha", From 96b5fa9b599ddcb3dcc37e31060c9ebbbaae285e Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Thu, 2 Jan 2014 15:35:26 -0800 Subject: [PATCH 59/76] update koality.yml to run the test case correctly --- koality.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/koality.yml b/koality.yml index b706abb..722d71d 100644 --- a/koality.yml +++ b/koality.yml @@ -9,6 +9,6 @@ test: machines: 1 scripts: - unit tests: - script: nosetests --with-xunit + script: nosetests --with-xunit glacier_test.py xunit: nosetests.xml timeout: 300 From 6c2ead14bbda587866c500e935c298e7d0a43fa3 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Thu, 2 Jan 2014 15:51:50 -0800 Subject: [PATCH 60/76] update tmpdir check to work on Mac, not expect /tmp --- glacier_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/glacier_test.py b/glacier_test.py index 13bf7e3..8e206f2 100755 --- a/glacier_test.py +++ b/glacier_test.py @@ -29,6 +29,7 @@ import mock from mock import Mock, patch, sentinel import nose.tools +from tempfile import gettempdir import glacier @@ -122,7 +123,7 @@ def test_archive_upload(self): call_args = mock_vault.create_archive_from_file.call_args uploaded_name = call_args[1]['file_obj'].name if encrypt: - nose.tools.assert_equals(uploaded_name.find('/tmp'), 0) + nose.tools.assert_equals(uploaded_name.find(gettempdir()), 0) else: nose.tools.assert_equals(uploaded_name, 'glacier_test.input') @@ -148,7 +149,7 @@ def test_archive_upload_concurrent(self): call_args = upload_mock.call_args if encrypt: - nose.tools.assert_equals(call_args[0][0].find('/tmp'), 0) + nose.tools.assert_equals(call_args[0][0].find(gettempdir()), 0) else: nose.tools.assert_equals(call_args[0][0], 'glacier_test.input') nose.tools.assert_equals(call_args[1]['description'], 'glacier_test.input') From efeee9faaaee4e6fdf7f79fd0ea021ea3eb12a15 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Thu, 2 Jan 2014 17:19:23 -0800 Subject: [PATCH 61/76] Configure the test case to use pre-generated testing GPNUPG keys. --- glacier_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/glacier_test.py b/glacier_test.py index 8e206f2..0b65435 100755 --- a/glacier_test.py +++ b/glacier_test.py @@ -23,6 +23,7 @@ from __future__ import print_function +import os import sys import unittest @@ -33,6 +34,7 @@ import glacier +os.environ['GNUPGHOME']='gnupg_test_home' EX_TEMPFAIL = 75 From 287fdabae0925b25b98fd24442876e95363d788b Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Thu, 2 Jan 2014 17:19:48 -0800 Subject: [PATCH 62/76] updated random seet for gnupg_test_home --- gnupg_test_home/random_seed | Bin 600 -> 600 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/gnupg_test_home/random_seed b/gnupg_test_home/random_seed index e8672bcbb1facfb944eadbbc9d5637f40ca1f57a..aa5ff6ffd0331b91e9a16fb9bb797ade36e911c7 100644 GIT binary patch literal 600 zcmV-e0;m0iYa6Okq3!gdK13=P9!R@Rkt62Gidj4B&EVhWfjsiZb5`hig-4?7+U1o< zJB(R$quz{b38DU%4nQq$mp-U=`5sF7d85HH8pT}(HojF#(MWOZ{J;_ojw15O;R4se z-y0r#?*nMcgb8&*id#b>9L@{7yQknkW%1|(`5!JB6e4esR{koBpG*4ny_J??37J^zqNqDQQDcZpNxjwFQC@1tgA>WRWK=2blG`T05LfSi^ z`d**Ag+t`}B@2&8*xI1r5K$i>ls@$B=_7 z#~PH%F2;;W?1~2~6ZFdXrh(xHUXV@zc94zgvzX--8``aB(sjpHpUI@%i&>usc@aFh zy)YFjRQYL45zCUoU~~Cb-QFkN{~^65pJMQUG{2GaAoEr475^UXpi8jJPNLL7*Li8% z=MU20L0}``7#qHx%s$PRs3^F|+zaSh-8`>XHJ4i!Xh3e7Cl}1M;JvucB3#iV%k8M` z(Bw>sq6)++a>~=~pUi!6{SH#5+Rtnb4MPYQ@BbWhbJS##P)N*) z*wf*!`_j#1pX1C(+qFCI31t}Knxr32Q|7#&`i?*E}U>?FH z&=ddnPr$Z?R?)y#w8xrm*QuMGp@KMv`7{6*Qz2>O=)-ssG#rQ|F#Ujkv|N&MQR4}c zO`LBrDq5~C9gLkRmD?tJLh)ZBE1ZKg zc~&EfS4Hd)ha^=;x@!dHwpWFMdDJEBfA>Rktv=GBqwZb&gFh@^1rpmr5-gaA44oyK{$1|H_z)wKCZOSGPTl^T^Ll;4`DmmrQB5KE??CW4k zA@-D?dp~BnG}55w=fGl)T;t&3e=P`-D(8Z67O1q%>#|(WdKyGQUO!967b2Id5s8qG z5{{g#@5mf>A}2jXQwffe7s=i;pZ4wj4Ri(KJ?-pScn#eSf_%h5+~-v&>E#b6fewiG zt+#Io1T);>C?pl7ciOeI_)~mW&5uGF2b^CFv|r5F9*XW+x`SIHm5Lt52IC-}g6zB% m0b5RkFY1qXH@KC|+dlImNY*IGpjd>H31qSfIodtHZwQiJLm Date: Tue, 7 Jan 2014 11:14:55 -0800 Subject: [PATCH 63/76] fix indentation bug preventing final output after decrypt --- glacier.py | 64 +++++++++++++++++++++++++++--------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/glacier.py b/glacier.py index c19fb4e..79d5fb8 100755 --- a/glacier.py +++ b/glacier.py @@ -563,39 +563,39 @@ def _write_archive_retrieval_job(f, job, multipart_size, else: destfile = f - if job.archive_size > multipart_size: - downloader = ConcurrentDownloader( - job=job, - part_size=DEFAULT_PART_SIZE, - num_threads=DEFAULT_NUM_THREADS - ) - downloader.download(destfile.name) - - """ - def fetch(start, end): - byte_range = start, end - 1 - destfile.write(job.get_output(byte_range).read()) - - whole_parts = job.archive_size // multipart_size - for first_byte in xrange(0, whole_parts * multipart_size, - multipart_size): - fetch(first_byte, first_byte + multipart_size) - remainder = job.archive_size % multipart_size - if remainder: - fetch(job.archive_size - remainder, job.archive_size) - """ - else: - destfile.write(job.get_output().read()) + if job.archive_size > multipart_size: + downloader = ConcurrentDownloader( + job=job, + part_size=DEFAULT_PART_SIZE, + num_threads=DEFAULT_NUM_THREADS + ) + downloader.download(destfile.name) + + """ + def fetch(start, end): + byte_range = start, end - 1 + destfile.write(job.get_output(byte_range).read()) + + whole_parts = job.archive_size // multipart_size + for first_byte in xrange(0, whole_parts * multipart_size, + multipart_size): + fetch(first_byte, first_byte + multipart_size) + remainder = job.archive_size % multipart_size + if remainder: + fetch(job.archive_size - remainder, job.archive_size) + """ + else: + destfile.write(job.get_output().read()) - # Make sure that the file now exactly matches the downloaded archive, - # even if the file existed before and was longer. - try: - destfile.truncate(job.archive_size) - except IOError, e: - # Allow ESPIPE, since the "file" couldn't have existed - # before in this case. - if e.errno != errno.ESPIPE: - raise + # Make sure that the file now exactly matches the downloaded archive, + # even if the file existed before and was longer. + try: + destfile.truncate(job.archive_size) + except IOError, e: + # Allow ESPIPE, since the "file" couldn't have existed + # before in this case. + if e.errno != errno.ESPIPE: + raise # Decrypt file if encryptor is given if encryptor: From 24b741aa3266b8f33217ff8211271c4b9753d4f1 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Tue, 7 Jan 2014 11:50:23 -0800 Subject: [PATCH 64/76] remove commented-out code --- glacier.py | 16 +--------------- 1 file changed, 1 insertion(+), 15 deletions(-) diff --git a/glacier.py b/glacier.py index 79d5fb8..6e82f18 100755 --- a/glacier.py +++ b/glacier.py @@ -566,24 +566,10 @@ def _write_archive_retrieval_job(f, job, multipart_size, if job.archive_size > multipart_size: downloader = ConcurrentDownloader( job=job, - part_size=DEFAULT_PART_SIZE, + part_size=multipart_size, num_threads=DEFAULT_NUM_THREADS ) downloader.download(destfile.name) - - """ - def fetch(start, end): - byte_range = start, end - 1 - destfile.write(job.get_output(byte_range).read()) - - whole_parts = job.archive_size // multipart_size - for first_byte in xrange(0, whole_parts * multipart_size, - multipart_size): - fetch(first_byte, first_byte + multipart_size) - remainder = job.archive_size % multipart_size - if remainder: - fetch(job.archive_size - remainder, job.archive_size) - """ else: destfile.write(job.get_output().read()) From 9c8146618799f95963ef43f5cc4579f93fb12148 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Tue, 7 Jan 2014 11:58:27 -0800 Subject: [PATCH 65/76] verify that incorrect combinations of new args raise exceptions --- glacier.py | 12 ++++++++---- glacier_test.py | 22 +++++++++++++--------- 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/glacier.py b/glacier.py index 6e82f18..4d6fbbb 100755 --- a/glacier.py +++ b/glacier.py @@ -823,11 +823,15 @@ def parse_args(self, args=None): parsed = parser.parse_args(args) if (parsed.func == archive_upload_func - and parsed.concurrent and parsed.file is sys.stdin): - raise ConsoleError( - "concurrent uploads do not support streaming from stdin" - ) + if parsed.concurrent: + raise ConsoleError( + "concurrent uploads do not support streaming from stdin" + ) + if parsed.encrypt: + raise ConsoleError( + "encrypted uploads do not support streaming from stdin" + ) return parsed diff --git a/glacier_test.py b/glacier_test.py index 0b65435..fb519fd 100755 --- a/glacier_test.py +++ b/glacier_test.py @@ -157,20 +157,24 @@ def test_archive_upload_concurrent(self): nose.tools.assert_equals(call_args[1]['description'], 'glacier_test.input') def test_archive_stdin_upload(self): - # NOTE: concurrency and encryption are not - # currently supported when streaming fron stdin. - for concurrent in (False, ): - for encrypt in (False, ): + for concurrent in (False, True): + for encrypt in (False, True): args = ['archive', 'upload', 'vault_name', '-'] if concurrent: args.append('--concurrent') if encrypt: args.append('--encrypt') - self.run_app(args) - self.connection.get_vault.assert_called_once_with('vault_name') - vault = self.connection.get_vault.return_value - vault.create_archive_from_file.assert_called_once_with( - file_obj=sys.stdin, description='') + if concurrent or encrypt: + # NOTE: concurrency and encryption are not + # currently supported when streaming fron stdin. + nose.tools.assert_raises(Exception, + self.run_app, args) + else: + self.run_app(args) + self.connection.get_vault.assert_called_once_with('vault_name') + vault = self.connection.get_vault.return_value + vault.create_archive_from_file.assert_called_once_with( + file_obj=sys.stdin, description='') def test_archive_retrieve_no_job(self): self.init_app(['archive', 'retrieve', 'vault_name', 'archive_name']) From 59f31bdbd84e4d109cbdcc1f87fd3115ace406ee Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Mon, 13 Jan 2014 16:36:16 -0800 Subject: [PATCH 66/76] removed extra line --- glacier.py | 1 - 1 file changed, 1 deletion(-) diff --git a/glacier.py b/glacier.py index 4d6fbbb..432bf00 100755 --- a/glacier.py +++ b/glacier.py @@ -797,7 +797,6 @@ def parse_args(self, args=None): archive_retrieve_subparser.add_argument( '--decrypt', default=False, action="store_true") - # Delete command archive_delete_subparser = archive_subparser.add_parser('delete') archive_delete_subparser.set_defaults(func=self.archive_delete) From 929abb15380c72fda8787d47c5ad08acef815914 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Mon, 13 Jan 2014 17:00:45 -0800 Subject: [PATCH 67/76] shorten new lines that are too long --- glacier.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/glacier.py b/glacier.py index 432bf00..1d33a52 100755 --- a/glacier.py +++ b/glacier.py @@ -717,7 +717,6 @@ def too_old(last_seen): print(self.args.name) - def parse_args(self, args=None): parser = argparse.ArgumentParser() parser.add_argument('--region', default=DEFAULT_REGION) @@ -825,11 +824,11 @@ def parse_args(self, args=None): and parsed.file is sys.stdin): if parsed.concurrent: raise ConsoleError( - "concurrent uploads do not support streaming from stdin" + "concurrent uploads do not support streaming stdin" ) if parsed.encrypt: raise ConsoleError( - "encrypted uploads do not support streaming from stdin" + "encrypted uploads do not support streaming stdin" ) return parsed From 2afd841af6c994ef0c64286a29f25161bb624859 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Mon, 13 Jan 2014 17:01:07 -0800 Subject: [PATCH 68/76] shorten new lines that are too long --- glacier_test.py | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/glacier_test.py b/glacier_test.py index fb519fd..bc3d4ee 100755 --- a/glacier_test.py +++ b/glacier_test.py @@ -34,7 +34,7 @@ import glacier -os.environ['GNUPGHOME']='gnupg_test_home' +os.environ['GNUPGHOME'] = 'gnupg_test_home' EX_TEMPFAIL = 75 @@ -110,26 +110,32 @@ def test_archive_list_force_ids(self): def test_archive_upload(self): for encrypt in (False, True): - args = ['archive', 'upload', 'vault_name', 'glacier_test.input','--name','glacier_test.input'] + args = ['archive', + 'upload', + 'vault_name', + 'glacier_test.input', + '--name', + 'glacier_test.input'] if encrypt: args.append('--encrypt') - + open_mock = Mock(wraps=open) with patch('__builtin__.open', open_mock): self.run_app(args) - + self.connection.get_vault.assert_called_with('vault_name') - + mock_vault = self.connection.get_vault.return_value - + call_args = mock_vault.create_archive_from_file.call_args uploaded_name = call_args[1]['file_obj'].name if encrypt: nose.tools.assert_equals(uploaded_name.find(gettempdir()), 0) else: nose.tools.assert_equals(uploaded_name, 'glacier_test.input') - - nose.tools.assert_equals(call_args[1]['description'], 'glacier_test.input') + + nose.tools.assert_equals(call_args[1]['description'], + 'glacier_test.input') def test_archive_upload_concurrent(self): for encrypt in (False, True): @@ -138,23 +144,22 @@ def test_archive_upload_concurrent(self): '--concurrent'] if encrypt: args.append('--encrypt') - + open_mock = Mock(wraps=open) upload_mock = Mock(return_value='fake_archive_id') with patch('__builtin__.open', open_mock), \ patch('boto.glacier.concurrent.ConcurrentUploader.upload', upload_mock): self.run_app(args) - - self.connection.get_vault.assert_called_with('vault_name') - mock_vault = self.connection.get_vault.return_value + self.connection.get_vault.assert_called_with('vault_name') call_args = upload_mock.call_args if encrypt: nose.tools.assert_equals(call_args[0][0].find(gettempdir()), 0) else: nose.tools.assert_equals(call_args[0][0], 'glacier_test.input') - nose.tools.assert_equals(call_args[1]['description'], 'glacier_test.input') + nose.tools.assert_equals(call_args[1]['description'], + 'glacier_test.input') def test_archive_stdin_upload(self): for concurrent in (False, True): @@ -165,13 +170,15 @@ def test_archive_stdin_upload(self): if encrypt: args.append('--encrypt') if concurrent or encrypt: - # NOTE: concurrency and encryption are not + # NOTE: concurrency and encryption are not # currently supported when streaming fron stdin. nose.tools.assert_raises(Exception, self.run_app, args) else: self.run_app(args) - self.connection.get_vault.assert_called_once_with('vault_name') + self.connection.get_vault.assert_called_once_with( + 'vault_name' + ) vault = self.connection.get_vault.return_value vault.create_archive_from_file.assert_called_once_with( file_obj=sys.stdin, description='') From 9211c64021b1b9a8abe85a5e70205e82e1145678 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Mon, 13 Jan 2014 17:15:38 -0800 Subject: [PATCH 69/76] rename --multipart-size to --part-size --- glacier.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/glacier.py b/glacier.py index 1d33a52..69453fb 100755 --- a/glacier.py +++ b/glacier.py @@ -555,7 +555,7 @@ def archive_upload(self, encryptor=None): self.cache.add_archive(self.args.vault, name, archive_id) @staticmethod - def _write_archive_retrieval_job(f, job, multipart_size, + def _write_archive_retrieval_job(f, job, part_size, encryptor=None): if encryptor: @@ -563,10 +563,10 @@ def _write_archive_retrieval_job(f, job, multipart_size, else: destfile = f - if job.archive_size > multipart_size: + if job.archive_size > part_size: downloader = ConcurrentDownloader( job=job, - part_size=multipart_size, + part_size=part_size, num_threads=DEFAULT_NUM_THREADS ) downloader.download(destfile.name) @@ -592,7 +592,7 @@ def _write_archive_retrieval_job(f, job, multipart_size, def _archive_retrieve_completed(cls, args, job, name, encryptor=None): if args.output_filename == '-': cls._write_archive_retrieval_job( - sys.stdout, job, args.multipart_size, + sys.stdout, job, args.part_size, encryptor=encryptor) else: if args.output_filename: @@ -600,7 +600,7 @@ def _archive_retrieve_completed(cls, args, job, name, encryptor=None): else: filename = os.path.basename(name) with open(filename, 'wb') as f: - cls._write_archive_retrieval_job(f, job, args.multipart_size, + cls._write_archive_retrieval_job(f, job, args.part_size, encryptor=encryptor) def archive_retrieve_one(self, name, encryptor=None): @@ -788,7 +788,7 @@ def parse_args(self, args=None): archive_retrieve_subparser.add_argument('vault') archive_retrieve_subparser.add_argument('names', nargs='+', metavar='name') - archive_retrieve_subparser.add_argument('--multipart-size', type=int, + archive_retrieve_subparser.add_argument('--part-size', type=int, default=(8 * 1024 * 1024)) archive_retrieve_subparser.add_argument('-o', dest='output_filename', metavar='OUTPUT_FILENAME') From d8bd0416af7d108c1a3656872723f339cca8af20 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Mon, 13 Jan 2014 17:34:52 -0800 Subject: [PATCH 70/76] update README.md to reflect new args --- README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index f289216..5971960 100644 --- a/README.md +++ b/README.md @@ -211,9 +211,8 @@ Commands * glacier vault create vault-name * glacier vault sync [--wait] [--fix] [--max-age hours] vault-name * glacier archive list vault-name -* glacier archive upload [--encrypt] [--name archive-name] vault-name filename -* glacier archive multipart_upload [--encrypt] [--name archive-name] [--part-size part-size] vault-name filename -* glacier archive retrieve [--wait] [--decrypt] [-o filename] [--multipart-size bytes] vault-name archive-name [archive-name...] +* glacier archive upload [--encrypt] [--concurrent [--part-size size] [--num-threads count] [--name archive-name] vault-name filename +* glacier archive retrieve [--wait] [--decrypt] [-o filename] [--part-size bytes] vault-name archive-name [archive-name...] * glacier archive delete vault-name archive-name * glacier job list From e13c63f6815c058c9b51348d6877c8c20f9584f3 Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Mon, 13 Jan 2014 17:36:28 -0800 Subject: [PATCH 71/76] fix typo in README.md update --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5971960..7584c34 100644 --- a/README.md +++ b/README.md @@ -211,7 +211,7 @@ Commands * glacier vault create vault-name * glacier vault sync [--wait] [--fix] [--max-age hours] vault-name * glacier archive list vault-name -* glacier archive upload [--encrypt] [--concurrent [--part-size size] [--num-threads count] [--name archive-name] vault-name filename +* glacier archive upload [--encrypt] [--concurrent] [--part-size size] [--num-threads count] [--name archive-name] vault-name filename * glacier archive retrieve [--wait] [--decrypt] [-o filename] [--part-size bytes] vault-name archive-name [archive-name...] * glacier archive delete vault-name archive-name * glacier job list From 12b2c3752d87fbdb3cc6c0b3fdbc0891e93affbd Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Mon, 13 Jan 2014 17:38:10 -0800 Subject: [PATCH 72/76] update [] around optional --concurrent arg --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7584c34..526381d 100644 --- a/README.md +++ b/README.md @@ -211,7 +211,7 @@ Commands * glacier vault create vault-name * glacier vault sync [--wait] [--fix] [--max-age hours] vault-name * glacier archive list vault-name -* glacier archive upload [--encrypt] [--concurrent] [--part-size size] [--num-threads count] [--name archive-name] vault-name filename +* glacier archive upload [--encrypt] [--concurrent [--part-size size] [--num-threads count]] [--name archive-name] vault-name filename * glacier archive retrieve [--wait] [--decrypt] [-o filename] [--part-size bytes] vault-name archive-name [archive-name...] * glacier archive delete vault-name archive-name * glacier job list From a3b68efec22824a18678a67996b561d24b2dc78c Mon Sep 17 00:00:00 2001 From: Scott Smith Date: Tue, 21 Jan 2014 14:45:52 -0800 Subject: [PATCH 73/76] allow glacier-cli to use newer sqlalchemy, but not require it --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index adafe9f..13d3b35 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ --index-url http://pypi.counsyl.com/counsyl/prod/+simple/ -SQLAlchemy==0.8.4 +SQLAlchemy>=0.8.4 boto==2.19.0 iso8601==0.1.8 python-gnupg==0.3.5 From 7dd297b869b140c67e30571aca659f34b9bb35e8 Mon Sep 17 00:00:00 2001 From: Chris Crutchfield Date: Tue, 4 Feb 2014 16:27:12 -0800 Subject: [PATCH 74/76] Update requirements-dev.txt --- requirements-dev.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 0faf58e..17390db 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,6 @@ --index-url http://pypi.counsyl.com/counsyl/prod/+simple/ -codequality==0.2-dev + +counsyl-codequality==0.2.2 mock==1.0.1 nose==1.2.1 pep8==1.4.4 From 9a70cf22dc09699a22735d34067fd61baae405da Mon Sep 17 00:00:00 2001 From: Chris Crutchfield Date: Fri, 7 Feb 2014 11:18:39 -0800 Subject: [PATCH 75/76] Update requirements-dev.txt --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 17390db..6955664 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,6 @@ --index-url http://pypi.counsyl.com/counsyl/prod/+simple/ -counsyl-codequality==0.2.2 +codequality==0.2.2.vcounsyl mock==1.0.1 nose==1.2.1 pep8==1.4.4 From 58721b46d89c4c987784c6d34212c5a3c261f037 Mon Sep 17 00:00:00 2001 From: Chris Crutchfield Date: Fri, 14 Feb 2014 10:39:19 -0800 Subject: [PATCH 76/76] Update version to 0.1.1.alpha --- .gitignore | 2 ++ setup.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 0d20b64..dca6ea3 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ *.pyc +*~ +*# diff --git a/setup.py b/setup.py index cc46beb..1129ab1 100644 --- a/setup.py +++ b/setup.py @@ -3,7 +3,7 @@ setup( name='counsyl-glacier-cli', - version='0.1.0', + version='0.1.1.alpha', author='Counsyl', author_email='root@counsyl.com', maintainer='Counsyl',