Merge pull request #2988 from nateprewitt/pre-commit

Add pre-commit config

.github/workflows/lint.yml

@@ -0,0 +1,19 @@
+name: Lint code
+
+on:
+  push:
+  pull_request:
+    branches-ignore: [ master ]
+
+jobs:
+  lint:
+    runs-on: ubuntu-20.04
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.9
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.9
+    - name: Run pre-commit
+      uses: pre-commit/action@v2.0.0

.github/workflows/run-tests.yml

@@ -1,4 +1,3 @@
-
 name: Run tests
 
 on:

.pre-commit-config.yaml

@@ -0,0 +1,12 @@
+exclude: ^(.github|.changes|docs/|boto3/compat.py|boto3/data)
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+    -   id: check-yaml
+    -   id: end-of-file-fixer
+    -   id: trailing-whitespace
+-   repo: https://github.com/pycqa/flake8
+    rev: 3.9.2
+    hooks:
+    -   id: flake8

CHANGELOG.rst

@@ -8249,4 +8249,3 @@ CHANGELOG
 
 * feature:Resources: Supports S3, EC2, SQS, SNS, and IAM resources
 * feature:Clients: Supports low-level clients for most services
-

CODE_OF_CONDUCT.md

@@ -2,4 +2,3 @@
 This project has adopted the [Amazon Open Source Code of Conduct](https://aws.github.io/code-of-conduct).
 For more information see the [Code of Conduct FAQ](https://aws.github.io/code-of-conduct-faq) or contact
 opensource-codeofconduct@amazon.com with any additional questions or comments.
-

README.rst

@@ -49,10 +49,10 @@ Assuming that you have Python and ``virtualenv`` installed, set up your environm
 
     $ python -m pip install boto3
 
-    
+
 Using Boto3
 ~~~~~~~~~~~~~~
-After installing boto3 
+After installing boto3
 
 Next, set up credentials (in e.g. ``~/.aws/credentials``):
 
@@ -68,7 +68,7 @@ Then, set up a default region (in e.g. ``~/.aws/config``):
 
    [default]
    region=us-east-1
-    
+
 Other credentials configuration method can be found `here <https://boto3.amazonaws.com/v1/documentation/api/latest/guide/credentials.html>`__
 
 Then, from a Python interpreter:
@@ -137,4 +137,3 @@ More Resources
 * `NOTICE <https://github.com/boto/boto3/blob/develop/NOTICE>`__
 * `Changelog <https://github.com/boto/boto3/blob/develop/CHANGELOG.rst>`__
 * `License <https://github.com/boto/boto3/blob/develop/LICENSE>`__
-

boto3/compat.py

@@ -75,7 +75,7 @@ def _warn_deprecated_python():
                      'aws-cli-v1/'
     }
     deprecated_versions = {
-        (2,7): py_27_params,
+        (2, 7): py_27_params,
     }
     py_version = sys.version_info[:2]
 

boto3/docs/collection.py

@@ -185,11 +185,13 @@ def document_collection_method(section, resource_name, action_name,
             'method_description': (
                 'Creates an iterable of all %s resources '
                 'in the collection filtered by kwargs passed to '
-                'method.' % collection_model.resource.type +
-                'A %s collection will include all resources by '
+                'method. A %s collection will include all resources by '
                 'default if no filters are provided, and extreme '
                 'caution should be taken when performing actions '
-                'on all resources.'% collection_model.resource.type),
+                'on all resources.' % (
+                    collection_model.resource.type,
+                    collection_model.resource.type
+                )),
             'example_prefix': '%s_iterator = %s.%s.filter' % (
                 xform_name(collection_model.resource.type),
                 example_resource_name, collection_model.name),

boto3/dynamodb/conditions.py

@@ -144,8 +144,9 @@ def __init__(self, *values):
         AttributeBase.__init__(self, values[0].name)
 
     def __eq__(self, other):
-        return ConditionBase.__eq__(self, other) and \
-               AttributeBase.__eq__(self, other)
+        return (
+            ConditionBase.__eq__(self, other) and AttributeBase.__eq__(self, other)
+        )
 
     def __ne__(self, other):
         return not self.__eq__(other)

scripts/ci/run-crt-tests

@@ -29,7 +29,7 @@ def run(command):
 
 
 try:
-    import awscrt
+    import awscrt # noqa
 except ImportError:
     print("MISSING DEPENDENCY: awscrt must be installed to run the crt tests.")
     sys.exit(1)

scripts/new-change

@@ -135,7 +135,7 @@ def replace_issue_references(parsed, repo_name):
             '`%s <https://github.com/%s/issues/%s>`__' % (
                 match.group(), repo_name, number))
 
-    new_description = re.sub('#\d+', linkify, description)
+    new_description = re.sub(r'#\d+', linkify, description)
     parsed['description'] = new_description
 
 

setup.cfg

@@ -9,3 +9,11 @@ requires_dist =
 
 [options.extras_require]
 crt = botocore[crt]>=1.21.0,<2.0a0
+
+[flake8]
+ignore = E203,E501,W503,W504
+exclude =
+    docs,
+    boto3/compat.py,
+    boto3/data,
+    .changes

tests/__init__.py

@@ -12,24 +12,12 @@
 # language governing permissions and limitations under the License.
 
 import random
-import sys
 import time
 
-from botocore.compat import six
-
-
 import unittest
 from unittest import mock
 
 
-# In python 3, order matters when calling assertEqual to
-# compare lists and dictionaries with lists. Therefore,
-# assertItemsEqual needs to be used but it is renamed to
-# assertCountEqual in python 3.
-if six.PY2:
-    unittest.TestCase.assertCountEqual = unittest.TestCase.assertItemsEqual
-
-
 def unique_id(name):
     """
     Generate a unique ID that includes the given name,

tests/functional/docs/test_ec2.py

@@ -19,17 +19,23 @@
 class TestInstanceDeleteTags(BaseDocsFunctionalTests):
     def setUp(self):
         self.documenter = ServiceDocumenter(
-            'ec2', session=Session(region_name='us-east-1'))
+            'ec2', session=Session(region_name='us-east-1')
+        )
         self.generated_contents = self.documenter.document_service()
         self.generated_contents = self.generated_contents.decode('utf-8')
 
     def test_delete_tags_method_is_documented(self):
         contents = self.get_class_document_block(
-                'EC2.Instance', self.generated_contents)
+            'EC2.Instance', self.generated_contents
+        )
         method_contents = self.get_method_document_block(
-                'delete_tags', contents)
-        self.assert_contains_lines_in_order([
-            'response = instance.delete_tags(',
-            'DryRun=True|False,',
-            'Tags=[',
-        ], method_contents)
+            'delete_tags', contents
+        )
+        self.assert_contains_lines_in_order(
+            [
+                'response = instance.delete_tags(',
+                'DryRun=True|False,',
+                'Tags=[',
+            ],
+            method_contents
+        )

tests/functional/docs/test_smoke.py

@@ -23,12 +23,13 @@
 def botocore_session():
     return botocore.session.get_session()
 
+
 @pytest.fixture
 def boto3_session():
     return boto3.Session(region_name='us-east-1')
 
+
 def all_services():
-    botocore_session = botocore.session.get_session()
     session = boto3.Session(region_name='us-east-1')
     for service_name in session.get_available_services():
         yield service_name
@@ -52,32 +53,28 @@ def test_documentation(
     # Check that all of the services have the appropriate title
     _assert_has_title(generated_docs, client)
 
-
     # Check that all services have the client documented.
     _assert_has_client_documentation(generated_docs, service_name, client)
 
-
-    #If the service has resources, make sure the service resource
-    #is at least documented.
+    # If the service has resources, make sure the service resource
+    # is at least documented.
     if service_name in available_resources:
-    
+
         resource = boto3.resource(service_name, 'us-east-1')
         _assert_has_resource_documentation(
             generated_docs, service_name, resource
         )
 
     # If the client can paginate, make sure the paginators are documented.
     try:
-        paginator_model = botocore_session.get_paginator_model(
-                service_name)
+        paginator_model = botocore_session.get_paginator_model(service_name)
         _assert_has_paginator_documentation(
             generated_docs, service_name, client,
             sorted(paginator_model._paginator_config)
         )
     except DataNotFoundError:
         pass
 
-
     # If the client has waiters, make sure the waiters are documented.
     if client.waiter_names:
         waiter_model = botocore_session.get_waiter_model(service_name)

tests/functional/dynamodb/test_stubber_conditions.py

@@ -31,11 +31,15 @@ def test_table_query_can_be_stubbed_with_expressions(self):
         )
 
         stubber = Stubber(table.meta.client)
-        stubber.add_response('query', dict(Items=list()), expected_params=dict(
+        stubber.add_response(
+            'query',
+            dict(Items=list()),
+            expected_params=dict(
                 TableName='mytable',
                 KeyConditionExpression=key_expr,
                 FilterExpression=filter_expr
-        ))
+            )
+        )
 
         with stubber:
             response = table.query(KeyConditionExpression=key_expr,
@@ -46,15 +50,20 @@ def test_table_query_can_be_stubbed_with_expressions(self):
 
     def test_table_scan_can_be_stubbed_with_expressions(self):
         table = self.resource.Table('mytable')
-        filter_expr = Attr('myattr').eq('foo') & (
-                Attr('myattr2').lte('buzz') | Attr('myattr2').gte('fizz')
+        filter_expr = (
+            Attr('myattr').eq('foo') &
+            (Attr('myattr2').lte('buzz') | Attr('myattr2').gte('fizz'))
         )
 
         stubber = Stubber(table.meta.client)
-        stubber.add_response('scan', dict(Items=list()), expected_params=dict(
+        stubber.add_response(
+            'scan',
+            dict(Items=list()),
+            expected_params=dict(
                 TableName='mytable',
                 FilterExpression=filter_expr
-        ))
+            )
+        )
 
         with stubber:
             response = table.scan(FilterExpression=filter_expr)

tests/functional/dynamodb/test_table.py

@@ -10,7 +10,7 @@
 # distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
 # ANY KIND, either express or implied. See the License for the specific
 # language governing permissions and limitations under the License.
-from tests import unittest, mock
+from tests import unittest
 
 import boto3
 from botocore.stub import Stubber

tests/functional/test_s3.py

@@ -377,7 +377,7 @@ def stub_get_object(self, full_contents, start_byte=0, end_byte=None):
         # If this is a ranged get, ContentRange needs to be returned,
         # contents needs to be pruned, and Range needs to be an expected param.
         if end_byte is not None:
-            contents = full_contents[start_byte:end_byte+1]
+            contents = full_contents[start_byte:end_byte + 1]
             part_range = 'bytes=%s-%s' % (start_byte, end_byte_range)
             content_range = 'bytes=%s-%s/%s' % (
                 start_byte, end_byte, len(full_contents))

tests/functional/test_smoke.py

@@ -17,6 +17,7 @@
 
 boto3_session = None
 
+
 def create_session():
     global boto3_session
     if boto3_session is None:
@@ -28,6 +29,7 @@ def create_session():
 
     return boto3_session
 
+
 def _all_resources():
     session = create_session()
     for service_name in session.get_available_resources():

tests/integration/test_dynamodb.py

@@ -171,16 +171,20 @@ def test_condition_attribute_type(self):
 
     def test_condition_and(self):
         r = self.scan(
-            filter_expression=(Attr('MyHashKey').eq('mykey') &
-                               Attr('MyString').eq('mystring')))
+            filter_expression=(
+                Attr('MyHashKey').eq('mykey') & Attr('MyString').eq('mystring')
+            )
+        )
         item = r['Items'][0]
         self.assertTrue(
             item['MyHashKey'] == 'mykey' and item['MyString'] == 'mystring')
 
     def test_condition_or(self):
         r = self.scan(
-            filter_expression=(Attr('MyHashKey').eq('mykey2') |
-                               Attr('MyString').eq('mystring')))
+            filter_expression=(
+                Attr('MyHashKey').eq('mykey2') | Attr('MyString').eq('mystring')
+            )
+        )
         item = r['Items'][0]
         self.assertTrue(
             item['MyHashKey'] == 'mykey2' or item['MyString'] == 'mystring')

tests/integration/test_s3.py

@@ -472,6 +472,7 @@ def test_callback_called_once_with_sigv4(self):
         # twice when using signature version 4.
         self.amount_seen = 0
         lock = threading.Lock()
+
         def progress_callback(amount):
             with lock:
                 self.amount_seen += amount
@@ -593,9 +594,11 @@ def test_download_above_threshold(self):
 
     def test_download_file_with_directory_not_exist(self):
         transfer = self.create_s3_transfer()
-        self.client.put_object(Bucket=self.bucket_name,
-                                Key='foo.txt',
-                                Body=b'foo')
+        self.client.put_object(
+            Bucket=self.bucket_name,
+            Key='foo.txt',
+            Body=b'foo'
+        )
         self.addCleanup(self.delete_object, 'foo.txt')
         download_path = os.path.join(self.files.rootdir, 'a', 'b', 'c',
                                      'downloaded.txt')
@@ -667,7 +670,7 @@ def test_transfer_methods_through_bucket(self):
         # This is just a sanity check to ensure that the bucket interface work.
         key = 'bucket.txt'
         bucket = self.session.resource('s3').Bucket(self.bucket_name)
-        filename = self.files.create_file_with_size(key, 1024*1024)
+        filename = self.files.create_file_with_size(key, 1024 * 1024)
         bucket.upload_file(Filename=filename, Key=key)
         self.addCleanup(self.delete_object, key)
         download_path = os.path.join(self.files.rootdir, unique_id('foo'))
@@ -678,7 +681,7 @@ def test_transfer_methods_through_object(self):
         # This is just a sanity check to ensure that the object interface work.
         key = 'object.txt'
         obj = self.session.resource('s3').Object(self.bucket_name, key)
-        filename = self.files.create_file_with_size(key, 1024*1024)
+        filename = self.files.create_file_with_size(key, 1024 * 1024)
         obj.upload_file(Filename=filename)
         self.addCleanup(self.delete_object, key)
         download_path = os.path.join(self.files.rootdir, unique_id('foo'))