Merge pull request #219 from Lorak-mmk/fix-integration-standard

Enable tests/integration/standard

.github/workflows/integration-tests.yml

@@ -21,5 +21,4 @@ jobs:
 
     - name: Test with pytest
       run: |
-        ./ci/run_integration_test.sh tests/integration/standard/test_authentication.py tests/integration/standard/test_cluster.py tests/integration/standard/test_concurrent.py tests/integration/standard/test_connection.py tests/integration/standard/test_control_connection.py tests/integration/standard/test_custom_payload.py tests/integration/standard/test_custom_protocol_handler.py tests/integration/standard/test_cython_protocol_handlers.py tests/integration/standard/test_scylla_cloud.py tests/integration/standard/test_use_keyspace.py tests/integration/standard/test_ip_change.py tests/integration/cqlengine/
-        # can't run this, cause only 2 cpus on github actions: tests/integration/standard/test_shard_aware.py
+        ./ci/run_integration_test.sh tests/integration/standard/ tests/integration/cqlengine/

ci/run_integration_test.sh

@@ -23,7 +23,7 @@ pip install -U pip wheel setuptools
 
 # install driver wheel
 pip install --ignore-installed -r test-requirements.txt pytest
-pip install .
+pip install -e .
 
 # download awscli
 pip install awscli

pytest.ini

@@ -1,4 +1,5 @@
 [pytest]
 log_format = %(asctime)s.%(msecs)03d %(levelname)s [%(module)s:%(lineno)s]: %(message)s
 log_level = DEBUG
-log_date_format = %Y-%m-%d %H:%M:%S
+log_date_format = %Y-%m-%d %H:%M:%S
+xfail_strict=true

tests/integration/__init__.py

@@ -34,6 +34,7 @@
 from itertools import groupby
 import six
 import shutil
+import pytest
 
 
 from cassandra import OperationTimedOut, ReadTimeout, ReadFailure, WriteTimeout, WriteFailure, AlreadyExists,\
@@ -343,6 +344,7 @@ def _id_and_mark(f):
 local = local_decorator_creator()
 notprotocolv1 = unittest.skipUnless(PROTOCOL_VERSION > 1, 'Protocol v1 not supported')
 lessthenprotocolv4 = unittest.skipUnless(PROTOCOL_VERSION < 4, 'Protocol versions 4 or greater not supported')
+lessthanprotocolv3 = unittest.skipUnless(PROTOCOL_VERSION < 3, 'Protocol versions 3 or greater not supported')
 greaterthanprotocolv3 = unittest.skipUnless(PROTOCOL_VERSION >= 4, 'Protocol versions less than 4 are not supported')
 protocolv6 = unittest.skipUnless(6 in get_supported_protocol_versions(), 'Protocol versions less than 6 are not supported')
 
@@ -366,8 +368,22 @@ def _id_and_mark(f):
 lessthandse51 = unittest.skipUnless(DSE_VERSION and DSE_VERSION < Version('5.1'), "DSE version less than 5.1 required")
 lessthandse60 = unittest.skipUnless(DSE_VERSION and DSE_VERSION < Version('6.0'), "DSE version less than 6.0 required")
 
-requirescollectionindexes = unittest.skipUnless(SCYLLA_VERSION is None or Version(SCYLLA_VERSION.split(':')[1]) >= Version('5.2'), 'Test requires Scylla >= 5.2 or Cassandra')
-requirescustomindexes = unittest.skipUnless(SCYLLA_VERSION is None, 'Currently, Scylla does not support SASI or any other CUSTOM INDEX class.')
+# pytest.mark.xfail instead of unittest.expectedFailure because
+# 1. unittest doesn't skip setUpClass when used on class and we need it sometimes
+# 2. unittest doesn't have conditional xfail, and I prefer to use pytest than custom decorator
+# 3. unittest doesn't have a reason argument, so you don't see the reason in pytest report
+requires_collection_indexes = pytest.mark.xfail(SCYLLA_VERSION is not None and Version(SCYLLA_VERSION.split(':')[1]) < Version('5.2'),
+                                              reason='Scylla supports collection indexes from 5.2 onwards') 
+requires_custom_indexes = pytest.mark.xfail(SCYLLA_VERSION is not None, 
+                                          reason='Scylla does not support SASI or any other CUSTOM INDEX class')
+requires_java_udf = pytest.mark.xfail(SCYLLA_VERSION is not None,
+                                    reason='Scylla does not support UDFs written in Java')
+requires_composite_type = pytest.mark.xfail(SCYLLA_VERSION is not None,
+                                            reason='Scylla does not support composite types')
+requires_custom_payload = pytest.mark.xfail(SCYLLA_VERSION is not None or PROTOCOL_VERSION < 4,
+                                            reason='Scylla does not support custom payloads. Cassandra requires native protocol v4.0+')
+xfail_scylla = lambda reason, *args, **kwargs: pytest.mark.xfail(SCYLLA_VERSION is not None, reason=reason, *args, **kwargs)
+incorrect_test = lambda reason='This test seems to be incorrect and should be fixed', *args, **kwargs: pytest.mark.xfail(reason=reason, *args, **kwargs)
 
 pypy = unittest.skipUnless(platform.python_implementation() == "PyPy", "Test is skipped unless it's on PyPy")
 notpy3 = unittest.skipIf(sys.version_info >= (3, 0), "Test not applicable for Python 3.x runtime")

tests/integration/cqlengine/management/test_management.py

@@ -24,7 +24,7 @@
 from cassandra.cqlengine.models import Model
 from cassandra.cqlengine import columns
 
-from tests.integration import DSE_VERSION, PROTOCOL_VERSION, greaterthancass20, requirescollectionindexes, MockLoggingHandler, CASSANDRA_VERSION
+from tests.integration import DSE_VERSION, PROTOCOL_VERSION, greaterthancass20, requires_collection_indexes, MockLoggingHandler, CASSANDRA_VERSION
 from tests.integration.cqlengine.base import BaseCassEngTestCase
 from tests.integration.cqlengine.query.test_queryset import TestModel
 from cassandra.cqlengine.usertype import UserType
@@ -427,7 +427,7 @@ def test_sync_index_case_sensitive(self):
         self.assertIsNotNone(management._get_index_name_by_column(table_meta, 'second_key'))
 
     @greaterthancass20
-    @requirescollectionindexes
+    @requires_collection_indexes
     def test_sync_indexed_set(self):
         """
         Tests that models that have container types with indices can be synced.

tests/integration/cqlengine/query/test_named.py

@@ -27,7 +27,7 @@
 from tests.integration.cqlengine.query.test_queryset import BaseQuerySetUsage
 
 
-from tests.integration import BasicSharedKeyspaceUnitTestCase, greaterthanorequalcass30, requirescollectionindexes
+from tests.integration import BasicSharedKeyspaceUnitTestCase, greaterthanorequalcass30, requires_collection_indexes
 
 
 class TestQuerySetOperation(BaseCassEngTestCase):
@@ -118,7 +118,7 @@ def test_query_expression_where_clause_generation(self):
         self.assertIsInstance(where.operator, GreaterThanOrEqualOperator)
         self.assertEqual(where.value, 1)
 
-@requirescollectionindexes
+@requires_collection_indexes
 class TestQuerySetCountSelectionAndIteration(BaseQuerySetUsage):
 
     @classmethod

tests/integration/cqlengine/query/test_queryset.py

@@ -39,7 +39,7 @@
 from cassandra.util import uuid_from_time
 from cassandra.cqlengine.connection import get_session
 from tests.integration import PROTOCOL_VERSION, CASSANDRA_VERSION, greaterthancass20, greaterthancass21, \
-    greaterthanorequalcass30, TestCluster, requirescollectionindexes
+    greaterthanorequalcass30, TestCluster, requires_collection_indexes
 from tests.integration.cqlengine import execute_count, DEFAULT_KEYSPACE
 
 
@@ -384,7 +384,7 @@ def tearDownClass(cls):
         drop_table(CustomIndexedTestModel)
         drop_table(TestMultiClusteringModel)
 
-@requirescollectionindexes
+@requires_collection_indexes
 class TestQuerySetCountSelectionAndIteration(BaseQuerySetUsage):
 
     @execute_count(2)
@@ -558,7 +558,7 @@ class NonEqualityFilteringModel(Model):
     num = qa.count()
     assert num == 1, num
 
-@requirescollectionindexes
+@requires_collection_indexes
 class TestQuerySetDistinct(BaseQuerySetUsage):
 
     @execute_count(1)
@@ -597,7 +597,7 @@ def test_distinct_with_explicit_count(self):
         self.assertEqual(q.count(), 2)
 
 
-@requirescollectionindexes
+@requires_collection_indexes
 class TestQuerySetOrdering(BaseQuerySetUsage):
     @execute_count(2)
     def test_order_by_success_case(self):
@@ -646,7 +646,7 @@ def test_ordering_on_multiple_clustering_columns(self):
         assert [r.three for r in results] == [1, 2, 3, 4, 5]
 
 
-@requirescollectionindexes
+@requires_collection_indexes
 class TestQuerySetSlicing(BaseQuerySetUsage):
 
     @execute_count(1)
@@ -701,7 +701,7 @@ def test_negative_slicing(self):
             self.assertEqual(model.attempt_id, expect)
 
 
-@requirescollectionindexes
+@requires_collection_indexes
 class TestQuerySetValidation(BaseQuerySetUsage):
 
     def test_primary_key_or_index_must_be_specified(self):
@@ -783,7 +783,7 @@ def test_custom_indexed_field_can_be_queried(self):
         list(CustomIndexedTestModel.objects.filter(test_id=1, description='test'))
 
 
-@requirescollectionindexes
+@requires_collection_indexes
 class TestQuerySetDelete(BaseQuerySetUsage):
 
     @execute_count(9)
@@ -942,7 +942,7 @@ def test_success_case(self):
         assert '4' in datas
 
 
-@requirescollectionindexes
+@requires_collection_indexes
 class TestInOperator(BaseQuerySetUsage):
     @execute_count(1)
     def test_kwarg_success_case(self):
@@ -1003,7 +1003,7 @@ class bool_model2(Model):
 
 
 @greaterthancass20
-@requirescollectionindexes
+@requires_collection_indexes
 class TestContainsOperator(BaseQuerySetUsage):
 
     @execute_count(6)
@@ -1069,7 +1069,7 @@ def test_query_expression_success_case(self):
             self.assertEqual(q.count(), 0)
 
 
-@requirescollectionindexes
+@requires_collection_indexes
 class TestValuesList(BaseQuerySetUsage):
 
     @execute_count(2)
@@ -1082,7 +1082,7 @@ def test_values_list(self):
         assert item == 10
 
 
-@requirescollectionindexes
+@requires_collection_indexes
 class TestObjectsProperty(BaseQuerySetUsage):
     @execute_count(1)
     def test_objects_property_returns_fresh_queryset(self):
@@ -1113,7 +1113,7 @@ class PagingTest(Model):
         assert len(results) == 2
 
 
-@requirescollectionindexes
+@requires_collection_indexes
 class ModelQuerySetTimeoutTestCase(BaseQuerySetUsage):
     def test_default_timeout(self):
         with mock.patch.object(Session, 'execute') as mock_execute:
@@ -1131,7 +1131,7 @@ def test_none_timeout(self):
             self.assertEqual(mock_execute.call_args[-1]['timeout'], None)
 
 
-@requirescollectionindexes
+@requires_collection_indexes
 class DMLQueryTimeoutTestCase(BaseQuerySetUsage):
     def setUp(self):
         self.model = TestModel(test_id=1, attempt_id=1, description='timeout test')

tests/integration/cqlengine/statements/test_base_statement.py

@@ -26,7 +26,7 @@
 
 from tests.integration.cqlengine.base import BaseCassEngTestCase, TestQueryUpdateModel
 from tests.integration.cqlengine import DEFAULT_KEYSPACE
-from tests.integration import greaterthanorequalcass3_10, requirescustomindexes, TestCluster
+from tests.integration import greaterthanorequalcass3_10, requires_custom_indexes, TestCluster
 
 from cassandra.cqlengine.connection import execute
 
@@ -102,7 +102,7 @@ def test_insert_statement_execute(self):
         self.assertEqual(TestQueryUpdateModel.objects.count(), 0)
 
     @greaterthanorequalcass3_10
-    @requirescustomindexes
+    @requires_custom_indexes
     def test_like_operator(self):
         """
         Test to verify the like operator works appropriately

tests/integration/standard/test_authentication_misconfiguration.py

@@ -19,12 +19,6 @@
 
 class MisconfiguredAuthenticationTests(unittest.TestCase):
     """ One node (not the contact point) has password auth. The rest of the nodes have no auth """
-    # TODO: 	Fix ccm to apply following options to scylla.yaml
-    # 	node3.set_configuration_options(values={
-    # 	'authenticator': 'PasswordAuthenticator',
-    # 	'authorizer': 'CassandraAuthorizer',
-    # 	})
-    # To make it working for scylla
     @classmethod
     def setUpClass(cls):
         if not USE_CASS_EXTERNAL:
@@ -38,7 +32,6 @@ def setUpClass(cls):
 
             cls.ccm_cluster = ccm_cluster
 
-    @unittest.expectedFailure
     def test_connect_no_auth_provider(self):
         cluster = TestCluster()
         cluster.connect()

tests/integration/standard/test_client_warnings.py

@@ -18,7 +18,8 @@
 import six
 from cassandra.query import BatchStatement
 
-from tests.integration import use_singledc, PROTOCOL_VERSION, local, TestCluster
+from tests.integration import (use_singledc, PROTOCOL_VERSION, local, TestCluster,
+                               requires_custom_payload, xfail_scylla)
 
 
 def setup_module():
@@ -27,7 +28,7 @@ def setup_module():
 
 # Failing with scylla because there is no warning message when changing the value of 'batch_size_warn_threshold_in_kb'
 # config")
-@unittest.expectedFailure
+@xfail_scylla('Empty warnings: TypeError: object of type \'NoneType\' has no len()')
 class ClientWarningTests(unittest.TestCase):
 
     @classmethod
@@ -94,6 +95,7 @@ def test_warning_with_trace(self):
         self.assertIsNotNone(future.get_query_trace())
 
     @local
+    @requires_custom_payload
     def test_warning_with_custom_payload(self):
         """
         Test to validate client warning with custom payload
@@ -113,6 +115,7 @@ def test_warning_with_custom_payload(self):
         self.assertDictEqual(future.custom_payload, payload)
 
     @local
+    @requires_custom_payload
     def test_warning_with_trace_and_custom_payload(self):
         """
         Test to validate client warning with tracing and client warning

tests/integration/standard/test_cluster.py

@@ -40,10 +40,10 @@
 from cassandra.connection import DefaultEndPoint
 
 from tests import notwindows
-from tests.integration import use_singledc, get_server_versions, CASSANDRA_VERSION, \
+from tests.integration import use_cluster, get_server_versions, CASSANDRA_VERSION, \
     execute_until_pass, execute_with_long_wait_retry, get_node, MockLoggingHandler, get_unsupported_lower_protocol, \
-    get_unsupported_upper_protocol, protocolv6, local, CASSANDRA_IP, greaterthanorequalcass30, lessthanorequalcass40, \
-    DSE_VERSION, TestCluster, PROTOCOL_VERSION
+    get_unsupported_upper_protocol, lessthanprotocolv3, protocolv6, local, CASSANDRA_IP, greaterthanorequalcass30, \
+    lessthanorequalcass40, DSE_VERSION, TestCluster, PROTOCOL_VERSION, xfail_scylla, incorrect_test
 from tests.integration.util import assert_quiescent_pool_state
 import sys
 
@@ -52,7 +52,7 @@
 
 def setup_module():
     os.environ['SCYLLA_EXT_OPTS'] = "--smp 1"
-    use_singledc()
+    use_cluster("cluster_tests", [3], start=True, workloads=None)
     warnings.simplefilter("always")
 
 
@@ -289,8 +289,7 @@ def test_protocol_negotiation(self):
 
         cluster.shutdown()
 
-    # "Failing with scylla because there is option to create a cluster with 'lower bound' protocol
-    @unittest.expectedFailure
+    @xfail_scylla("Failing with scylla because there is option to create a cluster with 'lower bound' protocol")
     def test_invalid_protocol_negotation(self):
         """
         Test for protocol negotiation when explicit versions are set
@@ -411,12 +410,11 @@ def test_connect_to_bad_hosts(self):
                               protocol_version=PROTOCOL_VERSION)
         self.assertRaises(NoHostAvailable, cluster.connect)
 
+    @lessthanprotocolv3
     def test_cluster_settings(self):
         """
         Test connection setting getters and setters
         """
-        if PROTOCOL_VERSION >= 3:
-            raise unittest.SkipTest("min/max requests and core/max conns aren't used with v3 protocol")
 
         cluster = TestCluster()
 
@@ -1228,8 +1226,7 @@ def test_replicas_are_queried(self):
 
     @greaterthanorequalcass30
     @lessthanorequalcass40
-    # The scylla failed because 'Unknown identifier column1'
-    @unittest.expectedFailure
+    @incorrect_test()
     def test_compact_option(self):
         """
         Test the driver can connect with the no_compact option and the results

tests/integration/standard/test_custom_cluster.py

@@ -30,7 +30,7 @@ def setup_module():
     # wait until all nodes are up
     wait_until_not_raised(lambda: TestCluster(contact_points=['127.0.0.1'], port=9046).connect().shutdown(), 1, 20)
     wait_until_not_raised(lambda: TestCluster(contact_points=['127.0.0.2'], port=9046).connect().shutdown(), 1, 20)
-    wait_until_not_raised(lambda: TestCluster(contact_points=['127.0.0.3'], port=9046).connect().shutdown(), 1, 20)
+    wait_until_not_raised(lambda: TestCluster(contact_points=['127.0.0.3'], port=9046).connect().shutdown(), 1, 120)
 
 
 def teardown_module():