From 91efffe5f1c601e285e8ebbc0d78989c5d1af87b Mon Sep 17 00:00:00 2001 From: Dmitry Kropachev Date: Thu, 15 Aug 2024 19:00:23 -0400 Subject: [PATCH] Remove files from scylla repo There are some files that we don't want to sync from upstream even in far future. This commit to remove them and add them to .gitignore --- .gitignore | 13 + Jenkinsfile | 688 -------------- Jenkinsfile.bak | 873 ------------------ build.yaml.bak | 264 ------ docs/.nav | 5 - .../cassandra/datastax/graph/fluent/index.rst | 24 - .../datastax/graph/fluent/predicates.rst | 14 - .../cassandra/datastax/graph/fluent/query.rst | 8 - docs/api/cassandra/datastax/graph/index.rst | 133 --- docs/api/cassandra/graph.rst | 121 --- docs/api/index.rst | 11 - docs/classic_graph.rst | 299 ------ docs/conf.py | 2 +- docs/core_graph.rst | 436 --------- docs/geo_types.rst | 41 - docs/graph.rst | 437 --------- docs/graph_fluent.rst | 417 --------- docs/installation.rst | 10 - docs/upgrading.rst | 23 - 19 files changed, 14 insertions(+), 3805 deletions(-) delete mode 100644 Jenkinsfile delete mode 100644 Jenkinsfile.bak delete mode 100644 build.yaml.bak delete mode 100644 docs/api/cassandra/datastax/graph/fluent/index.rst delete mode 100644 docs/api/cassandra/datastax/graph/fluent/predicates.rst delete mode 100644 docs/api/cassandra/datastax/graph/fluent/query.rst delete mode 100644 docs/api/cassandra/datastax/graph/index.rst delete mode 100644 docs/api/cassandra/graph.rst delete mode 100644 docs/classic_graph.rst delete mode 100644 docs/core_graph.rst delete mode 100644 docs/geo_types.rst delete mode 100644 docs/graph.rst delete mode 100644 docs/graph_fluent.rst diff --git a/.gitignore b/.gitignore index 4541d034f0..cea4dad9b0 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,16 @@ tests/unit/cython/bytesio_testhelper.c #iPython *.ipynb +# Files that exist on the upstream, but we want to explicitly drop from scylladb/python-driver +tox.ini +Jenkinsfile +Jenkinsfile.bak +.travis.yml +build.yaml.bak +docs/core_graph.rst +docs/classic_graph.rst +docs/geo_types.rst +docs/graph.rst +docs/graph_fluent.rst +docs/api/cassandra/graph.rst +docs/api/cassandra/datastax/graph/ diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 37b37ccb5e..0000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,688 +0,0 @@ -#!groovy -/* - -There are multiple combinations to test the python driver. - -Test Profiles: - - Full: Execute all unit and integration tests, including long tests. - Standard: Execute unit and integration tests. - Smoke Tests: Execute a small subset of tests. - EVENT_LOOP: Execute a small subset of tests selected to test EVENT_LOOPs. - -Matrix Types: - - Full: All server versions, python runtimes tested with and without Cython. - Develop: Smaller matrix for dev purpose. - Cassandra: All cassandra server versions. - Dse: All dse server versions. - -Parameters: - - EVENT_LOOP: 'LIBEV' (Default), 'GEVENT', 'EVENTLET', 'ASYNCIO', 'ASYNCORE', 'TWISTED' - CYTHON: Default, 'True', 'False' - -*/ - -@Library('dsdrivers-pipeline-lib@develop') -import com.datastax.jenkins.drivers.python.Slack - -slack = new Slack() - -// Define our predefined matrices -// -// Smoke tests are CI-friendly test configuration. Currently-supported Python version + modern C*/DSE instances. -// We also avoid cython since it's tested as part of the nightlies. -matrices = [ - "FULL": [ - "SERVER": ['2.1', '2.2', '3.0', '3.11', '4.0', 'dse-5.0.15', 'dse-5.1.35', 'dse-6.0.18', 'dse-6.7.17', 'dse-6.8.30'], - "RUNTIME": ['2.7.18', '3.5.9', '3.6.10', '3.7.7', '3.8.3'], - "CYTHON": ["True", "False"] - ], - "DEVELOP": [ - "SERVER": ['2.1', '3.11', 'dse-6.8.30'], - "RUNTIME": ['2.7.18', '3.6.10'], - "CYTHON": ["True", "False"] - ], - "CASSANDRA": [ - "SERVER": ['2.1', '2.2', '3.0', '3.11', '4.0'], - "RUNTIME": ['2.7.18', '3.5.9', '3.6.10', '3.7.7', '3.8.3'], - "CYTHON": ["True", "False"] - ], - "DSE": [ - "SERVER": ['dse-5.0.15', 'dse-5.1.35', 'dse-6.0.18', 'dse-6.7.17', 'dse-6.8.30'], - "RUNTIME": ['2.7.18', '3.5.9', '3.6.10', '3.7.7', '3.8.3'], - "CYTHON": ["True", "False"] - ], - "SMOKE": [ - "SERVER": ['3.11', '4.0', 'dse-6.8.30'], - "RUNTIME": ['3.7.7', '3.8.3'], - "CYTHON": ["False"] - ] -] - -def initializeSlackContext() { - /* - Based on git branch/commit, configure the build context and env vars. - */ - - def driver_display_name = 'Cassandra Python Driver' - if (env.GIT_URL.contains('riptano/python-driver')) { - driver_display_name = 'private ' + driver_display_name - } else if (env.GIT_URL.contains('python-dse-driver')) { - driver_display_name = 'DSE Python Driver' - } - env.DRIVER_DISPLAY_NAME = driver_display_name - env.GIT_SHA = "${env.GIT_COMMIT.take(7)}" - env.GITHUB_PROJECT_URL = "https://${GIT_URL.replaceFirst(/(git@|http:\/\/|https:\/\/)/, '').replace(':', '/').replace('.git', '')}" - env.GITHUB_BRANCH_URL = "${env.GITHUB_PROJECT_URL}/tree/${env.BRANCH_NAME}" - env.GITHUB_COMMIT_URL = "${env.GITHUB_PROJECT_URL}/commit/${env.GIT_COMMIT}" -} - -def getBuildContext() { - /* - Based on schedule and parameters, configure the build context and env vars. - */ - - def profile = "${params.PROFILE}" - def EVENT_LOOP = "${params.EVENT_LOOP.toLowerCase()}" - matrixType = "SMOKE" - developBranchPattern = ~"((dev|long)-)?python-.*" - - if (developBranchPattern.matcher(env.BRANCH_NAME).matches()) { - matrixType = "DEVELOP" - if (env.BRANCH_NAME.contains("long")) { - profile = "FULL" - } - } - - // Check if parameters were set explicitly - if (params.MATRIX != "DEFAULT") { - matrixType = params.MATRIX - } - - matrix = matrices[matrixType].clone() - if (params.CYTHON != "DEFAULT") { - matrix["CYTHON"] = [params.CYTHON] - } - - if (params.SERVER_VERSION != "DEFAULT") { - matrix["SERVER"] = [params.SERVER_VERSION] - } - - if (params.PYTHON_VERSION != "DEFAULT") { - matrix["RUNTIME"] = [params.PYTHON_VERSION] - } - - if (params.CI_SCHEDULE == "WEEKNIGHTS") { - matrix["SERVER"] = params.CI_SCHEDULE_SERVER_VERSION.split(' ') - matrix["RUNTIME"] = params.CI_SCHEDULE_PYTHON_VERSION.split(' ') - } - - context = [ - vars: [ - "PROFILE=${profile}", - "EVENT_LOOP=${EVENT_LOOP}" - ], - matrix: matrix - ] - - return context -} - -def buildAndTest(context) { - initializeEnvironment() - installDriverAndCompileExtensions() - - try { - executeTests() - } finally { - junit testResults: '*_results.xml' - } -} - -def getMatrixBuilds(buildContext) { - def tasks = [:] - matrix = buildContext.matrix - - matrix["SERVER"].each { serverVersion -> - matrix["RUNTIME"].each { runtimeVersion -> - matrix["CYTHON"].each { cythonFlag -> - def taskVars = [ - "CASSANDRA_VERSION=${serverVersion}", - "PYTHON_VERSION=${runtimeVersion}", - "CYTHON_ENABLED=${cythonFlag}" - ] - def cythonDesc = cythonFlag == "True" ? ", Cython": "" - tasks["${serverVersion}, py${runtimeVersion}${cythonDesc}"] = { - node("${OS_VERSION}") { - scm_variables = checkout scm - env.GIT_COMMIT = scm_variables.get('GIT_COMMIT') - env.GIT_URL = scm_variables.get('GIT_URL') - initializeSlackContext() - - if (env.BUILD_STATED_SLACK_NOTIFIED != 'true') { - slack.notifyChannel() - } - - withEnv(taskVars) { - buildAndTest(context) - } - } - } - } - } - } - return tasks -} - -def initializeEnvironment() { - sh label: 'Initialize the environment', script: '''#!/bin/bash -lex - pyenv global ${PYTHON_VERSION} - sudo apt-get install socat - pip install --upgrade pip - pip install -U setuptools - pip install ${HOME}/ccm - ''' - - // Determine if server version is Apache CassandraⓇ or DataStax Enterprise - if (env.CASSANDRA_VERSION.split('-')[0] == 'dse') { - sh label: 'Install DataStax Enterprise requirements', script: '''#!/bin/bash -lex - pip install -r test-datastax-requirements.txt - ''' - } else { - sh label: 'Install Apache CassandraⓇ requirements', script: '''#!/bin/bash -lex - pip install -r test-requirements.txt - ''' - - sh label: 'Uninstall the geomet dependency since it is not required for Cassandra', script: '''#!/bin/bash -lex - pip uninstall -y geomet - ''' - } - - sh label: 'Install unit test modules', script: '''#!/bin/bash -lex - pip install nose-ignore-docstring nose-exclude service_identity - ''' - - if (env.CYTHON_ENABLED == 'True') { - sh label: 'Install cython modules', script: '''#!/bin/bash -lex - pip install cython numpy - ''' - } - - sh label: 'Download Apache CassandraⓇ or DataStax Enterprise', script: '''#!/bin/bash -lex - . ${CCM_ENVIRONMENT_SHELL} ${CASSANDRA_VERSION} - ''' - - if (env.CASSANDRA_VERSION.split('-')[0] == 'dse') { - env.DSE_FIXED_VERSION = env.CASSANDRA_VERSION.split('-')[1] - sh label: 'Update environment for DataStax Enterprise', script: '''#!/bin/bash -le - cat >> ${HOME}/environment.txt << ENVIRONMENT_EOF -CCM_CASSANDRA_VERSION=${DSE_FIXED_VERSION} # maintain for backwards compatibility -CCM_VERSION=${DSE_FIXED_VERSION} -CCM_SERVER_TYPE=dse -DSE_VERSION=${DSE_FIXED_VERSION} -CCM_IS_DSE=true -CCM_BRANCH=${DSE_FIXED_VERSION} -DSE_BRANCH=${DSE_FIXED_VERSION} -ENVIRONMENT_EOF - ''' - } - - sh label: 'Display Python and environment information', script: '''#!/bin/bash -le - # Load CCM environment variables - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - python --version - pip --version - pip freeze - printenv | sort - ''' -} - -def installDriverAndCompileExtensions() { - if (env.CYTHON_ENABLED == 'True') { - sh label: 'Install the driver and compile with C extensions with Cython', script: '''#!/bin/bash -lex - python setup.py build_ext --inplace - ''' - } else { - sh label: 'Install the driver and compile with C extensions without Cython', script: '''#!/bin/bash -lex - python setup.py build_ext --inplace --no-cython - ''' - } -} - -def executeStandardTests() { - - sh label: 'Execute unit tests', script: '''#!/bin/bash -lex - # Load CCM environment variables - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - EVENT_LOOP=${EVENT_LOOP} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=unit_results.xml tests/unit/ || true - EVENT_LOOP=eventlet VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=unit_eventlet_results.xml tests/unit/io/test_eventletreactor.py || true - EVENT_LOOP=gevent VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=unit_gevent_results.xml tests/unit/io/test_geventreactor.py || true - ''' - - sh label: 'Execute Simulacron integration tests', script: '''#!/bin/bash -lex - # Load CCM environment variables - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - SIMULACRON_JAR="${HOME}/simulacron.jar" - SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP=${EVENT_LOOP} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_results.xml tests/integration/simulacron/ || true - - # Run backpressure tests separately to avoid memory issue - SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP=${EVENT_LOOP} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_backpressure_1_results.xml tests/integration/simulacron/test_backpressure.py:TCPBackpressureTests.test_paused_connections || true - SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP=${EVENT_LOOP} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_backpressure_2_results.xml tests/integration/simulacron/test_backpressure.py:TCPBackpressureTests.test_queued_requests_timeout || true - SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP=${EVENT_LOOP} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_backpressure_3_results.xml tests/integration/simulacron/test_backpressure.py:TCPBackpressureTests.test_cluster_busy || true - SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP=${EVENT_LOOP} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_backpressure_4_results.xml tests/integration/simulacron/test_backpressure.py:TCPBackpressureTests.test_node_busy || true - ''' - - sh label: 'Execute CQL engine integration tests', script: '''#!/bin/bash -lex - # Load CCM environment variables - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - EVENT_LOOP_MANAGER=${EVENT_LOOP_MANAGER} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=cqle_results.xml tests/integration/cqlengine/ || true - ''' - - sh label: 'Execute Apache CassandraⓇ integration tests', script: '''#!/bin/bash -lex - # Load CCM environment variables - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - EVENT_LOOP=${EVENT_LOOP} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=standard_results.xml tests/integration/standard/ || true - ''' - - if (env.CASSANDRA_VERSION.split('-')[0] == 'dse' && env.CASSANDRA_VERSION.split('-')[1] != '4.8') { - sh label: 'Execute DataStax Enterprise integration tests', script: '''#!/bin/bash -lex - # Load CCM environment variable - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - EVENT_LOOP_MANAGER=${EVENT_LOOP_MANAGER} CASSANDRA_DIR=${CCM_INSTALL_DIR} DSE_VERSION=${DSE_VERSION} ADS_HOME="${HOME}/" VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=dse_results.xml tests/integration/advanced/ || true - ''' - } - - sh label: 'Execute DataStax Constellation integration tests', script: '''#!/bin/bash -lex - # Load CCM environment variable - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - EVENT_LOOP=${EVENT_LOOP} CLOUD_PROXY_PATH="${HOME}/proxy/" CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=advanced_results.xml tests/integration/cloud/ || true - ''' - - if (env.PROFILE == 'FULL') { - sh label: 'Execute long running integration tests', script: '''#!/bin/bash -lex - # Load CCM environment variable - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - EVENT_LOOP=${EVENT_LOOP} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --exclude-dir=tests/integration/long/upgrade --with-ignore-docstrings --with-xunit --xunit-file=long_results.xml tests/integration/long/ || true - ''' - } -} - -def executeDseSmokeTests() { - sh label: 'Execute profile DataStax Enterprise smoke test integration tests', script: '''#!/bin/bash -lex - # Load CCM environment variable - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - EVENT_LOOP=${EVENT_LOOP} CCM_ARGS="${CCM_ARGS}" CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} DSE_VERSION=${DSE_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=standard_results.xml tests/integration/standard/test_dse.py || true - ''' -} - -def executeEventLoopTests() { - sh label: 'Execute profile event loop manager integration tests', script: '''#!/bin/bash -lex - # Load CCM environment variable - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - EVENT_LOOP_TESTS=( - "tests/integration/standard/test_cluster.py" - "tests/integration/standard/test_concurrent.py" - "tests/integration/standard/test_connection.py" - "tests/integration/standard/test_control_connection.py" - "tests/integration/standard/test_metrics.py" - "tests/integration/standard/test_query.py" - "tests/integration/simulacron/test_endpoint.py" - "tests/integration/long/test_ssl.py" - ) - EVENT_LOOP=${EVENT_LOOP} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=standard_results.xml ${EVENT_LOOP_TESTS[@]} || true - ''' -} - -def executeTests() { - switch(env.PROFILE) { - case 'DSE-SMOKE-TEST': - executeDseSmokeTests() - break - case 'EVENT_LOOP': - executeEventLoopTests() - break - default: - executeStandardTests() - break - } -} - - -// TODO move this in the shared lib -def getDriverMetricType() { - metric_type = 'oss' - if (env.GIT_URL.contains('riptano/python-driver')) { - metric_type = 'oss-private' - } else if (env.GIT_URL.contains('python-dse-driver')) { - metric_type = 'dse' - } - return metric_type -} - -def describeBuild(buildContext) { - script { - def runtimes = buildContext.matrix["RUNTIME"] - def serverVersions = buildContext.matrix["SERVER"] - def numBuilds = runtimes.size() * serverVersions.size() * buildContext.matrix["CYTHON"].size() - currentBuild.displayName = "${env.PROFILE} (${env.EVENT_LOOP} | ${numBuilds} builds)" - currentBuild.description = "${env.PROFILE} build testing servers (${serverVersions.join(', ')}) against Python (${runtimes.join(', ')}) using ${env.EVENT_LOOP} event loop manager" - } -} - -def scheduleTriggerJobName() { - "drivers/python/oss/master/disabled" -} - -pipeline { - agent none - - // Global pipeline timeout - options { - disableConcurrentBuilds() - timeout(time: 10, unit: 'HOURS') // TODO timeout should be per build - buildDiscarder(logRotator(artifactNumToKeepStr: '10', // Keep only the last 10 artifacts - numToKeepStr: '50')) // Keep only the last 50 build records - } - - parameters { - choice( - name: 'ADHOC_BUILD_TYPE', - choices: ['BUILD', 'BUILD-AND-EXECUTE-TESTS'], - description: '''

Perform a adhoc build operation

- - - - - - - - - - - - - - - -
ChoiceDescription
BUILDPerforms a Per-Commit build
BUILD-AND-EXECUTE-TESTSPerforms a build and executes the integration and unit tests
''') - choice( - name: 'PROFILE', - choices: ['STANDARD', 'FULL', 'DSE-SMOKE-TEST', 'EVENT_LOOP'], - description: '''

Profile to utilize for scheduled or adhoc builds

- - - - - - - - - - - - - - - - - - - - - - - -
ChoiceDescription
STANDARDExecute the standard tests for the driver
FULLExecute all tests for the driver, including long tests.
DSE-SMOKE-TESTExecute only the DataStax Enterprise smoke tests
EVENT_LOOPExecute only the event loop tests for the specified event loop manager (see: EVENT_LOOP)
''') - choice( - name: 'MATRIX', - choices: ['DEFAULT', 'SMOKE', 'FULL', 'DEVELOP', 'CASSANDRA', 'DSE'], - description: '''

The matrix for the build.

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ChoiceDescription
DEFAULTDefault to the build context.
SMOKEBasic smoke tests for current Python runtimes + C*/DSE versions, no Cython
FULLAll server versions, python runtimes tested with and without Cython.
DEVELOPSmaller matrix for dev purpose.
CASSANDRAAll cassandra server versions.
DSEAll dse server versions.
''') - choice( - name: 'PYTHON_VERSION', - choices: ['DEFAULT', '2.7.18', '3.5.9', '3.6.10', '3.7.7', '3.8.3'], - description: 'Python runtime version. Default to the build context.') - choice( - name: 'SERVER_VERSION', - choices: ['DEFAULT', - '2.1', // Legacy Apache CassandraⓇ - '2.2', // Legacy Apache CassandraⓇ - '3.0', // Previous Apache CassandraⓇ - '3.11', // Current Apache CassandraⓇ - '4.0', // Development Apache CassandraⓇ - 'dse-5.0.15', // Long Term Support DataStax Enterprise - 'dse-5.1.35', // Legacy DataStax Enterprise - 'dse-6.0.18', // Previous DataStax Enterprise - 'dse-6.7.17', // Previous DataStax Enterprise - 'dse-6.8.30', // Current DataStax Enterprise - ], - description: '''Apache CassandraⓇ and DataStax Enterprise server version to use for adhoc BUILD-AND-EXECUTE-TESTS ONLY! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ChoiceDescription
DEFAULTDefault to the build context.
2.1Apache CassandraⓇ; v2.1.x
2.2Apache CassandarⓇ; v2.2.x
3.0Apache CassandraⓇ v3.0.x
3.11Apache CassandraⓇ v3.11.x
4.0Apache CassandraⓇ v4.x (CURRENTLY UNDER DEVELOPMENT)
dse-5.0.15DataStax Enterprise v5.0.x (Long Term Support)
dse-5.1.35DataStax Enterprise v5.1.x
dse-6.0.18DataStax Enterprise v6.0.x
dse-6.7.17DataStax Enterprise v6.7.x
dse-6.8.30DataStax Enterprise v6.8.x (CURRENTLY UNDER DEVELOPMENT)
''') - choice( - name: 'CYTHON', - choices: ['DEFAULT', 'True', 'False'], - description: '''

Flag to determine if Cython should be enabled

- - - - - - - - - - - - - - - - - - - -
ChoiceDescription
DefaultDefault to the build context.
TrueEnable Cython
FalseDisable Cython
''') - choice( - name: 'EVENT_LOOP', - choices: ['LIBEV', 'GEVENT', 'EVENTLET', 'ASYNCIO', 'ASYNCORE', 'TWISTED'], - description: '''

Event loop manager to utilize for scheduled or adhoc builds

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ChoiceDescription
LIBEVA full-featured and high-performance event loop that is loosely modeled after libevent, but without its limitations and bugs
GEVENTA co-routine -based Python networking library that uses greenlet to provide a high-level synchronous API on top of the libev or libuv event loop
EVENTLETA concurrent networking library for Python that allows you to change how you run your code, not how you write it
ASYNCIOA library to write concurrent code using the async/await syntax
ASYNCOREA module provides the basic infrastructure for writing asynchronous socket service clients and servers
TWISTEDAn event-driven networking engine written in Python and licensed under the open source MIT license
''') - choice( - name: 'CI_SCHEDULE', - choices: ['DO-NOT-CHANGE-THIS-SELECTION', 'WEEKNIGHTS', 'WEEKENDS'], - description: 'CI testing schedule to execute periodically scheduled builds and tests of the driver (DO NOT CHANGE THIS SELECTION)') - string( - name: 'CI_SCHEDULE_PYTHON_VERSION', - defaultValue: 'DO-NOT-CHANGE-THIS-SELECTION', - description: 'CI testing python version to utilize for scheduled test runs of the driver (DO NOT CHANGE THIS SELECTION)') - string( - name: 'CI_SCHEDULE_SERVER_VERSION', - defaultValue: 'DO-NOT-CHANGE-THIS-SELECTION', - description: 'CI testing server version to utilize for scheduled test runs of the driver (DO NOT CHANGE THIS SELECTION)') - } - - triggers { - parameterizedCron((scheduleTriggerJobName() == env.JOB_NAME) ? """ - # Every weeknight (Monday - Friday) around 4:00 AM - # These schedules will run with and without Cython enabled for Python v2.7.18 and v3.5.9 - H 4 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;EVENT_LOOP=LIBEV;CI_SCHEDULE_PYTHON_VERSION=2.7.18 3.5.9;CI_SCHEDULE_SERVER_VERSION=2.2 3.11 dse-5.1.35 dse-6.0.18 dse-6.7.17 - """ : "") - } - - environment { - OS_VERSION = 'ubuntu/bionic64/python-driver' - CCM_ENVIRONMENT_SHELL = '/usr/local/bin/ccm_environment.sh' - CCM_MAX_HEAP_SIZE = '1536M' - } - - stages { - stage ('Build and Test') { - when { - beforeAgent true - allOf { - not { buildingTag() } - } - } - - steps { - script { - context = getBuildContext() - withEnv(context.vars) { - describeBuild(context) - - // build and test all builds - parallel getMatrixBuilds(context) - - slack.notifyChannel(currentBuild.currentResult) - } - } - } - } - - } -} diff --git a/Jenkinsfile.bak b/Jenkinsfile.bak deleted file mode 100644 index 87b20804ca..0000000000 --- a/Jenkinsfile.bak +++ /dev/null @@ -1,873 +0,0 @@ -#!groovy - -def initializeEnvironment() { - env.DRIVER_DISPLAY_NAME = 'Cassandra Python Driver' - env.DRIVER_METRIC_TYPE = 'oss' - if (env.GIT_URL.contains('riptano/python-driver')) { - env.DRIVER_DISPLAY_NAME = 'private ' + env.DRIVER_DISPLAY_NAME - env.DRIVER_METRIC_TYPE = 'oss-private' - } else if (env.GIT_URL.contains('python-dse-driver')) { - env.DRIVER_DISPLAY_NAME = 'DSE Python Driver' - env.DRIVER_METRIC_TYPE = 'dse' - } - - env.GIT_SHA = "${env.GIT_COMMIT.take(7)}" - env.GITHUB_PROJECT_URL = "https://${GIT_URL.replaceFirst(/(git@|http:\/\/|https:\/\/)/, '').replace(':', '/').replace('.git', '')}" - env.GITHUB_BRANCH_URL = "${GITHUB_PROJECT_URL}/tree/${env.BRANCH_NAME}" - env.GITHUB_COMMIT_URL = "${GITHUB_PROJECT_URL}/commit/${env.GIT_COMMIT}" - - sh label: 'Assign Python global environment', script: '''#!/bin/bash -lex - pyenv global ${PYTHON_VERSION} - ''' - - sh label: 'Install socat; required for unix socket tests', script: '''#!/bin/bash -lex - sudo apt-get install socat - ''' - - sh label: 'Install the latest setuptools', script: '''#!/bin/bash -lex - pip install --upgrade pip - pip install -U setuptools - ''' - - sh label: 'Install CCM', script: '''#!/bin/bash -lex - pip install ${HOME}/ccm - ''' - - // Determine if server version is Apache Cassandra� or DataStax Enterprise - if (env.CASSANDRA_VERSION.split('-')[0] == 'dse') { - sh label: 'Install DataStax Enterprise requirements', script: '''#!/bin/bash -lex - pip install -r test-datastax-requirements.txt - ''' - } else { - sh label: 'Install Apache CassandraⓇ requirements', script: '''#!/bin/bash -lex - pip install -r test-requirements.txt - ''' - - sh label: 'Uninstall the geomet dependency since it is not required for Cassandra', script: '''#!/bin/bash -lex - pip uninstall -y geomet - ''' - - } - - sh label: 'Install unit test modules', script: '''#!/bin/bash -lex - pip install nose-ignore-docstring nose-exclude service_identity - ''' - - if (env.CYTHON_ENABLED == 'True') { - sh label: 'Install cython modules', script: '''#!/bin/bash -lex - pip install cython numpy - ''' - } - - sh label: 'Download Apache CassandraⓇ or DataStax Enterprise', script: '''#!/bin/bash -lex - . ${CCM_ENVIRONMENT_SHELL} ${CASSANDRA_VERSION} - ''' - - sh label: 'Display Python and environment information', script: '''#!/bin/bash -le - # Load CCM environment variables - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - python --version - pip --version - printenv | sort - ''' -} - -def installDriverAndCompileExtensions() { - if (env.CYTHON_ENABLED == 'True') { - sh label: 'Install the driver and compile with C extensions with Cython', script: '''#!/bin/bash -lex - python setup.py build_ext --inplace - ''' - } else { - sh label: 'Install the driver and compile with C extensions without Cython', script: '''#!/bin/bash -lex - python setup.py build_ext --inplace --no-cython - ''' - } -} - -def executeStandardTests() { - - sh label: 'Execute unit tests', script: '''#!/bin/bash -lex - # Load CCM environment variables - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - EVENT_LOOP_MANAGER=${EVENT_LOOP_MANAGER} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=unit_results.xml tests/unit/ || true - EVENT_LOOP_MANAGER=eventlet VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=unit_eventlet_results.xml tests/unit/io/test_eventletreactor.py || true - EVENT_LOOP_MANAGER=gevent VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=unit_gevent_results.xml tests/unit/io/test_geventreactor.py || true - ''' - - sh label: 'Execute Simulacron integration tests', script: '''#!/bin/bash -lex - # Load CCM environment variables - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - SIMULACRON_JAR="${HOME}/simulacron.jar" - SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP_MANAGER=${EVENT_LOOP_MANAGER} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_results.xml tests/integration/simulacron/ || true - - # Run backpressure tests separately to avoid memory issue - SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP_MANAGER=${EVENT_LOOP_MANAGER} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_backpressure_1_results.xml tests/integration/simulacron/test_backpressure.py:TCPBackpressureTests.test_paused_connections || true - SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP_MANAGER=${EVENT_LOOP_MANAGER} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_backpressure_2_results.xml tests/integration/simulacron/test_backpressure.py:TCPBackpressureTests.test_queued_requests_timeout || true - SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP_MANAGER=${EVENT_LOOP_MANAGER} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_backpressure_3_results.xml tests/integration/simulacron/test_backpressure.py:TCPBackpressureTests.test_cluster_busy || true - SIMULACRON_JAR=${SIMULACRON_JAR} EVENT_LOOP_MANAGER=${EVENT_LOOP_MANAGER} CASSANDRA_DIR=${CCM_INSTALL_DIR} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --exclude test_backpressure.py --xunit-file=simulacron_backpressure_4_results.xml tests/integration/simulacron/test_backpressure.py:TCPBackpressureTests.test_node_busy || true - ''' - - sh label: 'Execute CQL engine integration tests', script: '''#!/bin/bash -lex - # Load CCM environment variables - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - EVENT_LOOP_MANAGER=${EVENT_LOOP_MANAGER} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=cqle_results.xml tests/integration/cqlengine/ || true - ''' - - sh label: 'Execute Apache CassandraⓇ integration tests', script: '''#!/bin/bash -lex - # Load CCM environment variables - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - EVENT_LOOP_MANAGER=${EVENT_LOOP_MANAGER} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=standard_results.xml tests/integration/standard/ || true - ''' - - if (env.CASSANDRA_VERSION.split('-')[0] == 'dse' && env.CASSANDRA_VERSION.split('-')[1] != '4.8') { - sh label: 'Execute DataStax Enterprise integration tests', script: '''#!/bin/bash -lex - # Load CCM environment variable - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - EVENT_LOOP_MANAGER=${EVENT_LOOP_MANAGER} CASSANDRA_DIR=${CCM_INSTALL_DIR} DSE_VERSION=${DSE_VERSION} ADS_HOME="${HOME}/" VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=dse_results.xml tests/integration/advanced/ || true - ''' - } - - sh label: 'Execute DataStax Constellation integration tests', script: '''#!/bin/bash -lex - # Load CCM environment variable - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - EVENT_LOOP_MANAGER=${EVENT_LOOP_MANAGER} CLOUD_PROXY_PATH="${HOME}/proxy/" CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=advanced_results.xml tests/integration/cloud/ || true - ''' - - if (env.EXECUTE_LONG_TESTS == 'True') { - sh label: 'Execute long running integration tests', script: '''#!/bin/bash -lex - # Load CCM environment variable - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - EVENT_LOOP_MANAGER=${EVENT_LOOP_MANAGER} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --exclude-dir=tests/integration/long/upgrade --with-ignore-docstrings --with-xunit --xunit-file=long_results.xml tests/integration/long/ || true - ''' - } -} - -def executeDseSmokeTests() { - sh label: 'Execute profile DataStax Enterprise smoke test integration tests', script: '''#!/bin/bash -lex - # Load CCM environment variable - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - EVENT_LOOP_MANAGER=${EVENT_LOOP_MANAGER} CCM_ARGS="${CCM_ARGS}" CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} DSE_VERSION=${DSE_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=standard_results.xml tests/integration/standard/test_dse.py || true - ''' -} - -def executeEventLoopTests() { - sh label: 'Execute profile event loop manager integration tests', script: '''#!/bin/bash -lex - # Load CCM environment variable - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - EVENT_LOOP_TESTS=( - "tests/integration/standard/test_cluster.py" - "tests/integration/standard/test_concurrent.py" - "tests/integration/standard/test_connection.py" - "tests/integration/standard/test_control_connection.py" - "tests/integration/standard/test_metrics.py" - "tests/integration/standard/test_query.py" - "tests/integration/simulacron/test_endpoint.py" - "tests/integration/long/test_ssl.py" - ) - EVENT_LOOP_MANAGER=${EVENT_LOOP_MANAGER} CCM_ARGS="${CCM_ARGS}" DSE_VERSION=${DSE_VERSION} CASSANDRA_VERSION=${CCM_CASSANDRA_VERSION} MAPPED_CASSANDRA_VERSION=${MAPPED_CASSANDRA_VERSION} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=standard_results.xml ${EVENT_LOOP_TESTS[@]} || true - ''' -} - -def executeUpgradeTests() { - sh label: 'Execute profile upgrade integration tests', script: '''#!/bin/bash -lex - # Load CCM environment variable - set -o allexport - . ${HOME}/environment.txt - set +o allexport - - EVENT_LOOP_MANAGER=${EVENT_LOOP_MANAGER} VERIFY_CYTHON=${CYTHON_ENABLED} nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=upgrade_results.xml tests/integration/upgrade || true - ''' -} - -def executeTests() { - switch(params.PROFILE) { - case 'DSE-SMOKE-TEST': - executeDseSmokeTests() - break - case 'EVENT-LOOP': - executeEventLoopTests() - break - case 'UPGRADE': - executeUpgradeTests() - break - default: - executeStandardTests() - break - } -} - -def notifySlack(status = 'started') { - // Set the global pipeline scoped environment (this is above each matrix) - env.BUILD_STATED_SLACK_NOTIFIED = 'true' - - def buildType = 'Commit' - if (params.CI_SCHEDULE != 'DO-NOT-CHANGE-THIS-SELECTION') { - buildType = "${params.CI_SCHEDULE.toLowerCase().capitalize()}" - } - - def color = 'good' // Green - if (status.equalsIgnoreCase('aborted')) { - color = '808080' // Grey - } else if (status.equalsIgnoreCase('unstable')) { - color = 'warning' // Orange - } else if (status.equalsIgnoreCase('failed')) { - color = 'danger' // Red - } - - def message = """Build ${status} for ${env.DRIVER_DISPLAY_NAME} [${buildType}] -<${env.GITHUB_BRANCH_URL}|${env.BRANCH_NAME}> - <${env.RUN_DISPLAY_URL}|#${env.BUILD_NUMBER}> - <${env.GITHUB_COMMIT_URL}|${env.GIT_SHA}>""" - if (params.CI_SCHEDULE != 'DO-NOT-CHANGE-THIS-SELECTION') { - message += " - ${params.CI_SCHEDULE_PYTHON_VERSION} - ${params.EVENT_LOOP_MANAGER}" - } - if (!status.equalsIgnoreCase('Started')) { - message += """ -${status} after ${currentBuild.durationString - ' and counting'}""" - } - - slackSend color: "${color}", - channel: "#python-driver-dev-bots", - message: "${message}" -} - -def submitCIMetrics(buildType) { - long durationMs = currentBuild.duration - long durationSec = durationMs / 1000 - long nowSec = (currentBuild.startTimeInMillis + durationMs) / 1000 - def branchNameNoPeriods = env.BRANCH_NAME.replaceAll('\\.', '_') - def durationMetric = "okr.ci.python.${env.DRIVER_METRIC_TYPE}.${buildType}.${branchNameNoPeriods} ${durationSec} ${nowSec}" - - timeout(time: 1, unit: 'MINUTES') { - withCredentials([string(credentialsId: 'lab-grafana-address', variable: 'LAB_GRAFANA_ADDRESS'), - string(credentialsId: 'lab-grafana-port', variable: 'LAB_GRAFANA_PORT')]) { - withEnv(["DURATION_METRIC=${durationMetric}"]) { - sh label: 'Send runtime metrics to labgrafana', script: '''#!/bin/bash -lex - echo "${DURATION_METRIC}" | nc -q 5 ${LAB_GRAFANA_ADDRESS} ${LAB_GRAFANA_PORT} - ''' - } - } - } -} - -def describePerCommitStage() { - script { - def type = 'standard' - def serverDescription = 'current Apache CassandaraⓇ and supported DataStax Enterprise versions' - if (env.BRANCH_NAME ==~ /long-python.*/) { - type = 'long' - } else if (env.BRANCH_NAME ==~ /dev-python.*/) { - type = 'dev' - } - - currentBuild.displayName = "Per-Commit (${env.EVENT_LOOP_MANAGER} | ${type.capitalize()})" - currentBuild.description = "Per-Commit build and ${type} testing of ${serverDescription} against Python v2.7.18 and v3.5.9 using ${env.EVENT_LOOP_MANAGER} event loop manager" - } - - sh label: 'Describe the python environment', script: '''#!/bin/bash -lex - python -V - pip freeze - ''' -} - -def describeScheduledTestingStage() { - script { - def type = params.CI_SCHEDULE.toLowerCase().capitalize() - def displayName = "${type} schedule (${env.EVENT_LOOP_MANAGER}" - if (env.CYTHON_ENABLED == 'True') { - displayName += " | Cython" - } - if (params.PROFILE != 'NONE') { - displayName += " | ${params.PROFILE}" - } - displayName += ")" - currentBuild.displayName = displayName - - def serverVersionDescription = "${params.CI_SCHEDULE_SERVER_VERSION.replaceAll(' ', ', ')} server version(s) in the matrix" - def pythonVersionDescription = "${params.CI_SCHEDULE_PYTHON_VERSION.replaceAll(' ', ', ')} Python version(s) in the matrix" - def description = "${type} scheduled testing using ${env.EVENT_LOOP_MANAGER} event loop manager" - if (env.CYTHON_ENABLED == 'True') { - description += ", with Cython enabled" - } - if (params.PROFILE != 'NONE') { - description += ", ${params.PROFILE} profile" - } - description += ", ${serverVersionDescription}, and ${pythonVersionDescription}" - currentBuild.description = description - } -} - -def describeAdhocTestingStage() { - script { - def serverType = params.ADHOC_BUILD_AND_EXECUTE_TESTS_SERVER_VERSION.split('-')[0] - def serverDisplayName = 'Apache CassandaraⓇ' - def serverVersion = " v${serverType}" - if (serverType == 'ALL') { - serverDisplayName = "all ${serverDisplayName} and DataStax Enterprise server versions" - serverVersion = '' - } else { - try { - serverVersion = " v${env.ADHOC_BUILD_AND_EXECUTE_TESTS_SERVER_VERSION.split('-')[1]}" - } catch (e) { - ;; // no-op - } - if (serverType == 'dse') { - serverDisplayName = 'DataStax Enterprise' - } - } - def displayName = "${params.ADHOC_BUILD_AND_EXECUTE_TESTS_SERVER_VERSION} for v${params.ADHOC_BUILD_AND_EXECUTE_TESTS_PYTHON_VERSION} (${env.EVENT_LOOP_MANAGER}" - if (env.CYTHON_ENABLED == 'True') { - displayName += " | Cython" - } - if (params.PROFILE != 'NONE') { - displayName += " | ${params.PROFILE}" - } - displayName += ")" - currentBuild.displayName = displayName - - def description = "Testing ${serverDisplayName} ${serverVersion} using ${env.EVENT_LOOP_MANAGER} against Python ${params.ADHOC_BUILD_AND_EXECUTE_TESTS_PYTHON_VERSION}" - if (env.CYTHON_ENABLED == 'True') { - description += ", with Cython" - } - if (params.PROFILE == 'NONE') { - if (params.EXECUTE_LONG_TESTS) { - description += ", with" - } else { - description += ", without" - } - description += " long tests executed" - } else { - description += ", ${params.PROFILE} profile" - } - currentBuild.description = description - } -} - -def branchPatternCron = ~"(master)" -def riptanoPatternCron = ~"(riptano)" - -pipeline { - agent none - - // Global pipeline timeout - options { - timeout(time: 10, unit: 'HOURS') - buildDiscarder(logRotator(artifactNumToKeepStr: '10', // Keep only the last 10 artifacts - numToKeepStr: '50')) // Keep only the last 50 build records - } - - parameters { - choice( - name: 'ADHOC_BUILD_TYPE', - choices: ['BUILD', 'BUILD-AND-EXECUTE-TESTS'], - description: '''

Perform a adhoc build operation

- - - - - - - - - - - - - - - -
ChoiceDescription
BUILDPerforms a Per-Commit build
BUILD-AND-EXECUTE-TESTSPerforms a build and executes the integration and unit tests
''') - choice( - name: 'ADHOC_BUILD_AND_EXECUTE_TESTS_PYTHON_VERSION', - choices: ['2.7.18', '3.4.10', '3.5.9', '3.6.10', '3.7.7', '3.8.3'], - description: 'Python version to use for adhoc BUILD-AND-EXECUTE-TESTS ONLY!') - choice( - name: 'ADHOC_BUILD_AND_EXECUTE_TESTS_SERVER_VERSION', - choices: ['2.1', // Legacy Apache CassandraⓇ - '2.2', // Legacy Apache CassandraⓇ - '3.0', // Previous Apache CassandraⓇ - '3.11', // Current Apache CassandraⓇ - '4.0', // Development Apache CassandraⓇ - 'dse-5.0', // Long Term Support DataStax Enterprise - 'dse-5.1', // Legacy DataStax Enterprise - 'dse-6.0', // Previous DataStax Enterprise - 'dse-6.7', // Previous DataStax Enterprise - 'dse-6.8', // Current DataStax Enterprise - 'ALL'], - description: '''Apache CassandraⓇ and DataStax Enterprise server version to use for adhoc BUILD-AND-EXECUTE-TESTS ONLY! - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ChoiceDescription
2.1Apache CassandaraⓇ; v2.1.x
2.2Apache CassandarⓇ; v2.2.x
3.0Apache CassandaraⓇ v3.0.x
3.11Apache CassandaraⓇ v3.11.x
4.0Apache CassandaraⓇ v4.x (CURRENTLY UNDER DEVELOPMENT)
dse-5.0DataStax Enterprise v5.0.x (Long Term Support)
dse-5.1DataStax Enterprise v5.1.x
dse-6.0DataStax Enterprise v6.0.x
dse-6.7DataStax Enterprise v6.7.x
dse-6.8DataStax Enterprise v6.8.x (CURRENTLY UNDER DEVELOPMENT)
''') - booleanParam( - name: 'CYTHON', - defaultValue: false, - description: 'Flag to determine if Cython should be enabled for scheduled or adhoc builds') - booleanParam( - name: 'EXECUTE_LONG_TESTS', - defaultValue: false, - description: 'Flag to determine if long integration tests should be executed for scheduled or adhoc builds') - choice( - name: 'EVENT_LOOP_MANAGER', - choices: ['LIBEV', 'GEVENT', 'EVENTLET', 'ASYNCIO', 'ASYNCORE', 'TWISTED'], - description: '''

Event loop manager to utilize for scheduled or adhoc builds

- - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
ChoiceDescription
LIBEVA full-featured and high-performance event loop that is loosely modeled after libevent, but without its limitations and bugs
GEVENTA co-routine -based Python networking library that uses greenlet to provide a high-level synchronous API on top of the libev or libuv event loop
EVENTLETA concurrent networking library for Python that allows you to change how you run your code, not how you write it
ASYNCIOA library to write concurrent code using the async/await syntax
ASYNCOREA module provides the basic infrastructure for writing asynchronous socket service clients and servers
TWISTEDAn event-driven networking engine written in Python and licensed under the open source MIT license
''') - choice( - name: 'PROFILE', - choices: ['NONE', 'DSE-SMOKE-TEST', 'EVENT-LOOP', 'UPGRADE'], - description: '''

Profile to utilize for scheduled or adhoc builds

- - - - - - - - - - - - - - - - - - - - - - - -
ChoiceDescription
NONEExecute the standard tests for the driver
DSE-SMOKE-TESTExecute only the DataStax Enterprise smoke tests
EVENT-LOOPExecute only the event loop tests for the specified event loop manager (see: EVENT_LOOP_MANAGER)
UPGRADEExecute only the upgrade tests
''') - choice( - name: 'CI_SCHEDULE', - choices: ['DO-NOT-CHANGE-THIS-SELECTION', 'WEEKNIGHTS', 'WEEKENDS'], - description: 'CI testing schedule to execute periodically scheduled builds and tests of the driver (DO NOT CHANGE THIS SELECTION)') - string( - name: 'CI_SCHEDULE_PYTHON_VERSION', - defaultValue: 'DO-NOT-CHANGE-THIS-SELECTION', - description: 'CI testing python version to utilize for scheduled test runs of the driver (DO NOT CHANGE THIS SELECTION)') - string( - name: 'CI_SCHEDULE_SERVER_VERSION', - defaultValue: 'DO-NOT-CHANGE-THIS-SELECTION', - description: 'CI testing server version to utilize for scheduled test runs of the driver (DO NOT CHANGE THIS SELECTION)') - } - - triggers { - parameterizedCron((branchPatternCron.matcher(env.BRANCH_NAME).matches() && !riptanoPatternCron.matcher(GIT_URL).find()) ? """ - # Every weeknight (Monday - Friday) around 4:00 AM - # These schedules will run with and without Cython enabled for Python v2.7.18 and v3.5.9 - H 4 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;EVENT_LOOP_MANAGER=LIBEV;CI_SCHEDULE_PYTHON_VERSION=2.7.18;CI_SCHEDULE_SERVER_VERSION=2.2 3.11 dse-5.1 dse-6.0 dse-6.7 - H 4 * * 1-5 %CI_SCHEDULE=WEEKNIGHTS;EVENT_LOOP_MANAGER=LIBEV;CI_SCHEDULE_PYTHON_VERSION=3.5.9;CI_SCHEDULE_SERVER_VERSION=2.2 3.11 dse-5.1 dse-6.0 dse-6.7 - - # Every Saturday around 12:00, 4:00 and 8:00 PM - # These schedules are for weekly libev event manager runs with and without Cython for most of the Python versions (excludes v3.5.9.x) - H 12 * * 6 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=LIBEV;CI_SCHEDULE_PYTHON_VERSION=2.7.18;CI_SCHEDULE_SERVER_VERSION=2.1 3.0 dse-5.1 dse-6.0 dse-6.7 - H 12 * * 6 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=LIBEV;CI_SCHEDULE_PYTHON_VERSION=3.4.10;CI_SCHEDULE_SERVER_VERSION=2.1 3.0 dse-5.1 dse-6.0 dse-6.7 - H 12 * * 6 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=LIBEV;CI_SCHEDULE_PYTHON_VERSION=3.6.10;CI_SCHEDULE_SERVER_VERSION=2.1 3.0 dse-5.1 dse-6.0 dse-6.7 - H 12 * * 6 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=LIBEV;CI_SCHEDULE_PYTHON_VERSION=3.7.7;CI_SCHEDULE_SERVER_VERSION=2.1 3.0 dse-5.1 dse-6.0 dse-6.7 - H 12 * * 6 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=LIBEV;CI_SCHEDULE_PYTHON_VERSION=3.8.3;CI_SCHEDULE_SERVER_VERSION=2.1 3.0 dse-5.1 dse-6.0 dse-6.7 - # These schedules are for weekly gevent event manager event loop only runs with and without Cython for most of the Python versions (excludes v3.4.10.x) - H 16 * * 6 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=GEVENT;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=2.7.18;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - H 16 * * 6 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=GEVENT;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=3.5.9;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - H 16 * * 6 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=GEVENT;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=3.6.10;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - H 16 * * 6 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=GEVENT;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=3.7.7;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - H 16 * * 6 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=GEVENT;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=3.8.3;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - # These schedules are for weekly eventlet event manager event loop only runs with and without Cython for most of the Python versions (excludes v3.4.10.x) - H 20 * * 6 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=EVENTLET;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=2.7.18;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - H 20 * * 6 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=EVENTLET;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=3.5.9;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - H 20 * * 6 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=EVENTLET;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=3.6.10;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - H 20 * * 6 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=EVENTLET;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=3.7.7;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - H 20 * * 6 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=EVENTLET;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=3.8.3;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - - # Every Sunday around 12:00 and 4:00 AM - # These schedules are for weekly asyncore event manager event loop only runs with and without Cython for most of the Python versions (excludes v3.4.10.x) - H 0 * * 7 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=ASYNCORE;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=2.7.18;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - H 0 * * 7 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=ASYNCORE;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=3.5.9;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - H 0 * * 7 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=ASYNCORE;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=3.6.10;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - H 0 * * 7 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=ASYNCORE;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=3.7.7;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - H 0 * * 7 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=ASYNCORE;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=3.8.3;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - # These schedules are for weekly twisted event manager event loop only runs with and without Cython for most of the Python versions (excludes v3.4.10.x) - H 4 * * 7 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=TWISTED;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=2.7.18;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - H 4 * * 7 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=TWISTED;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=3.5.9;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - H 4 * * 7 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=TWISTED;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=3.6.10;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - H 4 * * 7 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=TWISTED;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=3.7.7;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - H 4 * * 7 %CI_SCHEDULE=WEEKENDS;EVENT_LOOP_MANAGER=TWISTED;PROFILE=EVENT-LOOP;CI_SCHEDULE_PYTHON_VERSION=3.8.3;CI_SCHEDULE_SERVER_VERSION=2.1 2.2 3.0 3.11 dse-5.1 dse-6.0 dse-6.7 - """ : "") - } - - environment { - OS_VERSION = 'ubuntu/bionic64/python-driver' - CYTHON_ENABLED = "${params.CYTHON ? 'True' : 'False'}" - EVENT_LOOP_MANAGER = "${params.EVENT_LOOP_MANAGER.toLowerCase()}" - EXECUTE_LONG_TESTS = "${params.EXECUTE_LONG_TESTS ? 'True' : 'False'}" - CCM_ENVIRONMENT_SHELL = '/usr/local/bin/ccm_environment.sh' - CCM_MAX_HEAP_SIZE = '1536M' - } - - stages { - stage ('Per-Commit') { - options { - timeout(time: 2, unit: 'HOURS') - } - when { - beforeAgent true - branch pattern: '((dev|long)-)?python-.*', comparator: 'REGEXP' - allOf { - expression { params.ADHOC_BUILD_TYPE == 'BUILD' } - expression { params.CI_SCHEDULE == 'DO-NOT-CHANGE-THIS-SELECTION' } - not { buildingTag() } - } - } - - matrix { - axes { - axis { - name 'CASSANDRA_VERSION' - values '3.11', // Current Apache Cassandra - 'dse-6.8' // Current DataStax Enterprise - } - axis { - name 'PYTHON_VERSION' - values '2.7.18', '3.5.9' - } - axis { - name 'CYTHON_ENABLED' - values 'False' - } - } - - agent { - label "${OS_VERSION}" - } - - stages { - stage('Initialize-Environment') { - steps { - initializeEnvironment() - script { - if (env.BUILD_STATED_SLACK_NOTIFIED != 'true') { - notifySlack() - } - } - } - } - stage('Describe-Build') { - steps { - describePerCommitStage() - } - } - stage('Install-Driver-And-Compile-Extensions') { - steps { - installDriverAndCompileExtensions() - } - } - stage('Execute-Tests') { - steps { - - script { - if (env.BRANCH_NAME ==~ /long-python.*/) { - withEnv(["EXECUTE_LONG_TESTS=True"]) { - executeTests() - } - } - else { - executeTests() - } - } - } - post { - always { - junit testResults: '*_results.xml' - } - } - } - } - } - post { - always { - node('master') { - submitCIMetrics('commit') - } - } - aborted { - notifySlack('aborted') - } - success { - notifySlack('completed') - } - unstable { - notifySlack('unstable') - } - failure { - notifySlack('FAILED') - } - } - } - - stage ('Scheduled-Testing') { - when { - beforeAgent true - allOf { - expression { params.ADHOC_BUILD_TYPE == 'BUILD' } - expression { params.CI_SCHEDULE != 'DO-NOT-CHANGE-THIS-SELECTION' } - not { buildingTag() } - } - } - matrix { - axes { - axis { - name 'CASSANDRA_VERSION' - values '2.1', // Legacy Apache Cassandra - '2.2', // Legacy Apache Cassandra - '3.0', // Previous Apache Cassandra - '3.11', // Current Apache Cassandra - 'dse-5.1', // Legacy DataStax Enterprise - 'dse-6.0', // Previous DataStax Enterprise - 'dse-6.7' // Current DataStax Enterprise - } - axis { - name 'CYTHON_ENABLED' - values 'True', 'False' - } - } - when { - beforeAgent true - allOf { - expression { return params.CI_SCHEDULE_SERVER_VERSION.split(' ').any { it =~ /(ALL|${env.CASSANDRA_VERSION})/ } } - } - } - - environment { - PYTHON_VERSION = "${params.CI_SCHEDULE_PYTHON_VERSION}" - } - agent { - label "${OS_VERSION}" - } - - stages { - stage('Initialize-Environment') { - steps { - initializeEnvironment() - script { - if (env.BUILD_STATED_SLACK_NOTIFIED != 'true') { - notifySlack() - } - } - } - } - stage('Describe-Build') { - steps { - describeScheduledTestingStage() - } - } - stage('Install-Driver-And-Compile-Extensions') { - steps { - installDriverAndCompileExtensions() - } - } - stage('Execute-Tests') { - steps { - executeTests() - } - post { - always { - junit testResults: '*_results.xml' - } - } - } - } - } - post { - aborted { - notifySlack('aborted') - } - success { - notifySlack('completed') - } - unstable { - notifySlack('unstable') - } - failure { - notifySlack('FAILED') - } - } - } - - - stage('Adhoc-Testing') { - when { - beforeAgent true - allOf { - expression { params.ADHOC_BUILD_TYPE == 'BUILD-AND-EXECUTE-TESTS' } - not { buildingTag() } - } - } - - environment { - CYTHON_ENABLED = "${params.CYTHON ? 'True' : 'False'}" - PYTHON_VERSION = "${params.ADHOC_BUILD_AND_EXECUTE_TESTS_PYTHON_VERSION}" - } - - matrix { - axes { - axis { - name 'CASSANDRA_VERSION' - values '2.1', // Legacy Apache Cassandra - '2.2', // Legacy Apache Cassandra - '3.0', // Previous Apache Cassandra - '3.11', // Current Apache Cassandra - '4.0', // Development Apache Cassandra - 'dse-5.0', // Long Term Support DataStax Enterprise - 'dse-5.1', // Legacy DataStax Enterprise - 'dse-6.0', // Previous DataStax Enterprise - 'dse-6.7', // Current DataStax Enterprise - 'dse-6.8' // Development DataStax Enterprise - } - } - when { - beforeAgent true - allOf { - expression { params.ADHOC_BUILD_AND_EXECUTE_TESTS_SERVER_VERSION ==~ /(ALL|${env.CASSANDRA_VERSION})/ } - } - } - - agent { - label "${OS_VERSION}" - } - - stages { - stage('Describe-Build') { - steps { - describeAdhocTestingStage() - } - } - stage('Initialize-Environment') { - steps { - initializeEnvironment() - } - } - stage('Install-Driver-And-Compile-Extensions') { - steps { - installDriverAndCompileExtensions() - } - } - stage('Execute-Tests') { - steps { - executeTests() - } - post { - always { - junit testResults: '*_results.xml' - } - } - } - } - } - } - } -} diff --git a/build.yaml.bak b/build.yaml.bak deleted file mode 100644 index 100c86558a..0000000000 --- a/build.yaml.bak +++ /dev/null @@ -1,264 +0,0 @@ -schedules: - nightly_master: - schedule: nightly - disable_pull_requests: true - branches: - include: [master] - env_vars: | - EVENT_LOOP_MANAGER='libev' - matrix: - exclude: - - python: [3.6, 3.7, 3.8] - - cassandra: ['2.1', '3.0', '4.0', 'test-dse'] - - commit_long_test: - schedule: per_commit - disable_pull_requests: true - branches: - include: [/long-python.*/] - env_vars: | - EVENT_LOOP_MANAGER='libev' - matrix: - exclude: - - python: [3.6, 3.7, 3.8] - - cassandra: ['2.1', '3.0', 'test-dse'] - - commit_branches: - schedule: per_commit - disable_pull_requests: true - branches: - include: [/python.*/] - env_vars: | - EVENT_LOOP_MANAGER='libev' - EXCLUDE_LONG=1 - matrix: - exclude: - - python: [3.6, 3.7, 3.8] - - cassandra: ['2.1', '3.0', 'test-dse'] - - commit_branches_dev: - schedule: per_commit - disable_pull_requests: true - branches: - include: [/dev-python.*/] - env_vars: | - EVENT_LOOP_MANAGER='libev' - EXCLUDE_LONG=1 - matrix: - exclude: - - python: [2.7, 3.7, 3.6, 3.8] - - cassandra: ['2.0', '2.1', '2.2', '3.0', '4.0', 'test-dse', 'dse-4.8', 'dse-5.0', 'dse-6.0', 'dse-6.8'] - - release_test: - schedule: per_commit - disable_pull_requests: true - branches: - include: [/release-.+/] - env_vars: | - EVENT_LOOP_MANAGER='libev' - - weekly_master: - schedule: 0 10 * * 6 - disable_pull_requests: true - branches: - include: [master] - env_vars: | - EVENT_LOOP_MANAGER='libev' - matrix: - exclude: - - python: [3.5] - - cassandra: ['2.2', '3.1'] - - weekly_gevent: - schedule: 0 14 * * 6 - disable_pull_requests: true - branches: - include: [master] - env_vars: | - EVENT_LOOP_MANAGER='gevent' - JUST_EVENT_LOOP=1 - - weekly_eventlet: - schedule: 0 18 * * 6 - disable_pull_requests: true - branches: - include: [master] - env_vars: | - EVENT_LOOP_MANAGER='eventlet' - JUST_EVENT_LOOP=1 - - weekly_asyncio: - schedule: 0 22 * * 6 - disable_pull_requests: true - branches: - include: [master] - env_vars: | - EVENT_LOOP_MANAGER='asyncio' - JUST_EVENT_LOOP=1 - matrix: - exclude: - - python: [2.7] - - weekly_async: - schedule: 0 10 * * 7 - disable_pull_requests: true - branches: - include: [master] - env_vars: | - EVENT_LOOP_MANAGER='asyncore' - JUST_EVENT_LOOP=1 - - weekly_twister: - schedule: 0 14 * * 7 - disable_pull_requests: true - branches: - include: [master] - env_vars: | - EVENT_LOOP_MANAGER='twisted' - JUST_EVENT_LOOP=1 - - upgrade_tests: - schedule: adhoc - branches: - include: [master, python-546] - env_vars: | - EVENT_LOOP_MANAGER='libev' - JUST_UPGRADE=True - matrix: - exclude: - - python: [3.6, 3.7, 3.8] - - cassandra: ['2.0', '2.1', '2.2', '3.0', '4.0', 'test-dse'] - -python: - - 2.7 - - 3.5 - - 3.6 - - 3.7 - - 3.8 - -os: - - ubuntu/bionic64/python-driver - -cassandra: - - '2.1' - - '2.2' - - '3.0' - - '3.11' - - '4.0' - - 'dse-4.8' - - 'dse-5.0' - - 'dse-5.1' - - 'dse-6.0' - - 'dse-6.7' - - 'dse-6.8.0' - -env: - CYTHON: - - CYTHON - - NO_CYTHON - -build: - - script: | - export JAVA_HOME=$CCM_JAVA_HOME - export PATH=$JAVA_HOME/bin:$PATH - export PYTHONPATH="" - export CCM_MAX_HEAP_SIZE=1024M - - # Required for unix socket tests - sudo apt-get install socat - - # Install latest setuptools - pip install --upgrade pip - pip install -U setuptools - - pip install git+ssh://git@github.com/riptano/ccm-private.git@cassandra-7544-native-ports-with-dse-fix - - #pip install $HOME/ccm - - if [ -n "$CCM_IS_DSE" ]; then - pip install -r test-datastax-requirements.txt - else - pip install -r test-requirements.txt - fi - - pip install nose-ignore-docstring - pip install nose-exclude - pip install service_identity - - FORCE_CYTHON=False - if [[ $CYTHON == 'CYTHON' ]]; then - FORCE_CYTHON=True - pip install cython - pip install numpy - # Install the driver & compile C extensions - python setup.py build_ext --inplace - else - # Install the driver & compile C extensions with no cython - python setup.py build_ext --inplace --no-cython - fi - - echo "JUST_UPGRADE: $JUST_UPGRADE" - if [[ $JUST_UPGRADE == 'True' ]]; then - EVENT_LOOP_MANAGER=$EVENT_LOOP_MANAGER VERIFY_CYTHON=$FORCE_CYTHON nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=upgrade_results.xml tests/integration/upgrade || true - exit 0 - fi - - if [[ $JUST_SMOKE == 'true' ]]; then - # When we ONLY want to run the smoke tests - echo "JUST_SMOKE: $JUST_SMOKE" - echo "==========RUNNING SMOKE TESTS===========" - EVENT_LOOP_MANAGER=$EVENT_LOOP_MANAGER CCM_ARGS="$CCM_ARGS" CASSANDRA_VERSION=$CCM_CASSANDRA_VERSION DSE_VERSION='6.7.0' MAPPED_CASSANDRA_VERSION=$MAPPED_CASSANDRA_VERSION VERIFY_CYTHON=$FORCE_CYTHON nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=standard_results.xml tests/integration/standard/test_dse.py || true - exit 0 - fi - - # Run the unit tests, this is not done in travis because - # it takes too much time for the whole matrix to build with cython - if [[ $CYTHON == 'CYTHON' ]]; then - EVENT_LOOP_MANAGER=$EVENT_LOOP_MANAGER VERIFY_CYTHON=1 nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=unit_results.xml tests/unit/ || true - EVENT_LOOP_MANAGER=eventlet VERIFY_CYTHON=1 nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=unit_eventlet_results.xml tests/unit/io/test_eventletreactor.py || true - EVENT_LOOP_MANAGER=gevent VERIFY_CYTHON=1 nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=unit_gevent_results.xml tests/unit/io/test_geventreactor.py || true - fi - - if [ -n "$JUST_EVENT_LOOP" ]; then - echo "Running integration event loop subset with $EVENT_LOOP_MANAGER" - EVENT_LOOP_TESTS=( - "tests/integration/standard/test_cluster.py" - "tests/integration/standard/test_concurrent.py" - "tests/integration/standard/test_connection.py" - "tests/integration/standard/test_control_connection.py" - "tests/integration/standard/test_metrics.py" - "tests/integration/standard/test_query.py" - "tests/integration/simulacron/test_endpoint.py" - "tests/integration/long/test_ssl.py" - ) - EVENT_LOOP_MANAGER=$EVENT_LOOP_MANAGER CCM_ARGS="$CCM_ARGS" DSE_VERSION=$DSE_VERSION CASSANDRA_VERSION=$CCM_CASSANDRA_VERSION MAPPED_CASSANDRA_VERSION=$MAPPED_CASSANDRA_VERSION VERIFY_CYTHON=$FORCE_CYTHON nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=standard_results.xml ${EVENT_LOOP_TESTS[@]} || true - exit 0 - fi - - echo "Running with event loop manager: $EVENT_LOOP_MANAGER" - echo "==========RUNNING SIMULACRON TESTS==========" - SIMULACRON_JAR="$HOME/simulacron.jar" - SIMULACRON_JAR=$SIMULACRON_JAR EVENT_LOOP_MANAGER=$EVENT_LOOP_MANAGER CASSANDRA_DIR=$CCM_INSTALL_DIR CCM_ARGS="$CCM_ARGS" DSE_VERSION=$DSE_VERSION CASSANDRA_VERSION=$CCM_CASSANDRA_VERSION MAPPED_CASSANDRA_VERSION=$MAPPED_CASSANDRA_VERSION VERIFY_CYTHON=$FORCE_CYTHON nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=simulacron_results.xml tests/integration/simulacron/ || true - - echo "Running with event loop manager: $EVENT_LOOP_MANAGER" - echo "==========RUNNING CQLENGINE TESTS==========" - EVENT_LOOP_MANAGER=$EVENT_LOOP_MANAGER CCM_ARGS="$CCM_ARGS" DSE_VERSION=$DSE_VERSION CASSANDRA_VERSION=$CCM_CASSANDRA_VERSION MAPPED_CASSANDRA_VERSION=$MAPPED_CASSANDRA_VERSION VERIFY_CYTHON=$FORCE_CYTHON nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=cqle_results.xml tests/integration/cqlengine/ || true - - echo "==========RUNNING INTEGRATION TESTS==========" - EVENT_LOOP_MANAGER=$EVENT_LOOP_MANAGER CCM_ARGS="$CCM_ARGS" DSE_VERSION=$DSE_VERSION CASSANDRA_VERSION=$CCM_CASSANDRA_VERSION MAPPED_CASSANDRA_VERSION=$MAPPED_CASSANDRA_VERSION VERIFY_CYTHON=$FORCE_CYTHON nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=standard_results.xml tests/integration/standard/ || true - - if [ -n "$DSE_VERSION" ] && ! [[ $DSE_VERSION == "4.8"* ]]; then - echo "==========RUNNING DSE INTEGRATION TESTS==========" - EVENT_LOOP_MANAGER=$EVENT_LOOP_MANAGER CASSANDRA_DIR=$CCM_INSTALL_DIR DSE_VERSION=$DSE_VERSION ADS_HOME=$HOME/ VERIFY_CYTHON=$FORCE_CYTHON nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=dse_results.xml tests/integration/advanced/ || true - fi - - echo "==========RUNNING CLOUD TESTS==========" - EVENT_LOOP_MANAGER=$EVENT_LOOP_MANAGER CLOUD_PROXY_PATH="$HOME/proxy/" CASSANDRA_VERSION=$CCM_CASSANDRA_VERSION MAPPED_CASSANDRA_VERSION=$MAPPED_CASSANDRA_VERSION VERIFY_CYTHON=$FORCE_CYTHON nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --with-ignore-docstrings --with-xunit --xunit-file=advanced_results.xml tests/integration/cloud/ || true - - if [ -z "$EXCLUDE_LONG" ]; then - echo "==========RUNNING LONG INTEGRATION TESTS==========" - EVENT_LOOP_MANAGER=$EVENT_LOOP_MANAGER CCM_ARGS="$CCM_ARGS" DSE_VERSION=$DSE_VERSION CASSANDRA_VERSION=$CCM_CASSANDRA_VERSION MAPPED_CASSANDRA_VERSION=$MAPPED_CASSANDRA_VERSION VERIFY_CYTHON=$FORCE_CYTHON nosetests -s -v --logging-format="[%(levelname)s] %(asctime)s %(thread)d: %(message)s" --exclude-dir=tests/integration/long/upgrade --with-ignore-docstrings --with-xunit --xunit-file=long_results.xml tests/integration/long/ || true - fi - - - xunit: - - "*_results.xml" diff --git a/docs/.nav b/docs/.nav index af49594d99..e57bdd5bcc 100644 --- a/docs/.nav +++ b/docs/.nav @@ -4,16 +4,11 @@ scylla_specific execution_profiles lwt object_mapper -geo_types -graph -graph_fluent -classic_graph performance query_paging security upgrading user_defined_types dates-and-times -cloud faq api diff --git a/docs/api/cassandra/datastax/graph/fluent/index.rst b/docs/api/cassandra/datastax/graph/fluent/index.rst deleted file mode 100644 index 5547e0fdd7..0000000000 --- a/docs/api/cassandra/datastax/graph/fluent/index.rst +++ /dev/null @@ -1,24 +0,0 @@ -:mod:`cassandra.datastax.graph.fluent` -====================================== - -.. module:: cassandra.datastax.graph.fluent - -.. autoclass:: DseGraph - - .. autoattribute:: DSE_GRAPH_QUERY_LANGUAGE - - .. automethod:: create_execution_profile - - .. automethod:: query_from_traversal - - .. automethod:: traversal_source(session=None, graph_name=None, execution_profile=EXEC_PROFILE_GRAPH_DEFAULT, traversal_class=None) - - .. automethod:: batch(session=None, execution_profile=None) - -.. autoclass:: DSESessionRemoteGraphConnection(session[, graph_name, execution_profile]) - -.. autoclass:: BaseGraphRowFactory - -.. autoclass:: graph_traversal_row_factory - -.. autoclass:: graph_traversal_dse_object_row_factory diff --git a/docs/api/cassandra/datastax/graph/fluent/predicates.rst b/docs/api/cassandra/datastax/graph/fluent/predicates.rst deleted file mode 100644 index f6e86f6451..0000000000 --- a/docs/api/cassandra/datastax/graph/fluent/predicates.rst +++ /dev/null @@ -1,14 +0,0 @@ -:mod:`cassandra.datastax.graph.fluent.predicates` -================================================= - -.. module:: cassandra.datastax.graph.fluent.predicates - - -.. autoclass:: Search - :members: - -.. autoclass:: CqlCollection - :members: - -.. autoclass:: Geo - :members: diff --git a/docs/api/cassandra/datastax/graph/fluent/query.rst b/docs/api/cassandra/datastax/graph/fluent/query.rst deleted file mode 100644 index 3dd859f96e..0000000000 --- a/docs/api/cassandra/datastax/graph/fluent/query.rst +++ /dev/null @@ -1,8 +0,0 @@ -:mod:`cassandra.datastax.graph.fluent.query` -============================================ - -.. module:: cassandra.datastax.graph.fluent.query - - -.. autoclass:: TraversalBatch - :members: diff --git a/docs/api/cassandra/datastax/graph/index.rst b/docs/api/cassandra/datastax/graph/index.rst deleted file mode 100644 index a9b41cbdc2..0000000000 --- a/docs/api/cassandra/datastax/graph/index.rst +++ /dev/null @@ -1,133 +0,0 @@ -``cassandra.datastax.graph`` - Graph Statements, Options, and Row Factories -=========================================================================== - -.. _api-datastax-graph: - -.. module:: cassandra.datastax.graph - -.. autofunction:: single_object_row_factory - -.. autofunction:: graph_result_row_factory - -.. autofunction:: graph_object_row_factory - -.. autofunction:: graph_graphson2_row_factory - -.. autofunction:: graph_graphson3_row_factory - -.. function:: to_int(value) - - Wraps a value to be explicitly serialized as a graphson Int. - -.. function:: to_bigint(value) - - Wraps a value to be explicitly serialized as a graphson Bigint. - -.. function:: to_smallint(value) - - Wraps a value to be explicitly serialized as a graphson Smallint. - -.. function:: to_float(value) - - Wraps a value to be explicitly serialized as a graphson Float. - -.. function:: to_double(value) - - Wraps a value to be explicitly serialized as a graphson Double. - -.. autoclass:: GraphProtocol - :members: - :noindex: - -.. autoclass:: GraphOptions - :noindex: - - .. autoattribute:: graph_name - - .. autoattribute:: graph_source - - .. autoattribute:: graph_language - - .. autoattribute:: graph_read_consistency_level - - .. autoattribute:: graph_write_consistency_level - - .. autoattribute:: is_default_source - - .. autoattribute:: is_analytics_source - - .. autoattribute:: is_graph_source - - .. automethod:: set_source_default - - .. automethod:: set_source_analytics - - .. automethod:: set_source_graph - - -.. autoclass:: SimpleGraphStatement - :members: - :noindex: - -.. autoclass:: Result - :members: - :noindex: - -.. autoclass:: Vertex - :members: - :noindex: - -.. autoclass:: VertexProperty - :members: - :noindex: - -.. autoclass:: Edge - :members: - :noindex: - -.. autoclass:: Path - :members: - :noindex: - -.. autoclass:: T - :members: - :noindex: - -.. autoclass:: GraphSON1Serializer - :members: - :noindex: - -.. autoclass:: GraphSON1Deserializer - :noindex: - - .. automethod:: deserialize_date - - .. automethod:: deserialize_timestamp - - .. automethod:: deserialize_time - - .. automethod:: deserialize_duration - - .. automethod:: deserialize_int - - .. automethod:: deserialize_bigint - - .. automethod:: deserialize_double - - .. automethod:: deserialize_float - - .. automethod:: deserialize_uuid - - .. automethod:: deserialize_blob - - .. automethod:: deserialize_decimal - - .. automethod:: deserialize_point - - .. automethod:: deserialize_linestring - - .. automethod:: deserialize_polygon - -.. autoclass:: GraphSON2Reader - :members: - :noindex: diff --git a/docs/api/cassandra/graph.rst b/docs/api/cassandra/graph.rst deleted file mode 100644 index 43ddd3086c..0000000000 --- a/docs/api/cassandra/graph.rst +++ /dev/null @@ -1,121 +0,0 @@ -``cassandra.graph`` - Graph Statements, Options, and Row Factories -================================================================== - -.. note:: This module is only for backward compatibility for dse-driver users. Consider using :ref:`cassandra.datastax.graph `. - -.. module:: cassandra.graph - -.. autofunction:: single_object_row_factory - -.. autofunction:: graph_result_row_factory - -.. autofunction:: graph_object_row_factory - -.. autofunction:: graph_graphson2_row_factory - -.. autofunction:: graph_graphson3_row_factory - -.. function:: to_int(value) - - Wraps a value to be explicitly serialized as a graphson Int. - -.. function:: to_bigint(value) - - Wraps a value to be explicitly serialized as a graphson Bigint. - -.. function:: to_smallint(value) - - Wraps a value to be explicitly serialized as a graphson Smallint. - -.. function:: to_float(value) - - Wraps a value to be explicitly serialized as a graphson Float. - -.. function:: to_double(value) - - Wraps a value to be explicitly serialized as a graphson Double. - -.. autoclass:: GraphProtocol - :members: - -.. autoclass:: GraphOptions - - .. autoattribute:: graph_name - - .. autoattribute:: graph_source - - .. autoattribute:: graph_language - - .. autoattribute:: graph_read_consistency_level - - .. autoattribute:: graph_write_consistency_level - - .. autoattribute:: is_default_source - - .. autoattribute:: is_analytics_source - - .. autoattribute:: is_graph_source - - .. automethod:: set_source_default - - .. automethod:: set_source_analytics - - .. automethod:: set_source_graph - - -.. autoclass:: SimpleGraphStatement - :members: - -.. autoclass:: Result - :members: - -.. autoclass:: Vertex - :members: - -.. autoclass:: VertexProperty - :members: - -.. autoclass:: Edge - :members: - -.. autoclass:: Path - :members: - -.. autoclass:: GraphSON1Serializer - :members: - -.. autoclass:: GraphSON1Deserializer - - .. automethod:: deserialize_date - - .. automethod:: deserialize_timestamp - - .. automethod:: deserialize_time - - .. automethod:: deserialize_duration - - .. automethod:: deserialize_int - - .. automethod:: deserialize_bigint - - .. automethod:: deserialize_double - - .. automethod:: deserialize_float - - .. automethod:: deserialize_uuid - - .. automethod:: deserialize_blob - - .. automethod:: deserialize_decimal - - .. automethod:: deserialize_point - - .. automethod:: deserialize_linestring - - .. automethod:: deserialize_polygon - -.. autoclass:: GraphSON2Reader - :members: - -.. autoclass:: GraphSON3Reader - :members: diff --git a/docs/api/index.rst b/docs/api/index.rst index 9e778d508c..cf792283d0 100644 --- a/docs/api/index.rst +++ b/docs/api/index.rst @@ -10,7 +10,6 @@ Core Driver cassandra/cluster cassandra/policies cassandra/auth - cassandra/graph cassandra/metadata cassandra/metrics cassandra/query @@ -42,13 +41,3 @@ Object Mapper cassandra/cqlengine/connection cassandra/cqlengine/management cassandra/cqlengine/usertype - -DataStax Graph --------------- -.. toctree:: - :maxdepth: 1 - - cassandra/datastax/graph/index - cassandra/datastax/graph/fluent/index - cassandra/datastax/graph/fluent/query - cassandra/datastax/graph/fluent/predicates diff --git a/docs/classic_graph.rst b/docs/classic_graph.rst deleted file mode 100644 index ef68c86359..0000000000 --- a/docs/classic_graph.rst +++ /dev/null @@ -1,299 +0,0 @@ -DataStax Classic Graph Queries -============================== - -Getting Started -~~~~~~~~~~~~~~~ - -First, we need to create a graph in the system. To access the system API, we -use the system execution profile :: - - from cassandra.cluster import Cluster, EXEC_PROFILE_GRAPH_SYSTEM_DEFAULT - - cluster = Cluster() - session = cluster.connect() - - graph_name = 'movies' - session.execute_graph("system.graph(name).ifNotExists().engine(Classic).create()", {'name': graph_name}, - execution_profile=EXEC_PROFILE_GRAPH_SYSTEM_DEFAULT) - - -To execute requests on our newly created graph, we need to setup an execution -profile. Additionally, we also need to set the schema_mode to `development` -for the schema creation:: - - - from cassandra.cluster import Cluster, GraphExecutionProfile, EXEC_PROFILE_GRAPH_DEFAULT - from cassandra.graph import GraphOptions - - graph_name = 'movies' - ep = GraphExecutionProfile(graph_options=GraphOptions(graph_name=graph_name)) - - cluster = Cluster(execution_profiles={EXEC_PROFILE_GRAPH_DEFAULT: ep}) - session = cluster.connect() - - session.execute_graph("schema.config().option('graph.schema_mode').set('development')") - - -We are ready to configure our graph schema. We will create a simple one for movies:: - - # properties are used to define a vertex - properties = """ - schema.propertyKey("genreId").Text().create(); - schema.propertyKey("personId").Text().create(); - schema.propertyKey("movieId").Text().create(); - schema.propertyKey("name").Text().create(); - schema.propertyKey("title").Text().create(); - schema.propertyKey("year").Int().create(); - schema.propertyKey("country").Text().create(); - """ - - session.execute_graph(properties) # we can execute multiple statements in a single request - - # A Vertex represents a "thing" in the world. - vertices = """ - schema.vertexLabel("genre").properties("genreId","name").create(); - schema.vertexLabel("person").properties("personId","name").create(); - schema.vertexLabel("movie").properties("movieId","title","year","country").create(); - """ - - session.execute_graph(vertices) - - # An edge represents a relationship between two vertices - edges = """ - schema.edgeLabel("belongsTo").single().connection("movie","genre").create(); - schema.edgeLabel("actor").connection("movie","person").create(); - """ - - session.execute_graph(edges) - - # Indexes to execute graph requests efficiently - indexes = """ - schema.vertexLabel("genre").index("genresById").materialized().by("genreId").add(); - schema.vertexLabel("genre").index("genresByName").materialized().by("name").add(); - schema.vertexLabel("person").index("personsById").materialized().by("personId").add(); - schema.vertexLabel("person").index("personsByName").materialized().by("name").add(); - schema.vertexLabel("movie").index("moviesById").materialized().by("movieId").add(); - schema.vertexLabel("movie").index("moviesByTitle").materialized().by("title").add(); - schema.vertexLabel("movie").index("moviesByYear").secondary().by("year").add(); - """ - -Next, we'll add some data:: - - session.execute_graph(""" - g.addV('genre').property('genreId', 1).property('name', 'Action').next(); - g.addV('genre').property('genreId', 2).property('name', 'Drama').next(); - g.addV('genre').property('genreId', 3).property('name', 'Comedy').next(); - g.addV('genre').property('genreId', 4).property('name', 'Horror').next(); - """) - - session.execute_graph(""" - g.addV('person').property('personId', 1).property('name', 'Mark Wahlberg').next(); - g.addV('person').property('personId', 2).property('name', 'Leonardo DiCaprio').next(); - g.addV('person').property('personId', 3).property('name', 'Iggy Pop').next(); - """) - - session.execute_graph(""" - g.addV('movie').property('movieId', 1).property('title', 'The Happening'). - property('year', 2008).property('country', 'United States').next(); - g.addV('movie').property('movieId', 2).property('title', 'The Italian Job'). - property('year', 2003).property('country', 'United States').next(); - - g.addV('movie').property('movieId', 3).property('title', 'Revolutionary Road'). - property('year', 2008).property('country', 'United States').next(); - g.addV('movie').property('movieId', 4).property('title', 'The Man in the Iron Mask'). - property('year', 1998).property('country', 'United States').next(); - - g.addV('movie').property('movieId', 5).property('title', 'Dead Man'). - property('year', 1995).property('country', 'United States').next(); - """) - -Now that our genre, actor and movie vertices are added, we'll create the relationships (edges) between them:: - - session.execute_graph(""" - genre_horror = g.V().hasLabel('genre').has('name', 'Horror').next(); - genre_drama = g.V().hasLabel('genre').has('name', 'Drama').next(); - genre_action = g.V().hasLabel('genre').has('name', 'Action').next(); - - leo = g.V().hasLabel('person').has('name', 'Leonardo DiCaprio').next(); - mark = g.V().hasLabel('person').has('name', 'Mark Wahlberg').next(); - iggy = g.V().hasLabel('person').has('name', 'Iggy Pop').next(); - - the_happening = g.V().hasLabel('movie').has('title', 'The Happening').next(); - the_italian_job = g.V().hasLabel('movie').has('title', 'The Italian Job').next(); - rev_road = g.V().hasLabel('movie').has('title', 'Revolutionary Road').next(); - man_mask = g.V().hasLabel('movie').has('title', 'The Man in the Iron Mask').next(); - dead_man = g.V().hasLabel('movie').has('title', 'Dead Man').next(); - - the_happening.addEdge('belongsTo', genre_horror); - the_italian_job.addEdge('belongsTo', genre_action); - rev_road.addEdge('belongsTo', genre_drama); - man_mask.addEdge('belongsTo', genre_drama); - man_mask.addEdge('belongsTo', genre_action); - dead_man.addEdge('belongsTo', genre_drama); - - the_happening.addEdge('actor', mark); - the_italian_job.addEdge('actor', mark); - rev_road.addEdge('actor', leo); - man_mask.addEdge('actor', leo); - dead_man.addEdge('actor', iggy); - """) - -We are all set. You can now query your graph. Here are some examples:: - - # Find all movies of the genre Drama - for r in session.execute_graph(""" - g.V().has('genre', 'name', 'Drama').in('belongsTo').valueMap();"""): - print(r) - - # Find all movies of the same genre than the movie 'Dead Man' - for r in session.execute_graph(""" - g.V().has('movie', 'title', 'Dead Man').out('belongsTo').in('belongsTo').valueMap();"""): - print(r) - - # Find all movies of Mark Wahlberg - for r in session.execute_graph(""" - g.V().has('person', 'name', 'Mark Wahlberg').in('actor').valueMap();"""): - print(r) - -To see a more graph examples, see `DataStax Graph Examples `_. - -Graph Types -~~~~~~~~~~~ - -Here are the supported graph types with their python representations: - -========== ================ -DSE Graph Python -========== ================ -boolean bool -bigint long, int (PY3) -int int -smallint int -varint int -float float -double double -uuid uuid.UUID -Decimal Decimal -inet str -timestamp datetime.datetime -date datetime.date -time datetime.time -duration datetime.timedelta -point Point -linestring LineString -polygon Polygon -blob bytearray, buffer (PY2), memoryview (PY3), bytes (PY3) -========== ================ - -Graph Row Factory -~~~~~~~~~~~~~~~~~ - -By default (with :class:`.GraphExecutionProfile.row_factory` set to :func:`.graph.graph_object_row_factory`), known graph result -types are unpacked and returned as specialized types (:class:`.Vertex`, :class:`.Edge`). If the result is not one of these -types, a :class:`.graph.Result` is returned, containing the graph result parsed from JSON and removed from its outer dict. -The class has some accessor convenience methods for accessing top-level properties by name (`type`, `properties` above), -or lists by index:: - - # dicts with `__getattr__` or `__getitem__` - result = session.execute_graph("[[key_str: 'value', key_int: 3]]", execution_profile=EXEC_PROFILE_GRAPH_SYSTEM_DEFAULT)[0] # Using system exec just because there is no graph defined - result # dse.graph.Result({u'key_str': u'value', u'key_int': 3}) - result.value # {u'key_int': 3, u'key_str': u'value'} (dict) - result.key_str # u'value' - result.key_int # 3 - result['key_str'] # u'value' - result['key_int'] # 3 - - # lists with `__getitem__` - result = session.execute_graph('[[0, 1, 2]]', execution_profile=EXEC_PROFILE_GRAPH_SYSTEM_DEFAULT)[0] - result # dse.graph.Result([0, 1, 2]) - result.value # [0, 1, 2] (list) - result[1] # 1 (list[1]) - -You can use a different row factory by setting :attr:`.Session.default_graph_row_factory` or passing it to -:meth:`.Session.execute_graph`. For example, :func:`.graph.single_object_row_factory` returns the JSON result string`, -unparsed. :func:`.graph.graph_result_row_factory` returns parsed, but unmodified results (such that all metadata is retained, -unlike :func:`.graph.graph_object_row_factory`, which sheds some as attributes and properties are unpacked). These results -also provide convenience methods for converting to known types (:meth:`~.Result.as_vertex`, :meth:`~.Result.as_edge`, :meth:`~.Result.as_path`). - -Vertex and Edge properties are never unpacked since their types are unknown. If you know your graph schema and want to -deserialize properties, use the :class:`.GraphSON1Deserializer`. It provides convenient methods to deserialize by types (e.g. -deserialize_date, deserialize_uuid, deserialize_polygon etc.) Example:: - - # ... - from cassandra.graph import GraphSON1Deserializer - - row = session.execute_graph("g.V().toList()")[0] - value = row.properties['my_property_key'][0].value # accessing the VertexProperty value - value = GraphSON1Deserializer.deserialize_timestamp(value) - - print(value) # 2017-06-26 08:27:05 - print(type(value)) # - - -Named Parameters -~~~~~~~~~~~~~~~~ - -Named parameters are passed in a dict to :meth:`.cluster.Session.execute_graph`:: - - result_set = session.execute_graph('[a, b]', {'a': 1, 'b': 2}, execution_profile=EXEC_PROFILE_GRAPH_SYSTEM_DEFAULT) - [r.value for r in result_set] # [1, 2] - -All python types listed in `Graph Types`_ can be passed as named parameters and will be serialized -automatically to their graph representation: - -Example:: - - session.execute_graph(""" - g.addV('person'). - property('name', text_value). - property('age', integer_value). - property('birthday', timestamp_value). - property('house_yard', polygon_value).toList() - """, { - 'text_value': 'Mike Smith', - 'integer_value': 34, - 'timestamp_value': datetime.datetime(1967, 12, 30), - 'polygon_value': Polygon(((30, 10), (40, 40), (20, 40), (10, 20), (30, 10))) - }) - - -As with all Execution Profile parameters, graph options can be set in the cluster default (as shown in the first example) -or specified per execution:: - - ep = session.execution_profile_clone_update(EXEC_PROFILE_GRAPH_DEFAULT, - graph_options=GraphOptions(graph_name='something-else')) - session.execute_graph(statement, execution_profile=ep) - -Using GraphSON2 Protocol -~~~~~~~~~~~~~~~~~~~~~~~~ - -The default graph protocol used is GraphSON1. However GraphSON1 may -cause problems of type conversion happening during the serialization -of the query to the DSE Graph server, or the deserialization of the -responses back from a string Gremlin query. GraphSON2 offers better -support for the complex data types handled by DSE Graph. - -DSE >=5.0.4 now offers the possibility to use the GraphSON2 protocol -for graph queries. Enabling GraphSON2 can be done by `changing the -graph protocol of the execution profile` and `setting the graphson2 row factory`:: - - from cassandra.cluster import Cluster, GraphExecutionProfile, EXEC_PROFILE_GRAPH_DEFAULT - from cassandra.graph import GraphOptions, GraphProtocol, graph_graphson2_row_factory - - # Create a GraphSON2 execution profile - ep = GraphExecutionProfile(graph_options=GraphOptions(graph_name='types', - graph_protocol=GraphProtocol.GRAPHSON_2_0), - row_factory=graph_graphson2_row_factory) - - cluster = Cluster(execution_profiles={EXEC_PROFILE_GRAPH_DEFAULT: ep}) - session = cluster.connect() - session.execute_graph(...) - -Using GraphSON2, all properties will be automatically deserialized to -its Python representation. Note that it may bring significant -behavioral change at runtime. - -It is generally recommended to switch to GraphSON2 as it brings more -consistent support for complex data types in the Graph driver and will -be activated by default in the next major version (Python dse-driver -driver 3.0). diff --git a/docs/conf.py b/docs/conf.py index 2d576988ff..653625bd96 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -65,7 +65,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. -exclude_patterns = ['_build', 'cloud.rst', 'core_graph.rst', 'classic_graph.rst', 'geo_types.rst', 'graph.rst', 'graph_fluent.rst'] +exclude_patterns = ['_build'] # The name of the Pygments (syntax highlighting) style to use. pygments_style = 'sphinx' diff --git a/docs/core_graph.rst b/docs/core_graph.rst deleted file mode 100644 index c3fa8d8271..0000000000 --- a/docs/core_graph.rst +++ /dev/null @@ -1,436 +0,0 @@ -:orphan: - -DataStax Graph Queries -====================== - -The driver executes graph queries over the Cassandra native protocol. Use -:meth:`.Session.execute_graph` or :meth:`.Session.execute_graph_async` for -executing gremlin queries in DataStax Graph. - -The driver defines three Execution Profiles suitable for graph execution: - -* :data:`~.cluster.EXEC_PROFILE_GRAPH_DEFAULT` -* :data:`~.cluster.EXEC_PROFILE_GRAPH_SYSTEM_DEFAULT` -* :data:`~.cluster.EXEC_PROFILE_GRAPH_ANALYTICS_DEFAULT` - -See :doc:`getting-started` and :doc:`execution-profiles` -for more detail on working with profiles. - -In DSE 6.8.0, the Core graph engine has been introduced and is now the default. It -provides a better unified multi-model, performance and scale. This guide -is for graphs that use the core engine. If you work with previous versions of -DSE or existing graphs, see ``classic_graph``. - -Getting Started with Graph and the Core Engine -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -First, we need to create a graph in the system. To access the system API, we -use the system execution profile :: - - from cassandra.cluster import Cluster, EXEC_PROFILE_GRAPH_SYSTEM_DEFAULT - - cluster = Cluster() - session = cluster.connect() - - graph_name = 'movies' - session.execute_graph("system.graph(name).create()", {'name': graph_name}, - execution_profile=EXEC_PROFILE_GRAPH_SYSTEM_DEFAULT) - - -Graphs that use the core engine only support GraphSON3. Since they are Cassandra tables under -the hood, we can automatically configure the execution profile with the proper options -(row_factory and graph_protocol) when executing queries. You only need to make sure that -the `graph_name` is set and GraphSON3 will be automatically used:: - - from cassandra.cluster import Cluster, GraphExecutionProfile, EXEC_PROFILE_GRAPH_DEFAULT - - graph_name = 'movies' - ep = GraphExecutionProfile(graph_options=GraphOptions(graph_name=graph_name)) - cluster = Cluster(execution_profiles={EXEC_PROFILE_GRAPH_DEFAULT: ep}) - session = cluster.connect() - session.execute_graph("g.addV(...)") - - -Note that this graph engine detection is based on the metadata. You might experience -some query errors if the graph has been newly created and is not yet in the metadata. This -would result to a badly configured execution profile. If you really want to avoid that, -configure your execution profile explicitly:: - - from cassandra.cluster import Cluster, GraphExecutionProfile, EXEC_PROFILE_GRAPH_DEFAULT - from cassandra.graph import GraphOptions, GraphProtocol, graph_graphson3_row_factory - - graph_name = 'movies' - ep_graphson3 = GraphExecutionProfile( - row_factory=graph_graphson3_row_factory, - graph_options=GraphOptions( - graph_protocol=GraphProtocol.GRAPHSON_3_0, - graph_name=graph_name)) - - cluster = Cluster(execution_profiles={'core': ep_graphson3}) - session = cluster.connect() - session.execute_graph("g.addV(...)", execution_profile='core') - - -We are ready to configure our graph schema. We will create a simple one for movies:: - - # A Vertex represents a "thing" in the world. - # Create the genre vertex - query = """ - schema.vertexLabel('genre') - .partitionBy('genreId', Int) - .property('name', Text) - .create() - """ - session.execute_graph(query) - - # Create the person vertex - query = """ - schema.vertexLabel('person') - .partitionBy('personId', Int) - .property('name', Text) - .create() - """ - session.execute_graph(query) - - # Create the movie vertex - query = """ - schema.vertexLabel('movie') - .partitionBy('movieId', Int) - .property('title', Text) - .property('year', Int) - .property('country', Text) - .create() - """ - session.execute_graph(query) - - # An edge represents a relationship between two vertices - # Create our edges - queries = """ - schema.edgeLabel('belongsTo').from('movie').to('genre').create(); - schema.edgeLabel('actor').from('movie').to('person').create(); - """ - session.execute_graph(queries) - - # Indexes to execute graph requests efficiently - - # If you have a node with the search workload enabled (solr), use the following: - indexes = """ - schema.vertexLabel('genre').searchIndex() - .by("name") - .create(); - - schema.vertexLabel('person').searchIndex() - .by("name") - .create(); - - schema.vertexLabel('movie').searchIndex() - .by('title') - .by("year") - .create(); - """ - session.execute_graph(indexes) - - # Otherwise, use secondary indexes: - indexes = """ - schema.vertexLabel('genre') - .secondaryIndex('by_genre') - .by('name') - .create() - - schema.vertexLabel('person') - .secondaryIndex('by_name') - .by('name') - .create() - - schema.vertexLabel('movie') - .secondaryIndex('by_title') - .by('title') - .create() - """ - session.execute_graph(indexes) - -Add some edge indexes (materialized views):: - - indexes = """ - schema.edgeLabel('belongsTo') - .from('movie') - .to('genre') - .materializedView('movie__belongsTo__genre_by_in_genreId') - .ifNotExists() - .partitionBy(IN, 'genreId') - .clusterBy(OUT, 'movieId', Asc) - .create() - - schema.edgeLabel('actor') - .from('movie') - .to('person') - .materializedView('movie__actor__person_by_in_personId') - .ifNotExists() - .partitionBy(IN, 'personId') - .clusterBy(OUT, 'movieId', Asc) - .create() - """ - session.execute_graph(indexes) - -Next, we'll add some data:: - - session.execute_graph(""" - g.addV('genre').property('genreId', 1).property('name', 'Action').next(); - g.addV('genre').property('genreId', 2).property('name', 'Drama').next(); - g.addV('genre').property('genreId', 3).property('name', 'Comedy').next(); - g.addV('genre').property('genreId', 4).property('name', 'Horror').next(); - """) - - session.execute_graph(""" - g.addV('person').property('personId', 1).property('name', 'Mark Wahlberg').next(); - g.addV('person').property('personId', 2).property('name', 'Leonardo DiCaprio').next(); - g.addV('person').property('personId', 3).property('name', 'Iggy Pop').next(); - """) - - session.execute_graph(""" - g.addV('movie').property('movieId', 1).property('title', 'The Happening'). - property('year', 2008).property('country', 'United States').next(); - g.addV('movie').property('movieId', 2).property('title', 'The Italian Job'). - property('year', 2003).property('country', 'United States').next(); - - g.addV('movie').property('movieId', 3).property('title', 'Revolutionary Road'). - property('year', 2008).property('country', 'United States').next(); - g.addV('movie').property('movieId', 4).property('title', 'The Man in the Iron Mask'). - property('year', 1998).property('country', 'United States').next(); - - g.addV('movie').property('movieId', 5).property('title', 'Dead Man'). - property('year', 1995).property('country', 'United States').next(); - """) - -Now that our genre, actor and movie vertices are added, we'll create the relationships (edges) between them:: - - session.execute_graph(""" - genre_horror = g.V().hasLabel('genre').has('name', 'Horror').id().next(); - genre_drama = g.V().hasLabel('genre').has('name', 'Drama').id().next(); - genre_action = g.V().hasLabel('genre').has('name', 'Action').id().next(); - - leo = g.V().hasLabel('person').has('name', 'Leonardo DiCaprio').id().next(); - mark = g.V().hasLabel('person').has('name', 'Mark Wahlberg').id().next(); - iggy = g.V().hasLabel('person').has('name', 'Iggy Pop').id().next(); - - the_happening = g.V().hasLabel('movie').has('title', 'The Happening').id().next(); - the_italian_job = g.V().hasLabel('movie').has('title', 'The Italian Job').id().next(); - rev_road = g.V().hasLabel('movie').has('title', 'Revolutionary Road').id().next(); - man_mask = g.V().hasLabel('movie').has('title', 'The Man in the Iron Mask').id().next(); - dead_man = g.V().hasLabel('movie').has('title', 'Dead Man').id().next(); - - g.addE('belongsTo').from(__.V(the_happening)).to(__.V(genre_horror)).next(); - g.addE('belongsTo').from(__.V(the_italian_job)).to(__.V(genre_action)).next(); - g.addE('belongsTo').from(__.V(rev_road)).to(__.V(genre_drama)).next(); - g.addE('belongsTo').from(__.V(man_mask)).to(__.V(genre_drama)).next(); - g.addE('belongsTo').from(__.V(man_mask)).to(__.V(genre_action)).next(); - g.addE('belongsTo').from(__.V(dead_man)).to(__.V(genre_drama)).next(); - - g.addE('actor').from(__.V(the_happening)).to(__.V(mark)).next(); - g.addE('actor').from(__.V(the_italian_job)).to(__.V(mark)).next(); - g.addE('actor').from(__.V(rev_road)).to(__.V(leo)).next(); - g.addE('actor').from(__.V(man_mask)).to(__.V(leo)).next(); - g.addE('actor').from(__.V(dead_man)).to(__.V(iggy)).next(); - """) - -We are all set. You can now query your graph. Here are some examples:: - - # Find all movies of the genre Drama - for r in session.execute_graph(""" - g.V().has('genre', 'name', 'Drama').in('belongsTo').valueMap();"""): - print(r) - - # Find all movies of the same genre than the movie 'Dead Man' - for r in session.execute_graph(""" - g.V().has('movie', 'title', 'Dead Man').out('belongsTo').in('belongsTo').valueMap();"""): - print(r) - - # Find all movies of Mark Wahlberg - for r in session.execute_graph(""" - g.V().has('person', 'name', 'Mark Wahlberg').in('actor').valueMap();"""): - print(r) - -To see a more graph examples, see `DataStax Graph Examples `_. - -Graph Types for the Core Engine -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Here are the supported graph types with their python representations: - -============ ================= -DSE Graph Python Driver -============ ================= -text str -boolean bool -bigint long -int int -smallint int -varint long -double float -float float -uuid UUID -bigdecimal Decimal -duration Duration (cassandra.util) -inet str or IPV4Address/IPV6Address (if available) -timestamp datetime.datetime -date datetime.date -time datetime.time -polygon Polygon -point Point -linestring LineString -blob bytearray, buffer (PY2), memoryview (PY3), bytes (PY3) -list list -map dict -set set or list - (Can return a list due to numerical values returned by Java) -tuple tuple -udt class or namedtuple -============ ================= - -Named Parameters -~~~~~~~~~~~~~~~~ - -Named parameters are passed in a dict to :meth:`.cluster.Session.execute_graph`:: - - result_set = session.execute_graph('[a, b]', {'a': 1, 'b': 2}, execution_profile=EXEC_PROFILE_GRAPH_SYSTEM_DEFAULT) - [r.value for r in result_set] # [1, 2] - -All python types listed in `Graph Types for the Core Engine`_ can be passed as named parameters and will be serialized -automatically to their graph representation: - -Example:: - - session.execute_graph(""" - g.addV('person'). - property('name', text_value). - property('age', integer_value). - property('birthday', timestamp_value). - property('house_yard', polygon_value).next() - """, { - 'text_value': 'Mike Smith', - 'integer_value': 34, - 'timestamp_value': datetime.datetime(1967, 12, 30), - 'polygon_value': Polygon(((30, 10), (40, 40), (20, 40), (10, 20), (30, 10))) - }) - - -As with all Execution Profile parameters, graph options can be set in the cluster default (as shown in the first example) -or specified per execution:: - - ep = session.execution_profile_clone_update(EXEC_PROFILE_GRAPH_DEFAULT, - graph_options=GraphOptions(graph_name='something-else')) - session.execute_graph(statement, execution_profile=ep) - -CQL collections, Tuple and UDT -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This is a very interesting feature of the core engine: we can use all CQL data types, including -list, map, set, tuple and udt. Here is an example using all these types:: - - query = """ - schema.type('address') - .property('address', Text) - .property('city', Text) - .property('state', Text) - .create(); - """ - session.execute_graph(query) - - # It works the same way than normal CQL UDT, so we - # can create an udt class and register it - class Address(object): - def __init__(self, address, city, state): - self.address = address - self.city = city - self.state = state - - session.cluster.register_user_type(graph_name, 'address', Address) - - query = """ - schema.vertexLabel('person') - .partitionBy('personId', Int) - .property('address', typeOf('address')) - .property('friends', listOf(Text)) - .property('skills', setOf(Text)) - .property('scores', mapOf(Text, Int)) - .property('last_workout', tupleOf(Text, Date)) - .create() - """ - session.execute_graph(query) - - # insertion example - query = """ - g.addV('person') - .property('personId', pid) - .property('address', address) - .property('friends', friends) - .property('skills', skills) - .property('scores', scores) - .property('last_workout', last_workout) - .next() - """ - - session.execute_graph(query, { - 'pid': 3, - 'address': Address('42 Smith St', 'Quebec', 'QC'), - 'friends': ['Al', 'Mike', 'Cathy'], - 'skills': {'food', 'fight', 'chess'}, - 'scores': {'math': 98, 'french': 3}, - 'last_workout': ('CrossFit', datetime.date(2018, 11, 20)) - }) - -Limitations ------------ - -Since Python is not a strongly-typed language and the UDT/Tuple graphson representation is, you might -get schema errors when trying to write numerical data. Example:: - - session.execute_graph(""" - schema.vertexLabel('test_tuple').partitionBy('id', Int).property('t', tupleOf(Text, Bigint)).create() - """) - - session.execute_graph(""" - g.addV('test_tuple').property('id', 0).property('t', t) - """, - {'t': ('Test', 99))} - ) - - # error: [Invalid query] message="Value component 1 is of type int, not bigint" - -This is because the server requires the client to include a GraphSON schema definition -with every UDT or tuple query. In the general case, the driver can't determine what Graph type -is meant by, e.g., an int value, and so it can't serialize the value with the correct type in the schema. -The driver provides some numerical type-wrapper factories that you can use to specify types: - -* :func:`~cassandra.datastax.graph.to_int` -* :func:`~cassandra.datastax.graph.to_bigint` -* :func:`~cassandra.datastax.graph.to_smallint` -* :func:`~cassandra.datastax.graph.to_float` -* :func:`~cassandra.datastax.graph.to_double` - -Here's the working example of the case above:: - - from cassandra.graph import to_bigint - - session.execute_graph(""" - g.addV('test_tuple').property('id', 0).property('t', t) - """, - {'t': ('Test', to_bigint(99))} - ) - -Continuous Paging -~~~~~~~~~~~~~~~~~ - -This is another nice feature that comes with the core engine: continuous paging with -graph queries. If all nodes of the cluster are >= DSE 6.8.0, it is automatically -enabled under the hood to get the best performance. If you want to explicitly -enable/disable it, you can do it through the execution profile:: - - # Disable it - ep = GraphExecutionProfile(..., continuous_paging_options=None)) - cluster = Cluster(execution_profiles={EXEC_PROFILE_GRAPH_DEFAULT: ep}) - - # Enable with a custom max_pages option - ep = GraphExecutionProfile(..., - continuous_paging_options=ContinuousPagingOptions(max_pages=10))) - cluster = Cluster(execution_profiles={EXEC_PROFILE_GRAPH_DEFAULT: ep}) diff --git a/docs/geo_types.rst b/docs/geo_types.rst deleted file mode 100644 index d85e1d3c95..0000000000 --- a/docs/geo_types.rst +++ /dev/null @@ -1,41 +0,0 @@ -:orphan: - -DSE Geometry Types -================== -This section shows how to query and work with the geometric types provided by DSE. - -These types are enabled implicitly by creating the Session from :class:`cassandra.cluster.Cluster`. -This module implicitly registers these types for use in the driver. This extension provides -some simple representative types in :mod:`cassandra.util` for inserting and retrieving data:: - - from cassandra.cluster import Cluster - from cassandra.util import Point, LineString, Polygon - session = Cluster().connect() - - session.execute("INSERT INTO ks.geo (k, point, line, poly) VALUES (%s, %s, %s, %s)", - 0, Point(1, 2), LineString(((1, 2), (3, 4))), Polygon(((1, 2), (3, 4), (5, 6)))) - -Queries returning geometric types return the :mod:`dse.util` types. Note that these can easily be used to construct -types from third-party libraries using the common attributes:: - - from shapely.geometry import LineString - shapely_linestrings = [LineString(res.line.coords) for res in session.execute("SELECT line FROM ks.geo")] - -For prepared statements, shapely geometry types can be used interchangeably with the built-in types because their -defining attributes are the same:: - - from shapely.geometry import Point - prepared = session.prepare("UPDATE ks.geo SET point = ? WHERE k = ?") - session.execute(prepared, (0, Point(1.2, 3.4))) - -In order to use shapely types in a CQL-interpolated (non-prepared) query, one must update the encoder with those types, specifying -the same string encoder as set for the internal types:: - - from cassandra import util - from shapely.geometry import Point, LineString, Polygon - - encoder_func = session.encoder.mapping[util.Point] - for t in (Point, LineString, Polygon): - session.encoder.mapping[t] = encoder_func - - session.execute("UPDATE ks.geo SET point = %s where k = %s", (0, Point(1.2, 3.4))) diff --git a/docs/graph.rst b/docs/graph.rst deleted file mode 100644 index 1b61bbc713..0000000000 --- a/docs/graph.rst +++ /dev/null @@ -1,437 +0,0 @@ -:orphan: - -DataStax Graph Queries -====================== - -The driver executes graph queries over the Cassandra native protocol. Use -:meth:`.Session.execute_graph` or :meth:`.Session.execute_graph_async` for -executing gremlin queries in DataStax Graph. - -The driver defines three Execution Profiles suitable for graph execution: - -* :data:`~.cluster.EXEC_PROFILE_GRAPH_DEFAULT` -* :data:`~.cluster.EXEC_PROFILE_GRAPH_SYSTEM_DEFAULT` -* :data:`~.cluster.EXEC_PROFILE_GRAPH_ANALYTICS_DEFAULT` - -See :doc:`getting-started` and :doc:`execution-profiles` -for more detail on working with profiles. - -In DSE 6.8.0, the Core graph engine has been introduced and is now the default. It -provides a better unified multi-model, performance and scale. This guide -is for graphs that use the core engine. If you work with previous versions of -DSE or existing graphs, see :doc:`classic_graph`. - -Getting Started with Graph and the Core Engine -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -First, we need to create a graph in the system. To access the system API, we -use the system execution profile :: - - from cassandra.cluster import Cluster, EXEC_PROFILE_GRAPH_SYSTEM_DEFAULT - - cluster = Cluster() - session = cluster.connect() - - graph_name = 'movies' - session.execute_graph("system.graph(name).create()", {'name': graph_name}, - execution_profile=EXEC_PROFILE_GRAPH_SYSTEM_DEFAULT) - - -Graphs that use the core engine only support GraphSON3. Since they are Cassandra tables under -the hood, we can automatically configure the execution profile with the proper options -(row_factory and graph_protocol) when executing queries. You only need to make sure that -the `graph_name` is set and GraphSON3 will be automatically used:: - - from cassandra.cluster import Cluster, GraphExecutionProfile, EXEC_PROFILE_GRAPH_DEFAULT - - graph_name = 'movies' - ep = GraphExecutionProfile(graph_options=GraphOptions(graph_name=graph_name)) - cluster = Cluster(execution_profiles={EXEC_PROFILE_GRAPH_DEFAULT: ep}) - session = cluster.connect() - session.execute_graph("g.addV(...)") - - -Note that this graph engine detection is based on the metadata. You might experience -some query errors if the graph has been newly created and is not yet in the metadata. This -would result to a badly configured execution profile. If you really want to avoid that, -configure your execution profile explicitly:: - - from cassandra.cluster import Cluster, GraphExecutionProfile, EXEC_PROFILE_GRAPH_DEFAULT - from cassandra.graph import GraphOptions, GraphProtocol, graph_graphson3_row_factory - - graph_name = 'movies' - ep_graphson3 = GraphExecutionProfile( - row_factory=graph_graphson3_row_factory, - graph_options=GraphOptions( - graph_protocol=GraphProtocol.GRAPHSON_3_0, - graph_name=graph_name)) - - cluster = Cluster(execution_profiles={'core': ep_graphson3}) - session = cluster.connect() - session.execute_graph("g.addV(...)", execution_profile='core') - - -We are ready to configure our graph schema. We will create a simple one for movies:: - - # A Vertex represents a "thing" in the world. - # Create the genre vertex - query = """ - schema.vertexLabel('genre') - .partitionBy('genreId', Int) - .property('name', Text) - .create() - """ - session.execute_graph(query) - - # Create the person vertex - query = """ - schema.vertexLabel('person') - .partitionBy('personId', Int) - .property('name', Text) - .create() - """ - session.execute_graph(query) - - # Create the movie vertex - query = """ - schema.vertexLabel('movie') - .partitionBy('movieId', Int) - .property('title', Text) - .property('year', Int) - .property('country', Text) - .create() - """ - session.execute_graph(query) - - # An edge represents a relationship between two vertices - # Create our edges - queries = """ - schema.edgeLabel('belongsTo').from('movie').to('genre').create(); - schema.edgeLabel('actor').from('movie').to('person').create(); - """ - session.execute_graph(queries) - - # Indexes to execute graph requests efficiently - - # If you have a node with the search workload enabled (solr), use the following: - indexes = """ - schema.vertexLabel('genre').searchIndex() - .by("name") - .create(); - - schema.vertexLabel('person').searchIndex() - .by("name") - .create(); - - schema.vertexLabel('movie').searchIndex() - .by('title') - .by("year") - .create(); - """ - session.execute_graph(indexes) - - # Otherwise, use secondary indexes: - indexes = """ - schema.vertexLabel('genre') - .secondaryIndex('by_genre') - .by('name') - .create() - - schema.vertexLabel('person') - .secondaryIndex('by_name') - .by('name') - .create() - - schema.vertexLabel('movie') - .secondaryIndex('by_title') - .by('title') - .create() - """ - session.execute_graph(indexes) - -Add some edge indexes (materialized views):: - - indexes = """ - schema.edgeLabel('belongsTo') - .from('movie') - .to('genre') - .materializedView('movie__belongsTo__genre_by_in_genreId') - .ifNotExists() - .partitionBy(IN, 'genreId') - .clusterBy(OUT, 'movieId', Asc) - .create() - - schema.edgeLabel('actor') - .from('movie') - .to('person') - .materializedView('movie__actor__person_by_in_personId') - .ifNotExists() - .partitionBy(IN, 'personId') - .clusterBy(OUT, 'movieId', Asc) - .create() - """ - session.execute_graph(indexes) - -Next, we'll add some data:: - - session.execute_graph(""" - g.addV('genre').property('genreId', 1).property('name', 'Action').next(); - g.addV('genre').property('genreId', 2).property('name', 'Drama').next(); - g.addV('genre').property('genreId', 3).property('name', 'Comedy').next(); - g.addV('genre').property('genreId', 4).property('name', 'Horror').next(); - """) - - session.execute_graph(""" - g.addV('person').property('personId', 1).property('name', 'Mark Wahlberg').next(); - g.addV('person').property('personId', 2).property('name', 'Leonardo DiCaprio').next(); - g.addV('person').property('personId', 3).property('name', 'Iggy Pop').next(); - """) - - session.execute_graph(""" - g.addV('movie').property('movieId', 1).property('title', 'The Happening'). - property('year', 2008).property('country', 'United States').next(); - g.addV('movie').property('movieId', 2).property('title', 'The Italian Job'). - property('year', 2003).property('country', 'United States').next(); - - g.addV('movie').property('movieId', 3).property('title', 'Revolutionary Road'). - property('year', 2008).property('country', 'United States').next(); - g.addV('movie').property('movieId', 4).property('title', 'The Man in the Iron Mask'). - property('year', 1998).property('country', 'United States').next(); - - g.addV('movie').property('movieId', 5).property('title', 'Dead Man'). - property('year', 1995).property('country', 'United States').next(); - """) - -Now that our genre, actor and movie vertices are added, we'll create the relationships (edges) between them:: - - session.execute_graph(""" - genre_horror = g.V().hasLabel('genre').has('name', 'Horror').id().next(); - genre_drama = g.V().hasLabel('genre').has('name', 'Drama').id().next(); - genre_action = g.V().hasLabel('genre').has('name', 'Action').id().next(); - - leo = g.V().hasLabel('person').has('name', 'Leonardo DiCaprio').id().next(); - mark = g.V().hasLabel('person').has('name', 'Mark Wahlberg').id().next(); - iggy = g.V().hasLabel('person').has('name', 'Iggy Pop').id().next(); - - the_happening = g.V().hasLabel('movie').has('title', 'The Happening').id().next(); - the_italian_job = g.V().hasLabel('movie').has('title', 'The Italian Job').id().next(); - rev_road = g.V().hasLabel('movie').has('title', 'Revolutionary Road').id().next(); - man_mask = g.V().hasLabel('movie').has('title', 'The Man in the Iron Mask').id().next(); - dead_man = g.V().hasLabel('movie').has('title', 'Dead Man').id().next(); - - g.addE('belongsTo').from(__.V(the_happening)).to(__.V(genre_horror)).next(); - g.addE('belongsTo').from(__.V(the_italian_job)).to(__.V(genre_action)).next(); - g.addE('belongsTo').from(__.V(rev_road)).to(__.V(genre_drama)).next(); - g.addE('belongsTo').from(__.V(man_mask)).to(__.V(genre_drama)).next(); - g.addE('belongsTo').from(__.V(man_mask)).to(__.V(genre_action)).next(); - g.addE('belongsTo').from(__.V(dead_man)).to(__.V(genre_drama)).next(); - - g.addE('actor').from(__.V(the_happening)).to(__.V(mark)).next(); - g.addE('actor').from(__.V(the_italian_job)).to(__.V(mark)).next(); - g.addE('actor').from(__.V(rev_road)).to(__.V(leo)).next(); - g.addE('actor').from(__.V(man_mask)).to(__.V(leo)).next(); - g.addE('actor').from(__.V(dead_man)).to(__.V(iggy)).next(); - """) - -We are all set. You can now query your graph. Here are some examples:: - - # Find all movies of the genre Drama - for r in session.execute_graph(""" - g.V().has('genre', 'name', 'Drama').in('belongsTo').valueMap();"""): - print(r) - - # Find all movies of the same genre than the movie 'Dead Man' - for r in session.execute_graph(""" - g.V().has('movie', 'title', 'Dead Man').out('belongsTo').in('belongsTo').valueMap();"""): - print(r) - - # Find all movies of Mark Wahlberg - for r in session.execute_graph(""" - g.V().has('person', 'name', 'Mark Wahlberg').in('actor').valueMap();"""): - print(r) - -To see a more graph examples, see `DataStax Graph Examples `_. - -Graph Types for the Core Engine -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Here are the supported graph types with their python representations: - -============ ================= -DSE Graph Python Driver -============ ================= -text str -boolean bool -bigint long -int int -smallint int -varint long -double float -float float -uuid UUID -bigdecimal Decimal -duration Duration (cassandra.util) -inet str or IPV4Address/IPV6Address (if available) -timestamp datetime.datetime -date datetime.date -time datetime.time -polygon Polygon -point Point -linestring LineString -blob bytearray, buffer (PY2), memoryview (PY3), bytes (PY3) -list list -map dict -set set or list - (Can return a list due to numerical values returned by Java) -tuple tuple -udt class or namedtuple -============ ================= -======= - -Named Parameters -~~~~~~~~~~~~~~~~ - -Named parameters are passed in a dict to :meth:`.cluster.Session.execute_graph`:: - - result_set = session.execute_graph('[a, b]', {'a': 1, 'b': 2}, execution_profile=EXEC_PROFILE_GRAPH_SYSTEM_DEFAULT) - [r.value for r in result_set] # [1, 2] - -All python types listed in `Graph Types for the Core Engine`_ can be passed as named parameters and will be serialized -automatically to their graph representation: - -Example:: - - session.execute_graph(""" - g.addV('person'). - property('name', text_value). - property('age', integer_value). - property('birthday', timestamp_value). - property('house_yard', polygon_value).next() - """, { - 'text_value': 'Mike Smith', - 'integer_value': 34, - 'timestamp_value': datetime.datetime(1967, 12, 30), - 'polygon_value': Polygon(((30, 10), (40, 40), (20, 40), (10, 20), (30, 10))) - }) - - -As with all Execution Profile parameters, graph options can be set in the cluster default (as shown in the first example) -or specified per execution:: - - ep = session.execution_profile_clone_update(EXEC_PROFILE_GRAPH_DEFAULT, - graph_options=GraphOptions(graph_name='something-else')) - session.execute_graph(statement, execution_profile=ep) - -CQL collections, Tuple and UDT -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -This is a very interesting feature of the core engine: we can use all CQL data types, including -list, map, set, tuple and udt. Here is an example using all these types:: - - query = """ - schema.type('address') - .property('address', Text) - .property('city', Text) - .property('state', Text) - .create(); - """ - session.execute_graph(query) - - # It works the same way than normal CQL UDT, so we - # can create an udt class and register it - class Address(object): - def __init__(self, address, city, state): - self.address = address - self.city = city - self.state = state - - session.cluster.register_user_type(graph_name, 'address', Address) - - query = """ - schema.vertexLabel('person') - .partitionBy('personId', Int) - .property('address', typeOf('address')) - .property('friends', listOf(Text)) - .property('skills', setOf(Text)) - .property('scores', mapOf(Text, Int)) - .property('last_workout', tupleOf(Text, Date)) - .create() - """ - session.execute_graph(query) - - # insertion example - query = """ - g.addV('person') - .property('personId', pid) - .property('address', address) - .property('friends', friends) - .property('skills', skills) - .property('scores', scores) - .property('last_workout', last_workout) - .next() - """ - - session.execute_graph(query, { - 'pid': 3, - 'address': Address('42 Smith St', 'Quebec', 'QC'), - 'friends': ['Al', 'Mike', 'Cathy'], - 'skills': {'food', 'fight', 'chess'}, - 'scores': {'math': 98, 'french': 3}, - 'last_workout': ('CrossFit', datetime.date(2018, 11, 20)) - }) - -Limitations ------------ - -Since Python is not a strongly-typed language and the UDT/Tuple graphson representation is, you might -get schema errors when trying to write numerical data. Example:: - - session.execute_graph(""" - schema.vertexLabel('test_tuple').partitionBy('id', Int).property('t', tupleOf(Text, Bigint)).create() - """) - - session.execute_graph(""" - g.addV('test_tuple').property('id', 0).property('t', t) - """, - {'t': ('Test', 99))} - ) - - # error: [Invalid query] message="Value component 1 is of type int, not bigint" - -This is because the server requires the client to include a GraphSON schema definition -with every UDT or tuple query. In the general case, the driver can't determine what Graph type -is meant by, e.g., an int value, and so it can't serialize the value with the correct type in the schema. -The driver provides some numerical type-wrapper factories that you can use to specify types: - -* :func:`~.to_int` -* :func:`~.to_bigint` -* :func:`~.to_smallint` -* :func:`~.to_float` -* :func:`~.to_double` - -Here's the working example of the case above:: - - from cassandra.graph import to_bigint - - session.execute_graph(""" - g.addV('test_tuple').property('id', 0).property('t', t) - """, - {'t': ('Test', to_bigint(99))} - ) - -Continuous Paging -~~~~~~~~~~~~~~~~~ - -This is another nice feature that comes with the core engine: continuous paging with -graph queries. If all nodes of the cluster are >= DSE 6.8.0, it is automatically -enabled under the hood to get the best performance. If you want to explicitly -enable/disable it, you can do it through the execution profile:: - - # Disable it - ep = GraphExecutionProfile(..., continuous_paging_options=None)) - cluster = Cluster(execution_profiles={EXEC_PROFILE_GRAPH_DEFAULT: ep}) - - # Enable with a custom max_pages option - ep = GraphExecutionProfile(..., - continuous_paging_options=ContinuousPagingOptions(max_pages=10))) - cluster = Cluster(execution_profiles={EXEC_PROFILE_GRAPH_DEFAULT: ep}) diff --git a/docs/graph_fluent.rst b/docs/graph_fluent.rst deleted file mode 100644 index cada908f2f..0000000000 --- a/docs/graph_fluent.rst +++ /dev/null @@ -1,417 +0,0 @@ -:orphan: - -DataStax Graph Fluent API -========================= - -The fluent API adds graph features to the core driver: - -* A TinkerPop GraphTraversalSource builder to execute traversals on a DSE cluster -* The ability to execution traversal queries explicitly using execute_graph -* GraphSON serializers for all DSE Graph types. -* DSE Search predicates - -The Graph fluent API depends on Apache TinkerPop and is not installed by default. Make sure -you have the Graph requirements are properly :ref:`installed `. - -You might be interested in reading the :doc:`DataStax Graph Getting Started documentation ` to -understand the basics of creating a graph and its schema. - -Graph Traversal Queries -~~~~~~~~~~~~~~~~~~~~~~~ - -The driver provides :meth:`.Session.execute_graph`, which allows users to execute traversal -query strings. Here is a simple example:: - - session.execute_graph("g.addV('genre').property('genreId', 1).property('name', 'Action').next();") - -Since graph queries can be very complex, working with strings is not very convenient and is -hard to maintain. This fluent API allows you to build Gremlin traversals and write your graph -queries directly in Python. These native traversal queries can be executed explicitly, with -a `Session` object, or implicitly:: - - from cassandra.cluster import Cluster, EXEC_PROFILE_GRAPH_DEFAULT - from cassandra.datastax.graph import GraphProtocol - from cassandra.datastax.graph.fluent import DseGraph - - # Create an execution profile, using GraphSON3 for Core graphs - ep_graphson3 = DseGraph.create_execution_profile( - 'my_core_graph_name', - graph_protocol=GraphProtocol.GRAPHSON_3_0) - cluster = Cluster(execution_profiles={EXEC_PROFILE_GRAPH_DEFAULT: ep_graphson3}) - session = cluster.connect() - - # Execute a fluent graph query - g = DseGraph.traversal_source(session=session) - g.addV('genre').property('genreId', 1).property('name', 'Action').next() - - # implicit execution caused by iterating over results - for v in g.V().has('genre', 'name', 'Drama').in_('belongsTo').valueMap(): - print(v) - -These Python types are also supported transparently:: - - g.addV('person').property('name', 'Mike').property('birthday', datetime(1984, 3, 11)). \ - property('house_yard', Polygon(((30, 10), (40, 40), (20, 40), (10, 20), (30, 10))) - -More readings about Gremlin: - -* `DataStax Drivers Fluent API `_ -* `gremlin-python documentation `_ - -Configuring a Traversal Execution Profile -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -The fluent api takes advantage of *configuration profiles* to allow -different execution configurations for the various query handlers. Graph traversal -execution requires a custom execution profile to enable Gremlin-bytecode as -query language. With Core graphs, it is important to use GraphSON3. Here is how -to accomplish this configuration: - -.. code-block:: python - - from cassandra.cluster import Cluster, EXEC_PROFILE_GRAPH_DEFAULT - from cassandra.datastax.graph import GraphProtocol - from cassandra.datastax.graph.fluent import DseGraph - - # Using GraphSON3 as graph protocol is a requirement with Core graphs. - ep = DseGraph.create_execution_profile( - 'graph_name', - graph_protocol=GraphProtocol.GRAPHSON_3_0) - - # For Classic graphs, GraphSON1, GraphSON2 and GraphSON3 (DSE 6.8+) are supported. - ep_classic = DseGraph.create_execution_profile('classic_graph_name') # default is GraphSON2 - - cluster = Cluster(execution_profiles={EXEC_PROFILE_GRAPH_DEFAULT: ep, 'classic': ep_classic}) - session = cluster.connect() - - g = DseGraph.traversal_source(session) # Build the GraphTraversalSource - print g.V().toList() # Traverse the Graph - -Note that the execution profile created with :meth:`DseGraph.create_execution_profile <.datastax.graph.fluent.DseGraph.create_execution_profile>` cannot -be used for any groovy string queries. - -If you want to change execution property defaults, please see the :doc:`Execution Profile documentation ` -for a more generalized discussion of the API. Graph traversal queries use the same execution profile defined for DSE graph. If you -need to change the default properties, please refer to the :doc:`DSE Graph query documentation page ` - -Explicit Graph Traversal Execution with a DSE Session -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Traversal queries can be executed explicitly using `session.execute_graph` or `session.execute_graph_async`. These functions -return results as DSE graph types. If you are familiar with DSE queries or need async execution, you might prefer that way. -Below is an example of explicit execution. For this example, assume the schema has been generated as above: - -.. code-block:: python - - from cassandra.cluster import Cluster, EXEC_PROFILE_GRAPH_DEFAULT - from cassandra.datastax.graph import GraphProtocol - from cassandra.datastax.graph.fluent import DseGraph - from pprint import pprint - - ep = DseGraph.create_execution_profile( - 'graph_name', - graph_protocol=GraphProtocol.GRAPHSON_3_0) - cluster = Cluster(execution_profiles={EXEC_PROFILE_GRAPH_DEFAULT: ep}) - session = cluster.connect() - - g = DseGraph.traversal_source(session=session) - -Convert a traversal to a bytecode query for classic graphs:: - - addV_query = DseGraph.query_from_traversal( - g.addV('genre').property('genreId', 1).property('name', 'Action'), - graph_protocol=GraphProtocol.GRAPHSON_3_0 - ) - v_query = DseGraph.query_from_traversal( - g.V(), - graph_protocol=GraphProtocol.GRAPHSON_3_0) - - for result in session.execute_graph(addV_query): - pprint(result.value) - for result in session.execute_graph(v_query): - pprint(result.value) - -Converting a traversal to a bytecode query for core graphs require some more work, because we -need the cluster context for UDT and tuple types: - -.. code-block:: python - context = { - 'cluster': cluster, - 'graph_name': 'the_graph_for_the_query' - } - addV_query = DseGraph.query_from_traversal( - g.addV('genre').property('genreId', 1).property('name', 'Action'), - graph_protocol=GraphProtocol.GRAPHSON_3_0, - context=context - ) - - for result in session.execute_graph(addV_query): - pprint(result.value) - -Implicit Graph Traversal Execution with TinkerPop -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -Using the :class:`DseGraph <.datastax.graph.fluent.DseGraph>` class, you can build a GraphTraversalSource -that will execute queries on a DSE session without explicitly passing anything to -that session. We call this *implicit execution* because the `Session` is not -explicitly involved. Everything is managed internally by TinkerPop while -traversing the graph and the results are TinkerPop types as well. - -Synchronous Example -------------------- - -.. code-block:: python - - # Build the GraphTraversalSource - g = DseGraph.traversal_source(session) - # implicitly execute the query by traversing the TraversalSource - g.addV('genre').property('genreId', 1).property('name', 'Action').next() - - # blocks until the query is completed and return the results - results = g.V().toList() - pprint(results) - -Asynchronous Exemple --------------------- - -You can execute a graph traversal query asynchronously by using `.promise()`. It returns a -python `Future `_. - -.. code-block:: python - - # Build the GraphTraversalSource - g = DseGraph.traversal_source(session) - # implicitly execute the query by traversing the TraversalSource - g.addV('genre').property('genreId', 1).property('name', 'Action').next() # not async - - # get a future and wait - future = g.V().promise() - results = list(future.result()) - pprint(results) - - # or set a callback - def cb(f): - results = list(f.result()) - pprint(results) - future = g.V().promise() - future.add_done_callback(cb) - # do other stuff... - -Specify the Execution Profile explicitly -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -If you don't want to change the default graph execution profile (`EXEC_PROFILE_GRAPH_DEFAULT`), you can register a new -one as usual and use it explicitly. Here is an example: - -.. code-block:: python - - from cassandra.cluster import Cluster - from cassandra.datastax.graph.fluent import DseGraph - - cluster = Cluster() - ep = DseGraph.create_execution_profile('graph_name', graph_protocol=GraphProtocol.GRAPHSON_3_0) - cluster.add_execution_profile('graph_traversal', ep) - session = cluster.connect() - - g = DseGraph.traversal_source() - query = DseGraph.query_from_traversal(g.V()) - session.execute_graph(query, execution_profile='graph_traversal') - -You can also create multiple GraphTraversalSources and use them with -the same execution profile (for different graphs): - -.. code-block:: python - - g_movies = DseGraph.traversal_source(session, graph_name='movies', ep) - g_series = DseGraph.traversal_source(session, graph_name='series', ep) - - print(g_movies.V().toList()) # Traverse the movies Graph - print(g_series.V().toList()) # Traverse the series Graph - -Batch Queries -~~~~~~~~~~~~~ - -DSE Graph supports batch queries using a :class:`TraversalBatch <.datastax.graph.fluent.query.TraversalBatch>` object -instantiated with :meth:`DseGraph.batch <.datastax.graph.fluent.DseGraph.batch>`. A :class:`TraversalBatch <.datastax.graph.fluent.query.TraversalBatch>` allows -you to execute multiple graph traversals in a single atomic transaction. A -traversal batch is executed with :meth:`.Session.execute_graph` or using -:meth:`TraversalBatch.execute <.datastax.graph.fluent.query.TraversalBatch.execute>` if bounded to a DSE session. - -Either way you choose to execute the traversal batch, you need to configure -the execution profile accordingly. Here is a example:: - - from cassandra.cluster import Cluster - from cassandra.datastax.graph.fluent import DseGraph - - ep = DseGraph.create_execution_profile( - 'graph_name', - graph_protocol=GraphProtocol.GRAPHSON_3_0) - cluster = Cluster(execution_profiles={'graphson3': ep}) - session = cluster.connect() - - g = DseGraph.traversal_source() - -To execute the batch using :meth:`.Session.execute_graph`, you need to convert -the batch to a GraphStatement:: - - batch = DseGraph.batch() - - batch.add( - g.addV('genre').property('genreId', 1).property('name', 'Action')) - batch.add( - g.addV('genre').property('genreId', 2).property('name', 'Drama')) # Don't use `.next()` with a batch - - graph_statement = batch.as_graph_statement(graph_protocol=GraphProtocol.GRAPHSON_3_0) - graph_statement.is_idempotent = True # configure any Statement parameters if needed... - session.execute_graph(graph_statement, execution_profile='graphson3') - -To execute the batch using :meth:`TraversalBatch.execute <.datastax.graph.fluent.query.TraversalBatch.execute>`, you need to bound the batch to a DSE session:: - - batch = DseGraph.batch(session, 'graphson3') # bound the session and execution profile - - batch.add( - g.addV('genre').property('genreId', 1).property('name', 'Action')) - batch.add( - g.addV('genre').property('genreId', 2).property('name', 'Drama')) # Don't use `.next()` with a batch - - batch.execute() - -DSL (Domain Specific Languages) -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -DSL are very useful to write better domain-specific APIs and avoiding -code duplication. Let's say we have a graph of `People` and we produce -a lot of statistics based on age. All graph traversal queries of our -application would look like:: - - g.V().hasLabel("people").has("age", P.gt(21))... - - -which is not really verbose and quite annoying to repeat in a code base. Let's create a DSL:: - - from gremlin_python.process.graph_traversal import GraphTraversal, GraphTraversalSource - - class MyAppTraversal(GraphTraversal): - - def younger_than(self, age): - return self.has("age", P.lt(age)) - - def older_than(self, age): - return self.has("age", P.gt(age)) - - - class MyAppTraversalSource(GraphTraversalSource): - - def __init__(self, *args, **kwargs): - super(MyAppTraversalSource, self).__init__(*args, **kwargs) - self.graph_traversal = MyAppTraversal - - def people(self): - return self.get_graph_traversal().V().hasLabel("people") - -Now, we can use our DSL that is a lot cleaner:: - - from cassandra.datastax.graph.fluent import DseGraph - - # ... - g = DseGraph.traversal_source(session=session, traversal_class=MyAppTraversalsource) - - g.people().younger_than(21)... - g.people().older_than(30)... - -To see a more complete example of DSL, see the `Python killrvideo DSL app `_ - -Search -~~~~~~ - -DSE Graph can use search indexes that take advantage of DSE Search functionality for -efficient traversal queries. Here are the list of additional search predicates: - -Text tokenization: - -* :meth:`token <.datastax.graph.fluent.predicates.Search.token>` -* :meth:`token_prefix <.datastax.graph.fluent.predicates.Search.token_prefix>` -* :meth:`token_regex <.datastax.graph.fluent.predicates.Search.token_regex>` -* :meth:`token_fuzzy <.datastax.graph.fluent.predicates.Search.token_fuzzy>` - -Text match: - -* :meth:`prefix <.datastax.graph.fluent.predicates.Search.prefix>` -* :meth:`regex <.datastax.graph.fluent.predicates.Search.regex>` -* :meth:`fuzzy <.datastax.graph.fluent.predicates.Search.fuzzy>` -* :meth:`phrase <.datastax.graph.fluent.predicates.Search.phrase>` - -Geo: - -* :meth:`inside <.datastax.graph.fluent.predicates.Geo.inside>` - -Create search indexes ---------------------- - -For text tokenization: - -.. code-block:: python - - - s.execute_graph("schema.vertexLabel('my_vertex_label').index('search').search().by('text_field').asText().add()") - -For text match: - -.. code-block:: python - - - s.execute_graph("schema.vertexLabel('my_vertex_label').index('search').search().by('text_field').asString().add()") - - -For geospatial: - -You can create a geospatial index on Point and LineString fields. - -.. code-block:: python - - - s.execute_graph("schema.vertexLabel('my_vertex_label').index('search').search().by('point_field').add()") - - -Using search indexes --------------------- - -Token: - -.. code-block:: python - - from cassandra.datastax.graph.fluent.predicates import Search - # ... - - g = DseGraph.traversal_source() - query = DseGraph.query_from_traversal( - g.V().has('my_vertex_label','text_field', Search.token_regex('Hello.+World')).values('text_field')) - session.execute_graph(query) - -Text: - -.. code-block:: python - - from cassandra.datastax.graph.fluent.predicates import Search - # ... - - g = DseGraph.traversal_source() - query = DseGraph.query_from_traversal( - g.V().has('my_vertex_label','text_field', Search.prefix('Hello')).values('text_field')) - session.execute_graph(query) - -Geospatial: - -.. code-block:: python - - from cassandra.datastax.graph.fluent.predicates import Geo - from cassandra.util import Distance - # ... - - g = DseGraph.traversal_source() - query = DseGraph.query_from_traversal( - g.V().has('my_vertex_label','point_field', Geo.inside(Distance(46, 71, 100)).values('point_field')) - session.execute_graph(query) - - -For more details, please refer to the official `DSE Search Indexes Documentation `_ diff --git a/docs/installation.rst b/docs/installation.rst index 1cb67cf130..4a4db3b172 100644 --- a/docs/installation.rst +++ b/docs/installation.rst @@ -28,16 +28,6 @@ To check if the installation was successful, you can run:: It should print something like "3.22.0". -.. _installation-datastax-graph: - -(*Optional*) Graph ---------------------------- -The driver provides an optional fluent graph API that depends on Apache TinkerPop (gremlinpython). It is -not installed by default. To be able to build Gremlin traversals, you need to install -the `graph` requirements:: - - pip install scylla-driver[graph] - (*Optional*) Compression Support -------------------------------- Compression can optionally be used for communication between the driver and diff --git a/docs/upgrading.rst b/docs/upgrading.rst index bc963e6722..1dd3cc558d 100644 --- a/docs/upgrading.rst +++ b/docs/upgrading.rst @@ -12,10 +12,6 @@ are not required anymore:: pip install scylla-driver -If you need the Graph *Fluent* API (features provided by dse-graph):: - - pip install scylla-driver[graph] - See :doc:`installation` for more details. Import from the cassandra module @@ -39,24 +35,6 @@ need to change only the first module of your import statements, not the submodul Also note that the cassandra.hosts module doesn't exist in scylla-driver. This module is named cassandra.pool. -dse-graph -^^^^^^^^^ - -dse-graph features are now built into scylla-driver. The only change you need -to do is your import statements: - -.. code-block:: python - - from dse_graph import .. - from dse_graph.query import .. - - # becomes - - from cassandra.datastax.graph.fluent import .. - from cassandra.datastax.graph.fluent.query import .. - -See :mod:`~.datastax.graph.fluent`. - Session.execute and Session.execute_async API ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ @@ -72,7 +50,6 @@ Deprecations These changes are optional, but recommended: -* Importing from `cassandra.graph` is deprecated. Consider importing from `cassandra.datastax.graph`. * Use :class:`~.policies.DefaultLoadBalancingPolicy` instead of DSELoadBalancingPolicy. Upgrading to 3.0