Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set global variable overrides on the command line with --vars #640

Merged
merged 3 commits into from
Feb 10, 2018
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions dbt/main.py
Original file line number Diff line number Diff line change
Expand Up @@ -295,6 +295,16 @@ def parse_args(args):
help='Which target to load for the given profile'
)

base_subparser.add_argument(
'--vars',
type=str,
default='{}',
help="""
Supply variables to the project. This argument overrides
variables defined in your dbt_project.yml file. This argument
should be a YAML string, eg. '{my_variable: my_value}'"""
)

sub = subs.add_parser('init', parents=[base_subparser])
sub.add_argument('project_name', type=str, help='Name of the new project')
sub.set_defaults(cls=init_task.InitTask, which='init')
Expand Down
6 changes: 6 additions & 0 deletions dbt/project.py
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,12 @@ def __init__(self, cfg, profiles, profiles_dir, profile_to_load=None,
"Could not find profile named '{}'"
.format(self.profile_to_load), self)

global_vars = dbt.utils.parse_cli_vars(getattr(args, 'vars', '{}'))
if 'vars' not in self.cfg['models']:
self.cfg['models']['vars'] = {}

self.cfg['models']['vars'].update(global_vars)

def __str__(self):
return pprint.pformat({'project': self.cfg, 'profiles': self.profiles})

Expand Down
18 changes: 18 additions & 0 deletions dbt/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from dbt.compat import basestring
from dbt.logger import GLOBAL_LOGGER as logger
from dbt.node_types import NodeType
from dbt.clients import yaml_helper


DBTConfigKeys = [
Expand Down Expand Up @@ -375,3 +376,20 @@ def invalid_ref_fail_unless_test(node, target_model_name,
node,
target_model_name,
target_model_package)


def parse_cli_vars(var_string):
try:
cli_vars = yaml_helper.load_yaml_text(var_string)
var_type = type(cli_vars)
if var_type == dict:
return cli_vars
else:
type_name = var_type.__name__
dbt.exceptions.raise_compiler_error(
"The --vars argument must be a YAML dictionary, but was "
"of type '{}'".format(type_name))
except dbt.exceptions.ValidationException as e:
logger.error(
"The YAML provided in the --vars argument is not valid.\n")
raise
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@

select
'{{ var("variable_1") }}'::varchar as var_1,
'{{ var("variable_2")[0] }}'::varchar as var_2,
'{{ var("variable_3")["value"] }}'::varchar as var_3

7 changes: 7 additions & 0 deletions test/integration/028_cli_vars/models_complex/schema.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@

complex_model:
constraints:
accepted_values:
- {field: var_1, values: ["abc"]}
- {field: var_2, values: ["def"]}
- {field: var_3, values: ["jkl"]}
5 changes: 5 additions & 0 deletions test/integration/028_cli_vars/models_simple/schema.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@

simple_model:
constraints:
accepted_values:
- {field: simple, values: ["abc"]}
4 changes: 4 additions & 0 deletions test/integration/028_cli_vars/models_simple/simple_model.sql
Original file line number Diff line number Diff line change
@@ -0,0 +1,4 @@

select
'{{ var("simple") }}'::varchar as simple

54 changes: 54 additions & 0 deletions test/integration/028_cli_vars/test_cli_vars.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
from nose.plugins.attrib import attr
from test.integration.base import DBTIntegrationTest
import yaml


class TestCLIVars(DBTIntegrationTest):
@property
def schema(self):
return "cli_vars_028"

@property
def models(self):
return "test/integration/028_cli_vars/models_complex"

@attr(type='postgres')
def test__cli_vars_longform(self):
self.use_default_project()
self.use_profile('postgres')

cli_vars = {
"variable_1": "abc",
"variable_2": ["def", "ghi"],
"variable_3": {
"value": "jkl"
}
}
self.run_dbt(["run", "--vars", yaml.dump(cli_vars)])
self.run_dbt(["test"])


class TestCLIVarsSimple(DBTIntegrationTest):
@property
def schema(self):
return "cli_vars_028"

@property
def models(self):
return "test/integration/028_cli_vars/models_simple"

@attr(type='postgres')
def test__cli_vars_shorthand(self):
self.use_default_project()
self.use_profile('postgres')

self.run_dbt(["run", "--vars", "simple: abc"])
self.run_dbt(["test"])

@attr(type='postgres')
def test__cli_vars_longer(self):
self.use_default_project()
self.use_profile('postgres')

self.run_dbt(["run", "--vars", "{simple: abc, unused: def}"])
self.run_dbt(["test"])