From 1881c0b932a0ae4ddc4d3042980afadd5ee003b8 Mon Sep 17 00:00:00 2001 From: Jacob Beck Date: Mon, 3 Feb 2020 08:56:30 -0700 Subject: [PATCH] batching --- core/dbt/tracking.py | 45 +++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/core/dbt/tracking.py b/core/dbt/tracking.py index c1f45a5d82c..845f265cf5c 100644 --- a/core/dbt/tracking.py +++ b/core/dbt/tracking.py @@ -31,8 +31,13 @@ class TimeoutEmitter(Emitter): def __init__(self): - super().__init__(COLLECTOR_URL, protocol=COLLECTOR_PROTOCOL, - buffer_size=1, on_failure=self.handle_failure) + super().__init__( + COLLECTOR_URL, protocol=COLLECTOR_PROTOCOL, + buffer_size=30, on_failure=self.handle_failure, + method='post', + # don't set this. + byte_limit=None, + ) @staticmethod def handle_failure(num_ok, unsent): @@ -41,21 +46,43 @@ def handle_failure(num_ok, unsent): logger.warning('Error sending message, disabling tracking') do_not_track() - def http_get(self, payload): - sp_logger.info("Sending GET request to {}...".format(self.endpoint)) - sp_logger.debug("Payload: {}".format(payload)) - r = requests.get(self.endpoint, params=payload, timeout=5.0) + def _log_request(self, request, payload): + sp_logger.info(f"Sending {request} request to {self.endpoint}...") + sp_logger.debug(f"Payload: {payload}") - msg = "GET request finished with status code: " + str(r.status_code) - if self.is_good_status_code(r.status_code): + def _log_result(self, request, status_code): + msg = f"{request} request finished with status code: {status_code}" + if self.is_good_status_code(status_code): sp_logger.info(msg) else: sp_logger.warning(msg) + + def http_post(self, payload): + self._log_request('POST', payload) + + r = requests.post( + self.endpoint, + data=payload, + headers={'content-type': 'application/json; charset=utf-8'}, + timeout=5.0 + ) + + self._log_result('GET', r.status_code) + return r + + def http_get(self, payload): + self._log_request('GET', payload) + + r = requests.get(self.endpoint, params=payload, timeout=5.0) + + self._log_result('GET', r.status_code) return r emitter = TimeoutEmitter() -tracker = Tracker(emitter, namespace="cf", app_id="dbt") +tracker = Tracker( + emitter, namespace="cf", app_id="dbt", +) class User: