Execution table schema changes (#37)

* make total_rows_processed a big int
* make execution track total time in s not ms

Author: seanbudd

rdl/alembic/versions/955122a76711_create_an_execution_summary_table.py

@@ -25,8 +25,8 @@ def upgrade():
                     sa.Column('execution_started', sa.DateTime(timezone=True),
                               server_default=sa.text('now()'), nullable=True),
                     sa.Column('execution_ended', sa.DateTime(timezone=True), nullable=True),
-                    sa.Column('execution_time_ms', sa.Integer(), nullable=True),
-                    sa.Column('total_rows_processed', sa.Integer(), nullable=True),
+                    sa.Column('execution_time_s', sa.Integer(), nullable=True),
+                    sa.Column('total_rows_processed', sa.BigInteger(), nullable=True),
                     sa.Column('total_models_processed', sa.Integer(), nullable=True),
                     sa.Column('status', sa.String(length=25), nullable=False),
                     sa.PrimaryKeyConstraint('id'),
@@ -38,11 +38,11 @@ def upgrade():
         '''
         INSERT INTO rdl.execution (
             id, execution_started,
-            execution_ended, execution_time_ms, total_rows_processed, total_models_processed, status
+            execution_ended, execution_time_s, total_rows_processed, total_models_processed, status
             )
         SELECT
             execution_id, MAX(completed_on) - make_interval(secs := SUM(execution_time_ms)::decimal / 1000),
-            MAX(completed_on), SUM(execution_time_ms), SUM(rows_processed), COUNT(id), 'UNKNOWN'
+            MAX(completed_on), SUM(execution_time_ms) / 1000, SUM(rows_processed), COUNT(id), 'UNKNOWN'
         FROM rdl.data_load_execution
         GROUP BY execution_id
         ''')

rdl/data_load_tracking/DataLoadTrackerRepository.py

@@ -47,7 +47,7 @@ def complete_execution(self, execution_id, total_number_of_models,
         current_execution.total_models_processed = total_number_of_models
         current_execution.status = status
         current_execution.execution_ended = execution_end_time
-        current_execution.execution_time_ms = int(total_execution_seconds * 1000)
+        current_execution.execution_time_s = total_execution_seconds
         current_execution.total_rows_processed = self.get_execution_rows(current_execution.id)
         session.commit()
         self.logger.info(current_execution)

rdl/entities/execution_entity.py

@@ -1,6 +1,6 @@
 import uuid
 
-from sqlalchemy import Column, DateTime, Integer, String
+from sqlalchemy import Column, DateTime, Integer, String, BigInteger
 from sqlalchemy.sql import func
 from sqlalchemy.dialects.postgresql import UUID
 
@@ -14,16 +14,16 @@ class ExecutionEntity(Base):
     id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
     execution_started = Column(DateTime(timezone=True), server_default=func.now())
     execution_ended = Column(DateTime(timezone=True))
-    execution_time_ms = Column(Integer, nullable=True)
-    total_rows_processed = Column(Integer, nullable=True)
+    execution_time_s = Column(Integer, nullable=True)
+    total_rows_processed = Column(BigInteger, nullable=True)
     total_models_processed = Column(Integer, nullable=True)
     status = Column(String(25), nullable=False, default=Constants.ExecutionStatus.STARTED)
 
     def __str__(self):
         if self.status == Constants.ExecutionStatus.STARTED:
             return f"Started Execution ID: {self.id} at {self.execution_started}"
 
-        total_execution_seconds = self.execution_time_ms // 1000
+        total_execution_seconds = self.execution_time_s
         execution_hours = total_execution_seconds // 3600
         execution_minutes = (total_execution_seconds // 60) % 60
         execution_seconds = total_execution_seconds % 60