Skip to content

Keep Usage Event records of running & un-processed Apps and Services #4374

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 8 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 79 additions & 0 deletions app/access/app_usage_consumer_access.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
module VCAP::CloudController
class AppUsageConsumerAccess < BaseAccess
def create?(_object, _params=nil)
admin_user?
end

def read?(object)
return @ok_read if instance_variable_defined?(:@ok_read)

@ok_read = admin_user? || admin_read_only_user? || global_auditor? || object_is_visible_to_user?(object, context.user)
end

def read_for_update?(_object, _params=nil)
admin_user?
end

def can_remove_related_object?(object, params=nil)
read_for_update?(object, params)
end

def read_related_object_for_update?(object, params=nil)
read_for_update?(object, params)
end

def update?(_object, _params=nil)
admin_user?
end

def delete?(_object)
admin_user?
end

# These methods should be called first to determine if the user's token has the appropriate scope for the operation

def read_with_token?(_)
admin_user? || admin_read_only_user? || has_read_scope? || global_auditor?
end

def create_with_token?(_)
admin_user? || has_write_scope?
end

def read_for_update_with_token?(_)
admin_user? || has_write_scope?
end

def can_remove_related_object_with_token?(*)
read_for_update_with_token?(*)
end

def read_related_object_for_update_with_token?(*)
read_for_update_with_token?(*)
end

def update_with_token?(_)
admin_user? || has_write_scope?
end

def delete_with_token?(_)
admin_user? || has_write_scope?
end

def index_with_token?(_)
admin_user? || admin_read_only_user?
end

def index?(_object_class, _params=nil)
admin_user? || admin_read_only_user?
end

def reset?(_object_class)
admin_user?
end

def reset_with_token?(_object_class)
admin_user?
end
end
end
79 changes: 79 additions & 0 deletions app/access/service_usage_consumer_access.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,79 @@
module VCAP::CloudController
class ServiceUsageConsumerAccess < BaseAccess
def create?(_object, _params=nil)
admin_user?
end

def read?(object)
return @ok_read if instance_variable_defined?(:@ok_read)

@ok_read = admin_user? || admin_read_only_user? || global_auditor? || object_is_visible_to_user?(object, context.user)
end

def read_for_update?(_object, _params=nil)
admin_user?
end

def can_remove_related_object?(object, params=nil)
read_for_update?(object, params)
end

def read_related_object_for_update?(object, params=nil)
read_for_update?(object, params)
end

def update?(_object, _params=nil)
admin_user?
end

def delete?(_object)
admin_user?
end

# These methods should be called first to determine if the user's token has the appropriate scope for the operation

def read_with_token?(_)
admin_user? || admin_read_only_user? || has_read_scope? || global_auditor?
end

def create_with_token?(_)
admin_user? || has_write_scope?
end

def read_for_update_with_token?(_)
admin_user? || has_write_scope?
end

def can_remove_related_object_with_token?(*)
read_for_update_with_token?(*)
end

def read_related_object_for_update_with_token?(*)
read_for_update_with_token?(*)
end

def update_with_token?(_)
admin_user? || has_write_scope?
end

def delete_with_token?(_)
admin_user? || has_write_scope?
end

def index_with_token?(_)
admin_user? || admin_read_only_user?
end

def index?(_object_class, _params=nil)
admin_user? || admin_read_only_user?
end

def reset?(_object_class)
admin_user?
end

def reset_with_token?(_object_class)
admin_user?
end
end
end
9 changes: 9 additions & 0 deletions app/actions/app_usage_consumer_delete.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
module VCAP::CloudController
class AppUsageConsumerDelete
def delete(app_usage_consumer)
app_usage_consumer.destroy
rescue Sequel::Error => e
raise CloudController::Errors::ApiError.new_from_details('AppUsageConsumerDeleteError', e.message)
end
end
end
9 changes: 9 additions & 0 deletions app/actions/service_usage_consumer_delete.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
module VCAP::CloudController
class ServiceUsageConsumerDelete
def delete(service_usage_consumer)
service_usage_consumer.destroy
rescue Sequel::Error => e
raise CloudController::Errors::ApiError.new_from_details('ServiceUsageConsumerDeleteError', e.message)
end
end
end
41 changes: 41 additions & 0 deletions app/controllers/v3/app_usage_consumers_controller.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
require 'presenters/v3/app_usage_consumer_presenter'
require 'messages/app_usage_consumers_list_message'
require 'fetchers/app_usage_consumer_list_fetcher'
require 'actions/app_usage_consumer_delete'

class AppUsageConsumersController < ApplicationController
def index
message = AppUsageConsumersListMessage.from_params(query_params)
invalid_param!(message.errors.full_messages) unless message.valid?

app_usage_consumers = AppUsageConsumer.where(guid: [])

app_usage_consumers = AppUsageConsumerListFetcher.fetch_all(message, AppUsageConsumer.dataset) if permission_queryer.can_read_globally?

render status: :ok, json: Presenters::V3::PaginatedListPresenter.new(
presenter: Presenters::V3::AppUsageConsumerPresenter,
paginated_result: SequelPaginator.new.get_page(app_usage_consumers, message.try(:pagination_options)),
path: '/v3/app_usage_consumers',
message: message,
extra_presenter_args: {}
)
end

def show
app_usage_consumer = AppUsageConsumer.where(consumer_guid: params[:guid]).first
resource_not_found!(:app_usage_consumer) unless app_usage_consumer && permission_queryer.can_read_globally?

render status: :ok, json: Presenters::V3::AppUsageConsumerPresenter.new(app_usage_consumer)
end

def destroy
unauthorized! unless permission_queryer.can_write_globally?

app_usage_consumer = AppUsageConsumer.where(consumer_guid: params[:guid]).first
resource_not_found!(:app_usage_consumer) unless app_usage_consumer

AppUsageConsumerDelete.new.delete(app_usage_consumer)

head :no_content
end
end
14 changes: 14 additions & 0 deletions app/controllers/v3/app_usage_events_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,20 @@ def index

app_usage_events = AppUsageEventListFetcher.fetch_all(message, AppUsageEvent.dataset) if permission_queryer.can_read_globally?

if message.consumer_guid && message.after_guid&.first && permission_queryer.can_write_globally?
begin
consumer = AppUsageConsumer.find_or_create(consumer_guid: message.consumer_guid) do |c|
c.last_processed_guid = message.after_guid.first
end

consumer.update(last_processed_guid: message.after_guid.first) if !consumer.new? && consumer.last_processed_guid != message.after_guid.first
rescue Sequel::ValidationFailed => e
unprocessable!(e.message)
rescue Sequel::Error
error!('Failed to update consumer tracking', 500)
end
end

render status: :ok, json: Presenters::V3::PaginatedListPresenter.new(
presenter: Presenters::V3::AppUsageEventPresenter,
paginated_result: SequelPaginator.new.get_page(app_usage_events, message.try(:pagination_options)),
Expand Down
41 changes: 41 additions & 0 deletions app/controllers/v3/service_usage_consumers_controller.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
require 'presenters/v3/service_usage_consumer_presenter'
require 'messages/service_usage_consumers_list_message'
require 'fetchers/service_usage_consumer_list_fetcher'
require 'actions/service_usage_consumer_delete'

class ServiceUsageConsumersController < ApplicationController
def index
message = ServiceUsageConsumersListMessage.from_params(query_params)
invalid_param!(message.errors.full_messages) unless message.valid?

service_usage_consumers = ServiceUsageConsumer.where(guid: [])

service_usage_consumers = ServiceUsageConsumerListFetcher.fetch_all(message, ServiceUsageConsumer.dataset) if permission_queryer.can_read_globally?

render status: :ok, json: Presenters::V3::PaginatedListPresenter.new(
presenter: Presenters::V3::ServiceUsageConsumerPresenter,
paginated_result: SequelPaginator.new.get_page(service_usage_consumers, message.try(:pagination_options)),
path: '/v3/service_usage_consumers',
message: message,
extra_presenter_args: {}
)
end

def show
service_usage_consumer = ServiceUsageConsumer.where(consumer_guid: params[:guid]).first
resource_not_found!(:service_usage_consumer) unless service_usage_consumer && permission_queryer.can_read_globally?

render status: :ok, json: Presenters::V3::ServiceUsageConsumerPresenter.new(service_usage_consumer)
end

def destroy
unauthorized! unless permission_queryer.can_write_globally?

service_usage_consumer = ServiceUsageConsumer.where(consumer_guid: params[:guid]).first
resource_not_found!(:service_usage_consumer) unless service_usage_consumer

ServiceUsageConsumerDelete.new.delete(service_usage_consumer)

head :no_content
end
end
14 changes: 14 additions & 0 deletions app/controllers/v3/service_usage_events_controller.rb
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,20 @@ def index

service_usage_events = ServiceUsageEventListFetcher.fetch_all(message, ServiceUsageEvent.dataset) if permission_queryer.can_read_globally?

if message.consumer_guid && message.after_guid&.first && permission_queryer.can_write_globally?
begin
consumer = ServiceUsageConsumer.find_or_create(consumer_guid: message.consumer_guid) do |c|
c.last_processed_guid = message.after_guid.first
end

consumer.update(last_processed_guid: message.after_guid.first) if !consumer.new? && consumer.last_processed_guid != message.after_guid.first
rescue Sequel::ValidationFailed => e
unprocessable!(e.message)
rescue Sequel::Error
error!('Failed to update consumer tracking', 500)
end
end

render status: :ok, json: Presenters::V3::PaginatedListPresenter.new(
presenter: Presenters::V3::ServiceUsageEventPresenter,
paginated_result: SequelPaginator.new.get_page(service_usage_events, message.try(:pagination_options)),
Expand Down
17 changes: 17 additions & 0 deletions app/fetchers/app_usage_consumer_list_fetcher.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
require 'fetchers/base_list_fetcher'

module VCAP::CloudController
class AppUsageConsumerListFetcher < BaseListFetcher
class << self
def fetch_all(message, dataset)
dataset = filter(message, dataset, AppUsageConsumer)

dataset = dataset.where(consumer_guid: message.consumer_guids) if message.requested?(:consumer_guids)

dataset = dataset.where(last_processed_guid: message.last_processed_guids) if message.requested?(:last_processed_guids)

dataset
end
end
end
end
17 changes: 17 additions & 0 deletions app/fetchers/service_usage_consumer_list_fetcher.rb
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
require 'fetchers/base_list_fetcher'

module VCAP::CloudController
class ServiceUsageConsumerListFetcher < BaseListFetcher
class << self
def fetch_all(message, dataset)
dataset = filter(message, dataset, ServiceUsageConsumer)

dataset = dataset.where(consumer_guid: message.consumer_guids) if message.requested?(:consumer_guids)

dataset = dataset.where(last_processed_guid: message.last_processed_guids) if message.requested?(:last_processed_guids)

dataset
end
end
end
end
8 changes: 5 additions & 3 deletions app/jobs/runtime/app_usage_events_cleanup.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,20 @@ module VCAP::CloudController
module Jobs
module Runtime
class AppUsageEventsCleanup < VCAP::CloudController::Jobs::CCJob
attr_accessor :cutoff_age_in_days
attr_accessor :cutoff_age_in_days, :keep_unprocessed_records, :threshold_for_keeping_unprocessed_records

def initialize(cutoff_age_in_days)
def initialize(cutoff_age_in_days, keep_unprocessed_records, threshold_for_keeping_unprocessed_records)
@cutoff_age_in_days = cutoff_age_in_days
@keep_unprocessed_records = keep_unprocessed_records
@threshold_for_keeping_unprocessed_records = threshold_for_keeping_unprocessed_records
end

def perform
logger = Steno.logger('cc.background')
logger.info('Cleaning up old AppUsageEvent rows')

repository = Repositories::AppUsageEventRepository.new
repository.delete_events_older_than(cutoff_age_in_days)
repository.delete_events_older_than(cutoff_age_in_days, keep_unprocessed_records, threshold_for_keeping_unprocessed_records)
end

def job_name_in_configuration
Expand Down
2 changes: 1 addition & 1 deletion app/jobs/runtime/events_cleanup.rb
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ def initialize(cutoff_age_in_days)
end

def perform
Database::OldRecordCleanup.new(Event, cutoff_age_in_days).delete
Database::OldRecordCleanup.new(Event, cutoff_age_in_days:).delete
end

def job_name_in_configuration
Expand Down
8 changes: 5 additions & 3 deletions app/jobs/v2/services/service_usage_events_cleanup.rb
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,20 @@ module VCAP::CloudController
module Jobs
module Services
class ServiceUsageEventsCleanup < VCAP::CloudController::Jobs::CCJob
attr_accessor :cutoff_age_in_days
attr_accessor :cutoff_age_in_days, :keep_unprocessed_records, :threshold_for_keeping_unprocessed_records

def initialize(cutoff_age_in_days)
def initialize(cutoff_age_in_days, keep_unprocessed_records, threshold_for_keeping_unprocessed_records)
@cutoff_age_in_days = cutoff_age_in_days
@keep_unprocessed_records = keep_unprocessed_records
@threshold_for_keeping_unprocessed_records = threshold_for_keeping_unprocessed_records
end

def perform
logger = Steno.logger('cc.background')
logger.info('Cleaning up old ServiceUsageEvent rows')

repository = Repositories::ServiceUsageEventRepository.new
repository.delete_events_older_than(cutoff_age_in_days)
repository.delete_events_older_than(cutoff_age_in_days, keep_unprocessed_records, threshold_for_keeping_unprocessed_records)
end

def job_name_in_configuration
Expand Down
Loading
Loading