Skip to content

Commit

Permalink
Merge pull request #13919 from jrafanie/fix_missing_reason_contants
Browse files Browse the repository at this point in the history
Fix missing reason contants
  • Loading branch information
chrisarcand authored Feb 14, 2017
2 parents 87d8b72 + 3ea55fa commit 89dfba8
Show file tree
Hide file tree
Showing 3 changed files with 48 additions and 36 deletions.
4 changes: 2 additions & 2 deletions app/models/miq_server/worker_management/monitor.rb
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ def check_not_responding(class_name = nil)
end

def monitor_reason_not_responding?(w)
[Reason::NOT_RESPONDING, Reason::MEMORY_EXCEEDED].include?(worker_get_monitor_reason(w.pid)) || w.stopping_for_too_long?
[MiqServer::NOT_RESPONDING, MiqServer::MEMORY_EXCEEDED].include?(worker_get_monitor_reason(w.pid)) || w.stopping_for_too_long?
end

def do_system_limit_exceeded
Expand All @@ -113,7 +113,7 @@ def do_system_limit_exceeded
msg = "#{w.format_full_log_msg} is being stopped because system resources exceeded threshold, it will be restarted once memory has freed up"
_log.warn(msg)
MiqEvent.raise_evm_event_queue_in_region(w.miq_server, "evm_server_memory_exceeded", :event_details => msg, :type => w.class.name)
restart_worker(w, Reason::MEMORY_EXCEEDED)
restart_worker(w, MiqServer::MEMORY_EXCEEDED)
break
end
end
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ def validate_worker(w)
msg = "#{w.format_full_log_msg} has not responded in #{Time.now.utc - w.last_heartbeat} seconds, restarting worker"
_log.error(msg)
MiqEvent.raise_evm_event_queue(w.miq_server, "evm_worker_not_responding", :event_details => msg, :type => w.class.name)
restart_worker(w, Reason::NOT_RESPONDING)
restart_worker(w, MiqServer::NOT_RESPONDING)
return false
end

Expand Down
78 changes: 45 additions & 33 deletions spec/models/miq_server/worker_monitor_spec.rb
Original file line number Diff line number Diff line change
Expand Up @@ -351,48 +351,60 @@
end
end

context "with worker that is using a lot of memory" do
context "threshold validation" do
let(:worker) { FactoryGirl.create(:miq_worker, :miq_server_id => server.id, :pid => 42) }
let(:server) { @miq_server }

before(:each) do
@worker1 = FactoryGirl.create(:miq_worker, :miq_server_id => @miq_server.id, :memory_usage => 2.gigabytes, :pid => 42)
allow_any_instance_of(MiqServer).to receive(:get_time_threshold).and_return(2.minutes)
allow_any_instance_of(MiqServer).to receive(:get_memory_threshold).and_return(500.megabytes)
allow_any_instance_of(MiqServer).to receive(:get_restart_interval).and_return(0.hours)
@miq_server.setup_drb_variables
allow(server).to receive(:get_time_threshold).and_return(2.minutes)
allow(server).to receive(:get_memory_threshold).and_return(500.megabytes)
allow(server).to receive(:get_restart_interval).and_return(0.hours)
server.setup_drb_variables
end

it "should not trigger memory threshold if worker is creating" do
@worker1.status = MiqWorker::STATUS_CREATING
expect(@miq_server.validate_worker(@worker1)).to be_truthy
it "should mark not responding if not recently heartbeated" do
worker.update(:last_heartbeat => 20.minutes.ago)
expect(server.validate_worker(worker)).to be_falsey
expect(worker.reload.status).to eq(MiqWorker::STATUS_STOPPING)
end

it "should not trigger memory threshold if worker is starting" do
@worker1.status = MiqWorker::STATUS_STARTING
expect(@miq_server.validate_worker(@worker1)).to be_truthy
end
context "for excessive memory" do
before { worker.memory_usage = 2.gigabytes }

it "should trigger memory threshold if worker is started" do
@worker1.status = MiqWorker::STATUS_STARTED
expect(@miq_server).to receive(:worker_set_monitor_status).with(@worker1.pid, :waiting_for_stop_before_restart).once
@miq_server.validate_worker(@worker1)
end
it "should not trigger memory threshold if worker is creating" do
worker.status = MiqWorker::STATUS_CREATING
expect(server.validate_worker(worker)).to be_truthy
end

it "should trigger memory threshold if worker is ready" do
@worker1.status = MiqWorker::STATUS_READY
expect(@miq_server).to receive(:worker_set_monitor_status).with(@worker1.pid, :waiting_for_stop_before_restart).once
@miq_server.validate_worker(@worker1)
end
it "should not trigger memory threshold if worker is starting" do
worker.status = MiqWorker::STATUS_STARTING
expect(server.validate_worker(worker)).to be_truthy
end

it "should trigger memory threshold if worker is working" do
@worker1.status = MiqWorker::STATUS_WORKING
expect(@miq_server).to receive(:worker_set_monitor_status).with(@worker1.pid, :waiting_for_stop_before_restart).once
@miq_server.validate_worker(@worker1)
end
it "should trigger memory threshold if worker is started" do
worker.status = MiqWorker::STATUS_STARTED
expect(server).to receive(:worker_set_monitor_status).with(worker.pid, :waiting_for_stop_before_restart).once
server.validate_worker(worker)
end

it "should trigger memory threshold if worker is ready" do
worker.status = MiqWorker::STATUS_READY
expect(server).to receive(:worker_set_monitor_status).with(worker.pid, :waiting_for_stop_before_restart).once
server.validate_worker(worker)
end

it "should trigger memory threshold if worker is working" do
worker.status = MiqWorker::STATUS_WORKING
expect(server).to receive(:worker_set_monitor_status).with(worker.pid, :waiting_for_stop_before_restart).once
server.validate_worker(worker)
end

it "should return proper message on heartbeat" do
@worker1.status = MiqWorker::STATUS_READY
expect(@miq_server.worker_heartbeat(@worker1.pid)).to eq([])
@miq_server.validate_worker(@worker1) # Validation will populate message
expect(@miq_server.worker_heartbeat(@worker1.pid)).to eq([['exit']])
it "should return proper message on heartbeat" do
worker.status = MiqWorker::STATUS_READY
expect(server.worker_heartbeat(worker.pid)).to eq([])
server.validate_worker(worker) # Validation will populate message
expect(server.worker_heartbeat(worker.pid)).to eq([['exit']])
end
end
end
end
Expand Down

0 comments on commit 89dfba8

Please sign in to comment.