diff --git a/Rakefile b/Rakefile
index 35b1ae46..490ae118 100644
--- a/Rakefile
+++ b/Rakefile
@@ -21,7 +21,7 @@ require_relative 'lib/rubocop/cop/generator'
 
 desc 'Run tests'
 task :test do
-  system("bundle exec minitest-queue #{Dir.glob('test/**/*_test.rb').shelljoin}")
+  system("bundle exec minitest-queue #{Dir.glob('test/**/*_test.rb').shelljoin}", exception: true)
 end
 
 desc 'Run RuboCop over itself'
diff --git a/tasks/changelog.rake b/tasks/changelog.rake
index 0004e389..13bad1fd 100644
--- a/tasks/changelog.rake
+++ b/tasks/changelog.rake
@@ -9,7 +9,7 @@ namespace :changelog do
       ref_type = :pull if args[:id]
       path = Changelog::Entry.new(type: type, ref_id: args[:id], ref_type: ref_type).write
       cmd = "git add #{path}"
-      system cmd
+      system cmd, exception: true
       puts "Entry '#{path}' created and added to git index"
     end
   end
@@ -21,7 +21,7 @@ namespace :changelog do
     Changelog.new.merge!.and_delete!
     cmd = "git commit -a -m 'Update Changelog'"
     puts cmd
-    system cmd
+    system cmd, exception: true
   end
 
   task :check_clean do