diff --git a/lib/secure_headers/railtie.rb b/lib/secure_headers/railtie.rb
index ba255ac..64f9eec 100644
--- a/lib/secure_headers/railtie.rb
+++ b/lib/secure_headers/railtie.rb
@@ -22,9 +22,12 @@ class Railtie < Rails::Railtie
         ActiveSupport.on_load(:action_controller) do
           include SecureHeaders
 
-          unless Rails.application.config.action_dispatch.default_headers.nil?
-            conflicting_headers.each do |header|
-              Rails.application.config.action_dispatch.default_headers.delete(header)
+          default_headers = Rails.application.config.action_dispatch.default_headers
+          unless default_headers.nil?
+            default_headers.each_key do |header|
+              if conflicting_headers.include?(header.downcase)
+                default_headers.delete(header)
+              end
             end
           end
         end