Cleanup error logs on shutdown (#3592)

* Do not log normal RabbitMQ exceptions * Do not log async cancelled exception in gundeck * Fix nesting of codensity actions * Add CHANGELOG entry
wireapp · Sep 27, 2023 · eee936e · eee936e
1 parent dbfc6c1
commit eee936e
Show file tree

Hide file tree

Showing 4 changed files with 21 additions and 15 deletions.
diff --git a/changelog.d/5-internal/shutdown-cleanup b/changelog.d/5-internal/shutdown-cleanup
@@ -0,0 +1 @@
+Avoid unnecessary error logs on service shutdown
diff --git a/integration/test/Testlib/RunServices.hs b/integration/test/Testlib/RunServices.hs
@@ -3,7 +3,7 @@
 module Testlib.RunServices where
 
 import Control.Concurrent
-import Control.Monad.Codensity (lowerCodensity)
+import Control.Monad.Codensity
 import System.Directory
 import System.Environment (getArgs)
 import System.Exit (exitWith)
@@ -44,7 +44,6 @@ main = do
       pure $ joinPath [projectRoot, "services/integration.yaml"]
 
   genv <- createGlobalEnv cfg
-  env <- lowerCodensity $ mkEnv genv
 
   args <- getArgs
 
@@ -57,10 +56,11 @@ main = do
           (_, _, _, ph) <- createProcess cp
           exitWith =<< waitForProcess ph
 
-  runAppWithEnv env $ do
-    lowerCodensity $ do
-      _modifyEnv <-
-        traverseConcurrentlyCodensity
-          (`startDynamicBackend` def)
-          [backendA, backendB]
-      liftIO run
+  runCodensity (mkEnv genv) $ \env ->
+    runAppWithEnv env $
+      lowerCodensity $ do
+        _modifyEnv <-
+          traverseConcurrentlyCodensity
+            (`startDynamicBackend` def)
+            [backendA, backendB]
+        liftIO run
diff --git a/libs/extended/src/Network/AMQP/Extended.hs b/libs/extended/src/Network/AMQP/Extended.hs
@@ -144,7 +144,6 @@ openConnectionWithRetries l RabbitMqOpts {..} hooks = do
 
     chanExceptionHandler :: Q.Connection -> SomeException -> m ()
     chanExceptionHandler conn e = do
-      logException l "RabbitMQ channel closed" e
       hooks.onChannelException e `catch` logException l "onChannelException hook threw an exception"
       case (Q.isNormalChannelClose e, fromException e) of
         (True, _) ->
@@ -153,7 +152,9 @@ openConnectionWithRetries l RabbitMqOpts {..} hooks = do
         (_, Just (Q.ConnectionClosedException {})) ->
           Log.info l $
             Log.msg (Log.val "RabbitMQ connection is closed, not attempting to reopen channel")
-        _ -> openChan conn
+        _ -> do
+          logException l "RabbitMQ channel closed" e
+          openChan conn
 
 logException :: (MonadIO m) => Logger -> String -> SomeException -> m ()
 logException l m (SomeException e) = do

diff --git a/services/gundeck/src/Gundeck/Monad.hs b/services/gundeck/src/Gundeck/Monad.hs
@@ -45,6 +45,7 @@ where
 import Bilge hiding (Request, header, options, statusCode)
 import Bilge.RPC
 import Cassandra
+import Control.Concurrent.Async (AsyncCancelled)
 import Control.Error
 import Control.Exception (throwIO)
 import Control.Lens
@@ -170,10 +171,13 @@ runDirect :: Env -> Gundeck a -> IO a
 runDirect e m =
   runClient (e ^. cstate) (runReaderT (unGundeck m) e)
     `catch` ( \(exception :: SomeException) -> do
-                Log.err (e ^. applog) $
-                  Log.msg ("IO Exception occurred" :: ByteString)
-                    . Log.field "message" (displayException exception)
-                    . Log.field "request" (unRequestId (e ^. reqId))
+                case fromException exception of
+                  Nothing ->
+                    Log.err (e ^. applog) $
+                      Log.msg ("IO Exception occurred" :: ByteString)
+                        . Log.field "message" (displayException exception)
+                        . Log.field "request" (unRequestId (e ^. reqId))
+                  Just (_ :: AsyncCancelled) -> pure ()
                 throwIO exception
             )