diff --git a/src/lib.rs b/src/lib.rs
index a9862c4..4241c36 100644
--- a/src/lib.rs
+++ b/src/lib.rs
@@ -142,6 +142,7 @@ pub struct Builder<T> {
 
 /// Wraps errors from either the listener or the TLS Acceptor
 #[derive(Debug, Error)]
+#[non_exhaustive]
 pub enum Error<LE: std::error::Error, TE: std::error::Error> {
     /// An error that arose from the listener ([AsyncAccept::Error])
     #[error("{0}")]
@@ -149,6 +150,11 @@ pub enum Error<LE: std::error::Error, TE: std::error::Error> {
     /// An error that occurred during the TLS accept handshake
     #[error("{0}")]
     TlsAcceptError(#[source] TE),
+    // TODO: is there any way we could include thee original connection, or maybe some
+    // info about it here?
+    /// The TLS handshake timed out
+    #[error("Timeout during TLS handshake")]
+    HandshakeTimeout(),
 }
 
 impl<A: AsyncAccept, T> TlsListener<A, T>
@@ -226,7 +232,7 @@ where
                 }
                 // The handshake timed out, try getting another connection from the
                 // queue
-                Poll::Ready(Some(Err(_))) => continue,
+                Poll::Ready(Some(Err(_))) => Poll::Ready(Some(Err(Error::HandshakeTimeout()))),
                 _ => Poll::Pending,
             };
         }