diff --git a/server.go b/server.go
index c98b8d7..7d86181 100644
--- a/server.go
+++ b/server.go
@@ -150,6 +150,8 @@ func (srv *Server) Shutdown(ctx context.Context) error {
 		srv.closed = true
 		srv.shutdown = true
 
+		srv.tryCloseDoneLocked()
+
 		// Close all active listeners.
 		for _, cancel := range srv.listeners {
 			cancel()