diff --git a/sound/core/seq_device.c b/sound/core/seq_device.c
index c4acf17e9f5e50..e40a2cba5002a8 100644
--- a/sound/core/seq_device.c
+++ b/sound/core/seq_device.c
@@ -148,8 +148,10 @@ void snd_seq_device_load_drivers(void)
 	flush_work(&autoload_work);
 }
 EXPORT_SYMBOL(snd_seq_device_load_drivers);
+#define cancel_autoload_drivers()	cancel_work_sync(&autoload_work)
 #else
 #define queue_autoload_drivers() /* NOP */
+#define cancel_autoload_drivers() /* NOP */
 #endif
 
 /*
@@ -159,6 +161,7 @@ static int snd_seq_device_dev_free(struct snd_device *device)
 {
 	struct snd_seq_device *dev = device->device_data;
 
+	cancel_autoload_drivers();
 	put_device(&dev->dev);
 	return 0;
 }