backend/libinput: Fix SIGSEGV found in low-memory fuzzing

Stack trace:

    #0  0x00007f17081f5b99 in wl_list_insert (list=list@entry=0x2d8, elm=elm@entry=0x7ffe7f7e85d0)
        at ../wayland-1.21.0/src/wayland-util.c:48
    emersion#1  0x00007f17081f5f2e in wl_signal_emit_mutable (signal=signal@entry=0x2d8, data=data@entry=0x7ffe7f7e8660)
        at ../wayland-1.21.0/src/wayland-server.c:2167
    emersion#2  0x00007f170815a971 in handle_switch_toggle (wlr_switch=0x2a0, event=0x55d5ba13dc00)
        at ../backend/libinput/switch.c:50
    swaywm#3  handle_libinput_event (event=0x55d5ba13dc00, backend=0x55d5b975d740) at ../backend/libinput/events.c:234
    swaywm#4  handle_libinput_readable (fd=<optimized out>, mask=<optimized out>, _backend=<optimized out>)
        at ../backend/libinput/backend.c:58
    swaywm#5  handle_libinput_readable (fd=fd@entry=34, mask=mask@entry=1, _backend=_backend@entry=0x55d5b975d740)
        at ../backend/libinput/backend.c:48
    swaywm#6  0x00007f170815c110 in backend_start (wlr_backend=0x55d5b975d740) at ../backend/libinput/backend.c:109
    swaywm#7  0x00007f1708160996 in multi_backend_start (wlr_backend=0x55d5b97583d0) at ../backend/multi/backend.c:32

backend/libinput/events.c

@@ -133,19 +133,12 @@ static void handle_device_added(struct wlr_libinput_backend *backend,
 }
 
 static void handle_device_removed(struct wlr_libinput_backend *backend,
-		struct libinput_device *libinput_dev) {
-	int vendor = libinput_device_get_id_vendor(libinput_dev);
-	int product = libinput_device_get_id_product(libinput_dev);
-	const char *name = libinput_device_get_name(libinput_dev);
+		struct wlr_libinput_input_device *dev) {
+	int vendor = libinput_device_get_id_vendor(dev->handle);
+	int product = libinput_device_get_id_product(dev->handle);
+	const char *name = libinput_device_get_name(dev->handle);
 	wlr_log(WLR_DEBUG, "Removing %s [%d:%d]", name, vendor, product);
 
-	struct wlr_libinput_input_device *dev =
-		libinput_device_get_user_data(libinput_dev);
-	if (dev == NULL) {
-		wlr_log(WLR_ERROR, "libinput_device has no wlr_libinput_input_device");
-		return;
-	}
-
 	destroy_libinput_input_device(dev);
 }
 
@@ -155,12 +148,18 @@ void handle_libinput_event(struct wlr_libinput_backend *backend,
 	struct wlr_libinput_input_device *dev =
 		libinput_device_get_user_data(libinput_dev);
 	enum libinput_event_type event_type = libinput_event_get_type(event);
+
+	if (dev == NULL && event_type != LIBINPUT_EVENT_DEVICE_ADDED) {
+		wlr_log(WLR_ERROR, "libinput_device has no wlr_libinput_input_device");
+		return;
+	}
+
 	switch (event_type) {
 	case LIBINPUT_EVENT_DEVICE_ADDED:
 		handle_device_added(backend, libinput_dev);
 		break;
 	case LIBINPUT_EVENT_DEVICE_REMOVED:
-		handle_device_removed(backend, libinput_dev);
+		handle_device_removed(backend, dev);
 		break;
 	case LIBINPUT_EVENT_KEYBOARD_KEY:
 		handle_keyboard_key(event, &dev->keyboard);