diff --git a/libfprint/fp-device.c b/libfprint/fp-device.c
index a0f2172..bf40585 100644
--- a/libfprint/fp-device.c
+++ b/libfprint/fp-device.c
@@ -113,7 +113,8 @@ fp_device_cancelled_cb (GCancellable *cancellable, FpDevice *self)
                          fp_device_cancel_in_idle_cb,
                          self,
                          NULL);
-  g_source_attach (priv->current_idle_cancel_source, NULL);
+  g_source_attach (priv->current_idle_cancel_source,
+                   g_task_get_context (priv->current_task));
   g_source_unref (priv->current_idle_cancel_source);
 }
 
diff --git a/libfprint/fpi-device.c b/libfprint/fpi-device.c
index 2cd0637..78a7939 100644
--- a/libfprint/fpi-device.c
+++ b/libfprint/fpi-device.c
@@ -334,12 +334,18 @@ fpi_device_add_timeout (FpDevice      *device,
 {
   FpDevicePrivate *priv = fp_device_get_instance_private (device);
   FpDeviceTimeoutSource *source;
+  GMainContext *context;
 
   source = (FpDeviceTimeoutSource *) g_source_new (&timeout_funcs,
                                                    sizeof (FpDeviceTimeoutSource));
   source->device = device;
 
-  g_source_attach (&source->source, NULL);
+  if (priv->current_task)
+    context = g_task_get_context (priv->current_task);
+  else
+    context = g_main_context_get_thread_default ();
+
+  g_source_attach (&source->source, context);
   g_source_set_callback (&source->source, (GSourceFunc) func, user_data, destroy_notify);
   g_source_set_ready_time (&source->source,
                            g_source_get_time (&source->source) + interval * (guint64) 1000);
@@ -930,7 +936,8 @@ fpi_device_return_task_in_idle (FpDevice              *device,
                          data,
                          (GDestroyNotify) fpi_device_task_return_data_free);
 
-  g_source_attach (priv->current_task_idle_return_source, NULL);
+  g_source_attach (priv->current_task_idle_return_source,
+                   g_task_get_context (priv->current_task));
   g_source_unref (priv->current_task_idle_return_source);
 }