diff --git a/src/calls-call-record-row.c b/src/calls-call-record-row.c
index b850b58..ef7e453 100644
--- a/src/calls-call-record-row.c
+++ b/src/calls-call-record-row.c
@@ -330,6 +330,12 @@ static void
 contact_name_cb (CallsCallRecordRow *self)
 {
   const gchar *name = NULL;
+  g_autofree gchar *target = NULL;
+  GAction *act = g_action_map_lookup_action (self->action_map, "copy-number");
+
+  g_object_get (G_OBJECT (self->record),
+                "target", &target,
+                NULL);
 
   if (self->contact)
     {
@@ -342,21 +348,17 @@ contact_name_cb (CallsCallRecordRow *self)
     }
   else
     {
-      g_autofree gchar *target = NULL;
-
-      g_object_get (G_OBJECT (self->record),
-                    "target", &target,
-                    NULL);
-
       if (!g_strcmp0 (target, ""))
         {
           gtk_label_set_text (self->target, ANONYMOUS_CALLER);
           gtk_actionable_set_action_name (GTK_ACTIONABLE (self->button), NULL);
+          g_simple_action_set_enabled (G_SIMPLE_ACTION (act), FALSE);
         }
       else
         {
           gtk_label_set_text (self->target, target);
           gtk_actionable_set_action_name (GTK_ACTIONABLE (self->button), "app.dial");
+          g_simple_action_set_enabled (G_SIMPLE_ACTION (act), TRUE);
         }
     }
 }
@@ -648,9 +650,6 @@ calls_call_record_row_init (CallsCallRecordRow *self)
   act = g_action_map_lookup_action (self->action_map, "delete-call");
   g_simple_action_set_enabled (G_SIMPLE_ACTION (act), TRUE);
 
-  act = g_action_map_lookup_action (self->action_map, "copy-number");
-  g_simple_action_set_enabled (G_SIMPLE_ACTION (act), TRUE);
-
   self->gesture = gtk_gesture_long_press_new (GTK_WIDGET (self->event_box));
   gtk_gesture_single_set_touch_only (GTK_GESTURE_SINGLE (self->gesture), TRUE);
   g_signal_connect (self->gesture, "pressed", G_CALLBACK (long_pressed), self);
diff --git a/src/ui/call-record-row.ui b/src/ui/call-record-row.ui
index 929d465..7c7cb1c 100644
--- a/src/ui/call-record-row.ui
+++ b/src/ui/call-record-row.ui
@@ -116,6 +116,7 @@
         <!-- Translators: This is a phone number -->
         <attribute name="label" translatable="yes">_Copy number</attribute>
         <attribute name="action">copy-number</attribute>
+        <attribute name="hidden-when">action-disabled</attribute>
       </item>
     </section>
   </menu>