diff --git a/src/xfaces.c b/src/xfaces.c
index d2d88ce5021..7e1eb33521c 100644
--- a/src/xfaces.c
+++ b/src/xfaces.c
@@ -1631,9 +1631,12 @@ unload_color (f, pixel)
      unsigned long pixel;
 {
 #ifdef HAVE_X_WINDOWS
-  BLOCK_INPUT;
-  x_free_colors (f, &pixel, 1);
-  UNBLOCK_INPUT;
+  if (pixel != -1)
+    {
+      BLOCK_INPUT;
+      x_free_colors (f, &pixel, 1);
+      UNBLOCK_INPUT;
+    }
 #endif
 }