diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 6404e1928..33ebbf6b4 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -157,7 +157,7 @@ class FileDownloader(object):
             return
         speed = float(byte_counter) / elapsed
         if speed > rate_limit:
-            time.sleep((byte_counter / rate_limit) - elapsed)
+            time.sleep(max((byte_counter / rate_limit) - elapsed, 0))
 
     def temp_name(self, filename):
         """Returns a temporary filename for the given filename."""