diff --git a/poezio/config.py b/poezio/config.py
index b1d09810..c68e8452 100644
--- a/poezio/config.py
+++ b/poezio/config.py
@@ -519,6 +519,7 @@ def check_create_cache_dir():
 
     try:
         makedirs(CACHE_DIR)
+        makedirs(path.join(CACHE_DIR, 'avatars'))
         makedirs(path.join(CACHE_DIR, 'images'))
     except OSError:
         pass
diff --git a/poezio/core/handlers.py b/poezio/core/handlers.py
index c376b67f..e5d1f244 100644
--- a/poezio/core/handlers.py
+++ b/poezio/core/handlers.py
@@ -14,7 +14,7 @@ import sys
 import time
 from datetime import datetime
 from hashlib import sha1, sha512
-from os import path
+from os import path, makedirs
 
 from slixmpp import InvalidJID
 from slixmpp.xmlstream.stanzabase import StanzaBase, ElementBase
@@ -384,7 +384,21 @@ class HandlerCore:
         contact = roster[jid]
         if not contact:
             return
-        log.debug('Received vCard avatar update from %s', jid)
+        avatar_hash = pres['vcard_temp_update']['photo']
+        log.debug('Received vCard avatar update from %s: %s', jid, avatar_hash)
+
+        # First check whether we have it in cache.
+        cache_dir = path.join(CACHE_DIR, 'avatars', jid)
+        cached_path = path.join(cache_dir, avatar_hash)
+        try:
+            with open(cached_path, 'rb') as avatar_file:
+                contact.avatar = avatar_file.read()
+            log.debug('Using cached avatar')
+            return
+        except OSError:
+            pass
+
+        # If we didn’t have any, query the vCard instead.
         try:
             result = yield from self.core.xmpp['xep_0054'].get_vcard(jid,
                                                                      cached=True,
@@ -396,6 +410,14 @@ class HandlerCore:
             return
         log.debug('Received %s avatar: %s', jid, avatar['TYPE'])
 
+        # Now we save the data on the file system to not have to request it again.
+        try:
+            makedirs(cache_dir)
+            with open(cached_path, 'wb') as avatar_file:
+                avatar_file.write(contact.avatar)
+        except OSError:
+            pass
+
     def on_nick_received(self, message):
         """
         Called when a pep notification for an user nickname