diff --git a/src/seasidecache.cpp b/src/seasidecache.cpp
index 3aa7775..3e2104e 100644
--- a/src/seasidecache.cpp
+++ b/src/seasidecache.cpp
@@ -2173,7 +2173,8 @@ bool SeasideCache::updateContactIndexing(const QContact &oldContact, const QCont
                     resolveUnknownAddresses(address.first, address.second, item);
                 }
 
-                m_phoneNumberIds.insert(address.second, iid);
+                if (!m_phoneNumberIds.contains(address.second, iid))
+                    m_phoneNumberIds.insert(address.second, iid);
             }
         }