diff --git a/channels/chan_sip.c b/channels/chan_sip.c
index ff96898954cd8f3f4bbc7fbd48b88a82e349d198..319ce5769f9b09b1e080747b864dd0619dfa6155 100644
--- a/channels/chan_sip.c
+++ b/channels/chan_sip.c
@@ -27372,7 +27372,9 @@ static struct sip_peer *build_peer(const char *name, struct ast_variable *v, str
 		found++;
 		/* we've unlinked the peer from the peers container but not unlinked from the peers_by_ip container yet
 		  this leads to a wrong refcounter and the peer object is never destroyed */
-		ao2_t_unlink(peers_by_ip, peer, "ao2_unlink peer from peers_by_ip table");
+		if (!ast_sockaddr_isnull(&peer->addr)) {
+			ao2_t_unlink(peers_by_ip, peer, "ao2_unlink peer from peers_by_ip table");
+		}
 		if (!(peer->the_mark))
 			firstpass = 0;
 	} else {