diff --git a/channels/chan_iax2.c b/channels/chan_iax2.c
index 164b217a921fbf37ee70024ba08a02d94b54f2c1..70122bc3cd2cd468874a9a1999ee48d7e706a790 100644
--- a/channels/chan_iax2.c
+++ b/channels/chan_iax2.c
@@ -5560,8 +5560,11 @@ static int authenticate_reply(struct chan_iax2_pvt *p, struct sockaddr_in *sin,
 			ast_mutex_unlock(&iaxsl[callno]);
 			if ((peer = realtime_peer(peer_name, NULL))) {
 				ast_mutex_lock(&iaxsl[callno]);
-				if (!(p = iaxs[callno]))
+				if (!(p = iaxs[callno])) {
+					if (ast_test_flag(peer, IAX_TEMPONLY))
+						destroy_peer(peer);
 					return -1;
+				}
 				res = authenticate(p->challenge, peer->secret,peer->outkey, authmethods, &ied, sin, &p->ecx, &p->dcx);
 				if (ast_test_flag(peer, IAX_TEMPONLY))
 					destroy_peer(peer);