diff --git a/res/res_rtp_asterisk.c b/res/res_rtp_asterisk.c
index 1fb4e1ceb5dceb92b6608c55bcc55406581f7c10..3f65f1344d3131d8088cdc3760611498be51559e 100644
--- a/res/res_rtp_asterisk.c
+++ b/res/res_rtp_asterisk.c
@@ -4866,12 +4866,15 @@ static void ast_rtp_remote_address_set(struct ast_rtp_instance *instance, struct
 	 * by checking if we're passive. Without this, we only send the pending packets once a new SSL packet is
 	 * received in __rtp_recvfrom.  If rtp->ice, this is instead done on_ice_complete
 	 */
-	if (!rtp->ice && rtp->dtls.dtls_setup == AST_RTP_DTLS_SETUP_PASSIVE) {
+#ifdef HAVE_PJPROJECT
+	if (rtp->ice) {
+		return;
+	}
+#endif
+	if (rtp->dtls.dtls_setup == AST_RTP_DTLS_SETUP_PASSIVE) {
 		dtls_srtp_flush_pending(instance, rtp);
 	}
 #endif
-
-	return;
 }
 
 /*! \brief Write t140 redundacy frame