diff --git a/res/res_pjsip_refer.c b/res/res_pjsip_refer.c
index 8890d235b5689416eb405b1301071e03ffce8c52..a20bf6ba4de04da5cc7e68fb0ef9a5a3943e515e 100644
--- a/res/res_pjsip_refer.c
+++ b/res/res_pjsip_refer.c
@@ -1019,6 +1019,10 @@ static int refer_incoming_invite_request(struct ast_sip_session *session, struct
 	ast_debug(3, "INVITE with Replaces being attempted.  '%s' --> '%s'\n",
 		ast_channel_name(session->channel), ast_channel_name(invite.channel));
 
+	/* Unhold the channel now, as later we are not having access to it anymore */
+	ast_queue_unhold(session->channel);
+	ast_queue_frame(session->channel, &ast_null_frame);
+
 	if (!invite.bridge) {
 		struct ast_channel *chan = session->channel;