diff --git a/apps/app_chanspy.c b/apps/app_chanspy.c
index 580038326f1a506097b3b5ba5bc45cd5b993d566..c3d0f901e1ccf82f7275fcb192328f69302e9db8 100644
--- a/apps/app_chanspy.c
+++ b/apps/app_chanspy.c
@@ -455,6 +455,7 @@ static int start_spying(struct ast_autochan *autochan, const char *spychan_name,
 
 	ast_log(LOG_NOTICE, "Attaching %s to %s\n", spychan_name, autochan->chan->name);
 
+	ast_set_flag(audiohook, AST_AUDIOHOOK_TRIGGER_SYNC | AST_AUDIOHOOK_SMALL_QUEUE);
 	res = ast_audiohook_attach(autochan->chan, audiohook);
 
 	if (!res && ast_test_flag(autochan->chan, AST_FLAG_NBRIDGE) && (peer = ast_bridged_channel(autochan->chan))) {
diff --git a/include/asterisk/audiohook.h b/include/asterisk/audiohook.h
index 1012822b49f83662d8417f38b7c85e65f60b319e..b5e147d98f89ce2383622d2b6ab0b2867b28b34d 100644
--- a/include/asterisk/audiohook.h
+++ b/include/asterisk/audiohook.h
@@ -58,6 +58,10 @@ enum ast_audiohook_flags {
 	AST_AUDIOHOOK_TRIGGER_WRITE = (2 << 0), /*!< Audiohook wants to be triggered when writing audio out */
 	AST_AUDIOHOOK_WANTS_DTMF = (1 << 1),    /*!< Audiohook also wants to receive DTMF frames */
 	AST_AUDIOHOOK_TRIGGER_SYNC = (1 << 2),  /*!< Audiohook wants to be triggered when both sides have combined audio available */
+	/*! Audiohooks with this flag set will not allow for a large amount of samples to build up on its
+	 * slinfactories. We will flush the factories if they contain too many samples.
+	 */
+	AST_AUDIOHOOK_SMALL_QUEUE = (1 << 3),
 };
 
 #define AST_AUDIOHOOK_SYNC_TOLERANCE 100 /*< Tolerance in milliseconds for audiohooks synchronization */
diff --git a/main/audiohook.c b/main/audiohook.c
index a452abd9f290a68d0cc176aeb59ac37dd4b9823d..1527a2d8287cdff7d563a412b055cf951c1e7afb 100644
--- a/main/audiohook.c
+++ b/main/audiohook.c
@@ -123,6 +123,7 @@ int ast_audiohook_write_frame(struct ast_audiohook *audiohook, enum ast_audiohoo
 	struct ast_slinfactory *factory = (direction == AST_AUDIOHOOK_DIRECTION_READ ? &audiohook->read_factory : &audiohook->write_factory);
 	struct ast_slinfactory *other_factory = (direction == AST_AUDIOHOOK_DIRECTION_READ ? &audiohook->write_factory : &audiohook->read_factory);
 	struct timeval *rwtime = (direction == AST_AUDIOHOOK_DIRECTION_READ ? &audiohook->read_time : &audiohook->write_time), previous_time = *rwtime;
+	int our_factory_samples;
 	int our_factory_ms;
 	int other_factory_samples;
 	int other_factory_ms;
@@ -130,7 +131,8 @@ int ast_audiohook_write_frame(struct ast_audiohook *audiohook, enum ast_audiohoo
 	/* Update last feeding time to be current */
 	*rwtime = ast_tvnow();
 
-	our_factory_ms = ast_tvdiff_ms(*rwtime, previous_time) + (ast_slinfactory_available(factory) / 8);
+	our_factory_samples = ast_slinfactory_available(factory);
+	our_factory_ms = ast_tvdiff_ms(*rwtime, previous_time) + (our_factory_samples / 8);
 	other_factory_samples = ast_slinfactory_available(other_factory);
 	other_factory_ms = other_factory_samples / 8;
 
@@ -141,6 +143,14 @@ int ast_audiohook_write_frame(struct ast_audiohook *audiohook, enum ast_audiohoo
 		ast_slinfactory_flush(other_factory);
 	}
 
+	if (ast_test_flag(audiohook, AST_AUDIOHOOK_SMALL_QUEUE) && (our_factory_samples > 640 || other_factory_samples > 640)) {
+		if (option_debug) {
+			ast_log(LOG_DEBUG, "Audiohook %p has stale audio in its factories. Flushing them both\n", audiohook);
+		}
+		ast_slinfactory_flush(factory);
+		ast_slinfactory_flush(other_factory);
+	}
+
 	/* Write frame out to respective factory */
 	ast_slinfactory_feed(factory, frame);