diff --git a/jitterbuf.c b/jitterbuf.c
index e319776ca03f6601a8a3d7f66fe66f9a6041581e..2c31007402e63dec0d25311373db4a4fdf011107 100755
--- a/jitterbuf.c
+++ b/jitterbuf.c
@@ -58,7 +58,7 @@ void jb_reset(jitterbuf *jb)
 
 	/* initialize length */
 	jb->info.current = jb->info.target = 0; 
-	jb->info.silence = 1; 
+	jb->info.silence_begin_ts = -1; 
 }
 
 jitterbuf * jb_new() 
@@ -405,7 +405,7 @@ static void jb_dbginfo(jitterbuf *jb)
 		jb->info.frames_in, jb->info.frames_out, jb->info.frames_late, jb->info.frames_lost, jb->info.frames_dropped, jb->info.frames_cur);
 	
 	jb_dbg("jitter=%ld current=%ld target=%ld min=%ld sil=%d len=%d len/fcur=%ld\n",
-		jb->info.jitter, jb->info.current, jb->info.target, jb->info.min, jb->info.silence, jb->info.current - jb->info.min, 
+		jb->info.jitter, jb->info.current, jb->info.target, jb->info.min, jb->info.silence_begin_ts, jb->info.current - jb->info.min, 
 		jb->info.frames_cur ? (jb->info.current - jb->info.min)/jb->info.frames_cur : -8);
 	if (jb->info.frames_in > 0) 
 		jb_dbg("jb info: Loss PCT = %ld%%, Late PCT = %ld%%\n",
@@ -506,7 +506,7 @@ static int _jb_get(jitterbuf *jb, jb_frame *frameout, long now)
 	jb->info.last_voice_ts += jb->info.last_voice_ms;
 
 	/* let's work on non-silent case first */
-	if (!jb->info.silence) { 
+	if (!jb->info.silence_begin_ts) { 
 		/* we want to grow */
 		if ((diff > 0) && 
 			/* we haven't grown in the delay length */
@@ -528,7 +528,7 @@ static int _jb_get(jitterbuf *jb, jb_frame *frameout, long now)
 			jb->info.last_voice_ts -= jb->info.last_voice_ms;
 
 			if (frame->type == JB_TYPE_SILENCE) 
-				jb->info.silence = 1;
+				jb->info.silence_begin_ts = frame->ts;
 
 			*frameout = *frame;
 			jb->info.frames_out++;
@@ -625,31 +625,46 @@ static int _jb_get(jitterbuf *jb, jb_frame *frameout, long now)
 		 * here, plus handle last_voice_ts a bit differently */
       
 		/* to disable silent special case altogether, just uncomment this: */
-		/* jb->info.silence = 0; */
+		/* jb->info.silence_begin_ts = 0; */
 
 		frame = queue_get(jb, now - jb->info.current);
 		if (!frame) {
 			return JB_NOFRAME;
+		} else if (frame->type != JB_TYPE_VOICE) {
+			/* normal case; in silent mode, got a non-voice frame */
+			*frameout = *frame;
+			return JB_OK;
 		}
-		if (frame && frame->type == JB_TYPE_VOICE) {
+		if (frame->ts < jb->info.silence_begin_ts) {
+			/* voice frame is late */
+			*frameout = *frame;
+			/* rewind last_voice, since we're just dumping */
+			jb->info.last_voice_ts -= jb->info.last_voice_ms;
+			jb->info.frames_out++;
+			decrement_losspct(jb);
+			jb->info.frames_late++;
+			jb->info.frames_lost--;
+			jb_dbg("l");
+			/*jb_warn("\nlate: wanted=%ld, this=%ld, next=%ld\n", jb->info.last_voice_ts - jb->info.current, frame->ts, queue_next(jb));
+			jb_warninfo(jb); */
+			return JB_DROP;
+		} else {
+			/* voice frame */
 			/* try setting current to target right away here */
 			jb->info.current = jb->info.target;
-			jb->info.silence = 0;
+			jb->info.silence_begin_ts = 0;
 			jb->info.last_voice_ts = frame->ts + jb->info.current + frame->ms;
 			jb->info.last_voice_ms = frame->ms;
 			*frameout = *frame;
 			jb_dbg("V");
 			return JB_OK;
 		}
-		/* normal case; in silent mode, got a non-voice frame */
-		*frameout = *frame;
-		return JB_OK;
 	}
 }
 
 long jb_next(jitterbuf *jb) 
 {
-	if (jb->info.silence) {
+	if (jb->info.silence_begin_ts) {
 		long next = queue_next(jb);
 		if (next > 0) { 
 			history_get(jb);
diff --git a/jitterbuf.h b/jitterbuf.h
index aa73e805b8b529731683004c430819547f264753..224751630019ecec38f29f8bece22ce70cf06d76 100755
--- a/jitterbuf.h
+++ b/jitterbuf.h
@@ -65,7 +65,7 @@ typedef struct jb_info {
 	long losspct; 		/* recent lost frame percentage (* 1000) */
 	long last_voice_ts;	/* the last ts that was read from the jb - in receiver's time */
 	long last_voice_ms;	/* the duration of the last voice frame */
-	long silence;		/* we are presently playing out silence */
+	long silence_begin_ts;	/* the time of the last CNG frame, when in silence */
 	long last_adjustment;   /* the time of the last adjustment */
 
 	/* settings */