diff --git a/lib/tls/mbedtls/ssl.c b/lib/tls/mbedtls/ssl.c
index da25a2712b1d0fb6bb11b81448a6c660e4adc750..e0e134c58bc29e35862b5131c1c19cf7a22e7273 100644
--- a/lib/tls/mbedtls/ssl.c
+++ b/lib/tls/mbedtls/ssl.c
@@ -75,7 +75,7 @@ lws_ssl_capable_read(struct lws *wsi, unsigned char *buf, int len)
 	}
 #endif
 #if defined(LWS_WITH_STATS)
-	if (!wsi->seen_rx) {
+	if (!wsi->seen_rx && wsi->accept_start_us) {
                 lws_stats_atomic_bump(wsi->context, pt,
                 		      LWSSTATS_MS_SSL_RX_DELAY,
 				time_in_microseconds() - wsi->accept_start_us);
diff --git a/lib/tls/openssl/ssl.c b/lib/tls/openssl/ssl.c
index eb55b0937a850eb71aa4faa9f77509ebc58e391c..59250fbc1c9afccd144d39a0a8cae69219357c32 100644
--- a/lib/tls/openssl/ssl.c
+++ b/lib/tls/openssl/ssl.c
@@ -216,7 +216,7 @@ lws_ssl_capable_read(struct lws *wsi, unsigned char *buf, int len)
 	}
 #endif
 #if defined(LWS_WITH_STATS)
-	if (!wsi->seen_rx) {
+	if (!wsi->seen_rx && wsi->accept_start_us) {
                 lws_stats_atomic_bump(wsi->context, pt, LWSSTATS_MS_SSL_RX_DELAY,
 				time_in_microseconds() - wsi->accept_start_us);
                 lws_stats_atomic_bump(wsi->context, pt, LWSSTATS_C_SSL_CONNS_HAD_RX, 1);
diff --git a/lib/tls/tls-server.c b/lib/tls/tls-server.c
index 1f727cf792c51ac5ba2628bbbbb9e52b93e7a02b..95ff4eff0aa043bec6d7fac396c8d3974282e775 100644
--- a/lib/tls/tls-server.c
+++ b/lib/tls/tls-server.c
@@ -312,6 +312,7 @@ lws_server_socket_service_ssl(struct lws *wsi, lws_sockfd_type accept_fd)
 		/* normal SSL connection processing path */
 
 #if defined(LWS_WITH_STATS)
+		/* only set this the first time around */
 		if (!wsi->accept_start_us)
 			wsi->accept_start_us = time_in_microseconds();
 #endif
@@ -340,7 +341,8 @@ lws_server_socket_service_ssl(struct lws *wsi, lws_sockfd_type accept_fd)
 		lws_stats_atomic_bump(wsi->context, pt,
 				      LWSSTATS_C_SSL_CONNECTIONS_ACCEPTED, 1);
 #if defined(LWS_WITH_STATS)
-		lws_stats_atomic_bump(wsi->context, pt,
+		if (wsi->accept_start_us)
+			lws_stats_atomic_bump(wsi->context, pt,
 				      LWSSTATS_MS_SSL_CONNECTIONS_ACCEPTED_DELAY,
 				      time_in_microseconds() - wsi->accept_start_us);
 		wsi->accept_start_us = time_in_microseconds();