fix history cache entry accounting

src/history.c

@@ -36,7 +36,7 @@ int neigh_history_add_entry_json(struct json_object *jarray,
 		return -1;
 
 	hwaddr_ntoa(he->macaddr, macstr);
-	remtime = timer_remaining_ms(&he->delete_timer);
+	remtime = timer_remaining_ms(&he->delete_timer) / 1000;
 
 	e = json_object_new_object();
 	es = json_object_new_object();
@@ -45,7 +45,6 @@ int neigh_history_add_entry_json(struct json_object *jarray,
 	json_object_object_add(e, "first_seen", json_object_new_int(he->createtime));
 	json_object_object_add(e, "last_seen", json_object_new_int(he->lastseen));
 	json_object_object_add(e, "ageout", json_object_new_int(remtime));
-	json_object_object_add(e, "alive", json_object_new_boolean(he->alive));
 	json_object_object_add(e, "mediatype", json_object_new_int(he->type));
 	json_object_object_add(e, "wan_stats", es);
 	json_object_object_add(es, "tx_packets", json_object_new_int64(he->ws.ul_packets));
@@ -113,7 +112,7 @@ int json_get_key_value(struct json_object *jo, const char *key, void *val)
 		(*(double *)val) = json_object_get_double(o);
 		break;
 	case json_type_int:
-		(*(int *)val) = json_object_get_int(o);
+		(*(int *)val) = json_object_get_int(o);	//FIXME: (u)int64_t
 		break;
 	case json_type_string:
 		memcpy(val, json_object_get_string(o), json_object_get_string_len(o));
@@ -145,68 +144,61 @@ int neigh_history_load_from_json_file(void *priv, const char *file)
 	}
 
 	len = json_object_array_length(jarray);
-	dbg("Num history entries = %d\n", len);
+	dbg("Num history entries in '%s' = %d\n", file, len);
 
 	for (i = 0; i < len; i++) {
-		uint64_t txpkts = 0, txbytes = 0;
-		uint64_t rxpkts = 0, rxbytes = 0;
+		struct neigh_history_entry he = {0};
 		struct json_object *je, *je_ws;
-		struct neigh_wanstats ws = {0};
-		uint32_t firstseen = 0;
-		uint32_t lastseen = 0;
-		uint32_t remtime = 0;
-		char hostname[256] = {0};
 		uint8_t macaddr[6] = {0};
-		char macstr[24] = {0};
-		int mediatype = 0;
-		int alive = -1;
+		char macstr[18] = {0};
+		int createtime = 0;
+		int lastseen = 0;
+		int timeout = 0;
+		int type = 0;
 
 		je = json_object_array_get_idx(jarray, i);
-		json_get_key_value(je, "macaddress", &macstr);
-		if (!hwaddr_aton(macstr, macaddr)) {
+		json_get_key_value(je, "macaddress", macstr);
+		if (!hwaddr_aton(macstr, he.macaddr)) {
 			dbg("Invalid value in history!\n");
 			return -1;
 		}
-
-		json_get_key_value(je, "hostname", &hostname);
-		json_get_key_value(je, "first_seen", &firstseen);
+		json_get_key_value(je, "hostname", he.hostname);
+		json_get_key_value(je, "first_seen", &createtime);
 		json_get_key_value(je, "last_seen", &lastseen);
-		json_get_key_value(je, "ageout", &remtime);
-		json_get_key_value(je, "alive", &alive);
-		json_get_key_value(je, "mediatype", &mediatype);
+		json_get_key_value(je, "ageout", &timeout);
+		json_get_key_value(je, "mediatype", &type);
 		json_object_object_get_ex(je, "wan_stats", &je_ws);
-		json_get_key_value(je_ws, "tx_packets", &txpkts);
-		json_get_key_value(je_ws, "tx_bytes", &txbytes);
-		json_get_key_value(je_ws, "rx_packets", &rxpkts);
-		json_get_key_value(je_ws, "rx_bytes", &rxbytes);
-		ws.ul_packets = txpkts;
-		ws.ul_bytes = txbytes;
-		ws.dl_packets = rxpkts;
-		ws.dl_bytes = rxbytes;
-
-		dbg("macaddr = %s\n", macstr);
-		dbg("hostname = %s\n", hostname);
-		dbg("firstseen = %u\n", firstseen);
-		dbg("lastseen = %u\n", lastseen);
-		dbg("remtime = %u\n", remtime);
-		dbg("alive = %d\n", alive);
-		dbg("mediatype = %d\n", mediatype);
-		dbg("tx-packets = %" PRIu64"\n", txpkts);
-		dbg("tx-bytes = %" PRIu64"\n", txbytes);
-		dbg("rx-packets = %" PRIu64"\n", rxpkts);
-		dbg("rx-bytes = %" PRIu64"\n", rxbytes);
-
-		dbg("%s: Now = %jd, Entry-firstseen = %u, lastseen = %u, remtime = %u\n",
-			__func__, (intmax_t)now, firstseen, lastseen, remtime);
-
-		if (difftime(now, (time_t)lastseen) > remtime / 1000) {
+		json_get_key_value(je_ws, "tx_packets", &he.ws.ul_packets);
+		json_get_key_value(je_ws, "tx_bytes", &he.ws.ul_bytes);
+		json_get_key_value(je_ws, "rx_packets", &he.ws.dl_packets);
+		json_get_key_value(je_ws, "rx_bytes", &he.ws.dl_bytes);
+
+		he.createtime = createtime;
+		he.lastseen = lastseen;
+		he.timeout = timeout;
+		he.type = type;
+
+		dbg("macstr = '%s' (macaddr = " MACFMT" )\n", macstr, MAC2STR(he.macaddr));
+		dbg("hostname = %s\n", he.hostname);
+		dbg("firstseen = %jd\n", (intmax_t)he.createtime);
+		dbg("lastseen = %jd\n", (intmax_t)he.lastseen);
+		dbg("remtime = %u\n", he.timeout);
+		dbg("mediatype = %d\n", he.type);
+		dbg("tx-packets = %" PRIu64"\n", he.ws.ul_packets);
+		dbg("tx-bytes = %" PRIu64"\n", he.ws.ul_bytes);
+		dbg("rx-packets = %" PRIu64"\n", he.ws.dl_packets);
+		dbg("rx-bytes = %" PRIu64"\n", he.ws.dl_bytes);
+
+		dbg("%s: Now = %jd, lastseen = %jd, remtime = %u\n", __func__,
+		    (intmax_t)now, (intmax_t)he.lastseen, he.timeout);
+
+		if (difftime(now, (time_t)he.lastseen) > he.timeout) {
 			dbg("Dropping stale history entry %d\n", i);
 			continue;
 		}
 
-		dbg("Adding from history entry %d\n", i);
-		neigh_history_enqueue(priv, macaddr, hostname, mediatype, &ws,
-				      remtime, alive);
+		dbg("Adding history entry " MACFMT " from file\n", MAC2STR(he.macaddr));
+		neigh_history_entry_add(priv, &he);
 	}
 
 	return 0;

src/neigh.c

@@ -108,6 +108,8 @@ struct neigh_history_entry *neigh_history_entry_create(void *priv,
 						       uint8_t *macaddr,
 						       const char *hostname,
 						       enum neigh_type type,
+						       time_t firstseen,
+						       time_t lastseen,
 						       uint32_t timeout)
 {
 	struct neigh_history_entry *e;
@@ -125,19 +127,29 @@ struct neigh_history_entry *neigh_history_entry_create(void *priv,
 	if (hostname && hostname[0] != '\0')
 		strncpy(e->hostname, hostname, 255);
 
-	time(&e->createtime);
-	time(&e->lastseen);
 	e->timeout = timeout;
 	e->priv = priv;
-
 	timer_init(&e->delete_timer, neigh_history_entry_delete_timer_cb);
-	timer_set(&e->delete_timer, e->timeout);
+
+	if (!firstseen)
+		time(&e->createtime);
+	else
+		e->createtime = firstseen;
+
+	if (!lastseen)
+		time(&e->lastseen);
+	else
+		e->lastseen = lastseen;
+
+	if (e->timeout)
+		timer_set(&e->delete_timer, e->timeout * 1000);
 
 	dbg("CREATE history entry: " MACFMT "\n", MAC2STR(macaddr));
 
 	return e;
 }
 
+#if 0
 int neigh_history_entry_update_lastseen(void *priv, uint8_t *macaddr)
 {
 	struct topologyd_private *p = (struct topologyd_private *)priv;
@@ -149,10 +161,11 @@ int neigh_history_entry_update_lastseen(void *priv, uint8_t *macaddr)
 
 	return 0;
 }
+#endif
 
 int neigh_history_enqueue(void *priv, uint8_t *macaddr, const char *hostname,
 			  enum neigh_type type, struct neigh_wanstats *ws,
-			  uint32_t timeout, bool alive)
+			  uint32_t timeout)
 {
 	struct topologyd_private *p = (struct topologyd_private *)priv;
 	struct neigh_history_entry *e = NULL;
@@ -160,9 +173,10 @@ int neigh_history_enqueue(void *priv, uint8_t *macaddr, const char *hostname,
 
 	e = neigh_history_lookup(&p->neigh_q, macaddr);
 	if (e) {
+		/* stop ageing history as neigh is live */
 		e->timeout = timeout;
-		timer_set(&e->delete_timer, e->timeout);
-		e->alive = alive;
+		timer_del(&e->delete_timer);
+		e->alive = true;
 
 		/* set neigh type iff wifi type determined */
 		if (type == NEIGH_TYPE_WIFI)
@@ -183,27 +197,65 @@ int neigh_history_enqueue(void *priv, uint8_t *macaddr, const char *hostname,
 		return 0;
 	}
 
-	dbg("No history of " MACFMT " found! Add new..\n", MAC2STR(macaddr));
+	dbg("No history for " MACFMT " found! Add new..\n", MAC2STR(macaddr));
 
-	e = neigh_history_entry_create(priv, macaddr, hostname, type, timeout);
+	e = neigh_history_entry_create(priv, macaddr, hostname, type, 0, 0, timeout);
 	if (e) {
 		int idx = neigh_hash(macaddr);
 
-		e->alive = alive;
-		hlist_add_head(&e->hlist, &p->neigh_q.history[idx]);
-		p->neigh_q.num_history++;
+		/* stop ageing history as neigh is live */
+		e->timeout = timeout;
+		timer_del(&e->delete_timer);
+		e->alive = true;
 		if (ws) {
-			e->ws.ul_packets += ws->ul_packets;
-			e->ws.ul_bytes += ws->ul_bytes;
-			e->ws.dl_packets += ws->dl_packets;
-			e->ws.dl_bytes += ws->dl_bytes;
+			e->ws.ul_packets = ws->ul_packets;
+			e->ws.ul_bytes = ws->ul_bytes;
+			e->ws.dl_packets = ws->dl_packets;
+			e->ws.dl_bytes = ws->dl_bytes;
 		}
+
+		hlist_add_head(&e->hlist, &p->neigh_q.history[idx]);
+		p->neigh_q.num_history++;
 		return 1;
 	}
 
 	return -1;
 }
 
+/* This function is only for loading history cache from file */
+int neigh_history_entry_add(void *priv, struct neigh_history_entry *he)
+{
+	struct topologyd_private *p = (struct topologyd_private *)priv;
+	struct neigh_history_entry *e = NULL;
+	int key = neigh_hash(he->macaddr);
+
+
+	hlist_for_each_entry(e, &p->neigh_q.history[key], hlist) {
+		if (!memcmp(e->macaddr, he->macaddr, 6)) {
+			/* history entry already present */
+			return -1;
+		}
+	}
+
+	dbg("Adding history for " MACFMT "\n", MAC2STR(he->macaddr));
+	e = neigh_history_entry_create(priv, he->macaddr, he->hostname,
+				       he->type, he->createtime, he->lastseen,
+				       he->timeout);
+	if (!e)
+		return -1;
+
+	e->alive = false;	/* because no neigh entry yet for this */
+	e->ws.ul_packets = he->ws.ul_packets;
+	e->ws.ul_bytes = he->ws.ul_bytes;
+	e->ws.dl_packets = he->ws.dl_packets;
+	e->ws.dl_bytes = he->ws.dl_bytes;
+
+	hlist_add_head(&e->hlist, &p->neigh_q.history[key]);
+	p->neigh_q.num_history++;
+
+	return 0;
+}
+
 int neigh_history_dequeue(void *priv, uint8_t *macaddr)
 {
 	struct topologyd_private *p = (struct topologyd_private *)priv;
@@ -286,13 +338,25 @@ struct neigh_entry *neigh_entry_create(void *priv, uint8_t *macaddr, uint16_t st
 static void neigh_entry_delete(struct neigh_entry *e)
 {
 	if (e) {
-		//TODO: neigh_history_entry_update
-		dbg("Update history for " MACFMT" before delete\n", MAC2STR(e->macaddr));
-		neigh_history_entry_update_lastseen(e->priv, e->macaddr);
-		neigh_history_enqueue(e->priv, e->macaddr, e->hostname,
-				      e->type, &e->ws,
-				      NEIGH_HISTORY_AGEOUT_DEFAULT * 1000,
-				      e->unreachable ? false : true);
+		struct neigh_history_entry *he;
+		struct topologyd_private *priv = (struct topologyd_private *)e->priv;
+
+		dbg("Sync history for " MACFMT" before delete\n", MAC2STR(e->macaddr));
+		he = neigh_history_lookup(&priv->neigh_q, e->macaddr);
+		if (he) {
+			dbg("update history: +ul-pkts = %llu, +dl-pkts = %llu\n",
+			    e->ws.ul_packets, e->ws.dl_packets);
+			he->alive = false;
+			time(&he->lastseen);
+			he->ws.ul_packets += e->ws.ul_packets;
+			he->ws.ul_bytes += e->ws.ul_bytes;
+			he->ws.dl_packets += e->ws.dl_packets;
+			he->ws.dl_bytes += e->ws.dl_bytes;
+
+			/* (re)start history ageing timer */
+			if (he->timeout)
+				timer_set(&he->delete_timer, he->timeout * 1000);
+		}
 
 		neigh_flush_ip_entry_stats(e->priv, &e->ipv4);
 
@@ -587,18 +651,20 @@ void neigh_set_unreachable(void *nq, const char *ifname, int val)
 
 		hlist_for_each_entry(e, &q->table[idx], hlist) {
 			if (!strncmp(e->ifname, ifname, sizeof(e->ifname))) {
-				struct neigh_history_entry *he = NULL;
+				//struct neigh_history_entry *he = NULL;
 
 				e->unreachable = val;
 				dbg(MACFMT " marking it %sreachable\n",
 				    MAC2STR(e->macaddr), val ? "un" : "");
 
+				/*
 				he = neigh_history_lookup(nq, e->macaddr);
 				if (he) {
 					dbg("Update history last-seen time\n");
 					time(&he->lastseen);
 					he->alive = false;
 				}
+				*/
 			}
 		}
 	}
@@ -805,7 +871,7 @@ struct neigh_entry *neigh_enqueue(void *nq, uint8_t *macaddr, uint16_t state,
 		/* add/update history entry for this neigh */
 		neigh_history_enqueue(priv, e->macaddr, e->hostname, e->type,
 				      &e->ws,
-				      NEIGH_HISTORY_AGEOUT_DEFAULT * 1000);
+				      NEIGH_HISTORY_AGEOUT_DEFAULT);
 #endif
 
 		return NULL;

src/netlink.c

@@ -213,8 +213,7 @@ static int topologyd_handle_neigh_tbl_change(struct topologyd_private *priv, boo
 		/* add/update history entry for this neigh */
 		neigh_history_enqueue(priv, new->macaddr, new->hostname,
 				      new->type, &new->ws,
-				      NEIGH_HISTORY_AGEOUT_DEFAULT * 1000,
-				      true);
+				      NEIGH_HISTORY_AGEOUT_DEFAULT);
 	}
 
 	/* update bridge port_nos on which the hosts are last seen */
@@ -638,8 +637,7 @@ int topologyd_get_known_neighbors(struct topologyd_private *priv, char *ifname)
 				/* add/update history entry for this neigh */
 				neigh_history_enqueue(priv, new->macaddr, new->hostname,
 						      new->type, &new->ws,
-						      NEIGH_HISTORY_AGEOUT_DEFAULT * 1000,
-						      true);
+						      NEIGH_HISTORY_AGEOUT_DEFAULT);
 			}
 
 			nl_object_put((struct nl_object *)neigh);

src/topology.h

@@ -312,9 +312,10 @@ struct node *lookup_node(struct topologyd_private *priv, struct list_head *list,
 int nfct_get_entries_nolo(struct topologyd_private *priv);
 
 
+int neigh_history_entry_add(void *priv, struct neigh_history_entry *he);
 int neigh_history_enqueue(void *priv, uint8_t *macaddr, const char *hostname,
 			  enum neigh_type type, struct neigh_wanstats *ws,
-			  uint32_t timeout, bool alive);
+			  uint32_t timeout);
 
 int neigh_history_load_from_json_file(void *priv, const char *file);
 int neigh_history_store_to_json_file(void *priv, const char *file);

src/ubus.c

@@ -775,8 +775,7 @@ static void topologyd_wifi_sta_event_handler(struct topologyd_private *p,
 						      new->hostname,
 						      new->type,
 						      &new->ws,
-						      NEIGH_HISTORY_AGEOUT_DEFAULT * 1000,
-						      true);
+						      NEIGH_HISTORY_AGEOUT_DEFAULT);
 			}
 		} else if (del) {
 			dbg("%s: neigh_dequeue " MACFMT " (priv = %p) >>>\n",