diff --git a/net/ipv4/netfilter/ip_tables.c b/net/ipv4/netfilter/ip_tables.c
index e25b4469ffc976e5db7e7252edece6cfc5b320bc..9574df2f7e61d251e8bb6e2b155abf951af0e9ef 100644
--- a/net/ipv4/netfilter/ip_tables.c
+++ b/net/ipv4/netfilter/ip_tables.c
@@ -61,9 +61,9 @@ ip_packet_match(const struct iphdr *ip,
 	if (ipinfo->flags & IPT_F_NO_DEF_MATCH)
 		return true;
 
-	if (NF_INVF(ipinfo, IPT_INV_SRCIP,
+	if (NF_INVF(ipinfo, IPT_INV_SRCIP, ipinfo->smsk.s_addr &&
 		    (ip->saddr & ipinfo->smsk.s_addr) != ipinfo->src.s_addr) ||
-	    NF_INVF(ipinfo, IPT_INV_DSTIP,
+	    NF_INVF(ipinfo, IPT_INV_DSTIP, ipinfo->dmsk.s_addr &&
 		    (ip->daddr & ipinfo->dmsk.s_addr) != ipinfo->dst.s_addr))
 		return false;