diff --git a/include/net/ip_vs.h b/include/net/ip_vs.h index 9a83bff32569..8dc80d5d42dc 100644 --- a/include/net/ip_vs.h +++ b/include/net/ip_vs.h @@ -1021,7 +1021,9 @@ struct ip_vs_dest { struct rcu_head rcu_head; struct list_head t_list; /* in dest_trash */ - unsigned int in_rs_table:1; /* we are in rs_table */ + unsigned long rcu_state; /* GP cookie on unlink (trash or rehash) */ + unsigned int in_rs_table:1, /* we are in rs_table */ + need_gp:1; /* Need GP on rehash */ }; /* The scheduler object */ @@ -1685,7 +1687,6 @@ static inline bool __ip_vs_conn_get(struct ip_vs_conn *cp) /* put back the conn without restarting its timer */ static inline void __ip_vs_conn_put(struct ip_vs_conn *cp) { - smp_mb__before_atomic(); refcount_dec(&cp->refcnt); } void ip_vs_conn_put(struct ip_vs_conn *cp); @@ -1923,7 +1924,7 @@ static inline void ip_vs_dest_put_and_free(struct ip_vs_dest *dest) kfree(dest); } -void ip_vs_dest_update_overload(struct ip_vs_dest *dest, bool overload); +void ip_vs_dest_update_overload(struct ip_vs_dest *dest); /* Check if dest is in overloaded state */ static inline bool ip_vs_dest_is_overloaded(const struct ip_vs_dest *dest) diff --git a/net/netfilter/ipvs/ip_vs_conn.c b/net/netfilter/ipvs/ip_vs_conn.c index 94f25fb7d87e..83d8a9153a34 100644 --- a/net/netfilter/ipvs/ip_vs_conn.c +++ b/net/netfilter/ipvs/ip_vs_conn.c @@ -293,7 +293,6 @@ static inline int ip_vs_conn_hash(struct ip_vs_conn *cp) cp->flags |= IP_VS_CONN_F_HASHED; WRITE_ONCE(cp->hn0.hash_key, hash_key); WRITE_ONCE(cp->hn1.hash_key, hash_key2); - refcount_inc(&cp->refcnt); hlist_bl_add_head_rcu(&cp->hn0.node, head); if (use2) hlist_bl_add_head_rcu(&cp->hn1.node, head2); @@ -319,11 +318,27 @@ static inline bool ip_vs_conn_unlink(struct ip_vs_conn *cp) struct hlist_bl_head *head, *head2; u32 hash_key, hash_key2; struct ip_vs_rht *t; - bool ret = false; bool use2; + /* No more users and no running/pending timer except us? */ + if (!refcount_dec_if_one(&cp->refcnt)) + return false; + + /* The ONE_PACKET connection is special: + * - the flag is present only for normal connections, not for templates + * - its timer is never started + * - it is never hashed, so you can not find it and delete it + * - it is not synced + * - it has only one user + * - it is freed after use immediately + */ if (cp->flags & IP_VS_CONN_F_ONE_PACKET) - return refcount_dec_if_one(&cp->refcnt); + return true; + + /* Add the needed ACQUIRE barrier between the above + * refcount_dec_if_one() call and the read of cp fields + */ + smp_acquire__after_ctrl_dep(); rcu_read_lock(); local_bh_disable(); @@ -337,15 +352,11 @@ static inline bool ip_vs_conn_unlink(struct ip_vs_conn *cp) false /* new_hash2 */, &head, &head2); if (cp->flags & IP_VS_CONN_F_HASHED) { - /* Decrease refcnt and unlink conn only if we are last user */ - if (use2 == ip_vs_conn_use_hash2(cp) && - refcount_dec_if_one(&cp->refcnt)) { - hlist_bl_del_rcu(&cp->hn0.node); - if (use2) - hlist_bl_del_rcu(&cp->hn1.node); - cp->flags &= ~IP_VS_CONN_F_HASHED; - ret = true; - } + /* Unlink conn as we are the last user */ + hlist_bl_del_rcu(&cp->hn0.node); + if (use2) + hlist_bl_del_rcu(&cp->hn1.node); + cp->flags &= ~IP_VS_CONN_F_HASHED; } conn_tab_unlock(head, head2); @@ -353,7 +364,7 @@ static inline bool ip_vs_conn_unlink(struct ip_vs_conn *cp) local_bh_enable(); rcu_read_unlock(); - return ret; + return true; } @@ -579,6 +590,8 @@ struct ip_vs_conn *ip_vs_conn_out_get(const struct ip_vs_conn_param *p) ip_vs_addr_equal(p->af, p->vaddr, &cp->caddr) && ip_vs_addr_equal(p->af, p->caddr, saddr) && + (!p->vport ^ + (!(cp->flags & IP_VS_CONN_F_NO_CPORT))) && p->protocol == cp->protocol) { if (__ip_vs_conn_get(cp)) goto out; @@ -621,9 +634,12 @@ static void __ip_vs_conn_put_timer(struct ip_vs_conn *cp) { unsigned long t = (cp->flags & IP_VS_CONN_F_ONE_PACKET) ? 0 : cp->timeout; - mod_timer(&cp->timer, jiffies+t); - __ip_vs_conn_put(cp); + /* Handoff our reference to the timer if it was just armed, + * put the reference if timer was already pending. + */ + if (mod_timer(&cp->timer, jiffies + t)) + __ip_vs_conn_put(cp); } void ip_vs_conn_put(struct ip_vs_conn *cp) @@ -1154,7 +1170,7 @@ ip_vs_bind_dest(struct ip_vs_conn *cp, struct ip_vs_dest *dest) atomic_inc(&dest->activeconns); tc = atomic_inc_return(&dest->totalconns); if (tc >= READ_ONCE(dest->u_threshold_val)) - ip_vs_dest_update_overload(dest, true); + ip_vs_dest_update_overload(dest); } else { /* It is a persistent connection/template, so increase the persistent connection counter */ @@ -1247,7 +1263,7 @@ static inline void ip_vs_unbind_dest(struct ip_vs_conn *cp) atomic_dec(&dest->activeconns); tc = atomic_fetch_dec(&dest->totalconns); if (tc <= READ_ONCE(dest->l_threshold_val)) - ip_vs_dest_update_overload(dest, false); + ip_vs_dest_update_overload(dest); } else { /* It is a persistent connection/template, so decrease the persistent connection counter */ @@ -1322,36 +1338,48 @@ static void ip_vs_conn_rcu_free(struct rcu_head *head) kmem_cache_free(ip_vs_conn_cachep, cp); } -/* Try to delete connection while not holding reference */ +/* Try to steal the reference from the pending timer. + * It can be called concurrently and always under RCU lock + * to protect from connection deletion from others. + */ +static bool ip_vs_conn_del_reference(struct ip_vs_conn *cp) +{ + /* Steal reference from the pending timer */ + if (!timer_delete(&cp->timer)) + return false; + + /* Drop cp->control chain too */ + if (cp->control) + cp->timeout = 0; + return true; +} + +/* Try to delete connection while not holding reference. + * Used when we need to remove the connection immediately + * and without overloading the timer processing. + * OTOH, we use ip_vs_conn_expire_now() when we prefer to + * leave the current context (packet processing) and to + * remove the connection from timer. + */ static void ip_vs_conn_del(struct ip_vs_conn *cp) { - if (timer_delete(&cp->timer)) { - /* Drop cp->control chain too */ - if (cp->control) - cp->timeout = 0; + if (ip_vs_conn_del_reference(cp)) ip_vs_conn_expire(&cp->timer); - } -} - -/* Try to delete connection while holding reference */ -static void ip_vs_conn_del_put(struct ip_vs_conn *cp) -{ - if (timer_delete(&cp->timer)) { - /* Drop cp->control chain too */ - if (cp->control) - cp->timeout = 0; - __ip_vs_conn_put(cp); - ip_vs_conn_expire(&cp->timer); - } else { - __ip_vs_conn_put(cp); - } } +/* Connection is removed in the following steps: + * - timer expires (with ref) or connection is deleted (with stolen ref) + * - there should be no more users except us (pending/running timer, + * n_control>0 or refcnt>1) + */ static void ip_vs_conn_expire(struct timer_list *t) { struct ip_vs_conn *cp = timer_container_of(cp, t, timer); struct netns_ipvs *ipvs = cp->ipvs; + rcu_read_lock(); + +repeat: /* * do I control anybody? */ @@ -1361,23 +1389,17 @@ static void ip_vs_conn_expire(struct timer_list *t) /* Unlink conn if not referenced anymore */ if (likely(ip_vs_conn_unlink(cp))) { struct ip_vs_conn *ct = cp->control; - - /* delete the timer if it is activated by other users */ - timer_delete(&cp->timer); + bool next = false; /* does anybody control me? */ if (ct) { - bool has_ref = !cp->timeout && __ip_vs_conn_get(ct); - ip_vs_control_del(cp); /* Drop CTL or non-assured TPL if not used anymore */ - if (has_ref && !atomic_read(&ct->n_control) && + if (!cp->timeout && !atomic_read(&ct->n_control) && (!(ct->flags & IP_VS_CONN_F_TEMPLATE) || !(ct->state & IP_VS_CTPL_S_ASSURED))) { IP_VS_DBG(4, "drop controlling connection\n"); - ip_vs_conn_del_put(ct); - } else if (has_ref) { - __ip_vs_conn_put(ct); + next = ip_vs_conn_del_reference(ct); } } @@ -1405,7 +1427,11 @@ static void ip_vs_conn_expire(struct timer_list *t) else call_rcu(&cp->rcu_head, ip_vs_conn_rcu_free); atomic_dec(&ipvs->conn_count); - return; + if (next) { + cp = ct; + goto repeat; + } + goto out; } expire_later: @@ -1413,13 +1439,16 @@ static void ip_vs_conn_expire(struct timer_list *t) refcount_read(&cp->refcnt), atomic_read(&cp->n_control)); - refcount_inc(&cp->refcnt); - cp->timeout = 60*HZ; + if (cp->timeout || atomic_read(&cp->n_control)) + cp->timeout = 60 * HZ; if (ipvs->sync_state & IP_VS_STATE_MASTER) ip_vs_sync_conn(ipvs, cp, sysctl_sync_threshold(ipvs)); __ip_vs_conn_put_timer(cp); + +out: + rcu_read_unlock(); } /* Modify timer, so that it expires as soon as possible. @@ -1432,7 +1461,7 @@ static void ip_vs_conn_expire(struct timer_list *t) void ip_vs_conn_expire_now(struct ip_vs_conn *cp) { /* Using mod_timer_pending will ensure the timer is not - * modified after the final timer_delete in ip_vs_conn_expire. + * started without giving it new reference. */ if (timer_pending(&cp->timer) && time_after(cp->timer.expires, jiffies)) @@ -1500,9 +1529,9 @@ ip_vs_conn_new(const struct ip_vs_conn_param *p, int dest_af, spin_lock_init(&cp->lock); /* - * Set the entry is referenced by the current thread before hashing - * it in the table, so that other thread run ip_vs_random_dropentry - * but cannot drop this entry. + * Set the entry as referenced by the current thread before starting + * the timer and giving it our reference to allow other threads to + * run ip_vs_random_dropentry() or any deletion method safely. */ refcount_set(&cp->refcnt, 1); diff --git a/net/netfilter/ipvs/ip_vs_core.c b/net/netfilter/ipvs/ip_vs_core.c index 8a3c880fdb26..f3eb72e95024 100644 --- a/net/netfilter/ipvs/ip_vs_core.c +++ b/net/netfilter/ipvs/ip_vs_core.c @@ -1305,6 +1305,7 @@ static inline bool is_new_conn_expected(const struct ip_vs_conn *cp, * In fwmark-VS actual vaddr and vport are unknown to IPVS * 2) Real Server and Virtual Service were NOT configured without port: * This is to allow match of different VS to the same RS ip-addr + * 3) Client port is not 0 */ struct ip_vs_conn *ip_vs_new_conn_out(struct ip_vs_service *svc, struct ip_vs_dest *dest, @@ -1328,7 +1329,7 @@ struct ip_vs_conn *ip_vs_new_conn_out(struct ip_vs_service *svc, /* check pre-requisites are satisfied */ if (svc->fwmark) return NULL; - if (!vport || !dport) + if (!vport || !dport || !cport) return NULL; /* for persistent service first create connection template */ diff --git a/net/netfilter/ipvs/ip_vs_ctl.c b/net/netfilter/ipvs/ip_vs_ctl.c index ce9fe20d6106..be818bf7d380 100644 --- a/net/netfilter/ipvs/ip_vs_ctl.c +++ b/net/netfilter/ipvs/ip_vs_ctl.c @@ -64,7 +64,8 @@ int ip_vs_get_debug_level(void) /* Protos */ -static void __ip_vs_del_service(struct ip_vs_service *svc, bool cleanup); +static void __ip_vs_del_service(struct ip_vs_service *svc, + struct list_head *dest_trash, bool cleanup); #ifdef CONFIG_IP_VS_IPV6 @@ -918,6 +919,11 @@ static void ip_vs_rs_hash(struct netns_ipvs *ipvs, struct ip_vs_dest *dest) */ hash = ip_vs_rs_hashkey(dest->af, &dest->addr, port); + /* Need RCU grace period when rehashing */ + if (dest->need_gp) { + cond_synchronize_rcu(dest->rcu_state); + dest->need_gp = 0; + } hlist_add_head_rcu(&dest->d_list, &ipvs->rs_table[hash]); dest->in_rs_table = 1; } @@ -1111,6 +1117,8 @@ ip_vs_trash_get_dest(struct ip_vs_service *svc, int dest_af, struct ip_vs_dest *dest; struct netns_ipvs *ipvs = svc->ipvs; + lockdep_assert_held(&ipvs->service_mutex); + /* * Find the destination in trash */ @@ -1144,24 +1152,57 @@ out: return dest; } -/* Put destination in trash */ -static void ip_vs_trash_put_dest(struct netns_ipvs *ipvs, +/* Add destination in temp list for trash */ +static void ip_vs_trash_add_dest(struct netns_ipvs *ipvs, struct ip_vs_dest *dest, unsigned long istart, - bool cleanup) + struct list_head *dest_trash) { - spin_lock_bh(&ipvs->dest_trash_lock); IP_VS_DBG_BUF(3, "Moving dest %s:%u into trash, dest->refcnt=%d\n", IP_VS_DBG_ADDR(dest->af, &dest->addr), ntohs(dest->port), refcount_read(&dest->refcnt)); + /* dest lives in trash with reference */ + list_add(&dest->t_list, dest_trash); + dest->idle_start = istart; +} + +/* Put destinations in trash */ +static void ip_vs_trash_put_dests(struct netns_ipvs *ipvs, + struct list_head *dest_trash, bool cleanup) +{ + spin_lock_bh(&ipvs->dest_trash_lock); if (list_empty(&ipvs->dest_trash) && !cleanup) mod_timer(&ipvs->dest_trash_timer, jiffies + (IP_VS_DEST_TRASH_PERIOD >> 1)); - /* dest lives in trash with reference */ - list_add(&dest->t_list, &ipvs->dest_trash); - dest->idle_start = istart; + list_splice(dest_trash, &ipvs->dest_trash); spin_unlock_bh(&ipvs->dest_trash_lock); } +/* Put destination back in trash */ +static void ip_vs_trash_put_back(struct netns_ipvs *ipvs, + struct ip_vs_dest *dest) +{ + LIST_HEAD(dest_trash); + + ip_vs_trash_add_dest(ipvs, dest, dest->idle_start, &dest_trash); + ip_vs_trash_put_dests(ipvs, &dest_trash, false); +} + +/* Put destinations in trash and save the RCU state */ +static void ip_vs_trash_save_dests(struct netns_ipvs *ipvs, + struct list_head *dest_trash, bool cleanup) +{ + struct ip_vs_dest *dest; + unsigned long rcu_state; + + if (!list_empty(dest_trash)) { + /* Remember when dests were removed and added to dest_trash */ + rcu_state = get_state_synchronize_rcu(); + list_for_each_entry(dest, dest_trash, t_list) + dest->rcu_state = rcu_state; + ip_vs_trash_put_dests(ipvs, dest_trash, cleanup); + } +} + static void ip_vs_dest_rcu_free(struct rcu_head *head) { struct ip_vs_dest *dest; @@ -1304,12 +1345,12 @@ void ip_vs_stats_free(struct ip_vs_stats *stats) } } -/* Update overload flag based on number of dest conns and lower/upper +/* Update overload state based on number of dest conns and lower/upper * connection thresholds: - * - conns reach u_threshold and exceed it: set the flag - * - conns go below l_threshold (or 75% of u_threshold): clear the flag + * - conns reach u_threshold and exceed it: set the overload state + * - conns go below l_threshold (or 75% of u_threshold): clear the state */ -static void __ip_vs_dest_update_overload(struct ip_vs_dest *dest, bool overload) +static void __ip_vs_dest_update_overload(struct ip_vs_dest *dest) { int conns; u32 l, u; @@ -1320,10 +1361,18 @@ static void __ip_vs_dest_update_overload(struct ip_vs_dest *dest, bool overload) /* Low threshold defaults to 75% of upper threshold */ l = READ_ONCE(dest->l_threshold) ? : (u - (u >> 2)); conns = atomic_read(&dest->totalconns); - if (conns >= (overload ? l : u)) { + if (conns >= (ip_vs_dest_is_overloaded(dest) ? l : u)) { WRITE_ONCE(dest->u_threshold_val, INT_MAX); WRITE_ONCE(dest->l_threshold_val, l); - return; + /* Commit thresholds before repeating the check, + * it is paired with atomic_fetch_dec() in + * ip_vs_unbind_dest() to avoid entering the + * overload state forever. + */ + smp_mb(); + conns = atomic_read(&dest->totalconns); + if (conns >= l) + return; } } @@ -1331,10 +1380,10 @@ static void __ip_vs_dest_update_overload(struct ip_vs_dest *dest, bool overload) WRITE_ONCE(dest->l_threshold_val, 0); } -void ip_vs_dest_update_overload(struct ip_vs_dest *dest, bool overload) +void ip_vs_dest_update_overload(struct ip_vs_dest *dest) { spin_lock_bh(&dest->dst_lock); - __ip_vs_dest_update_overload(dest, overload); + __ip_vs_dest_update_overload(dest); spin_unlock_bh(&dest->dst_lock); } @@ -1369,8 +1418,16 @@ __ip_vs_update_dest(struct ip_vs_service *svc, struct ip_vs_dest *dest, if ((udest->conn_flags & IP_VS_CONN_F_FWD_MASK) != IP_VS_DFWD_METHOD(dest) || udest->tun_type != dest->tun_type || - udest->tun_port != dest->tun_port) + udest->tun_port != dest->tun_port) { + bool unlinked = dest->in_rs_table; + ip_vs_rs_unhash(dest); + if (unlinked) { + dest->rcu_state = get_state_synchronize_rcu(); + /* Need grace period in this or next dest update */ + dest->need_gp = 1; + } + } /* set the tunnel info */ dest->tun_type = udest->tun_type; @@ -1406,13 +1463,10 @@ __ip_vs_update_dest(struct ip_vs_service *svc, struct ip_vs_dest *dest, if (READ_ONCE(dest->u_threshold) != udest->u_threshold || READ_ONCE(dest->l_threshold) != udest->l_threshold) { - bool overload; - spin_lock_bh(&dest->dst_lock); WRITE_ONCE(dest->u_threshold, udest->u_threshold); WRITE_ONCE(dest->l_threshold, udest->l_threshold); - overload = ip_vs_dest_is_overloaded(dest); - __ip_vs_dest_update_overload(dest, overload); + __ip_vs_dest_update_overload(dest); spin_unlock_bh(&dest->dst_lock); } @@ -1477,6 +1531,7 @@ ip_vs_new_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) if (ret < 0) goto err_stats; + dest->u_threshold_val = INT_MAX; dest->af = udest->af; dest->protocol = svc->protocol; dest->vaddr = svc->addr; @@ -1556,6 +1611,11 @@ ip_vs_add_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) dest = ip_vs_trash_get_dest(svc, udest->af, &daddr, dport); if (dest != NULL) { + /* Readers may still follow the n_list, d_list or estimator list + * links. Wait before reusing any of them. + */ + cond_synchronize_rcu(dest->rcu_state); + IP_VS_DBG_BUF(3, "Get destination %s:%u from trash, " "dest->refcnt=%d, service %u/%s:%u\n", IP_VS_DBG_ADDR(udest->af, &daddr), ntohs(dport), @@ -1566,11 +1626,14 @@ ip_vs_add_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) ret = ip_vs_start_estimator(svc->ipvs, &dest->stats); /* On error put back dest into the trash */ - if (ret < 0) - ip_vs_trash_put_dest(svc->ipvs, dest, dest->idle_start, - false); - else + if (ret < 0) { + ip_vs_trash_put_back(svc->ipvs, dest); + } else { + /* need_gp matters only on dest edit, reinit it */ + dest->need_gp = 0; + __ip_vs_update_dest(svc, dest, udest, 1); + } } else { /* * Allocate and initialize the dest structure @@ -1634,7 +1697,7 @@ ip_vs_edit_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) * Delete a destination (must be already unlinked from the service) */ static void __ip_vs_del_dest(struct netns_ipvs *ipvs, struct ip_vs_dest *dest, - bool cleanup) + struct list_head *dest_trash, bool cleanup) { ip_vs_stop_estimator(ipvs, &dest->stats); @@ -1643,7 +1706,7 @@ static void __ip_vs_del_dest(struct netns_ipvs *ipvs, struct ip_vs_dest *dest, */ ip_vs_rs_unhash(dest); - ip_vs_trash_put_dest(ipvs, dest, 0, cleanup); + ip_vs_trash_add_dest(ipvs, dest, 0, dest_trash); /* Queue up delayed work to expire all no destination connections. * No-op when CONFIG_SYSCTL is disabled. @@ -1693,6 +1756,7 @@ ip_vs_del_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) { struct ip_vs_dest *dest; __be16 dport = udest->port; + LIST_HEAD(dest_trash); /* We use function that requires RCU lock */ rcu_read_lock(); @@ -1712,7 +1776,9 @@ ip_vs_del_dest(struct ip_vs_service *svc, struct ip_vs_dest_user_kern *udest) /* * Delete the destination */ - __ip_vs_del_dest(svc->ipvs, dest, false); + __ip_vs_del_dest(svc->ipvs, dest, &dest_trash, false); + + ip_vs_trash_save_dests(svc->ipvs, &dest_trash, false); return 0; } @@ -2057,7 +2123,8 @@ out: * - The service must be unlinked, unlocked and not referenced! * - We are called under _bh lock */ -static void __ip_vs_del_service(struct ip_vs_service *svc, bool cleanup) +static void __ip_vs_del_service(struct ip_vs_service *svc, + struct list_head *dest_trash, bool cleanup) { struct ip_vs_dest *dest, *nxt; struct ip_vs_scheduler *old_sched; @@ -2091,7 +2158,7 @@ static void __ip_vs_del_service(struct ip_vs_service *svc, bool cleanup) */ list_for_each_entry_safe(dest, nxt, &svc->destinations, n_list) { __ip_vs_unlink_dest(svc, dest, 0); - __ip_vs_del_dest(svc->ipvs, dest, cleanup); + __ip_vs_del_dest(svc->ipvs, dest, dest_trash, cleanup); } /* @@ -2114,7 +2181,8 @@ static void __ip_vs_del_service(struct ip_vs_service *svc, bool cleanup) /* * Unlink a service from list and try to delete it if its refcnt reached 0 */ -static void ip_vs_unlink_service(struct ip_vs_service *svc, bool cleanup) +static void ip_vs_unlink_service(struct ip_vs_service *svc, + struct list_head *dest_trash, bool cleanup) { ip_vs_unregister_conntrack(svc); /* Hold svc to avoid double release from dest_trash */ @@ -2124,7 +2192,7 @@ static void ip_vs_unlink_service(struct ip_vs_service *svc, bool cleanup) */ ip_vs_svc_unhash(svc); - __ip_vs_del_service(svc, cleanup); + __ip_vs_del_service(svc, dest_trash, cleanup); } /* @@ -2134,12 +2202,14 @@ static int ip_vs_del_service(struct ip_vs_service *svc) { struct netns_ipvs *ipvs; struct ip_vs_rht *t, *p; + LIST_HEAD(dest_trash); int ns; if (svc == NULL) return -EEXIST; ipvs = svc->ipvs; - ip_vs_unlink_service(svc, false); + ip_vs_unlink_service(svc, &dest_trash, false); + ip_vs_trash_save_dests(ipvs, &dest_trash, false); /* Drop the table if no more services */ ns = ip_vs_get_num_services(ipvs); @@ -2190,6 +2260,7 @@ static int ip_vs_flush(struct netns_ipvs *ipvs, bool cleanup) struct hlist_bl_node *ne; struct hlist_bl_node *e; struct ip_vs_rht *t, *p; + LIST_HEAD(dest_trash); /* Stop the resizer and drop the tables */ if (!test_and_set_bit(IP_VS_WORK_SVC_NORESIZE, &ipvs->work_flags)) @@ -2199,8 +2270,9 @@ static int ip_vs_flush(struct netns_ipvs *ipvs, bool cleanup) if (ip_vs_get_num_services(ipvs)) { ip_vs_rht_walk_buckets(ipvs->svc_table, head) { hlist_bl_for_each_entry_safe(svc, e, ne, head, s_list) - ip_vs_unlink_service(svc, cleanup); + ip_vs_unlink_service(svc, &dest_trash, cleanup); } + ip_vs_trash_save_dests(ipvs, &dest_trash, cleanup); } /* Unregister the hash table and release it after RCU grace period */ diff --git a/net/netfilter/ipvs/ip_vs_ftp.c b/net/netfilter/ipvs/ip_vs_ftp.c index 9e3e005a8263..44de99f4d187 100644 --- a/net/netfilter/ipvs/ip_vs_ftp.c +++ b/net/netfilter/ipvs/ip_vs_ftp.c @@ -62,6 +62,17 @@ static unsigned short ports[IP_VS_APP_MAX_PORTS] = {21, 0}; module_param_array(ports, ushort, &ports_count, 0444); MODULE_PARM_DESC(ports, "Ports to monitor for FTP control commands"); +static bool is_control_port(u16 port) +{ + unsigned int i; + + for (i = 0; i < ports_count; i++) { + if (ports[i] == port) + return true; + } + return false; +} + static char *ip_vs_ftp_data_ptr(struct sk_buff *skb, struct ip_vs_iphdr *ipvsh) { @@ -319,6 +330,10 @@ static int ip_vs_ftp_out(struct ip_vs_app *app, struct ip_vs_conn *cp, return 1; } + /* Do not redirect data to control ports */ + if (!port || is_control_port(ntohs(port))) + return 0; + /* Now update or create a connection entry for it */ { struct ip_vs_conn_param p; @@ -368,7 +383,7 @@ static int ip_vs_ftp_out(struct ip_vs_app *app, struct ip_vs_conn *cp, buf_len = strlen(buf); ct = nf_ct_get(skb, &ctinfo); - if (ct) { + if (ct && !nf_ct_is_template(ct)) { bool mangled; /* If mangling fails this function will return 0 @@ -529,6 +544,9 @@ static int ip_vs_ftp_in(struct ip_vs_app *app, struct ip_vs_conn *cp, return 1; } + if (!port || is_control_port(ntohs(cp->vport) - 1)) + return 0; + /* Passive mode off */ cp->app_data = (void *) IP_VS_FTP_ACTIVE; diff --git a/net/netfilter/ipvs/ip_vs_lblc.c b/net/netfilter/ipvs/ip_vs_lblc.c index e1111c3b2721..f5ce1f2afb86 100644 --- a/net/netfilter/ipvs/ip_vs_lblc.c +++ b/net/netfilter/ipvs/ip_vs_lblc.c @@ -467,10 +467,11 @@ is_overloaded(struct ip_vs_dest *dest, struct ip_vs_service *svc) struct ip_vs_dest *d; list_for_each_entry_rcu(d, &svc->destinations, n_list) { - if (atomic_read(&d->activeconns)*2 - < atomic_read(&d->weight)) { + if (ip_vs_dest_is_overloaded(d)) + continue; + if (atomic_read(&d->activeconns) * 2 < + atomic_read(&d->weight)) return 1; - } } } return 0; @@ -507,6 +508,7 @@ ip_vs_lblc_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, dest = en->dest; if ((dest->cflags & IP_VS_DEST_CF_AVAILABLE) && + !ip_vs_dest_is_overloaded(dest) && atomic_read(&dest->weight) > 0 && !is_overloaded(dest, svc)) goto out; } diff --git a/net/netfilter/ipvs/ip_vs_lblcr.c b/net/netfilter/ipvs/ip_vs_lblcr.c index dfb5bf83750d..4046608922e7 100644 --- a/net/netfilter/ipvs/ip_vs_lblcr.c +++ b/net/netfilter/ipvs/ip_vs_lblcr.c @@ -630,10 +630,11 @@ is_overloaded(struct ip_vs_dest *dest, struct ip_vs_service *svc) struct ip_vs_dest *d; list_for_each_entry_rcu(d, &svc->destinations, n_list) { - if (atomic_read(&d->activeconns)*2 - < atomic_read(&d->weight)) { + if (ip_vs_dest_is_overloaded(d)) + continue; + if (atomic_read(&d->activeconns) * 2 < + atomic_read(&d->weight)) return 1; - } } } return 0; @@ -658,9 +659,6 @@ ip_vs_lblcr_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, if (en) { en->lastuse = jiffies; - /* Get the least loaded destination */ - dest = ip_vs_dest_set_min(&en->set); - /* More than one destination + enough time passed by, cleanup */ if (atomic_read(&en->set.size) > 1 && time_after(jiffies, en->set.lastmod + @@ -676,6 +674,9 @@ ip_vs_lblcr_schedule(struct ip_vs_service *svc, const struct sk_buff *skb, spin_unlock_bh(&svc->sched_lock); } + /* Get the least loaded destination */ + dest = ip_vs_dest_set_min(&en->set); + /* If the destination is not overloaded, use it */ if (dest && !is_overloaded(dest, svc)) goto out; diff --git a/net/netfilter/ipvs/ip_vs_proto_sctp.c b/net/netfilter/ipvs/ip_vs_proto_sctp.c index fb8af6b15a39..cb34bcceb4f6 100644 --- a/net/netfilter/ipvs/ip_vs_proto_sctp.c +++ b/net/netfilter/ipvs/ip_vs_proto_sctp.c @@ -30,6 +30,7 @@ sctp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, sch = skb_header_pointer(skb, iph->len + sizeof(_sctph), sizeof(_schunkh), &_schunkh); if (sch) { + /* Accept packet without scheduling? */ if (sch->type == SCTP_CID_ABORT || !(sysctl_sloppy_sctp(ipvs) || sch->type == SCTP_CID_INIT)) @@ -56,6 +57,12 @@ sctp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, if (svc) { int ignored; + /* Do not create connection for port 0 */ + if (!ports[0] || !ports[1]) { + *verdict = NF_DROP; + return 0; + } + if (ip_vs_todrop(ipvs)) { /* * It seems that we are very loaded. diff --git a/net/netfilter/ipvs/ip_vs_proto_tcp.c b/net/netfilter/ipvs/ip_vs_proto_tcp.c index fec0e8b47b71..25d4576d0e12 100644 --- a/net/netfilter/ipvs/ip_vs_proto_tcp.c +++ b/net/netfilter/ipvs/ip_vs_proto_tcp.c @@ -48,6 +48,7 @@ tcp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, if (likely(!ip_vs_iph_icmp(iph))) { th = skb_header_pointer(skb, iph->len, sizeof(_tcph), &_tcph); if (th) { + /* Accept packet without scheduling? */ if (th->rst || !(sysctl_sloppy_tcp(ipvs) || th->syn)) return 1; ports = &th->source; @@ -74,6 +75,12 @@ tcp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, if (svc) { int ignored; + /* Do not create connection for port 0 */ + if (!ports[0] || !ports[1]) { + *verdict = NF_DROP; + return 0; + } + if (ip_vs_todrop(ipvs)) { /* * It seems that we are very loaded. diff --git a/net/netfilter/ipvs/ip_vs_proto_udp.c b/net/netfilter/ipvs/ip_vs_proto_udp.c index f32785682402..ab3ab9524168 100644 --- a/net/netfilter/ipvs/ip_vs_proto_udp.c +++ b/net/netfilter/ipvs/ip_vs_proto_udp.c @@ -62,6 +62,12 @@ udp_conn_schedule(struct netns_ipvs *ipvs, int af, struct sk_buff *skb, if (svc) { int ignored; + /* Do not create connection for port 0 */ + if (!ports[0] || !ports[1]) { + *verdict = NF_DROP; + return 0; + } + if (ip_vs_todrop(ipvs)) { /* * It seems that we are very loaded. diff --git a/net/netfilter/ipvs/ip_vs_sync.c b/net/netfilter/ipvs/ip_vs_sync.c index 157f5c8361ab..7e562e745ab5 100644 --- a/net/netfilter/ipvs/ip_vs_sync.c +++ b/net/netfilter/ipvs/ip_vs_sync.c @@ -539,6 +539,7 @@ set: static void ip_vs_sync_conn_v0(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, int pkts) { + struct ip_vs_sync_conn_options opts; struct ip_vs_sync_mesg_v0 *m; struct ip_vs_sync_conn_v0 *s; struct ip_vs_sync_buff *buff; @@ -557,6 +558,14 @@ static void ip_vs_sync_conn_v0(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, if (!ip_vs_sync_conn_needed(ipvs, cp, pkts)) return; + if (flags & IP_VS_CONN_F_SEQ_MASK) { + /* Take atomic snapshot of flags and seqs */ + spin_lock_bh(&cp->lock); + flags = cp->flags; + memcpy(&opts, &cp->sync_conn_opt, sizeof(opts)); + spin_unlock_bh(&cp->lock); + } + spin_lock_bh(&ipvs->sync_buff_lock); if (!(ipvs->sync_state & IP_VS_STATE_MASTER)) { spin_unlock_bh(&ipvs->sync_buff_lock); @@ -604,7 +613,7 @@ static void ip_vs_sync_conn_v0(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, if (seq_mask) { struct ip_vs_sync_conn_options *opt = (struct ip_vs_sync_conn_options *)&s[1]; - memcpy(opt, &cp->sync_conn_opt, sizeof(*opt)); + memcpy(opt, &opts, sizeof(*opt)); } m->nr_conns++; @@ -630,6 +639,7 @@ static void ip_vs_sync_conn_v0(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, */ void ip_vs_sync_conn(struct netns_ipvs *ipvs, struct ip_vs_conn *cp, int pkts) { + struct ip_vs_sync_conn_options opts; struct ip_vs_sync_mesg *m; union ip_vs_sync_conn *s; struct ip_vs_sync_buff *buff; @@ -651,6 +661,13 @@ sloop: if (!ip_vs_sync_conn_needed(ipvs, cp, pkts)) goto control; flags = READ_ONCE(cp->flags); + if (flags & IP_VS_CONN_F_SEQ_MASK) { + /* Take atomic snapshot of flags and seqs */ + spin_lock_bh(&cp->lock); + flags = cp->flags; + memcpy(&opts, &cp->sync_conn_opt, sizeof(opts)); + spin_unlock_bh(&cp->lock); + } /* Sanity checks */ pe_name_len = 0; @@ -732,7 +749,7 @@ sloop: s->v4.vport = cp->vport; s->v4.dport = cp->dport; s->v4.fwmark = htonl(cp->fwmark); - s->v4.timeout = htonl(cp->timeout / HZ); + s->v4.timeout = htonl(max(1UL, cp->timeout / HZ)); m->nr_conns++; #ifdef CONFIG_IP_VS_IPV6 @@ -752,9 +769,9 @@ sloop: if (seq_mask) { *(p++) = IPVS_OPT_SEQ_DATA; *(p++) = sizeof(struct ip_vs_sync_conn_options); - hton_seq(&cp->in_seq, (struct ip_vs_seq *)p); + hton_seq(&opts.in_seq, (struct ip_vs_seq *)p); p += sizeof(struct ip_vs_seq); - hton_seq(&cp->out_seq, (struct ip_vs_seq *)p); + hton_seq(&opts.out_seq, (struct ip_vs_seq *)p); p += sizeof(struct ip_vs_seq); } /* Handle pe data */ @@ -850,9 +867,29 @@ static void ip_vs_proc_conn(struct netns_ipvs *ipvs, struct ip_vs_conn_param *pa unsigned long timeout, __u32 fwmark, struct ip_vs_sync_conn_options *opt) { + struct ip_vs_conn *cp_new = NULL; struct ip_vs_dest *dest; struct ip_vs_conn *cp; + /* Templates require cport 0. Ordinary connections require + * NO_CPORT exactly when cport is 0. + */ + if (flags & IP_VS_CONN_F_TEMPLATE) { + if (param->cport) { + IP_VS_DBG(2, "BACKUP, template with cport dropped\n"); + kfree(param->pe_data); + return; + } + } else if (!param->cport && !(flags & IP_VS_CONN_F_NO_CPORT)) { + IP_VS_DBG(2, "BACKUP, conn without cport dropped\n"); + kfree(param->pe_data); + return; + } else if (param->cport && (flags & IP_VS_CONN_F_NO_CPORT)) { + IP_VS_DBG(2, "BACKUP, conn with cport and NO_CPORT dropped\n"); + kfree(param->pe_data); + return; + } + if (!(flags & IP_VS_CONN_F_TEMPLATE)) { cp = ip_vs_conn_in_get(param); if (cp && ((cp->dport != dport) || @@ -891,9 +928,6 @@ static void ip_vs_proc_conn(struct netns_ipvs *ipvs, struct ip_vs_conn_param *pa flags &= IP_VS_CONN_F_BACKUP_UPD_MASK; flags |= cp->flags & ~IP_VS_CONN_F_BACKUP_UPD_MASK; cp->flags = flags; - spin_unlock_bh(&cp->lock); - if (!dest) - ip_vs_try_bind_dest(cp); } else { /* * Find the appropriate destination for the connection. @@ -920,6 +954,8 @@ static void ip_vs_proc_conn(struct netns_ipvs *ipvs, struct ip_vs_conn_param *pa } if (!(flags & IP_VS_CONN_F_TEMPLATE)) kfree(param->pe_data); + cp_new = cp; + spin_lock_bh(&cp->lock); } if (opt) { @@ -951,6 +987,9 @@ static void ip_vs_proc_conn(struct netns_ipvs *ipvs, struct ip_vs_conn_param *pa else cp->timeout = (3*60*HZ); } + spin_unlock_bh(&cp->lock); + if (!dest && !cp_new) + ip_vs_try_bind_dest(cp); ip_vs_conn_put(cp); }