forked from Qortal/Brooklyn
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
240 lines
7.5 KiB
240 lines
7.5 KiB
// SPDX-License-Identifier: GPL-2.0 |
|
/* |
|
* Copyright (C) 2015-2019 Jason A. Donenfeld <[email protected]>. All Rights Reserved. |
|
*/ |
|
|
|
#include "peer.h" |
|
#include "device.h" |
|
#include "queueing.h" |
|
#include "timers.h" |
|
#include "peerlookup.h" |
|
#include "noise.h" |
|
|
|
#include <linux/kref.h> |
|
#include <linux/lockdep.h> |
|
#include <linux/rcupdate.h> |
|
#include <linux/list.h> |
|
|
|
static struct kmem_cache *peer_cache; |
|
static atomic64_t peer_counter = ATOMIC64_INIT(0); |
|
|
|
struct wg_peer *wg_peer_create(struct wg_device *wg, |
|
const u8 public_key[NOISE_PUBLIC_KEY_LEN], |
|
const u8 preshared_key[NOISE_SYMMETRIC_KEY_LEN]) |
|
{ |
|
struct wg_peer *peer; |
|
int ret = -ENOMEM; |
|
|
|
lockdep_assert_held(&wg->device_update_lock); |
|
|
|
if (wg->num_peers >= MAX_PEERS_PER_DEVICE) |
|
return ERR_PTR(ret); |
|
|
|
peer = kmem_cache_zalloc(peer_cache, GFP_KERNEL); |
|
if (unlikely(!peer)) |
|
return ERR_PTR(ret); |
|
if (unlikely(dst_cache_init(&peer->endpoint_cache, GFP_KERNEL))) |
|
goto err; |
|
|
|
peer->device = wg; |
|
wg_noise_handshake_init(&peer->handshake, &wg->static_identity, |
|
public_key, preshared_key, peer); |
|
peer->internal_id = atomic64_inc_return(&peer_counter); |
|
peer->serial_work_cpu = nr_cpumask_bits; |
|
wg_cookie_init(&peer->latest_cookie); |
|
wg_timers_init(peer); |
|
wg_cookie_checker_precompute_peer_keys(peer); |
|
spin_lock_init(&peer->keypairs.keypair_update_lock); |
|
INIT_WORK(&peer->transmit_handshake_work, wg_packet_handshake_send_worker); |
|
INIT_WORK(&peer->transmit_packet_work, wg_packet_tx_worker); |
|
wg_prev_queue_init(&peer->tx_queue); |
|
wg_prev_queue_init(&peer->rx_queue); |
|
rwlock_init(&peer->endpoint_lock); |
|
kref_init(&peer->refcount); |
|
skb_queue_head_init(&peer->staged_packet_queue); |
|
wg_noise_reset_last_sent_handshake(&peer->last_sent_handshake); |
|
set_bit(NAPI_STATE_NO_BUSY_POLL, &peer->napi.state); |
|
netif_napi_add(wg->dev, &peer->napi, wg_packet_rx_poll, |
|
NAPI_POLL_WEIGHT); |
|
napi_enable(&peer->napi); |
|
list_add_tail(&peer->peer_list, &wg->peer_list); |
|
INIT_LIST_HEAD(&peer->allowedips_list); |
|
wg_pubkey_hashtable_add(wg->peer_hashtable, peer); |
|
++wg->num_peers; |
|
pr_debug("%s: Peer %llu created\n", wg->dev->name, peer->internal_id); |
|
return peer; |
|
|
|
err: |
|
kmem_cache_free(peer_cache, peer); |
|
return ERR_PTR(ret); |
|
} |
|
|
|
struct wg_peer *wg_peer_get_maybe_zero(struct wg_peer *peer) |
|
{ |
|
RCU_LOCKDEP_WARN(!rcu_read_lock_bh_held(), |
|
"Taking peer reference without holding the RCU read lock"); |
|
if (unlikely(!peer || !kref_get_unless_zero(&peer->refcount))) |
|
return NULL; |
|
return peer; |
|
} |
|
|
|
static void peer_make_dead(struct wg_peer *peer) |
|
{ |
|
/* Remove from configuration-time lookup structures. */ |
|
list_del_init(&peer->peer_list); |
|
wg_allowedips_remove_by_peer(&peer->device->peer_allowedips, peer, |
|
&peer->device->device_update_lock); |
|
wg_pubkey_hashtable_remove(peer->device->peer_hashtable, peer); |
|
|
|
/* Mark as dead, so that we don't allow jumping contexts after. */ |
|
WRITE_ONCE(peer->is_dead, true); |
|
|
|
/* The caller must now synchronize_net() for this to take effect. */ |
|
} |
|
|
|
static void peer_remove_after_dead(struct wg_peer *peer) |
|
{ |
|
WARN_ON(!peer->is_dead); |
|
|
|
/* No more keypairs can be created for this peer, since is_dead protects |
|
* add_new_keypair, so we can now destroy existing ones. |
|
*/ |
|
wg_noise_keypairs_clear(&peer->keypairs); |
|
|
|
/* Destroy all ongoing timers that were in-flight at the beginning of |
|
* this function. |
|
*/ |
|
wg_timers_stop(peer); |
|
|
|
/* The transition between packet encryption/decryption queues isn't |
|
* guarded by is_dead, but each reference's life is strictly bounded by |
|
* two generations: once for parallel crypto and once for serial |
|
* ingestion, so we can simply flush twice, and be sure that we no |
|
* longer have references inside these queues. |
|
*/ |
|
|
|
/* a) For encrypt/decrypt. */ |
|
flush_workqueue(peer->device->packet_crypt_wq); |
|
/* b.1) For send (but not receive, since that's napi). */ |
|
flush_workqueue(peer->device->packet_crypt_wq); |
|
/* b.2.1) For receive (but not send, since that's wq). */ |
|
napi_disable(&peer->napi); |
|
/* b.2.1) It's now safe to remove the napi struct, which must be done |
|
* here from process context. |
|
*/ |
|
netif_napi_del(&peer->napi); |
|
|
|
/* Ensure any workstructs we own (like transmit_handshake_work or |
|
* clear_peer_work) no longer are in use. |
|
*/ |
|
flush_workqueue(peer->device->handshake_send_wq); |
|
|
|
/* After the above flushes, a peer might still be active in a few |
|
* different contexts: 1) from xmit(), before hitting is_dead and |
|
* returning, 2) from wg_packet_consume_data(), before hitting is_dead |
|
* and returning, 3) from wg_receive_handshake_packet() after a point |
|
* where it has processed an incoming handshake packet, but where |
|
* all calls to pass it off to timers fails because of is_dead. We won't |
|
* have new references in (1) eventually, because we're removed from |
|
* allowedips; we won't have new references in (2) eventually, because |
|
* wg_index_hashtable_lookup will always return NULL, since we removed |
|
* all existing keypairs and no more can be created; we won't have new |
|
* references in (3) eventually, because we're removed from the pubkey |
|
* hash table, which allows for a maximum of one handshake response, |
|
* via the still-uncleared index hashtable entry, but not more than one, |
|
* and in wg_cookie_message_consume, the lookup eventually gets a peer |
|
* with a refcount of zero, so no new reference is taken. |
|
*/ |
|
|
|
--peer->device->num_peers; |
|
wg_peer_put(peer); |
|
} |
|
|
|
/* We have a separate "remove" function make sure that all active places where |
|
* a peer is currently operating will eventually come to an end and not pass |
|
* their reference onto another context. |
|
*/ |
|
void wg_peer_remove(struct wg_peer *peer) |
|
{ |
|
if (unlikely(!peer)) |
|
return; |
|
lockdep_assert_held(&peer->device->device_update_lock); |
|
|
|
peer_make_dead(peer); |
|
synchronize_net(); |
|
peer_remove_after_dead(peer); |
|
} |
|
|
|
void wg_peer_remove_all(struct wg_device *wg) |
|
{ |
|
struct wg_peer *peer, *temp; |
|
LIST_HEAD(dead_peers); |
|
|
|
lockdep_assert_held(&wg->device_update_lock); |
|
|
|
/* Avoid having to traverse individually for each one. */ |
|
wg_allowedips_free(&wg->peer_allowedips, &wg->device_update_lock); |
|
|
|
list_for_each_entry_safe(peer, temp, &wg->peer_list, peer_list) { |
|
peer_make_dead(peer); |
|
list_add_tail(&peer->peer_list, &dead_peers); |
|
} |
|
synchronize_net(); |
|
list_for_each_entry_safe(peer, temp, &dead_peers, peer_list) |
|
peer_remove_after_dead(peer); |
|
} |
|
|
|
static void rcu_release(struct rcu_head *rcu) |
|
{ |
|
struct wg_peer *peer = container_of(rcu, struct wg_peer, rcu); |
|
|
|
dst_cache_destroy(&peer->endpoint_cache); |
|
WARN_ON(wg_prev_queue_peek(&peer->tx_queue) || wg_prev_queue_peek(&peer->rx_queue)); |
|
|
|
/* The final zeroing takes care of clearing any remaining handshake key |
|
* material and other potentially sensitive information. |
|
*/ |
|
memzero_explicit(peer, sizeof(*peer)); |
|
kmem_cache_free(peer_cache, peer); |
|
} |
|
|
|
static void kref_release(struct kref *refcount) |
|
{ |
|
struct wg_peer *peer = container_of(refcount, struct wg_peer, refcount); |
|
|
|
pr_debug("%s: Peer %llu (%pISpfsc) destroyed\n", |
|
peer->device->dev->name, peer->internal_id, |
|
&peer->endpoint.addr); |
|
|
|
/* Remove ourself from dynamic runtime lookup structures, now that the |
|
* last reference is gone. |
|
*/ |
|
wg_index_hashtable_remove(peer->device->index_hashtable, |
|
&peer->handshake.entry); |
|
|
|
/* Remove any lingering packets that didn't have a chance to be |
|
* transmitted. |
|
*/ |
|
wg_packet_purge_staged_packets(peer); |
|
|
|
/* Free the memory used. */ |
|
call_rcu(&peer->rcu, rcu_release); |
|
} |
|
|
|
void wg_peer_put(struct wg_peer *peer) |
|
{ |
|
if (unlikely(!peer)) |
|
return; |
|
kref_put(&peer->refcount, kref_release); |
|
} |
|
|
|
int __init wg_peer_init(void) |
|
{ |
|
peer_cache = KMEM_CACHE(wg_peer, 0); |
|
return peer_cache ? 0 : -ENOMEM; |
|
} |
|
|
|
void wg_peer_uninit(void) |
|
{ |
|
kmem_cache_destroy(peer_cache); |
|
}
|
|
|