diff options
Diffstat (limited to 'fs/smb/client/smbdirect.c')
| -rw-r--r-- | fs/smb/client/smbdirect.c | 561 |
1 files changed, 561 insertions, 0 deletions
diff --git a/fs/smb/client/smbdirect.c b/fs/smb/client/smbdirect.c new file mode 100644 index 000000000..563ef488a --- /dev/null +++ b/fs/smb/client/smbdirect.c @@ -0,0 +1,561 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * Copyright (C) 2017, Microsoft Corporation. + * + * Author(s): Long Li <longli@microsoft.com> + */ + +#include "smbdirect.h" +#include "cifs_debug.h" +#include "cifsproto.h" +#include "smb2proto.h" + +/* Port numbers for SMBD transport */ +#define SMB_PORT 445 +#define SMBD_PORT 5445 + +/* Address lookup and resolve timeout in ms */ +#define RDMA_RESOLVE_TIMEOUT 5000 + +/* SMBD negotiation timeout in seconds */ +#define SMBD_NEGOTIATE_TIMEOUT 120 + +/* The timeout to wait for a keepalive message from peer in seconds */ +#define KEEPALIVE_RECV_TIMEOUT 5 + +/* + * Default maximum number of RDMA read/write outstanding on this connection + * This value is possibly decreased during QP creation on hardware limit + */ +#define SMBD_CM_RESPONDER_RESOURCES 32 + +/* + * User configurable initial values per SMBD transport connection + * as defined in [MS-SMBD] 3.1.1.1 + * Those may change after a SMBD negotiation + */ +/* The local peer's maximum number of credits to grant to the peer */ +int smbd_receive_credit_max = 255; + +/* The remote peer's credit request of local peer */ +int smbd_send_credit_target = 255; + +/* The maximum single message size can be sent to remote peer */ +int smbd_max_send_size = 1364; + +/* + * The maximum fragmented upper-layer payload receive size supported + * + * Assume max_payload_per_credit is + * smbd_max_receive_size - 24 = 1340 + * + * The maximum number would be + * smbd_receive_credit_max * max_payload_per_credit + * + * 1340 * 255 = 341700 (0x536C4) + * + * The minimum value from the spec is 131072 (0x20000) + * + * For now we use the logic we used in ksmbd before: + * (1364 * 255) / 2 = 173910 (0x2A756) + */ +int smbd_max_fragmented_recv_size = (1364 * 255) / 2; + +/* The maximum single-message size which can be received */ +int smbd_max_receive_size = 1364; + +/* The timeout to initiate send of a keepalive message on idle */ +int smbd_keep_alive_interval = 120; + +/* + * User configurable initial values for RDMA transport + * The actual values used may be lower and are limited to hardware capabilities + */ +/* Default maximum number of pages in a single RDMA write/read */ +int smbd_max_frmr_depth = 2048; + +/* If payload is less than this byte, use RDMA send/recv not read/write */ +int rdma_readwrite_threshold = 4096; + +/* Transport logging functions + * Logging are defined as classes. They can be OR'ed to define the actual + * logging level via module parameter smbd_logging_class + * e.g. cifs.smbd_logging_class=0xa0 will log all log_rdma_recv() and + * log_rdma_event() + */ +#define LOG_OUTGOING 0x1 +#define LOG_INCOMING 0x2 +#define LOG_READ 0x4 +#define LOG_WRITE 0x8 +#define LOG_RDMA_SEND 0x10 +#define LOG_RDMA_RECV 0x20 +#define LOG_KEEP_ALIVE 0x40 +#define LOG_RDMA_EVENT 0x80 +#define LOG_RDMA_MR 0x100 +static unsigned int smbd_logging_class; +module_param(smbd_logging_class, uint, 0644); +MODULE_PARM_DESC(smbd_logging_class, + "Logging class for SMBD transport 0x0 to 0x100"); + +#define ERR 0x0 +#define INFO 0x1 +static unsigned int smbd_logging_level = ERR; +module_param(smbd_logging_level, uint, 0644); +MODULE_PARM_DESC(smbd_logging_level, + "Logging level for SMBD transport, 0 (default): error, 1: info"); + +static bool smbd_logging_needed(struct smbdirect_socket *sc, + void *private_ptr, + unsigned int lvl, + unsigned int cls) +{ +#define BUILD_BUG_SAME(x) BUILD_BUG_ON(x != SMBDIRECT_LOG_ ##x) + BUILD_BUG_SAME(ERR); + BUILD_BUG_SAME(INFO); +#undef BUILD_BUG_SAME +#define BUILD_BUG_SAME(x) BUILD_BUG_ON(x != SMBDIRECT_ ##x) + BUILD_BUG_SAME(LOG_OUTGOING); + BUILD_BUG_SAME(LOG_INCOMING); + BUILD_BUG_SAME(LOG_READ); + BUILD_BUG_SAME(LOG_WRITE); + BUILD_BUG_SAME(LOG_RDMA_SEND); + BUILD_BUG_SAME(LOG_RDMA_RECV); + BUILD_BUG_SAME(LOG_KEEP_ALIVE); + BUILD_BUG_SAME(LOG_RDMA_EVENT); + BUILD_BUG_SAME(LOG_RDMA_MR); +#undef BUILD_BUG_SAME + + if (lvl <= smbd_logging_level || cls & smbd_logging_class) + return true; + return false; +} + +static void smbd_logging_vaprintf(struct smbdirect_socket *sc, + const char *func, + unsigned int line, + void *private_ptr, + unsigned int lvl, + unsigned int cls, + struct va_format *vaf) +{ + cifs_dbg(VFS, "%s:%u %pV", func, line, vaf); +} + +#define log_rdma(level, class, fmt, args...) \ +do { \ + if (level <= smbd_logging_level || class & smbd_logging_class) \ + cifs_dbg(VFS, "%s:%d " fmt, __func__, __LINE__, ##args);\ +} while (0) + +#define log_outgoing(level, fmt, args...) \ + log_rdma(level, LOG_OUTGOING, fmt, ##args) +#define log_incoming(level, fmt, args...) \ + log_rdma(level, LOG_INCOMING, fmt, ##args) +#define log_read(level, fmt, args...) log_rdma(level, LOG_READ, fmt, ##args) +#define log_write(level, fmt, args...) log_rdma(level, LOG_WRITE, fmt, ##args) +#define log_rdma_send(level, fmt, args...) \ + log_rdma(level, LOG_RDMA_SEND, fmt, ##args) +#define log_rdma_recv(level, fmt, args...) \ + log_rdma(level, LOG_RDMA_RECV, fmt, ##args) +#define log_keep_alive(level, fmt, args...) \ + log_rdma(level, LOG_KEEP_ALIVE, fmt, ##args) +#define log_rdma_event(level, fmt, args...) \ + log_rdma(level, LOG_RDMA_EVENT, fmt, ##args) +#define log_rdma_mr(level, fmt, args...) \ + log_rdma(level, LOG_RDMA_MR, fmt, ##args) + +static int smbd_post_send_full_iter(struct smbdirect_socket *sc, + struct smbdirect_send_batch *batch, + struct iov_iter *iter, + u32 remaining_data_length) +{ + int bytes = 0; + + /* + * smbdirect_connection_send_single_iter() respects the + * negotiated max_send_size, so we need to + * loop until the full iter is posted + */ + + while (iov_iter_count(iter) > 0) { + int rc; + + rc = smbdirect_connection_send_single_iter(sc, + batch, + iter, + 0, /* flags */ + remaining_data_length); + if (rc < 0) + return rc; + remaining_data_length -= rc; + bytes += rc; + } + + return bytes; +} + +/* + * Destroy the transport and related RDMA and memory resources + * Need to go through all the pending counters and make sure on one is using + * the transport while it is destroyed + */ +void smbd_destroy(struct TCP_Server_Info *server) +{ + struct smbd_connection *info = server->smbd_conn; + + if (!info) { + log_rdma_event(INFO, "rdma session already destroyed\n"); + return; + } + + smbdirect_socket_release(info->socket); + + kfree(info); + server->smbd_conn = NULL; +} + +/* + * Reconnect this SMBD connection, called from upper layer + * return value: 0 on success, or actual error code + */ +int smbd_reconnect(struct TCP_Server_Info *server) +{ + log_rdma_event(INFO, "reconnecting rdma session\n"); + + if (!server->smbd_conn) { + log_rdma_event(INFO, "rdma session already destroyed\n"); + goto create_conn; + } + + /* + * This is possible if transport is disconnected and we haven't received + * notification from RDMA, but upper layer has detected timeout + */ + log_rdma_event(INFO, "disconnecting transport\n"); + smbd_destroy(server); + +create_conn: + log_rdma_event(INFO, "creating rdma session\n"); + server->smbd_conn = smbd_get_connection( + server, (struct sockaddr *) &server->dstaddr); + + if (server->smbd_conn) { + cifs_dbg(VFS, "RDMA transport re-established\n"); + trace_smb3_smbd_connect_done(server->hostname, server->conn_id, &server->dstaddr); + return 0; + } + trace_smb3_smbd_connect_err(server->hostname, server->conn_id, &server->dstaddr); + return -ENOENT; +} + +/* Create a SMBD connection, called by upper layer */ +static struct smbd_connection *_smbd_get_connection( + struct TCP_Server_Info *server, struct sockaddr *dstaddr, int port) +{ + struct net *net = cifs_net_ns(server); + struct smbd_connection *info; + struct smbdirect_socket *sc; + struct smbdirect_socket_parameters init_params = {}; + struct smbdirect_socket_parameters *sp; + __be16 *sport; + u64 port_flags = 0; + int ret; + + switch (port) { + case SMBD_PORT: + /* + * only allow iWarp devices + * for port 5445. + */ + port_flags |= SMBDIRECT_FLAG_PORT_RANGE_ONLY_IW; + break; + case SMB_PORT: + /* + * only allow InfiniBand, RoCEv1 or RoCEv2 + * devices for port 445. + * + * (Basically don't allow iWarp devices) + */ + port_flags |= SMBDIRECT_FLAG_PORT_RANGE_ONLY_IB; + break; + } + + /* + * Create the initial parameters + */ + sp = &init_params; + sp->flags = port_flags; + sp->resolve_addr_timeout_msec = RDMA_RESOLVE_TIMEOUT; + sp->resolve_route_timeout_msec = RDMA_RESOLVE_TIMEOUT; + sp->rdma_connect_timeout_msec = RDMA_RESOLVE_TIMEOUT; + sp->negotiate_timeout_msec = SMBD_NEGOTIATE_TIMEOUT * 1000; + sp->initiator_depth = 1; + sp->responder_resources = SMBD_CM_RESPONDER_RESOURCES; + sp->recv_credit_max = smbd_receive_credit_max; + sp->send_credit_target = smbd_send_credit_target; + sp->max_send_size = smbd_max_send_size; + sp->max_fragmented_recv_size = smbd_max_fragmented_recv_size; + sp->max_recv_size = smbd_max_receive_size; + sp->max_frmr_depth = smbd_max_frmr_depth; + sp->keepalive_interval_msec = smbd_keep_alive_interval * 1000; + sp->keepalive_timeout_msec = KEEPALIVE_RECV_TIMEOUT * 1000; + + info = kzalloc_obj(*info); + if (!info) + return NULL; + ret = smbdirect_socket_create_kern(net, &sc); + if (ret) + goto socket_init_failed; + smbdirect_socket_set_logging(sc, NULL, smbd_logging_needed, smbd_logging_vaprintf); + ret = smbdirect_socket_set_initial_parameters(sc, sp); + if (ret) + goto set_params_failed; + ret = smbdirect_socket_set_kernel_settings(sc, IB_POLL_SOFTIRQ, GFP_KERNEL); + if (ret) + goto set_settings_failed; + + if (dstaddr->sa_family == AF_INET6) + sport = &((struct sockaddr_in6 *)dstaddr)->sin6_port; + else + sport = &((struct sockaddr_in *)dstaddr)->sin_port; + + *sport = htons(port); + + ret = smbdirect_connect_sync(sc, dstaddr); + if (ret) { + log_rdma_event(ERR, "connect to %pISpsfc failed: %1pe\n", + dstaddr, ERR_PTR(ret)); + goto connect_failed; + } + + info->socket = sc; + return info; + +connect_failed: +set_settings_failed: +set_params_failed: + smbdirect_socket_release(sc); +socket_init_failed: + kfree(info); + return NULL; +} + +const struct smbdirect_socket_parameters *smbd_get_parameters(struct smbd_connection *conn) +{ + if (unlikely(!conn->socket)) { + static const struct smbdirect_socket_parameters zero_params; + + return &zero_params; + } + + return smbdirect_socket_get_current_parameters(conn->socket); +} + +struct smbd_connection *smbd_get_connection( + struct TCP_Server_Info *server, struct sockaddr *dstaddr) +{ + struct smbd_connection *ret; + const struct smbdirect_socket_parameters *sp; + int port = SMBD_PORT; + +try_again: + ret = _smbd_get_connection(server, dstaddr, port); + + /* Try SMB_PORT if SMBD_PORT doesn't work */ + if (!ret && port == SMBD_PORT) { + port = SMB_PORT; + goto try_again; + } + if (!ret) + return NULL; + + sp = smbd_get_parameters(ret); + + server->rdma_readwrite_threshold = + rdma_readwrite_threshold > sp->max_fragmented_send_size ? + sp->max_fragmented_send_size : + rdma_readwrite_threshold; + + return ret; +} + +/* + * Receive data from the transport's receive reassembly queue + * All the incoming data packets are placed in reassembly queue + * iter: the buffer to read data into + * size: the length of data to read + * return value: actual data read + * + * Note: this implementation copies the data from reassembly queue to receive + * buffers used by upper layer. This is not the optimal code path. A better way + * to do it is to not have upper layer allocate its receive buffers but rather + * borrow the buffer from reassembly queue, and return it after data is + * consumed. But this will require more changes to upper layer code, and also + * need to consider packet boundaries while they still being reassembled. + */ +int smbd_recv(struct smbd_connection *info, struct msghdr *msg) +{ + struct smbdirect_socket *sc = info->socket; + + if (!smbdirect_connection_is_connected(sc)) + return -ENOTCONN; + + return smbdirect_connection_recvmsg(sc, msg, 0); +} + +/* + * Send data to transport + * Each rqst is transported as a SMBDirect payload + * rqst: the data to write + * return value: 0 if successfully write, otherwise error code + */ +int smbd_send(struct TCP_Server_Info *server, + int num_rqst, struct smb_rqst *rqst_array) +{ + struct smbd_connection *info = server->smbd_conn; + struct smbdirect_socket *sc = info->socket; + const struct smbdirect_socket_parameters *sp = smbd_get_parameters(info); + struct smb_rqst *rqst; + struct iov_iter iter; + struct smbdirect_send_batch_storage bstorage; + struct smbdirect_send_batch *batch; + unsigned int remaining_data_length, klen; + int rc, i, rqst_idx; + int error = 0; + + if (!smbdirect_connection_is_connected(sc)) + return -EAGAIN; + + /* + * Add in the page array if there is one. The caller needs to set + * rq_tailsz to PAGE_SIZE when the buffer has multiple pages and + * ends at page boundary + */ + remaining_data_length = 0; + for (i = 0; i < num_rqst; i++) + remaining_data_length += smb_rqst_len(server, &rqst_array[i]); + + if (unlikely(remaining_data_length > sp->max_fragmented_send_size)) { + /* assertion: payload never exceeds negotiated maximum */ + log_write(ERR, "payload size %d > max size %d\n", + remaining_data_length, sp->max_fragmented_send_size); + return -EINVAL; + } + + log_write(INFO, "num_rqst=%d total length=%u\n", + num_rqst, remaining_data_length); + + rqst_idx = 0; + batch = smbdirect_init_send_batch_storage(&bstorage, false, 0); + do { + rqst = &rqst_array[rqst_idx]; + + cifs_dbg(FYI, "Sending smb (RDMA): idx=%d smb_len=%lu\n", + rqst_idx, smb_rqst_len(server, rqst)); + for (i = 0; i < rqst->rq_nvec; i++) + dump_smb(rqst->rq_iov[i].iov_base, rqst->rq_iov[i].iov_len); + + log_write(INFO, "RDMA-WR[%u] nvec=%d len=%u iter=%zu rqlen=%lu\n", + rqst_idx, rqst->rq_nvec, remaining_data_length, + iov_iter_count(&rqst->rq_iter), smb_rqst_len(server, rqst)); + + /* Send the metadata pages. */ + klen = 0; + for (i = 0; i < rqst->rq_nvec; i++) + klen += rqst->rq_iov[i].iov_len; + iov_iter_kvec(&iter, ITER_SOURCE, rqst->rq_iov, rqst->rq_nvec, klen); + + rc = smbd_post_send_full_iter(sc, batch, &iter, remaining_data_length); + if (rc < 0) { + error = rc; + break; + } + remaining_data_length -= rc; + + if (iov_iter_count(&rqst->rq_iter) > 0) { + /* And then the data pages if there are any */ + rc = smbd_post_send_full_iter(sc, batch, &rqst->rq_iter, + remaining_data_length); + if (rc < 0) { + error = rc; + break; + } + remaining_data_length -= rc; + } + + } while (++rqst_idx < num_rqst); + + rc = smbdirect_connection_send_batch_flush(sc, batch, true); + if (unlikely(!rc && error)) + rc = error; + + /* + * As an optimization, we don't wait for individual I/O to finish + * before sending the next one. + * Send them all and wait for pending send count to get to 0 + * that means all the I/Os have been out and we are good to return + */ + + error = rc; + rc = smbdirect_connection_send_wait_zero_pending(sc); + if (unlikely(rc && !error)) + error = -EAGAIN; + + if (unlikely(error)) + return error; + + return 0; +} + +/* + * Register memory for RDMA read/write + * iter: the buffer to register memory with + * writing: true if this is a RDMA write (SMB read), false for RDMA read + * need_invalidate: true if this MR needs to be locally invalidated after I/O + * return value: the MR registered, NULL if failed. + */ +struct smbdirect_mr_io *smbd_register_mr(struct smbd_connection *info, + struct iov_iter *iter, + bool writing, bool need_invalidate) +{ + struct smbdirect_socket *sc = info->socket; + + if (!smbdirect_connection_is_connected(sc)) + return NULL; + + return smbdirect_connection_register_mr_io(sc, iter, writing, need_invalidate); +} + +void smbd_mr_fill_buffer_descriptor(struct smbdirect_mr_io *mr, + struct smbdirect_buffer_descriptor_v1 *v1) +{ + smbdirect_mr_io_fill_buffer_descriptor(mr, v1); +} + +/* + * Deregister a MR after I/O is done + * This function may wait if remote invalidation is not used + * and we have to locally invalidate the buffer to prevent data is being + * modified by remote peer after upper layer consumes it + */ +void smbd_deregister_mr(struct smbdirect_mr_io *mr) +{ + smbdirect_connection_deregister_mr_io(mr); +} + +void smbd_debug_proc_show(struct TCP_Server_Info *server, struct seq_file *m) +{ + if (!server->rdma) + return; + + if (!server->smbd_conn) { + seq_puts(m, "\nSMBDirect transport not available"); + return; + } + + smbdirect_connection_legacy_debug_proc_show(server->smbd_conn->socket, + server->rdma_readwrite_threshold, + m); +} + +MODULE_IMPORT_NS("SMBDIRECT"); |