rtems/rtems-libbsd
1
0
Fork 0
mirror of https://git.rtems.org/rtems-libbsd/ synced 2025-07-21 03:05:23 +08:00
Code Issues Packages Projects Releases Wiki Activity

NVME(4): Import from FreeBSD

Browse Source 
Update #3821.
This commit is contained in:
Sebastian Huber 2018-09-03 14:03:26 +02:00
parent b5f802ef43
commit 5ac41dcb21
10 changed files with 5570 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

385
freebsd/sys/dev/nvme/nvme.c Normal file
View File

@ -0,0 +1,385 @@
#include <machine/rtems-bsd-kernel-space.h>
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (C) 2012-2014 Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/bus.h>
#include <sys/conf.h>
#include <sys/module.h>
#include <vm/uma.h>
#include "nvme_private.h"
struct nvme_consumer {
uint32_t id;
nvme_cons_ns_fn_t ns_fn;
nvme_cons_ctrlr_fn_t ctrlr_fn;
nvme_cons_async_fn_t async_fn;
nvme_cons_fail_fn_t fail_fn;
};
struct nvme_consumer nvme_consumer[NVME_MAX_CONSUMERS];
#define INVALID_CONSUMER_ID 0xFFFF
uma_zone_t nvme_request_zone;
int32_t nvme_retry_count;
MALLOC_DEFINE(M_NVME, "nvme", "nvme(4) memory allocations");
devclass_t nvme_devclass;
static void
nvme_init(void)
{
uint32_t i;
nvme_request_zone = uma_zcreate("nvme_request",
sizeof(struct nvme_request), NULL, NULL, NULL, NULL, 0, 0);
for (i = 0; i < NVME_MAX_CONSUMERS; i++)
nvme_consumer[i].id = INVALID_CONSUMER_ID;
}
SYSINIT(nvme_register, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_init, NULL);
static void
nvme_uninit(void)
{
uma_zdestroy(nvme_request_zone);
}
SYSUNINIT(nvme_unregister, SI_SUB_DRIVERS, SI_ORDER_SECOND, nvme_uninit, NULL);
int
nvme_shutdown(device_t dev)
{
struct nvme_controller *ctrlr;
ctrlr = DEVICE2SOFTC(dev);
nvme_ctrlr_shutdown(ctrlr);
return (0);
}
void
nvme_dump_command(struct nvme_command *cmd)
{
printf(
"opc:%x f:%x cid:%x nsid:%x r2:%x r3:%x mptr:%jx prp1:%jx prp2:%jx cdw:%x %x %x %x %x %x\n",
cmd->opc, cmd->fuse, cmd->cid, le32toh(cmd->nsid),
cmd->rsvd2, cmd->rsvd3,
(uintmax_t)le64toh(cmd->mptr), (uintmax_t)le64toh(cmd->prp1), (uintmax_t)le64toh(cmd->prp2),
le32toh(cmd->cdw10), le32toh(cmd->cdw11), le32toh(cmd->cdw12),
le32toh(cmd->cdw13), le32toh(cmd->cdw14), le32toh(cmd->cdw15));
}
void
nvme_dump_completion(struct nvme_completion *cpl)
{
uint8_t p, sc, sct, m, dnr;
uint16_t status;
status = le16toh(cpl->status);
p = NVME_STATUS_GET_P(status);
sc = NVME_STATUS_GET_SC(status);
sct = NVME_STATUS_GET_SCT(status);
m = NVME_STATUS_GET_M(status);
dnr = NVME_STATUS_GET_DNR(status);
printf("cdw0:%08x sqhd:%04x sqid:%04x "
"cid:%04x p:%x sc:%02x sct:%x m:%x dnr:%x\n",
le32toh(cpl->cdw0), le16toh(cpl->sqhd), le16toh(cpl->sqid),
cpl->cid, p, sc, sct, m, dnr);
}
int
nvme_attach(device_t dev)
{
struct nvme_controller *ctrlr = DEVICE2SOFTC(dev);
int status;
status = nvme_ctrlr_construct(ctrlr, dev);
if (status != 0) {
nvme_ctrlr_destruct(ctrlr, dev);
return (status);
}
/*
* Reset controller twice to ensure we do a transition from cc.en==1 to
* cc.en==0. This is because we don't really know what status the
* controller was left in when boot handed off to OS. Linux doesn't do
* this, however. If we adopt that policy, see also nvme_ctrlr_resume().
*/
status = nvme_ctrlr_hw_reset(ctrlr);
if (status != 0) {
nvme_ctrlr_destruct(ctrlr, dev);
return (status);
}
status = nvme_ctrlr_hw_reset(ctrlr);
if (status != 0) {
nvme_ctrlr_destruct(ctrlr, dev);
return (status);
}
ctrlr->config_hook.ich_func = nvme_ctrlr_start_config_hook;
ctrlr->config_hook.ich_arg = ctrlr;
config_intrhook_establish(&ctrlr->config_hook);
return (0);
}
int
nvme_detach (device_t dev)
{
struct nvme_controller *ctrlr = DEVICE2SOFTC(dev);
nvme_ctrlr_destruct(ctrlr, dev);
return (0);
}
static void
nvme_notify(struct nvme_consumer *cons,
struct nvme_controller *ctrlr)
{
struct nvme_namespace *ns;
void *ctrlr_cookie;
int cmpset, ns_idx;
/*
* The consumer may register itself after the nvme devices
* have registered with the kernel, but before the
* driver has completed initialization. In that case,
* return here, and when initialization completes, the
* controller will make sure the consumer gets notified.
*/
if (!ctrlr->is_initialized)
return;
cmpset = atomic_cmpset_32(&ctrlr->notification_sent, 0, 1);
if (cmpset == 0)
return;
if (cons->ctrlr_fn != NULL)
ctrlr_cookie = (*cons->ctrlr_fn)(ctrlr);
else
ctrlr_cookie = (void *)(uintptr_t)0xdeadc0dedeadc0de;
ctrlr->cons_cookie[cons->id] = ctrlr_cookie;
/* ctrlr_fn has failed. Nothing to notify here any more. */
if (ctrlr_cookie == NULL)
return;
if (ctrlr->is_failed) {
ctrlr->cons_cookie[cons->id] = NULL;
if (cons->fail_fn != NULL)
(*cons->fail_fn)(ctrlr_cookie);
/*
* Do not notify consumers about the namespaces of a
* failed controller.
*/
return;
}
for (ns_idx = 0; ns_idx < min(ctrlr->cdata.nn, NVME_MAX_NAMESPACES); ns_idx++) {
ns = &ctrlr->ns[ns_idx];
if (ns->data.nsze == 0)
continue;
if (cons->ns_fn != NULL)
ns->cons_cookie[cons->id] =
(*cons->ns_fn)(ns, ctrlr_cookie);
}
}
void
nvme_notify_new_controller(struct nvme_controller *ctrlr)
{
int i;
for (i = 0; i < NVME_MAX_CONSUMERS; i++) {
if (nvme_consumer[i].id != INVALID_CONSUMER_ID) {
nvme_notify(&nvme_consumer[i], ctrlr);
}
}
}
static void
nvme_notify_new_consumer(struct nvme_consumer *cons)
{
device_t *devlist;
struct nvme_controller *ctrlr;
int dev_idx, devcount;
if (devclass_get_devices(nvme_devclass, &devlist, &devcount))
return;
for (dev_idx = 0; dev_idx < devcount; dev_idx++) {
ctrlr = DEVICE2SOFTC(devlist[dev_idx]);
nvme_notify(cons, ctrlr);
}
free(devlist, M_TEMP);
}
void
nvme_notify_async_consumers(struct nvme_controller *ctrlr,
const struct nvme_completion *async_cpl,
uint32_t log_page_id, void *log_page_buffer,
uint32_t log_page_size)
{
struct nvme_consumer *cons;
void *ctrlr_cookie;
uint32_t i;
for (i = 0; i < NVME_MAX_CONSUMERS; i++) {
cons = &nvme_consumer[i];
if (cons->id != INVALID_CONSUMER_ID && cons->async_fn != NULL &&
(ctrlr_cookie = ctrlr->cons_cookie[i]) != NULL) {
(*cons->async_fn)(ctrlr_cookie, async_cpl,
log_page_id, log_page_buffer, log_page_size);
}
}
}
void
nvme_notify_fail_consumers(struct nvme_controller *ctrlr)
{
struct nvme_consumer *cons;
void *ctrlr_cookie;
uint32_t i;
/*
* This controller failed during initialization (i.e. IDENTIFY
* command failed or timed out). Do not notify any nvme
* consumers of the failure here, since the consumer does not
* even know about the controller yet.
*/
if (!ctrlr->is_initialized)
return;
for (i = 0; i < NVME_MAX_CONSUMERS; i++) {
cons = &nvme_consumer[i];
if (cons->id != INVALID_CONSUMER_ID &&
(ctrlr_cookie = ctrlr->cons_cookie[i]) != NULL) {
ctrlr->cons_cookie[i] = NULL;
if (cons->fail_fn != NULL)
cons->fail_fn(ctrlr_cookie);
}
}
}
void
nvme_notify_ns(struct nvme_controller *ctrlr, int nsid)
{
struct nvme_consumer *cons;
struct nvme_namespace *ns = &ctrlr->ns[nsid - 1];
void *ctrlr_cookie;
uint32_t i;
if (!ctrlr->is_initialized)
return;
for (i = 0; i < NVME_MAX_CONSUMERS; i++) {
cons = &nvme_consumer[i];
if (cons->id != INVALID_CONSUMER_ID && cons->ns_fn != NULL &&
(ctrlr_cookie = ctrlr->cons_cookie[i]) != NULL)
ns->cons_cookie[i] = (*cons->ns_fn)(ns, ctrlr_cookie);
}
}
struct nvme_consumer *
nvme_register_consumer(nvme_cons_ns_fn_t ns_fn, nvme_cons_ctrlr_fn_t ctrlr_fn,
nvme_cons_async_fn_t async_fn,
nvme_cons_fail_fn_t fail_fn)
{
int i;
/*
* TODO: add locking around consumer registration.
*/
for (i = 0; i < NVME_MAX_CONSUMERS; i++)
if (nvme_consumer[i].id == INVALID_CONSUMER_ID) {
nvme_consumer[i].id = i;
nvme_consumer[i].ns_fn = ns_fn;
nvme_consumer[i].ctrlr_fn = ctrlr_fn;
nvme_consumer[i].async_fn = async_fn;
nvme_consumer[i].fail_fn = fail_fn;
nvme_notify_new_consumer(&nvme_consumer[i]);
return (&nvme_consumer[i]);
}
printf("nvme(4): consumer not registered - no slots available\n");
return (NULL);
}
void
nvme_unregister_consumer(struct nvme_consumer *consumer)
{
consumer->id = INVALID_CONSUMER_ID;
}
void
nvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl)
{
struct nvme_completion_poll_status *status = arg;
/*
* Copy status into the argument passed by the caller, so that
* the caller can check the status to determine if the
* the request passed or failed.
*/
memcpy(&status->cpl, cpl, sizeof(*cpl));
atomic_store_rel_int(&status->done, 1);
}
static int
nvme_modevent(module_t mod __unused, int type __unused, void *argp __unused)
{
return (0);
}
static moduledata_t nvme_mod = {
"nvme",
nvme_modevent,
0
};
DECLARE_MODULE(nvme, nvme_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST);
MODULE_VERSION(nvme, 1);
MODULE_DEPEND(nvme, cam, 1, 1, 1);

1402
freebsd/sys/dev/nvme/nvme_ctrlr.c Normal file
View File

File diff suppressed because it is too large Load Diff

329
freebsd/sys/dev/nvme/nvme_ctrlr_cmd.c Normal file
View File

@ -0,0 +1,329 @@
#include <machine/rtems-bsd-kernel-space.h>
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (C) 2012-2013 Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "nvme_private.h"
void
nvme_ctrlr_cmd_identify_controller(struct nvme_controller *ctrlr, void *payload,
nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
struct nvme_command *cmd;
req = nvme_allocate_request_vaddr(payload,
sizeof(struct nvme_controller_data), cb_fn, cb_arg);
cmd = &req->cmd;
cmd->opc = NVME_OPC_IDENTIFY;
/*
* TODO: create an identify command data structure, which
* includes this CNS bit in cdw10.
*/
cmd->cdw10 = htole32(1);
nvme_ctrlr_submit_admin_request(ctrlr, req);
}
void
nvme_ctrlr_cmd_identify_namespace(struct nvme_controller *ctrlr, uint32_t nsid,
void *payload, nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
struct nvme_command *cmd;
req = nvme_allocate_request_vaddr(payload,
sizeof(struct nvme_namespace_data), cb_fn, cb_arg);
cmd = &req->cmd;
cmd->opc = NVME_OPC_IDENTIFY;
/*
* TODO: create an identify command data structure
*/
cmd->nsid = htole32(nsid);
nvme_ctrlr_submit_admin_request(ctrlr, req);
}
void
nvme_ctrlr_cmd_create_io_cq(struct nvme_controller *ctrlr,
struct nvme_qpair *io_que, uint16_t vector, nvme_cb_fn_t cb_fn,
void *cb_arg)
{
struct nvme_request *req;
struct nvme_command *cmd;
req = nvme_allocate_request_null(cb_fn, cb_arg);
cmd = &req->cmd;
cmd->opc = NVME_OPC_CREATE_IO_CQ;
/*
* TODO: create a create io completion queue command data
* structure.
*/
cmd->cdw10 = htole32(((io_que->num_entries-1) << 16) | io_que->id);
/* 0x3 = interrupts enabled | physically contiguous */
cmd->cdw11 = htole32((vector << 16) | 0x3);
cmd->prp1 = htole64(io_que->cpl_bus_addr);
nvme_ctrlr_submit_admin_request(ctrlr, req);
}
void
nvme_ctrlr_cmd_create_io_sq(struct nvme_controller *ctrlr,
struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
struct nvme_command *cmd;
req = nvme_allocate_request_null(cb_fn, cb_arg);
cmd = &req->cmd;
cmd->opc = NVME_OPC_CREATE_IO_SQ;
/*
* TODO: create a create io submission queue command data
* structure.
*/
cmd->cdw10 = htole32(((io_que->num_entries-1) << 16) | io_que->id);
/* 0x1 = physically contiguous */
cmd->cdw11 = htole32((io_que->id << 16) | 0x1);
cmd->prp1 = htole64(io_que->cmd_bus_addr);
nvme_ctrlr_submit_admin_request(ctrlr, req);
}
void
nvme_ctrlr_cmd_delete_io_cq(struct nvme_controller *ctrlr,
struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
struct nvme_command *cmd;
req = nvme_allocate_request_null(cb_fn, cb_arg);
cmd = &req->cmd;
cmd->opc = NVME_OPC_DELETE_IO_CQ;
/*
* TODO: create a delete io completion queue command data
* structure.
*/
cmd->cdw10 = htole32(io_que->id);
nvme_ctrlr_submit_admin_request(ctrlr, req);
}
void
nvme_ctrlr_cmd_delete_io_sq(struct nvme_controller *ctrlr,
struct nvme_qpair *io_que, nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
struct nvme_command *cmd;
req = nvme_allocate_request_null(cb_fn, cb_arg);
cmd = &req->cmd;
cmd->opc = NVME_OPC_DELETE_IO_SQ;
/*
* TODO: create a delete io submission queue command data
* structure.
*/
cmd->cdw10 = htole32(io_que->id);
nvme_ctrlr_submit_admin_request(ctrlr, req);
}
void
nvme_ctrlr_cmd_set_feature(struct nvme_controller *ctrlr, uint8_t feature,
uint32_t cdw11, void *payload, uint32_t payload_size,
nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
struct nvme_command *cmd;
req = nvme_allocate_request_null(cb_fn, cb_arg);
cmd = &req->cmd;
cmd->opc = NVME_OPC_SET_FEATURES;
cmd->cdw10 = htole32(feature);
cmd->cdw11 = htole32(cdw11);
nvme_ctrlr_submit_admin_request(ctrlr, req);
}
void
nvme_ctrlr_cmd_get_feature(struct nvme_controller *ctrlr, uint8_t feature,
uint32_t cdw11, void *payload, uint32_t payload_size,
nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
struct nvme_command *cmd;
req = nvme_allocate_request_null(cb_fn, cb_arg);
cmd = &req->cmd;
cmd->opc = NVME_OPC_GET_FEATURES;
cmd->cdw10 = htole32(feature);
cmd->cdw11 = htole32(cdw11);
nvme_ctrlr_submit_admin_request(ctrlr, req);
}
void
nvme_ctrlr_cmd_set_num_queues(struct nvme_controller *ctrlr,
uint32_t num_queues, nvme_cb_fn_t cb_fn, void *cb_arg)
{
uint32_t cdw11;
cdw11 = ((num_queues - 1) << 16) | (num_queues - 1);
nvme_ctrlr_cmd_set_feature(ctrlr, NVME_FEAT_NUMBER_OF_QUEUES, cdw11,
NULL, 0, cb_fn, cb_arg);
}
void
nvme_ctrlr_cmd_set_async_event_config(struct nvme_controller *ctrlr,
uint32_t state, nvme_cb_fn_t cb_fn, void *cb_arg)
{
uint32_t cdw11;
cdw11 = state;
nvme_ctrlr_cmd_set_feature(ctrlr,
NVME_FEAT_ASYNC_EVENT_CONFIGURATION, cdw11, NULL, 0, cb_fn,
cb_arg);
}
void
nvme_ctrlr_cmd_set_interrupt_coalescing(struct nvme_controller *ctrlr,
uint32_t microseconds, uint32_t threshold, nvme_cb_fn_t cb_fn, void *cb_arg)
{
uint32_t cdw11;
if ((microseconds/100) >= 0x100) {
nvme_printf(ctrlr, "invalid coal time %d, disabling\n",
microseconds);
microseconds = 0;
threshold = 0;
}
if (threshold >= 0x100) {
nvme_printf(ctrlr, "invalid threshold %d, disabling\n",
threshold);
threshold = 0;
microseconds = 0;
}
cdw11 = ((microseconds/100) << 8) | threshold;
nvme_ctrlr_cmd_set_feature(ctrlr, NVME_FEAT_INTERRUPT_COALESCING, cdw11,
NULL, 0, cb_fn, cb_arg);
}
void
nvme_ctrlr_cmd_get_log_page(struct nvme_controller *ctrlr, uint8_t log_page,
uint32_t nsid, void *payload, uint32_t payload_size, nvme_cb_fn_t cb_fn,
void *cb_arg)
{
struct nvme_request *req;
struct nvme_command *cmd;
req = nvme_allocate_request_vaddr(payload, payload_size, cb_fn, cb_arg);
cmd = &req->cmd;
cmd->opc = NVME_OPC_GET_LOG_PAGE;
cmd->nsid = htole32(nsid);
cmd->cdw10 = ((payload_size/sizeof(uint32_t)) - 1) << 16;
cmd->cdw10 |= log_page;
cmd->cdw10 = htole32(cmd->cdw10);
nvme_ctrlr_submit_admin_request(ctrlr, req);
}
void
nvme_ctrlr_cmd_get_error_page(struct nvme_controller *ctrlr,
struct nvme_error_information_entry *payload, uint32_t num_entries,
nvme_cb_fn_t cb_fn, void *cb_arg)
{
KASSERT(num_entries > 0, ("%s called with num_entries==0\n", __func__));
/* Controller's error log page entries is 0-based. */
KASSERT(num_entries <= (ctrlr->cdata.elpe + 1),
("%s called with num_entries=%d but (elpe+1)=%d\n", __func__,
num_entries, ctrlr->cdata.elpe + 1));
if (num_entries > (ctrlr->cdata.elpe + 1))
num_entries = ctrlr->cdata.elpe + 1;
nvme_ctrlr_cmd_get_log_page(ctrlr, NVME_LOG_ERROR,
NVME_GLOBAL_NAMESPACE_TAG, payload, sizeof(*payload) * num_entries,
cb_fn, cb_arg);
}
void
nvme_ctrlr_cmd_get_health_information_page(struct nvme_controller *ctrlr,
uint32_t nsid, struct nvme_health_information_page *payload,
nvme_cb_fn_t cb_fn, void *cb_arg)
{
nvme_ctrlr_cmd_get_log_page(ctrlr, NVME_LOG_HEALTH_INFORMATION,
nsid, payload, sizeof(*payload), cb_fn, cb_arg);
}
void
nvme_ctrlr_cmd_get_firmware_page(struct nvme_controller *ctrlr,
struct nvme_firmware_page *payload, nvme_cb_fn_t cb_fn, void *cb_arg)
{
nvme_ctrlr_cmd_get_log_page(ctrlr, NVME_LOG_FIRMWARE_SLOT,
NVME_GLOBAL_NAMESPACE_TAG, payload, sizeof(*payload), cb_fn,
cb_arg);
}
void
nvme_ctrlr_cmd_abort(struct nvme_controller *ctrlr, uint16_t cid,
uint16_t sqid, nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
struct nvme_command *cmd;
req = nvme_allocate_request_null(cb_fn, cb_arg);
cmd = &req->cmd;
cmd->opc = NVME_OPC_ABORT;
cmd->cdw10 = htole32((cid << 16) | sqid);
nvme_ctrlr_submit_admin_request(ctrlr, req);
}

627
freebsd/sys/dev/nvme/nvme_ns.c Normal file
View File

@ -0,0 +1,627 @@
#include <machine/rtems-bsd-kernel-space.h>
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (C) 2012-2013 Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/bio.h>
#include <sys/bus.h>
#include <sys/conf.h>
#include <sys/disk.h>
#include <sys/fcntl.h>
#include <sys/ioccom.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/proc.h>
#include <sys/systm.h>
#include <dev/pci/pcivar.h>
#include <geom/geom.h>
#include "nvme_private.h"
static void nvme_bio_child_inbed(struct bio *parent, int bio_error);
static void nvme_bio_child_done(void *arg,
const struct nvme_completion *cpl);
static uint32_t nvme_get_num_segments(uint64_t addr, uint64_t size,
uint32_t alignment);
static void nvme_free_child_bios(int num_bios,
struct bio **child_bios);
static struct bio ** nvme_allocate_child_bios(int num_bios);
static struct bio ** nvme_construct_child_bios(struct bio *bp,
uint32_t alignment,
int *num_bios);
static int nvme_ns_split_bio(struct nvme_namespace *ns,
struct bio *bp,
uint32_t alignment);
static int
nvme_ns_ioctl(struct cdev *cdev, u_long cmd, caddr_t arg, int flag,
struct thread *td)
{
struct nvme_namespace *ns;
struct nvme_controller *ctrlr;
struct nvme_pt_command *pt;
ns = cdev->si_drv1;
ctrlr = ns->ctrlr;
switch (cmd) {
case NVME_IO_TEST:
case NVME_BIO_TEST:
nvme_ns_test(ns, cmd, arg);
break;
case NVME_PASSTHROUGH_CMD:
pt = (struct nvme_pt_command *)arg;
return (nvme_ctrlr_passthrough_cmd(ctrlr, pt, ns->id,
1 /* is_user_buffer */, 0 /* is_admin_cmd */));
case NVME_GET_NSID:
{
struct nvme_get_nsid *gnsid = (struct nvme_get_nsid *)arg;
strncpy(gnsid->cdev, device_get_nameunit(ctrlr->dev),
sizeof(gnsid->cdev));
gnsid->nsid = ns->id;
break;
}
case DIOCGMEDIASIZE:
*(off_t *)arg = (off_t)nvme_ns_get_size(ns);
break;
case DIOCGSECTORSIZE:
*(u_int *)arg = nvme_ns_get_sector_size(ns);
break;
default:
return (ENOTTY);
}
return (0);
}
static int
nvme_ns_open(struct cdev *dev __unused, int flags, int fmt __unused,
struct thread *td)
{
int error = 0;
if (flags & FWRITE)
error = securelevel_gt(td->td_ucred, 0);
return (error);
}
static int
nvme_ns_close(struct cdev *dev __unused, int flags, int fmt __unused,
struct thread *td)
{
return (0);
}
static void
nvme_ns_strategy_done(void *arg, const struct nvme_completion *cpl)
{
struct bio *bp = arg;
/*
* TODO: add more extensive translation of NVMe status codes
* to different bio error codes (i.e. EIO, EINVAL, etc.)
*/
if (nvme_completion_is_error(cpl)) {
bp->bio_error = EIO;
bp->bio_flags |= BIO_ERROR;
bp->bio_resid = bp->bio_bcount;
} else
bp->bio_resid = 0;
biodone(bp);
}
static void
nvme_ns_strategy(struct bio *bp)
{
struct nvme_namespace *ns;
int err;
ns = bp->bio_dev->si_drv1;
err = nvme_ns_bio_process(ns, bp, nvme_ns_strategy_done);
if (err) {
bp->bio_error = err;
bp->bio_flags |= BIO_ERROR;
bp->bio_resid = bp->bio_bcount;
biodone(bp);
}
}
static struct cdevsw nvme_ns_cdevsw = {
.d_version = D_VERSION,
.d_flags = D_DISK,
.d_read = physread,
.d_write = physwrite,
.d_open = nvme_ns_open,
.d_close = nvme_ns_close,
.d_strategy = nvme_ns_strategy,
.d_ioctl = nvme_ns_ioctl
};
uint32_t
nvme_ns_get_max_io_xfer_size(struct nvme_namespace *ns)
{
return ns->ctrlr->max_xfer_size;
}
uint32_t
nvme_ns_get_sector_size(struct nvme_namespace *ns)
{
uint8_t flbas_fmt, lbads;
flbas_fmt = (ns->data.flbas >> NVME_NS_DATA_FLBAS_FORMAT_SHIFT) &
NVME_NS_DATA_FLBAS_FORMAT_MASK;
lbads = (ns->data.lbaf[flbas_fmt] >> NVME_NS_DATA_LBAF_LBADS_SHIFT) &
NVME_NS_DATA_LBAF_LBADS_MASK;
return (1 << lbads);
}
uint64_t
nvme_ns_get_num_sectors(struct nvme_namespace *ns)
{
return (ns->data.nsze);
}
uint64_t
nvme_ns_get_size(struct nvme_namespace *ns)
{
return (nvme_ns_get_num_sectors(ns) * nvme_ns_get_sector_size(ns));
}
uint32_t
nvme_ns_get_flags(struct nvme_namespace *ns)
{
return (ns->flags);
}
const char *
nvme_ns_get_serial_number(struct nvme_namespace *ns)
{
return ((const char *)ns->ctrlr->cdata.sn);
}
const char *
nvme_ns_get_model_number(struct nvme_namespace *ns)
{
return ((const char *)ns->ctrlr->cdata.mn);
}
const struct nvme_namespace_data *
nvme_ns_get_data(struct nvme_namespace *ns)
{
return (&ns->data);
}
uint32_t
nvme_ns_get_stripesize(struct nvme_namespace *ns)
{
if (((ns->data.nsfeat >> NVME_NS_DATA_NSFEAT_NPVALID_SHIFT) &
NVME_NS_DATA_NSFEAT_NPVALID_MASK) != 0 && ns->data.npwg != 0) {
return ((ns->data.npwg + 1) * nvme_ns_get_sector_size(ns));
}
return (ns->boundary);
}
static void
nvme_ns_bio_done(void *arg, const struct nvme_completion *status)
{
struct bio *bp = arg;
nvme_cb_fn_t bp_cb_fn;
bp_cb_fn = bp->bio_driver1;
if (bp->bio_driver2)
free(bp->bio_driver2, M_NVME);
if (nvme_completion_is_error(status)) {
bp->bio_flags |= BIO_ERROR;
if (bp->bio_error == 0)
bp->bio_error = EIO;
}
if ((bp->bio_flags & BIO_ERROR) == 0)
bp->bio_resid = 0;
else
bp->bio_resid = bp->bio_bcount;
bp_cb_fn(bp, status);
}
static void
nvme_bio_child_inbed(struct bio *parent, int bio_error)
{
struct nvme_completion parent_cpl;
int children, inbed;
if (bio_error != 0) {
parent->bio_flags |= BIO_ERROR;
parent->bio_error = bio_error;
}
/*
* atomic_fetchadd will return value before adding 1, so we still
* must add 1 to get the updated inbed number. Save bio_children
* before incrementing to guard against race conditions when
* two children bios complete on different queues.
*/
children = atomic_load_acq_int(&parent->bio_children);
inbed = atomic_fetchadd_int(&parent->bio_inbed, 1) + 1;
if (inbed == children) {
bzero(&parent_cpl, sizeof(parent_cpl));
if (parent->bio_flags & BIO_ERROR) {
parent_cpl.status &= ~(NVME_STATUS_SC_MASK << NVME_STATUS_SC_SHIFT);
parent_cpl.status |= (NVME_SC_DATA_TRANSFER_ERROR) << NVME_STATUS_SC_SHIFT;
}
nvme_ns_bio_done(parent, &parent_cpl);
}
}
static void
nvme_bio_child_done(void *arg, const struct nvme_completion *cpl)
{
struct bio *child = arg;
struct bio *parent;
int bio_error;
parent = child->bio_parent;
g_destroy_bio(child);
bio_error = nvme_completion_is_error(cpl) ? EIO : 0;
nvme_bio_child_inbed(parent, bio_error);
}
static uint32_t
nvme_get_num_segments(uint64_t addr, uint64_t size, uint32_t align)
{
uint32_t num_segs, offset, remainder;
if (align == 0)
return (1);
KASSERT((align & (align - 1)) == 0, ("alignment not power of 2\n"));
num_segs = size / align;
remainder = size & (align - 1);
offset = addr & (align - 1);
if (remainder > 0 || offset > 0)
num_segs += 1 + (remainder + offset - 1) / align;
return (num_segs);
}
static void
nvme_free_child_bios(int num_bios, struct bio **child_bios)
{
int i;
for (i = 0; i < num_bios; i++) {
if (child_bios[i] != NULL)
g_destroy_bio(child_bios[i]);
}
free(child_bios, M_NVME);
}
static struct bio **
nvme_allocate_child_bios(int num_bios)
{
struct bio **child_bios;
int err = 0, i;
child_bios = malloc(num_bios * sizeof(struct bio *), M_NVME, M_NOWAIT);
if (child_bios == NULL)
return (NULL);
for (i = 0; i < num_bios; i++) {
child_bios[i] = g_new_bio();
if (child_bios[i] == NULL)
err = ENOMEM;
}
if (err == ENOMEM) {
nvme_free_child_bios(num_bios, child_bios);
return (NULL);
}
return (child_bios);
}
static struct bio **
nvme_construct_child_bios(struct bio *bp, uint32_t alignment, int *num_bios)
{
struct bio **child_bios;
struct bio *child;
uint64_t cur_offset;
caddr_t data;
uint32_t rem_bcount;
int i;
struct vm_page **ma;
uint32_t ma_offset;
*num_bios = nvme_get_num_segments(bp->bio_offset, bp->bio_bcount,
alignment);
child_bios = nvme_allocate_child_bios(*num_bios);
if (child_bios == NULL)
return (NULL);
bp->bio_children = *num_bios;
bp->bio_inbed = 0;
cur_offset = bp->bio_offset;
rem_bcount = bp->bio_bcount;
data = bp->bio_data;
ma_offset = bp->bio_ma_offset;
ma = bp->bio_ma;
for (i = 0; i < *num_bios; i++) {
child = child_bios[i];
child->bio_parent = bp;
child->bio_cmd = bp->bio_cmd;
child->bio_offset = cur_offset;
child->bio_bcount = min(rem_bcount,
alignment - (cur_offset & (alignment - 1)));
child->bio_flags = bp->bio_flags;
if (bp->bio_flags & BIO_UNMAPPED) {
child->bio_ma_offset = ma_offset;
child->bio_ma = ma;
child->bio_ma_n =
nvme_get_num_segments(child->bio_ma_offset,
child->bio_bcount, PAGE_SIZE);
ma_offset = (ma_offset + child->bio_bcount) &
PAGE_MASK;
ma += child->bio_ma_n;
if (ma_offset != 0)
ma -= 1;
} else {
child->bio_data = data;
data += child->bio_bcount;
}
cur_offset += child->bio_bcount;
rem_bcount -= child->bio_bcount;
}
return (child_bios);
}
static int
nvme_ns_split_bio(struct nvme_namespace *ns, struct bio *bp,
uint32_t alignment)
{
struct bio *child;
struct bio **child_bios;
int err, i, num_bios;
child_bios = nvme_construct_child_bios(bp, alignment, &num_bios);
if (child_bios == NULL)
return (ENOMEM);
for (i = 0; i < num_bios; i++) {
child = child_bios[i];
err = nvme_ns_bio_process(ns, child, nvme_bio_child_done);
if (err != 0) {
nvme_bio_child_inbed(bp, err);
g_destroy_bio(child);
}
}
free(child_bios, M_NVME);
return (0);
}
int
nvme_ns_bio_process(struct nvme_namespace *ns, struct bio *bp,
nvme_cb_fn_t cb_fn)
{
struct nvme_dsm_range *dsm_range;
uint32_t num_bios;
int err;
bp->bio_driver1 = cb_fn;
if (ns->boundary > 0 &&
(bp->bio_cmd == BIO_READ || bp->bio_cmd == BIO_WRITE)) {
num_bios = nvme_get_num_segments(bp->bio_offset,
bp->bio_bcount, ns->boundary);
if (num_bios > 1)
return (nvme_ns_split_bio(ns, bp, ns->boundary));
}
switch (bp->bio_cmd) {
case BIO_READ:
err = nvme_ns_cmd_read_bio(ns, bp, nvme_ns_bio_done, bp);
break;
case BIO_WRITE:
err = nvme_ns_cmd_write_bio(ns, bp, nvme_ns_bio_done, bp);
break;
case BIO_FLUSH:
err = nvme_ns_cmd_flush(ns, nvme_ns_bio_done, bp);
break;
case BIO_DELETE:
dsm_range =
malloc(sizeof(struct nvme_dsm_range), M_NVME,
M_ZERO | M_WAITOK);
if (!dsm_range) {
err = ENOMEM;
break;
}
dsm_range->length =
htole32(bp->bio_bcount/nvme_ns_get_sector_size(ns));
dsm_range->starting_lba =
htole64(bp->bio_offset/nvme_ns_get_sector_size(ns));
bp->bio_driver2 = dsm_range;
err = nvme_ns_cmd_deallocate(ns, dsm_range, 1,
nvme_ns_bio_done, bp);
if (err != 0)
free(dsm_range, M_NVME);
break;
default:
err = EIO;
break;
}
return (err);
}
int
nvme_ns_ioctl_process(struct nvme_namespace *ns, u_long cmd, caddr_t arg,
int flag, struct thread *td)
{
return (nvme_ns_ioctl(ns->cdev, cmd, arg, flag, td));
}
int
nvme_ns_construct(struct nvme_namespace *ns, uint32_t id,
struct nvme_controller *ctrlr)
{
struct make_dev_args md_args;
struct nvme_completion_poll_status status;
int res;
int unit;
uint8_t flbas_fmt;
uint8_t vwc_present;
ns->ctrlr = ctrlr;
ns->id = id;
/*
* Namespaces are reconstructed after a controller reset, so check
* to make sure we only call mtx_init once on each mtx.
*
* TODO: Move this somewhere where it gets called at controller
* construction time, which is not invoked as part of each
* controller reset.
*/
if (!mtx_initialized(&ns->lock))
mtx_init(&ns->lock, "nvme ns lock", NULL, MTX_DEF);
status.done = 0;
nvme_ctrlr_cmd_identify_namespace(ctrlr, id, &ns->data,
nvme_completion_poll_cb, &status);
nvme_completion_poll(&status);
if (nvme_completion_is_error(&status.cpl)) {
nvme_printf(ctrlr, "nvme_identify_namespace failed\n");
return (ENXIO);
}
/* Convert data to host endian */
nvme_namespace_data_swapbytes(&ns->data);
/*
* If the size of is zero, chances are this isn't a valid
* namespace (eg one that's not been configured yet). The
* standard says the entire id will be zeros, so this is a
* cheap way to test for that.
*/
if (ns->data.nsze == 0)
return (ENXIO);
flbas_fmt = (ns->data.flbas >> NVME_NS_DATA_FLBAS_FORMAT_SHIFT) &
NVME_NS_DATA_FLBAS_FORMAT_MASK;
/*
* Note: format is a 0-based value, so > is appropriate here,
* not >=.
*/
if (flbas_fmt > ns->data.nlbaf) {
printf("lba format %d exceeds number supported (%d)\n",
flbas_fmt, ns->data.nlbaf + 1);
return (ENXIO);
}
/*
* Older Intel devices advertise in vendor specific space an alignment
* that improves performance. If present use for the stripe size. NVMe
* 1.3 standardized this as NOIOB, and newer Intel drives use that.
*/
switch (pci_get_devid(ctrlr->dev)) {
case 0x09538086: /* Intel DC PC3500 */
case 0x0a538086: /* Intel DC PC3520 */
case 0x0a548086: /* Intel DC PC4500 */
case 0x0a558086: /* Dell Intel P4600 */
if (ctrlr->cdata.vs[3] != 0)
ns->boundary =
(1 << ctrlr->cdata.vs[3]) * ctrlr->min_page_size;
else
ns->boundary = 0;
break;
default:
ns->boundary = ns->data.noiob * nvme_ns_get_sector_size(ns);
break;
}
if (nvme_ctrlr_has_dataset_mgmt(&ctrlr->cdata))
ns->flags |= NVME_NS_DEALLOCATE_SUPPORTED;
vwc_present = (ctrlr->cdata.vwc >> NVME_CTRLR_DATA_VWC_PRESENT_SHIFT) &
NVME_CTRLR_DATA_VWC_PRESENT_MASK;
if (vwc_present)
ns->flags |= NVME_NS_FLUSH_SUPPORTED;
/*
* cdev may have already been created, if we are reconstructing the
* namespace after a controller-level reset.
*/
if (ns->cdev != NULL)
return (0);
/*
* Namespace IDs start at 1, so we need to subtract 1 to create a
* correct unit number.
*/
unit = device_get_unit(ctrlr->dev) * NVME_MAX_NAMESPACES + ns->id - 1;
make_dev_args_init(&md_args);
md_args.mda_devsw = &nvme_ns_cdevsw;
md_args.mda_unit = unit;
md_args.mda_mode = 0600;
md_args.mda_si_drv1 = ns;
res = make_dev_s(&md_args, &ns->cdev, "nvme%dns%d",
device_get_unit(ctrlr->dev), ns->id);
if (res != 0)
return (ENXIO);
ns->cdev->si_flags |= SI_UNMAPPED;
return (0);
}
void nvme_ns_destruct(struct nvme_namespace *ns)
{
if (ns->cdev != NULL)
destroy_dev(ns->cdev);
}

208
freebsd/sys/dev/nvme/nvme_ns_cmd.c Normal file
View File

@ -0,0 +1,208 @@
#include <machine/rtems-bsd-kernel-space.h>
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (C) 2012 Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include "nvme_private.h"
int
nvme_ns_cmd_read(struct nvme_namespace *ns, void *payload, uint64_t lba,
uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
req = nvme_allocate_request_vaddr(payload,
lba_count*nvme_ns_get_sector_size(ns), cb_fn, cb_arg);
if (req == NULL)
return (ENOMEM);
nvme_ns_read_cmd(&req->cmd, ns->id, lba, lba_count);
nvme_ctrlr_submit_io_request(ns->ctrlr, req);
return (0);
}
int
nvme_ns_cmd_read_bio(struct nvme_namespace *ns, struct bio *bp,
nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
uint64_t lba;
uint64_t lba_count;
req = nvme_allocate_request_bio(bp, cb_fn, cb_arg);
if (req == NULL)
return (ENOMEM);
lba = bp->bio_offset / nvme_ns_get_sector_size(ns);
lba_count = bp->bio_bcount / nvme_ns_get_sector_size(ns);
nvme_ns_read_cmd(&req->cmd, ns->id, lba, lba_count);
nvme_ctrlr_submit_io_request(ns->ctrlr, req);
return (0);
}
int
nvme_ns_cmd_write(struct nvme_namespace *ns, void *payload, uint64_t lba,
uint32_t lba_count, nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
req = nvme_allocate_request_vaddr(payload,
lba_count*nvme_ns_get_sector_size(ns), cb_fn, cb_arg);
if (req == NULL)
return (ENOMEM);
nvme_ns_write_cmd(&req->cmd, ns->id, lba, lba_count);
nvme_ctrlr_submit_io_request(ns->ctrlr, req);
return (0);
}
int
nvme_ns_cmd_write_bio(struct nvme_namespace *ns, struct bio *bp,
nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
uint64_t lba;
uint64_t lba_count;
req = nvme_allocate_request_bio(bp, cb_fn, cb_arg);
if (req == NULL)
return (ENOMEM);
lba = bp->bio_offset / nvme_ns_get_sector_size(ns);
lba_count = bp->bio_bcount / nvme_ns_get_sector_size(ns);
nvme_ns_write_cmd(&req->cmd, ns->id, lba, lba_count);
nvme_ctrlr_submit_io_request(ns->ctrlr, req);
return (0);
}
int
nvme_ns_cmd_deallocate(struct nvme_namespace *ns, void *payload,
uint8_t num_ranges, nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
struct nvme_command *cmd;
req = nvme_allocate_request_vaddr(payload,
num_ranges * sizeof(struct nvme_dsm_range), cb_fn, cb_arg);
if (req == NULL)
return (ENOMEM);
cmd = &req->cmd;
cmd->opc = NVME_OPC_DATASET_MANAGEMENT;
cmd->nsid = htole32(ns->id);
/* TODO: create a delete command data structure */
cmd->cdw10 = htole32(num_ranges - 1);
cmd->cdw11 = htole32(NVME_DSM_ATTR_DEALLOCATE);
nvme_ctrlr_submit_io_request(ns->ctrlr, req);
return (0);
}
int
nvme_ns_cmd_flush(struct nvme_namespace *ns, nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
req = nvme_allocate_request_null(cb_fn, cb_arg);
if (req == NULL)
return (ENOMEM);
nvme_ns_flush_cmd(&req->cmd, ns->id);
nvme_ctrlr_submit_io_request(ns->ctrlr, req);
return (0);
}
/* Timeout = 1 sec */
#define NVD_DUMP_TIMEOUT 200000
int
nvme_ns_dump(struct nvme_namespace *ns, void *virt, off_t offset, size_t len)
{
struct nvme_completion_poll_status status;
struct nvme_request *req;
struct nvme_command *cmd;
uint64_t lba, lba_count;
int i;
status.done = FALSE;
req = nvme_allocate_request_vaddr(virt, len, nvme_completion_poll_cb,
&status);
if (req == NULL)
return (ENOMEM);
cmd = &req->cmd;
if (len > 0) {
lba = offset / nvme_ns_get_sector_size(ns);
lba_count = len / nvme_ns_get_sector_size(ns);
nvme_ns_write_cmd(cmd, ns->id, lba, lba_count);
} else
nvme_ns_flush_cmd(cmd, ns->id);
nvme_ctrlr_submit_io_request(ns->ctrlr, req);
if (req->qpair == NULL)
return (ENXIO);
i = 0;
while ((i++ < NVD_DUMP_TIMEOUT) && (status.done == FALSE)) {
DELAY(5);
nvme_qpair_process_completions(req->qpair);
}
/*
* Normally, when using the polling interface, we can't return a
* timeout error because we don't know when the completion routines
* will be called if the command later completes. However, in this
* case we're running a system dump, so all interrupts are turned
* off, the scheduler isn't running so there's nothing to complete
* the transaction.
*/
if (status.done == FALSE)
return (ETIMEDOUT);
return (0);
}

358
freebsd/sys/dev/nvme/nvme_pci.c Normal file
View File

@ -0,0 +1,358 @@
#include <machine/rtems-bsd-kernel-space.h>
/*-
* Copyright (C) 2012-2016 Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/buf.h>
#include <sys/bus.h>
#include <sys/conf.h>
#include <sys/proc.h>
#include <sys/smp.h>
#include <dev/pci/pcireg.h>
#include <dev/pci/pcivar.h>
#include "nvme_private.h"
static int nvme_pci_probe(device_t);
static int nvme_pci_attach(device_t);
static int nvme_pci_detach(device_t);
static int nvme_pci_suspend(device_t);
static int nvme_pci_resume(device_t);
static void nvme_ctrlr_setup_interrupts(struct nvme_controller *ctrlr);
static device_method_t nvme_pci_methods[] = {
/* Device interface */
DEVMETHOD(device_probe, nvme_pci_probe),
DEVMETHOD(device_attach, nvme_pci_attach),
DEVMETHOD(device_detach, nvme_pci_detach),
DEVMETHOD(device_suspend, nvme_pci_suspend),
DEVMETHOD(device_resume, nvme_pci_resume),
DEVMETHOD(device_shutdown, nvme_shutdown),
{ 0, 0 }
};
static driver_t nvme_pci_driver = {
"nvme",
nvme_pci_methods,
sizeof(struct nvme_controller),
};
DRIVER_MODULE(nvme, pci, nvme_pci_driver, nvme_devclass, NULL, 0);
static struct _pcsid
{
uint32_t devid;
int match_subdevice;
uint16_t subdevice;
const char *desc;
uint32_t quirks;
} pci_ids[] = {
{ 0x01118086, 0, 0, "NVMe Controller" },
{ IDT32_PCI_ID, 0, 0, "IDT NVMe Controller (32 channel)" },
{ IDT8_PCI_ID, 0, 0, "IDT NVMe Controller (8 channel)" },
{ 0x09538086, 1, 0x3702, "DC P3700 SSD" },
{ 0x09538086, 1, 0x3703, "DC P3700 SSD [2.5\" SFF]" },
{ 0x09538086, 1, 0x3704, "DC P3500 SSD [Add-in Card]" },
{ 0x09538086, 1, 0x3705, "DC P3500 SSD [2.5\" SFF]" },
{ 0x09538086, 1, 0x3709, "DC P3600 SSD [Add-in Card]" },
{ 0x09538086, 1, 0x370a, "DC P3600 SSD [2.5\" SFF]" },
{ 0x00031c58, 0, 0, "HGST SN100", QUIRK_DELAY_B4_CHK_RDY },
{ 0x00231c58, 0, 0, "WDC SN200", QUIRK_DELAY_B4_CHK_RDY },
{ 0x05401c5f, 0, 0, "Memblaze Pblaze4", QUIRK_DELAY_B4_CHK_RDY },
{ 0xa821144d, 0, 0, "Samsung PM1725", QUIRK_DELAY_B4_CHK_RDY },
{ 0xa822144d, 0, 0, "Samsung PM1725a", QUIRK_DELAY_B4_CHK_RDY },
{ 0x00000000, 0, 0, NULL }
};
static int
nvme_match(uint32_t devid, uint16_t subdevice, struct _pcsid *ep)
{
if (devid != ep->devid)
return 0;
if (!ep->match_subdevice)
return 1;
if (subdevice == ep->subdevice)
return 1;
else
return 0;
}
static int
nvme_pci_probe (device_t device)
{
struct nvme_controller *ctrlr = DEVICE2SOFTC(device);
struct _pcsid *ep;
uint32_t devid;
uint16_t subdevice;
devid = pci_get_devid(device);
subdevice = pci_get_subdevice(device);
ep = pci_ids;
while (ep->devid) {
if (nvme_match(devid, subdevice, ep))
break;
++ep;
}
if (ep->devid)
ctrlr->quirks = ep->quirks;
if (ep->desc) {
device_set_desc(device, ep->desc);
return (BUS_PROBE_DEFAULT);
}
#if defined(PCIS_STORAGE_NVM)
if (pci_get_class(device) == PCIC_STORAGE &&
pci_get_subclass(device) == PCIS_STORAGE_NVM &&
pci_get_progif(device) == PCIP_STORAGE_NVM_ENTERPRISE_NVMHCI_1_0) {
device_set_desc(device, "Generic NVMe Device");
return (BUS_PROBE_GENERIC);
}
#endif
return (ENXIO);
}
static int
nvme_ctrlr_allocate_bar(struct nvme_controller *ctrlr)
{
ctrlr->resource_id = PCIR_BAR(0);
ctrlr->resource = bus_alloc_resource_any(ctrlr->dev, SYS_RES_MEMORY,
&ctrlr->resource_id, RF_ACTIVE);
if(ctrlr->resource == NULL) {
nvme_printf(ctrlr, "unable to allocate pci resource\n");
return (ENOMEM);
}
ctrlr->bus_tag = rman_get_bustag(ctrlr->resource);
ctrlr->bus_handle = rman_get_bushandle(ctrlr->resource);
ctrlr->regs = (struct nvme_registers *)ctrlr->bus_handle;
/*
* The NVMe spec allows for the MSI-X table to be placed behind
* BAR 4/5, separate from the control/doorbell registers. Always
* try to map this bar, because it must be mapped prior to calling
* pci_alloc_msix(). If the table isn't behind BAR 4/5,
* bus_alloc_resource() will just return NULL which is OK.
*/
ctrlr->bar4_resource_id = PCIR_BAR(4);
ctrlr->bar4_resource = bus_alloc_resource_any(ctrlr->dev, SYS_RES_MEMORY,
&ctrlr->bar4_resource_id, RF_ACTIVE);
return (0);
}
static int
nvme_pci_attach(device_t dev)
{
struct nvme_controller*ctrlr = DEVICE2SOFTC(dev);
int status;
ctrlr->dev = dev;
status = nvme_ctrlr_allocate_bar(ctrlr);
if (status != 0)
goto bad;
pci_enable_busmaster(dev);
nvme_ctrlr_setup_interrupts(ctrlr);
return nvme_attach(dev);
bad:
if (ctrlr->resource != NULL) {
bus_release_resource(dev, SYS_RES_MEMORY,
ctrlr->resource_id, ctrlr->resource);
}
if (ctrlr->bar4_resource != NULL) {
bus_release_resource(dev, SYS_RES_MEMORY,
ctrlr->bar4_resource_id, ctrlr->bar4_resource);
}
if (ctrlr->tag)
bus_teardown_intr(dev, ctrlr->res, ctrlr->tag);
if (ctrlr->res)
bus_release_resource(dev, SYS_RES_IRQ,
rman_get_rid(ctrlr->res), ctrlr->res);
if (ctrlr->msix_enabled)
pci_release_msi(dev);
return status;
}
static int
nvme_pci_detach(device_t dev)
{
struct nvme_controller*ctrlr = DEVICE2SOFTC(dev);
int rv;
rv = nvme_detach(dev);
if (ctrlr->msix_enabled)
pci_release_msi(dev);
pci_disable_busmaster(dev);
return (rv);
}
static int
nvme_ctrlr_configure_intx(struct nvme_controller *ctrlr)
{
ctrlr->msix_enabled = 0;
ctrlr->num_io_queues = 1;
ctrlr->num_cpus_per_ioq = mp_ncpus;
ctrlr->rid = 0;
ctrlr->res = bus_alloc_resource_any(ctrlr->dev, SYS_RES_IRQ,
&ctrlr->rid, RF_SHAREABLE | RF_ACTIVE);
if (ctrlr->res == NULL) {
nvme_printf(ctrlr, "unable to allocate shared IRQ\n");
return (ENOMEM);
}
bus_setup_intr(ctrlr->dev, ctrlr->res,
INTR_TYPE_MISC | INTR_MPSAFE, NULL, nvme_ctrlr_intx_handler,
ctrlr, &ctrlr->tag);
if (ctrlr->tag == NULL) {
nvme_printf(ctrlr, "unable to setup intx handler\n");
return (ENOMEM);
}
return (0);
}
static void
nvme_ctrlr_setup_interrupts(struct nvme_controller *ctrlr)
{
device_t dev;
int per_cpu_io_queues;
int min_cpus_per_ioq;
int num_vectors_requested, num_vectors_allocated;
int num_vectors_available;
dev = ctrlr->dev;
min_cpus_per_ioq = 1;
TUNABLE_INT_FETCH("hw.nvme.min_cpus_per_ioq", &min_cpus_per_ioq);
if (min_cpus_per_ioq < 1) {
min_cpus_per_ioq = 1;
} else if (min_cpus_per_ioq > mp_ncpus) {
min_cpus_per_ioq = mp_ncpus;
}
per_cpu_io_queues = 1;
TUNABLE_INT_FETCH("hw.nvme.per_cpu_io_queues", &per_cpu_io_queues);
if (per_cpu_io_queues == 0) {
min_cpus_per_ioq = mp_ncpus;
}
ctrlr->force_intx = 0;
TUNABLE_INT_FETCH("hw.nvme.force_intx", &ctrlr->force_intx);
/*
* FreeBSD currently cannot allocate more than about 190 vectors at
* boot, meaning that systems with high core count and many devices
* requesting per-CPU interrupt vectors will not get their full
* allotment. So first, try to allocate as many as we may need to
* understand what is available, then immediately release them.
* Then figure out how many of those we will actually use, based on
* assigning an equal number of cores to each I/O queue.
*/
/* One vector for per core I/O queue, plus one vector for admin queue. */
num_vectors_available = min(pci_msix_count(dev), mp_ncpus + 1);
if (pci_alloc_msix(dev, &num_vectors_available) != 0) {
num_vectors_available = 0;
}
pci_release_msi(dev);
if (ctrlr->force_intx || num_vectors_available < 2) {
nvme_ctrlr_configure_intx(ctrlr);
return;
}
/*
* Do not use all vectors for I/O queues - one must be saved for the
* admin queue.
*/
ctrlr->num_cpus_per_ioq = max(min_cpus_per_ioq,
howmany(mp_ncpus, num_vectors_available - 1));
ctrlr->num_io_queues = howmany(mp_ncpus, ctrlr->num_cpus_per_ioq);
num_vectors_requested = ctrlr->num_io_queues + 1;
num_vectors_allocated = num_vectors_requested;
/*
* Now just allocate the number of vectors we need. This should
* succeed, since we previously called pci_alloc_msix()
* successfully returning at least this many vectors, but just to
* be safe, if something goes wrong just revert to INTx.
*/
if (pci_alloc_msix(dev, &num_vectors_allocated) != 0) {
nvme_ctrlr_configure_intx(ctrlr);
return;
}
if (num_vectors_allocated < num_vectors_requested) {
pci_release_msi(dev);
nvme_ctrlr_configure_intx(ctrlr);
return;
}
ctrlr->msix_enabled = 1;
}
static int
nvme_pci_suspend(device_t dev)
{
struct nvme_controller *ctrlr;
ctrlr = DEVICE2SOFTC(dev);
return (nvme_ctrlr_suspend(ctrlr));
}
static int
nvme_pci_resume(device_t dev)
{
struct nvme_controller *ctrlr;
ctrlr = DEVICE2SOFTC(dev);
return (nvme_ctrlr_resume(ctrlr));
}

562
freebsd/sys/dev/nvme/nvme_private.h Normal file
View File

@ -0,0 +1,562 @@
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (C) 2012-2014 Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#ifndef __NVME_PRIVATE_H__
#define __NVME_PRIVATE_H__
#include <sys/param.h>
#include <sys/bio.h>
#include <sys/bus.h>
#include <sys/kernel.h>
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/module.h>
#include <sys/mutex.h>
#include <sys/rman.h>
#include <sys/systm.h>
#include <sys/taskqueue.h>
#include <vm/uma.h>
#include <machine/bus.h>
#include "nvme.h"
#define DEVICE2SOFTC(dev) ((struct nvme_controller *) device_get_softc(dev))
MALLOC_DECLARE(M_NVME);
#define IDT32_PCI_ID 0x80d0111d /* 32 channel board */
#define IDT8_PCI_ID 0x80d2111d /* 8 channel board */
/*
* For commands requiring more than 2 PRP entries, one PRP will be
* embedded in the command (prp1), and the rest of the PRP entries
* will be in a list pointed to by the command (prp2). This means
* that real max number of PRP entries we support is 32+1, which
* results in a max xfer size of 32*PAGE_SIZE.
*/
#define NVME_MAX_PRP_LIST_ENTRIES (NVME_MAX_XFER_SIZE / PAGE_SIZE)
#define NVME_ADMIN_TRACKERS (16)
#define NVME_ADMIN_ENTRIES (128)
/* min and max are defined in admin queue attributes section of spec */
#define NVME_MIN_ADMIN_ENTRIES (2)
#define NVME_MAX_ADMIN_ENTRIES (4096)
/*
* NVME_IO_ENTRIES defines the size of an I/O qpair's submission and completion
* queues, while NVME_IO_TRACKERS defines the maximum number of I/O that we
* will allow outstanding on an I/O qpair at any time. The only advantage in
* having IO_ENTRIES > IO_TRACKERS is for debugging purposes - when dumping
* the contents of the submission and completion queues, it will show a longer
* history of data.
*/
#define NVME_IO_ENTRIES (256)
#define NVME_IO_TRACKERS (128)
#define NVME_MIN_IO_TRACKERS (4)
#define NVME_MAX_IO_TRACKERS (1024)
/*
* NVME_MAX_IO_ENTRIES is not defined, since it is specified in CC.MQES
* for each controller.
*/
#define NVME_INT_COAL_TIME (0) /* disabled */
#define NVME_INT_COAL_THRESHOLD (0) /* 0-based */
#define NVME_MAX_NAMESPACES (16)
#define NVME_MAX_CONSUMERS (2)
#define NVME_MAX_ASYNC_EVENTS (8)
#define NVME_DEFAULT_TIMEOUT_PERIOD (30) /* in seconds */
#define NVME_MIN_TIMEOUT_PERIOD (5)
#define NVME_MAX_TIMEOUT_PERIOD (120)
#define NVME_DEFAULT_RETRY_COUNT (4)
/* Maximum log page size to fetch for AERs. */
#define NVME_MAX_AER_LOG_SIZE (4096)
/*
* Define CACHE_LINE_SIZE here for older FreeBSD versions that do not define
* it.
*/
#ifndef CACHE_LINE_SIZE
#define CACHE_LINE_SIZE (64)
#endif
extern uma_zone_t nvme_request_zone;
extern int32_t nvme_retry_count;
extern bool nvme_verbose_cmd_dump;
struct nvme_completion_poll_status {
struct nvme_completion cpl;
int done;
};
extern devclass_t nvme_devclass;
#define NVME_REQUEST_VADDR 1
#define NVME_REQUEST_NULL 2 /* For requests with no payload. */
#define NVME_REQUEST_UIO 3
#define NVME_REQUEST_BIO 4
#define NVME_REQUEST_CCB 5
struct nvme_request {
struct nvme_command cmd;
struct nvme_qpair *qpair;
union {
void *payload;
struct bio *bio;
} u;
uint32_t type;
uint32_t payload_size;
boolean_t timeout;
nvme_cb_fn_t cb_fn;
void *cb_arg;
int32_t retries;
STAILQ_ENTRY(nvme_request) stailq;
};
struct nvme_async_event_request {
struct nvme_controller *ctrlr;
struct nvme_request *req;
struct nvme_completion cpl;
uint32_t log_page_id;
uint32_t log_page_size;
uint8_t log_page_buffer[NVME_MAX_AER_LOG_SIZE];
};
struct nvme_tracker {
TAILQ_ENTRY(nvme_tracker) tailq;
struct nvme_request *req;
struct nvme_qpair *qpair;
struct callout timer;
bus_dmamap_t payload_dma_map;
uint16_t cid;
uint64_t *prp;
bus_addr_t prp_bus_addr;
};
struct nvme_qpair {
struct nvme_controller *ctrlr;
uint32_t id;
uint32_t phase;
uint16_t vector;
int rid;
struct resource *res;
void *tag;
uint32_t num_entries;
uint32_t num_trackers;
uint32_t sq_tdbl_off;
uint32_t cq_hdbl_off;
uint32_t sq_head;
uint32_t sq_tail;
uint32_t cq_head;
int64_t num_cmds;
int64_t num_intr_handler_calls;
int64_t num_retries;
int64_t num_failures;
struct nvme_command *cmd;
struct nvme_completion *cpl;
bus_dma_tag_t dma_tag;
bus_dma_tag_t dma_tag_payload;
bus_dmamap_t queuemem_map;
uint64_t cmd_bus_addr;
uint64_t cpl_bus_addr;
TAILQ_HEAD(, nvme_tracker) free_tr;
TAILQ_HEAD(, nvme_tracker) outstanding_tr;
STAILQ_HEAD(, nvme_request) queued_req;
struct nvme_tracker **act_tr;
boolean_t is_enabled;
struct mtx lock __aligned(CACHE_LINE_SIZE);
} __aligned(CACHE_LINE_SIZE);
struct nvme_namespace {
struct nvme_controller *ctrlr;
struct nvme_namespace_data data;
uint32_t id;
uint32_t flags;
struct cdev *cdev;
void *cons_cookie[NVME_MAX_CONSUMERS];
uint32_t boundary;
struct mtx lock;
};
/*
* One of these per allocated PCI device.
*/
struct nvme_controller {
device_t dev;
struct mtx lock;
uint32_t ready_timeout_in_ms;
uint32_t quirks;
#define QUIRK_DELAY_B4_CHK_RDY 1 /* Can't touch MMIO on disable */
#define QUIRK_DISABLE_TIMEOUT 2 /* Disable broken completion timeout feature */
bus_space_tag_t bus_tag;
bus_space_handle_t bus_handle;
int resource_id;
struct resource *resource;
/*
* The NVMe spec allows for the MSI-X table to be placed in BAR 4/5,
* separate from the control registers which are in BAR 0/1. These
* members track the mapping of BAR 4/5 for that reason.
*/
int bar4_resource_id;
struct resource *bar4_resource;
uint32_t msix_enabled;
uint32_t force_intx;
uint32_t enable_aborts;
uint32_t num_io_queues;
uint32_t num_cpus_per_ioq;
uint32_t max_hw_pend_io;
/* Fields for tracking progress during controller initialization. */
struct intr_config_hook config_hook;
uint32_t ns_identified;
uint32_t queues_created;
struct task reset_task;
struct task fail_req_task;
struct taskqueue *taskqueue;
/* For shared legacy interrupt. */
int rid;
struct resource *res;
void *tag;
bus_dma_tag_t hw_desc_tag;
bus_dmamap_t hw_desc_map;
/** maximum i/o size in bytes */
uint32_t max_xfer_size;
/** minimum page size supported by this controller in bytes */
uint32_t min_page_size;
/** interrupt coalescing time period (in microseconds) */
uint32_t int_coal_time;
/** interrupt coalescing threshold */
uint32_t int_coal_threshold;
/** timeout period in seconds */
uint32_t timeout_period;
struct nvme_qpair adminq;
struct nvme_qpair *ioq;
struct nvme_registers *regs;
struct nvme_controller_data cdata;
struct nvme_namespace ns[NVME_MAX_NAMESPACES];
struct cdev *cdev;
/** bit mask of event types currently enabled for async events */
uint32_t async_event_config;
uint32_t num_aers;
struct nvme_async_event_request aer[NVME_MAX_ASYNC_EVENTS];
void *cons_cookie[NVME_MAX_CONSUMERS];
uint32_t is_resetting;
uint32_t is_initialized;
uint32_t notification_sent;
boolean_t is_failed;
STAILQ_HEAD(, nvme_request) fail_req;
};
#define nvme_mmio_offsetof(reg) \
offsetof(struct nvme_registers, reg)
#define nvme_mmio_read_4(sc, reg) \
bus_space_read_4((sc)->bus_tag, (sc)->bus_handle, \
nvme_mmio_offsetof(reg))
#define nvme_mmio_write_4(sc, reg, val) \
bus_space_write_4((sc)->bus_tag, (sc)->bus_handle, \
nvme_mmio_offsetof(reg), val)
#define nvme_mmio_write_8(sc, reg, val) \
do { \
bus_space_write_4((sc)->bus_tag, (sc)->bus_handle, \
nvme_mmio_offsetof(reg), val & 0xFFFFFFFF); \
bus_space_write_4((sc)->bus_tag, (sc)->bus_handle, \
nvme_mmio_offsetof(reg)+4, \
(val & 0xFFFFFFFF00000000ULL) >> 32); \
} while (0);
#define nvme_printf(ctrlr, fmt, args...) \
device_printf(ctrlr->dev, fmt, ##args)
void nvme_ns_test(struct nvme_namespace *ns, u_long cmd, caddr_t arg);
void nvme_ctrlr_cmd_identify_controller(struct nvme_controller *ctrlr,
void *payload,
nvme_cb_fn_t cb_fn, void *cb_arg);
void nvme_ctrlr_cmd_identify_namespace(struct nvme_controller *ctrlr,
uint32_t nsid, void *payload,
nvme_cb_fn_t cb_fn, void *cb_arg);
void nvme_ctrlr_cmd_set_interrupt_coalescing(struct nvme_controller *ctrlr,
uint32_t microseconds,
uint32_t threshold,
nvme_cb_fn_t cb_fn,
void *cb_arg);
void nvme_ctrlr_cmd_get_error_page(struct nvme_controller *ctrlr,
struct nvme_error_information_entry *payload,
uint32_t num_entries, /* 0 = max */
nvme_cb_fn_t cb_fn,
void *cb_arg);
void nvme_ctrlr_cmd_get_health_information_page(struct nvme_controller *ctrlr,
uint32_t nsid,
struct nvme_health_information_page *payload,
nvme_cb_fn_t cb_fn,
void *cb_arg);
void nvme_ctrlr_cmd_get_firmware_page(struct nvme_controller *ctrlr,
struct nvme_firmware_page *payload,
nvme_cb_fn_t cb_fn,
void *cb_arg);
void nvme_ctrlr_cmd_create_io_cq(struct nvme_controller *ctrlr,
struct nvme_qpair *io_que, uint16_t vector,
nvme_cb_fn_t cb_fn, void *cb_arg);
void nvme_ctrlr_cmd_create_io_sq(struct nvme_controller *ctrlr,
struct nvme_qpair *io_que,
nvme_cb_fn_t cb_fn, void *cb_arg);
void nvme_ctrlr_cmd_delete_io_cq(struct nvme_controller *ctrlr,
struct nvme_qpair *io_que,
nvme_cb_fn_t cb_fn, void *cb_arg);
void nvme_ctrlr_cmd_delete_io_sq(struct nvme_controller *ctrlr,
struct nvme_qpair *io_que,
nvme_cb_fn_t cb_fn, void *cb_arg);
void nvme_ctrlr_cmd_set_num_queues(struct nvme_controller *ctrlr,
uint32_t num_queues, nvme_cb_fn_t cb_fn,
void *cb_arg);
void nvme_ctrlr_cmd_set_async_event_config(struct nvme_controller *ctrlr,
uint32_t state,
nvme_cb_fn_t cb_fn, void *cb_arg);
void nvme_ctrlr_cmd_abort(struct nvme_controller *ctrlr, uint16_t cid,
uint16_t sqid, nvme_cb_fn_t cb_fn, void *cb_arg);
void nvme_completion_poll_cb(void *arg, const struct nvme_completion *cpl);
int nvme_ctrlr_construct(struct nvme_controller *ctrlr, device_t dev);
void nvme_ctrlr_destruct(struct nvme_controller *ctrlr, device_t dev);
void nvme_ctrlr_shutdown(struct nvme_controller *ctrlr);
int nvme_ctrlr_hw_reset(struct nvme_controller *ctrlr);
void nvme_ctrlr_reset(struct nvme_controller *ctrlr);
/* ctrlr defined as void * to allow use with config_intrhook. */
void nvme_ctrlr_start_config_hook(void *ctrlr_arg);
void nvme_ctrlr_submit_admin_request(struct nvme_controller *ctrlr,
struct nvme_request *req);
void nvme_ctrlr_submit_io_request(struct nvme_controller *ctrlr,
struct nvme_request *req);
void nvme_ctrlr_post_failed_request(struct nvme_controller *ctrlr,
struct nvme_request *req);
int nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id,
uint16_t vector, uint32_t num_entries,
uint32_t num_trackers,
struct nvme_controller *ctrlr);
void nvme_qpair_submit_tracker(struct nvme_qpair *qpair,
struct nvme_tracker *tr);
bool nvme_qpair_process_completions(struct nvme_qpair *qpair);
void nvme_qpair_submit_request(struct nvme_qpair *qpair,
struct nvme_request *req);
void nvme_qpair_reset(struct nvme_qpair *qpair);
void nvme_qpair_fail(struct nvme_qpair *qpair);
void nvme_qpair_manual_complete_request(struct nvme_qpair *qpair,
struct nvme_request *req,
uint32_t sct, uint32_t sc);
void nvme_admin_qpair_enable(struct nvme_qpair *qpair);
void nvme_admin_qpair_disable(struct nvme_qpair *qpair);
void nvme_admin_qpair_destroy(struct nvme_qpair *qpair);
void nvme_io_qpair_enable(struct nvme_qpair *qpair);
void nvme_io_qpair_disable(struct nvme_qpair *qpair);
void nvme_io_qpair_destroy(struct nvme_qpair *qpair);
int nvme_ns_construct(struct nvme_namespace *ns, uint32_t id,
struct nvme_controller *ctrlr);
void nvme_ns_destruct(struct nvme_namespace *ns);
void nvme_sysctl_initialize_ctrlr(struct nvme_controller *ctrlr);
void nvme_dump_command(struct nvme_command *cmd);
void nvme_dump_completion(struct nvme_completion *cpl);
int nvme_attach(device_t dev);
int nvme_shutdown(device_t dev);
int nvme_detach(device_t dev);
/*
* Wait for a command to complete using the nvme_completion_poll_cb.
* Used in limited contexts where the caller knows it's OK to block
* briefly while the command runs. The ISR will run the callback which
* will set status->done to true.usually within microseconds. A 1s
* pause means something is seriously AFU and we should panic to
* provide the proper context to diagnose.
*/
static __inline
void
nvme_completion_poll(struct nvme_completion_poll_status *status)
{
int sanity = hz * 1;
while (!atomic_load_acq_int(&status->done) && --sanity > 0)
pause("nvme", 1);
if (sanity <= 0)
panic("NVME polled command failed to complete within 1s.");
}
static __inline void
nvme_single_map(void *arg, bus_dma_segment_t *seg, int nseg, int error)
{
uint64_t *bus_addr = (uint64_t *)arg;
if (error != 0)
printf("nvme_single_map err %d\n", error);
*bus_addr = seg[0].ds_addr;
}
static __inline struct nvme_request *
_nvme_allocate_request(nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
req = uma_zalloc(nvme_request_zone, M_NOWAIT | M_ZERO);
if (req != NULL) {
req->cb_fn = cb_fn;
req->cb_arg = cb_arg;
req->timeout = TRUE;
}
return (req);
}
static __inline struct nvme_request *
nvme_allocate_request_vaddr(void *payload, uint32_t payload_size,
nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
req = _nvme_allocate_request(cb_fn, cb_arg);
if (req != NULL) {
req->type = NVME_REQUEST_VADDR;
req->u.payload = payload;
req->payload_size = payload_size;
}
return (req);
}
static __inline struct nvme_request *
nvme_allocate_request_null(nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
req = _nvme_allocate_request(cb_fn, cb_arg);
if (req != NULL)
req->type = NVME_REQUEST_NULL;
return (req);
}
static __inline struct nvme_request *
nvme_allocate_request_bio(struct bio *bio, nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
req = _nvme_allocate_request(cb_fn, cb_arg);
if (req != NULL) {
req->type = NVME_REQUEST_BIO;
req->u.bio = bio;
}
return (req);
}
static __inline struct nvme_request *
nvme_allocate_request_ccb(union ccb *ccb, nvme_cb_fn_t cb_fn, void *cb_arg)
{
struct nvme_request *req;
req = _nvme_allocate_request(cb_fn, cb_arg);
if (req != NULL) {
req->type = NVME_REQUEST_CCB;
req->u.payload = ccb;
}
return (req);
}
#define nvme_free_request(req) uma_zfree(nvme_request_zone, req)
void nvme_notify_async_consumers(struct nvme_controller *ctrlr,
const struct nvme_completion *async_cpl,
uint32_t log_page_id, void *log_page_buffer,
uint32_t log_page_size);
void nvme_notify_fail_consumers(struct nvme_controller *ctrlr);
void nvme_notify_new_controller(struct nvme_controller *ctrlr);
void nvme_notify_ns(struct nvme_controller *ctrlr, int nsid);
void nvme_ctrlr_intx_handler(void *arg);
void nvme_ctrlr_poll(struct nvme_controller *ctrlr);
int nvme_ctrlr_suspend(struct nvme_controller *ctrlr);
int nvme_ctrlr_resume(struct nvme_controller *ctrlr);
#endif /* __NVME_PRIVATE_H__ */

1266
freebsd/sys/dev/nvme/nvme_qpair.c Normal file
View File

File diff suppressed because it is too large Load Diff

368
freebsd/sys/dev/nvme/nvme_sysctl.c Normal file
View File

@ -0,0 +1,368 @@
#include <machine/rtems-bsd-kernel-space.h>
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (C) 2012-2016 Intel Corporation
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <rtems/bsd/local/opt_nvme.h>
#include <sys/param.h>
#include <sys/bus.h>
#include <sys/sysctl.h>
#include "nvme_private.h"
#ifndef NVME_USE_NVD
#define NVME_USE_NVD 1
#endif
int nvme_use_nvd = NVME_USE_NVD;
bool nvme_verbose_cmd_dump = false;
SYSCTL_NODE(_hw, OID_AUTO, nvme, CTLFLAG_RD, 0, "NVMe sysctl tunables");
SYSCTL_INT(_hw_nvme, OID_AUTO, use_nvd, CTLFLAG_RDTUN,
&nvme_use_nvd, 1, "1 = Create NVD devices, 0 = Create NDA devices");
SYSCTL_BOOL(_hw_nvme, OID_AUTO, verbose_cmd_dump, CTLFLAG_RWTUN,
&nvme_verbose_cmd_dump, 0,
"enable verbose command printting when a command fails");
/*
* CTLTYPE_S64 and sysctl_handle_64 were added in r217616. Define these
* explicitly here for older kernels that don't include the r217616
* changeset.
*/
#ifndef CTLTYPE_S64
#define CTLTYPE_S64 CTLTYPE_QUAD
#define sysctl_handle_64 sysctl_handle_quad
#endif
static void
nvme_dump_queue(struct nvme_qpair *qpair)
{
struct nvme_completion *cpl;
struct nvme_command *cmd;
int i;
printf("id:%04Xh phase:%d\n", qpair->id, qpair->phase);
printf("Completion queue:\n");
for (i = 0; i < qpair->num_entries; i++) {
cpl = &qpair->cpl[i];
printf("%05d: ", i);
nvme_dump_completion(cpl);
}
printf("Submission queue:\n");
for (i = 0; i < qpair->num_entries; i++) {
cmd = &qpair->cmd[i];
printf("%05d: ", i);
nvme_dump_command(cmd);
}
}
static int
nvme_sysctl_dump_debug(SYSCTL_HANDLER_ARGS)
{
struct nvme_qpair *qpair = arg1;
uint32_t val = 0;
int error = sysctl_handle_int(oidp, &val, 0, req);
if (error)
return (error);
if (val != 0)
nvme_dump_queue(qpair);
return (0);
}
static int
nvme_sysctl_int_coal_time(SYSCTL_HANDLER_ARGS)
{
struct nvme_controller *ctrlr = arg1;
uint32_t oldval = ctrlr->int_coal_time;
int error = sysctl_handle_int(oidp, &ctrlr->int_coal_time, 0,
req);
if (error)
return (error);
if (oldval != ctrlr->int_coal_time)
nvme_ctrlr_cmd_set_interrupt_coalescing(ctrlr,
ctrlr->int_coal_time, ctrlr->int_coal_threshold, NULL,
NULL);
return (0);
}
static int
nvme_sysctl_int_coal_threshold(SYSCTL_HANDLER_ARGS)
{
struct nvme_controller *ctrlr = arg1;
uint32_t oldval = ctrlr->int_coal_threshold;
int error = sysctl_handle_int(oidp, &ctrlr->int_coal_threshold, 0,
req);
if (error)
return (error);
if (oldval != ctrlr->int_coal_threshold)
nvme_ctrlr_cmd_set_interrupt_coalescing(ctrlr,
ctrlr->int_coal_time, ctrlr->int_coal_threshold, NULL,
NULL);
return (0);
}
static int
nvme_sysctl_timeout_period(SYSCTL_HANDLER_ARGS)
{
struct nvme_controller *ctrlr = arg1;
uint32_t oldval = ctrlr->timeout_period;
int error = sysctl_handle_int(oidp, &ctrlr->timeout_period, 0, req);
if (error)
return (error);
if (ctrlr->timeout_period > NVME_MAX_TIMEOUT_PERIOD ||
ctrlr->timeout_period < NVME_MIN_TIMEOUT_PERIOD) {
ctrlr->timeout_period = oldval;
return (EINVAL);
}
return (0);
}
static void
nvme_qpair_reset_stats(struct nvme_qpair *qpair)
{
qpair->num_cmds = 0;
qpair->num_intr_handler_calls = 0;
qpair->num_retries = 0;
qpair->num_failures = 0;
}
static int
nvme_sysctl_num_cmds(SYSCTL_HANDLER_ARGS)
{
struct nvme_controller *ctrlr = arg1;
int64_t num_cmds = 0;
int i;
num_cmds = ctrlr->adminq.num_cmds;
for (i = 0; i < ctrlr->num_io_queues; i++)
num_cmds += ctrlr->ioq[i].num_cmds;
return (sysctl_handle_64(oidp, &num_cmds, 0, req));
}
static int
nvme_sysctl_num_intr_handler_calls(SYSCTL_HANDLER_ARGS)
{
struct nvme_controller *ctrlr = arg1;
int64_t num_intr_handler_calls = 0;
int i;
num_intr_handler_calls = ctrlr->adminq.num_intr_handler_calls;
for (i = 0; i < ctrlr->num_io_queues; i++)
num_intr_handler_calls += ctrlr->ioq[i].num_intr_handler_calls;
return (sysctl_handle_64(oidp, &num_intr_handler_calls, 0, req));
}
static int
nvme_sysctl_num_retries(SYSCTL_HANDLER_ARGS)
{
struct nvme_controller *ctrlr = arg1;
int64_t num_retries = 0;
int i;
num_retries = ctrlr->adminq.num_retries;
for (i = 0; i < ctrlr->num_io_queues; i++)
num_retries += ctrlr->ioq[i].num_retries;
return (sysctl_handle_64(oidp, &num_retries, 0, req));
}
static int
nvme_sysctl_num_failures(SYSCTL_HANDLER_ARGS)
{
struct nvme_controller *ctrlr = arg1;
int64_t num_failures = 0;
int i;
num_failures = ctrlr->adminq.num_failures;
for (i = 0; i < ctrlr->num_io_queues; i++)
num_failures += ctrlr->ioq[i].num_failures;
return (sysctl_handle_64(oidp, &num_failures, 0, req));
}
static int
nvme_sysctl_reset_stats(SYSCTL_HANDLER_ARGS)
{
struct nvme_controller *ctrlr = arg1;
uint32_t i, val = 0;
int error = sysctl_handle_int(oidp, &val, 0, req);
if (error)
return (error);
if (val != 0) {
nvme_qpair_reset_stats(&ctrlr->adminq);
for (i = 0; i < ctrlr->num_io_queues; i++)
nvme_qpair_reset_stats(&ctrlr->ioq[i]);
}
return (0);
}
static void
nvme_sysctl_initialize_queue(struct nvme_qpair *qpair,
struct sysctl_ctx_list *ctrlr_ctx, struct sysctl_oid *que_tree)
{
struct sysctl_oid_list *que_list = SYSCTL_CHILDREN(que_tree);
SYSCTL_ADD_UINT(ctrlr_ctx, que_list, OID_AUTO, "num_entries",
CTLFLAG_RD, &qpair->num_entries, 0,
"Number of entries in hardware queue");
SYSCTL_ADD_UINT(ctrlr_ctx, que_list, OID_AUTO, "num_trackers",
CTLFLAG_RD, &qpair->num_trackers, 0,
"Number of trackers pre-allocated for this queue pair");
SYSCTL_ADD_UINT(ctrlr_ctx, que_list, OID_AUTO, "sq_head",
CTLFLAG_RD, &qpair->sq_head, 0,
"Current head of submission queue (as observed by driver)");
SYSCTL_ADD_UINT(ctrlr_ctx, que_list, OID_AUTO, "sq_tail",
CTLFLAG_RD, &qpair->sq_tail, 0,
"Current tail of submission queue (as observed by driver)");
SYSCTL_ADD_UINT(ctrlr_ctx, que_list, OID_AUTO, "cq_head",
CTLFLAG_RD, &qpair->cq_head, 0,
"Current head of completion queue (as observed by driver)");
SYSCTL_ADD_QUAD(ctrlr_ctx, que_list, OID_AUTO, "num_cmds",
CTLFLAG_RD, &qpair->num_cmds, "Number of commands submitted");
SYSCTL_ADD_QUAD(ctrlr_ctx, que_list, OID_AUTO, "num_intr_handler_calls",
CTLFLAG_RD, &qpair->num_intr_handler_calls,
"Number of times interrupt handler was invoked (will typically be "
"less than number of actual interrupts generated due to "
"coalescing)");
SYSCTL_ADD_QUAD(ctrlr_ctx, que_list, OID_AUTO, "num_retries",
CTLFLAG_RD, &qpair->num_retries, "Number of commands retried");
SYSCTL_ADD_QUAD(ctrlr_ctx, que_list, OID_AUTO, "num_failures",
CTLFLAG_RD, &qpair->num_failures,
"Number of commands ending in failure after all retries");
SYSCTL_ADD_PROC(ctrlr_ctx, que_list, OID_AUTO,
"dump_debug", CTLTYPE_UINT | CTLFLAG_RW, qpair, 0,
nvme_sysctl_dump_debug, "IU", "Dump debug data");
}
void
nvme_sysctl_initialize_ctrlr(struct nvme_controller *ctrlr)
{
struct sysctl_ctx_list *ctrlr_ctx;
struct sysctl_oid *ctrlr_tree, *que_tree;
struct sysctl_oid_list *ctrlr_list;
#define QUEUE_NAME_LENGTH 16
char queue_name[QUEUE_NAME_LENGTH];
int i;
ctrlr_ctx = device_get_sysctl_ctx(ctrlr->dev);
ctrlr_tree = device_get_sysctl_tree(ctrlr->dev);
ctrlr_list = SYSCTL_CHILDREN(ctrlr_tree);
SYSCTL_ADD_UINT(ctrlr_ctx, ctrlr_list, OID_AUTO, "num_cpus_per_ioq",
CTLFLAG_RD, &ctrlr->num_cpus_per_ioq, 0,
"Number of CPUs assigned per I/O queue pair");
SYSCTL_ADD_PROC(ctrlr_ctx, ctrlr_list, OID_AUTO,
"int_coal_time", CTLTYPE_UINT | CTLFLAG_RW, ctrlr, 0,
nvme_sysctl_int_coal_time, "IU",
"Interrupt coalescing timeout (in microseconds)");
SYSCTL_ADD_PROC(ctrlr_ctx, ctrlr_list, OID_AUTO,
"int_coal_threshold", CTLTYPE_UINT | CTLFLAG_RW, ctrlr, 0,
nvme_sysctl_int_coal_threshold, "IU",
"Interrupt coalescing threshold");
SYSCTL_ADD_PROC(ctrlr_ctx, ctrlr_list, OID_AUTO,
"timeout_period", CTLTYPE_UINT | CTLFLAG_RW, ctrlr, 0,
nvme_sysctl_timeout_period, "IU",
"Timeout period (in seconds)");
SYSCTL_ADD_PROC(ctrlr_ctx, ctrlr_list, OID_AUTO,
"num_cmds", CTLTYPE_S64 | CTLFLAG_RD,
ctrlr, 0, nvme_sysctl_num_cmds, "IU",
"Number of commands submitted");
SYSCTL_ADD_PROC(ctrlr_ctx, ctrlr_list, OID_AUTO,
"num_intr_handler_calls", CTLTYPE_S64 | CTLFLAG_RD,
ctrlr, 0, nvme_sysctl_num_intr_handler_calls, "IU",
"Number of times interrupt handler was invoked (will "
"typically be less than number of actual interrupts "
"generated due to coalescing)");
SYSCTL_ADD_PROC(ctrlr_ctx, ctrlr_list, OID_AUTO,
"num_retries", CTLTYPE_S64 | CTLFLAG_RD,
ctrlr, 0, nvme_sysctl_num_retries, "IU",
"Number of commands retried");
SYSCTL_ADD_PROC(ctrlr_ctx, ctrlr_list, OID_AUTO,
"num_failures", CTLTYPE_S64 | CTLFLAG_RD,
ctrlr, 0, nvme_sysctl_num_failures, "IU",
"Number of commands ending in failure after all retries");
SYSCTL_ADD_PROC(ctrlr_ctx, ctrlr_list, OID_AUTO,
"reset_stats", CTLTYPE_UINT | CTLFLAG_RW, ctrlr, 0,
nvme_sysctl_reset_stats, "IU", "Reset statistics to zero");
que_tree = SYSCTL_ADD_NODE(ctrlr_ctx, ctrlr_list, OID_AUTO, "adminq",
CTLFLAG_RD, NULL, "Admin Queue");
nvme_sysctl_initialize_queue(&ctrlr->adminq, ctrlr_ctx, que_tree);
for (i = 0; i < ctrlr->num_io_queues; i++) {
snprintf(queue_name, QUEUE_NAME_LENGTH, "ioq%d", i);
que_tree = SYSCTL_ADD_NODE(ctrlr_ctx, ctrlr_list, OID_AUTO,
queue_name, CTLFLAG_RD, NULL, "IO Queue");
nvme_sysctl_initialize_queue(&ctrlr->ioq[i], ctrlr_ctx,
que_tree);
}
}

65
freebsd/sys/dev/nvme/nvme_util.c Normal file
View File

@ -0,0 +1,65 @@
#include <machine/rtems-bsd-kernel-space.h>
/*-
* SPDX-License-Identifier: BSD-2-Clause-FreeBSD
*
* Copyright (C) 2013 Intel Corporation
* Copyright (C) 1997 Justin T. Gibbs
* All rights reserved.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
*
* THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*/
#include <sys/cdefs.h>
__FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <dev/nvme/nvme.h>
void
nvme_strvis(uint8_t *dst, const uint8_t *src, int dstlen, int srclen)
{
uint8_t *cur_pos;
/* Trim leading/trailing spaces, nulls. */
while (srclen > 0 && src[0] == ' ')
src++, srclen--;
while (srclen > 0
&& (src[srclen - 1] == ' ' || src[srclen - 1] == '\0'))
srclen--;
while (srclen > 0 && dstlen > 1) {
cur_pos = dst;
/* Show '?' for non-printable characters. */
if (*src < 0x20 || *src >= 0x7F)
*cur_pos++ = '?';
else
*cur_pos++ = *src;
src++;
srclen--;
dstlen -= cur_pos - dst;
dst = cur_pos;
}
*dst = '\0';
}