usr/src/cmd/nvmeadm/nvmeadm.c
608
nvme_t *nvme = npa->npa_nvme;
usr/src/cmd/nvmeadm/nvmeadm.c
613
nvme_errmsg(nvme), nvme_errtostr(nvme, nvme_err(nvme)),
usr/src/cmd/nvmeadm/nvmeadm.c
614
nvme_err(nvme), nvme_syserr(nvme));
usr/src/cmd/nvmeadm/nvmeadm.c
844
nvmeadm_ctrl_disc_cb(nvme_t *nvme, const nvme_ctrl_disc_t *disc, void *arg)
usr/src/lib/fm/topo/modules/common/disk/disk_nvme.c
511
tnode_t *nvme;
usr/src/lib/fm/topo/modules/common/disk/disk_nvme.c
559
if ((nvme = topo_node_bind(mod, nvme_info->nei_parent, NVME, 0,
usr/src/lib/fm/topo/modules/common/disk/disk_nvme.c
566
nvme_info->nei_nvme = nvme;
usr/src/lib/fm/topo/modules/common/disk/disk_nvme.c
579
if (topo_node_fru_set(nvme, fru, 0, &err) != 0) {
usr/src/lib/fm/topo/modules/common/disk/disk_nvme.c
593
topo_node_label_set(nvme, label, &err) != 0) {
usr/src/lib/fm/topo/modules/common/disk/disk_nvme.c
608
if (topo_pgroup_create(nvme, &nvme_pgroup, &err) != 0) {
usr/src/lib/fm/topo/modules/common/disk/disk_nvme.c
621
if (topo_prop_set_string(nvme, TOPO_PGROUP_NVME, TOPO_PROP_NVME_VER,
usr/src/lib/fm/topo/modules/common/disk/disk_nvme.c
629
if (topo_pgroup_create(nvme, &io_pgroup, &err) != 0) {
usr/src/lib/fm/topo/modules/common/disk/disk_nvme.c
636
if (!disk_nvme_common_io(mod, nvme, nvme_info->nei_dinode)) {
usr/src/lib/fm/topo/modules/common/disk/disk_nvme.c
643
if (topo_node_range_create(mod, nvme, DISK, 0,
usr/src/lib/libnvme/common/libnvme.c
1210
nvme_ctrl_ns_init(nvme_t *nvme, const char *name, nvme_ctrl_t **ctrlp,
usr/src/lib/libnvme/common/libnvme.c
1221
return (nvme_error(nvme, NVME_ERR_BAD_PTR, 0, "encountered "
usr/src/lib/libnvme/common/libnvme.c
1230
return (nvme_error(nvme, NVME_ERR_BAD_PTR, 0, "encountered "
usr/src/lib/libnvme/common/libnvme.c
1240
return (nvme_error(nvme, NVME_ERR_BAD_PTR, 0,
usr/src/lib/libnvme/common/libnvme.c
1256
return (nvme_error(nvme, NVME_ERR_BAD_CONTROLLER, 0, "unable "
usr/src/lib/libnvme/common/libnvme.c
1272
return (nvme_error(nvme, NVME_ERR_BAD_CONTROLLER, 0,
usr/src/lib/libnvme/common/libnvme.c
1278
return (nvme_error(nvme, NVME_ERR_BAD_CONTROLLER, 0,
usr/src/lib/libnvme/common/libnvme.c
1286
return (nvme_error(nvme, NVME_ERR_BAD_CONTROLLER, 0,
usr/src/lib/libnvme/common/libnvme.c
1292
return (nvme_error(nvme, NVME_ERR_ILLEGAL_INSTANCE, 0,
usr/src/lib/libnvme/common/libnvme.c
1297
if (!nvme_ctrl_init_by_instance(nvme, (int32_t)inst, &ctrl)) {
usr/src/lib/libnvme/common/libnvme.c
1303
return (nvme_success(nvme));
usr/src/lib/libnvme/common/libnvme.c
1310
nvme_err_set(nvme, &err);
usr/src/lib/libnvme/common/libnvme.c
1318
return (nvme_success(nvme));
usr/src/lib/libnvme/common/libnvme.c
445
nvme_fini(nvme_t *nvme)
usr/src/lib/libnvme/common/libnvme.c
447
if (nvme == NULL)
usr/src/lib/libnvme/common/libnvme.c
450
if (nvme->nh_devinfo != DI_NODE_NIL) {
usr/src/lib/libnvme/common/libnvme.c
451
di_fini(nvme->nh_devinfo);
usr/src/lib/libnvme/common/libnvme.c
454
free(nvme);
usr/src/lib/libnvme/common/libnvme.c
460
nvme_t *nvme;
usr/src/lib/libnvme/common/libnvme.c
462
nvme = calloc(1, sizeof (nvme_t));
usr/src/lib/libnvme/common/libnvme.c
463
if (nvme == NULL) {
usr/src/lib/libnvme/common/libnvme.c
467
nvme->nh_devinfo = di_init("/", DINFOCPYALL);
usr/src/lib/libnvme/common/libnvme.c
468
if (nvme->nh_devinfo == DI_NODE_NIL) {
usr/src/lib/libnvme/common/libnvme.c
469
nvme_fini(nvme);
usr/src/lib/libnvme/common/libnvme.c
473
return (nvme);
usr/src/lib/libnvme/common/libnvme.c
527
nvme_ctrl_discover_init(nvme_t *nvme, nvme_ctrl_iter_t **iterp)
usr/src/lib/libnvme/common/libnvme.c
532
return (nvme_error(nvme, NVME_ERR_BAD_PTR, 0, "encountered "
usr/src/lib/libnvme/common/libnvme.c
539
return (nvme_error(nvme, NVME_ERR_NO_MEM, e, "failed to "
usr/src/lib/libnvme/common/libnvme.c
543
iter->ni_nvme = nvme;
usr/src/lib/libnvme/common/libnvme.c
545
return (nvme_success(nvme));
usr/src/lib/libnvme/common/libnvme.c
549
nvme_ctrl_discover(nvme_t *nvme, nvme_ctrl_disc_f func, void *arg)
usr/src/lib/libnvme/common/libnvme.c
556
return (nvme_error(nvme, NVME_ERR_BAD_PTR, 0, "encountered "
usr/src/lib/libnvme/common/libnvme.c
560
if (!nvme_ctrl_discover_init(nvme, &iter)) {
usr/src/lib/libnvme/common/libnvme.c
566
if (!func(nvme, disc, arg))
usr/src/lib/libnvme/common/libnvme.c
575
return (nvme_success(nvme));
usr/src/lib/libnvme/common/libnvme.c
618
nvme_ctrl_init(nvme_t *nvme, di_node_t di, nvme_ctrl_t **outp)
usr/src/lib/libnvme/common/libnvme.c
628
return (nvme_error(nvme, NVME_ERR_BAD_PTR, 0, "encountered "
usr/src/lib/libnvme/common/libnvme.c
633
return (nvme_error(nvme, NVME_ERR_BAD_PTR, 0, "encountered "
usr/src/lib/libnvme/common/libnvme.c
641
return (nvme_error(nvme, NVME_ERR_BAD_DEVI, 0, "devi %s has "
usr/src/lib/libnvme/common/libnvme.c
646
return (nvme_error(nvme, NVME_ERR_BAD_DEVI, 0, "devi %s isn't "
usr/src/lib/libnvme/common/libnvme.c
663
return (nvme_error(nvme, NVME_ERR_BAD_DEVI, 0, "devi %s isn't "
usr/src/lib/libnvme/common/libnvme.c
670
return (nvme_error(nvme, NVME_ERR_LIBDEVINFO, e, "failed to "
usr/src/lib/libnvme/common/libnvme.c
677
return (nvme_error(nvme, NVME_ERR_INTERNAL, 0, "failed to "
usr/src/lib/libnvme/common/libnvme.c
686
return (nvme_error(nvme, NVME_ERR_NO_MEM, e, "failed to "
usr/src/lib/libnvme/common/libnvme.c
690
ctrl->nc_nvme = nvme;
usr/src/lib/libnvme/common/libnvme.c
698
return (nvme_error(nvme, NVME_ERR_OPEN_DEV, e, "failed to open "
usr/src/lib/libnvme/common/libnvme.c
706
return (nvme_error(nvme, NVME_ERR_LIBDEVINFO, e, "failed to "
usr/src/lib/libnvme/common/libnvme.c
715
nvme_err_set(nvme, &err);
usr/src/lib/libnvme/common/libnvme.c
726
return (nvme_success(nvme));
usr/src/lib/libnvme/common/libnvme.c
737
nvme_ctrl_init_by_instance_cb(nvme_t *nvme, const nvme_ctrl_disc_t *disc,
usr/src/lib/libnvme/common/libnvme.c
752
if (!nvme_ctrl_init(nvme, disc->ncd_devi, &init->ncia_ctrl)) {
usr/src/lib/libnvme/common/libnvme.c
753
nvme_err_save(nvme, &init->ncia_err);
usr/src/lib/libnvme/common/libnvme.c
760
nvme_ctrl_init_by_instance(nvme_t *nvme, int32_t inst, nvme_ctrl_t **outp)
usr/src/lib/libnvme/common/libnvme.c
765
return (nvme_error(nvme, NVME_ERR_ILLEGAL_INSTANCE, 0,
usr/src/lib/libnvme/common/libnvme.c
770
return (nvme_error(nvme, NVME_ERR_BAD_PTR, 0, "encountered "
usr/src/lib/libnvme/common/libnvme.c
778
if (!nvme_ctrl_discover(nvme, nvme_ctrl_init_by_instance_cb, &init)) {
usr/src/lib/libnvme/common/libnvme.c
783
return (nvme_error(nvme, NVME_ERR_BAD_CONTROLLER, 0,
usr/src/lib/libnvme/common/libnvme.c
794
nvme_err_set(nvme, &init.ncia_err);
usr/src/lib/libnvme/common/libnvme.c
799
return (nvme_success(nvme));
usr/src/lib/libnvme/common/libnvme.h
553
typedef struct nvme nvme_t;
usr/src/lib/libnvme/common/libnvme_ctrl_info.c
166
nvme_ctrl_info_restore(nvme_t *nvme, nvlist_t *nvl, nvme_ctrl_info_t **outp)
usr/src/lib/libnvme/common/libnvme_ctrl_info.c
176
return (nvme_error(nvme, NVME_ERR_BAD_PTR, 0, "encountered "
usr/src/lib/libnvme/common/libnvme_ctrl_info.c
181
return (nvme_error(nvme, NVME_ERR_BAD_PTR, 0, "encountered "
usr/src/lib/libnvme/common/libnvme_ctrl_info.c
189
return (nvme_error(nvme, NVME_ERR_NO_MEM, e, "failed to "
usr/src/lib/libnvme/common/libnvme_ctrl_info.c
195
(void) nvme_error(nvme, NVME_ERR_BAD_RESTORE, ret, "failed "
usr/src/lib/libnvme/common/libnvme_ctrl_info.c
201
(void) nvme_error(nvme, NVME_ERR_BAD_RESTORE, 0,
usr/src/lib/libnvme/common/libnvme_ctrl_info.c
215
(void) nvme_error(nvme, NVME_ERR_BAD_RESTORE, ret,
usr/src/lib/libnvme/common/libnvme_ctrl_info.c
221
(void) nvme_error(nvme, NVME_ERR_BAD_RESTORE, 0,
usr/src/lib/libnvme/common/libnvme_ctrl_info.c
227
(void) nvme_error(nvme, NVME_ERR_BAD_RESTORE, 0, "identify "
usr/src/lib/libnvme/common/libnvme_ctrl_info.c
234
(void) nvme_error(nvme, NVME_ERR_BAD_RESTORE, 0, "identify "
usr/src/lib/libnvme/common/libnvme_ctrl_info.c
245
(void) nvme_error(nvme, NVME_ERR_BAD_RESTORE, 0, "device "
usr/src/lib/libnvme/common/libnvme_ctrl_info.c
251
(void) nvme_error(nvme, NVME_ERR_BAD_RESTORE, 0, "found "
usr/src/lib/libnvme/common/libnvme_ctrl_info.c
266
(void) nvme_error(nvme, NVME_ERR_BAD_RESTORE, ret,
usr/src/lib/libnvme/common/libnvme_error.c
536
nvme_err(nvme_t *nvme)
usr/src/lib/libnvme/common/libnvme_error.c
538
return (nvme->nh_err.ne_err);
usr/src/lib/libnvme/common/libnvme_error.c
542
nvme_syserr(nvme_t *nvme)
usr/src/lib/libnvme/common/libnvme_error.c
544
return (nvme->nh_err.ne_syserr);
usr/src/lib/libnvme/common/libnvme_error.c
548
nvme_errmsg(nvme_t *nvme)
usr/src/lib/libnvme/common/libnvme_error.c
550
return (nvme->nh_err.ne_errmsg);
usr/src/lib/libnvme/common/libnvme_error.c
554
nvme_errlen(nvme_t *nvme)
usr/src/lib/libnvme/common/libnvme_error.c
556
return (nvme->nh_err.ne_errlen);
usr/src/lib/libnvme/common/libnvme_error.c
560
nvme_errtostr(nvme_t *nvme, nvme_err_t err)
usr/src/lib/libnvme/common/libnvme_error.c
861
nvme_error(nvme_t *nvme, nvme_err_t err, int32_t sys, const char *fmt, ...)
usr/src/lib/libnvme/common/libnvme_error.c
866
nvme_error_common(&nvme->nh_err, err, sys, fmt, ap);
usr/src/lib/libnvme/common/libnvme_error.c
899
nvme_success(nvme_t *nvme)
usr/src/lib/libnvme/common/libnvme_error.c
901
return (nvme_success_common(&nvme->nh_err));
usr/src/lib/libnvme/common/libnvme_error.c
911
nvme_err_save(const nvme_t *nvme, nvme_err_data_t *out)
usr/src/lib/libnvme/common/libnvme_error.c
913
*out = nvme->nh_err;
usr/src/lib/libnvme/common/libnvme_error.c
917
nvme_err_set(nvme_t *nvme, const nvme_err_data_t *err)
usr/src/lib/libnvme/common/libnvme_error.c
919
nvme->nh_err = *err;
usr/src/test/nvme-tests/tests/libnvme/bad-req-nsmgmt.c
105
libnvme_test_init(&nvme, &ctrl);
usr/src/test/nvme-tests/tests/libnvme/bad-req-nsmgmt.c
151
nvme_fini(nvme);
usr/src/test/nvme-tests/tests/libnvme/bad-req-nsmgmt.c
99
nvme_t *nvme;
usr/src/test/nvme-tests/tests/libnvme/bad-req.c
228
nvme_t *nvme;
usr/src/test/nvme-tests/tests/libnvme/bad-req.c
237
libnvme_test_init(&nvme, &ctrl);
usr/src/test/nvme-tests/tests/libnvme/bad-req.c
361
nvme_fini(nvme);
usr/src/test/nvme-tests/tests/libnvme/check-destruct.c
106
libnvme_test_init(&nvme, &ctrl);
usr/src/test/nvme-tests/tests/libnvme/check-destruct.c
150
nvme_fini(nvme);
usr/src/test/nvme-tests/tests/libnvme/check-destruct.c
95
nvme_t *nvme;
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
113
libnvme_test_hdl_warn(nvme, "failed to iterate controllers");
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
130
ctrl_disc_nop_cb(nvme_t *nvme, const nvme_ctrl_disc_t *disc, void *arg)
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
136
ctrl_disc_bad_disc_init(nvme_t *nvme, nvme_ctrl_iter_t **iterp,
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
139
if (nvme_ctrl_discover_init(nvme, iterp)) {
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
143
} else if (nvme_err(nvme) != exp_err) {
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
146
nvme_errtostr(nvme, nvme_err(nvme)), nvme_err(nvme),
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
147
nvme_errtostr(nvme, exp_err), exp_err);
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
157
ctrl_disc_bad_disc(nvme_t *nvme, nvme_ctrl_disc_f func, nvme_err_t exp_err,
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
160
if (nvme_ctrl_discover(nvme, func, NULL)) {
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
164
} else if (nvme_err(nvme) != exp_err) {
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
167
nvme_errtostr(nvme, nvme_err(nvme)), nvme_err(nvme),
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
168
nvme_errtostr(nvme, exp_err), exp_err);
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
181
nvme_t *nvme;
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
186
libnvme_test_init(&nvme, &ctrl);
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
188
if (!nvme_ctrl_discover(nvme, ctrl_disc_count_cb, &nctrl)) {
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
189
libnvme_test_hdl_warn(nvme, "failed to discover controllers");
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
199
if (!ctrl_match(nvme, ctrl)) {
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
203
if (!ctrl_disc_bad_disc_init(nvme, NULL, NVME_ERR_BAD_PTR,
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
208
if (!ctrl_disc_bad_disc(nvme, NULL, NVME_ERR_BAD_PTR,
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
214
if (!ctrl_disc_bad_disc_init(nvme, &iter, NVME_ERR_NO_MEM,
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
219
if (!ctrl_disc_bad_disc(nvme, ctrl_disc_nop_cb, NVME_ERR_NO_MEM,
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
230
nvme_fini(nvme);
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
31
ctrl_disc_count_cb(nvme_t *nvme, const nvme_ctrl_disc_t *disc, void *arg)
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
81
ctrl_match(nvme_t *nvme, nvme_ctrl_t *targ)
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
95
if (!nvme_ctrl_discover_init(nvme, &iter)) {
usr/src/test/nvme-tests/tests/libnvme/ctrl-disc.c
96
libnvme_test_hdl_warn(nvme, "failed to initialize controller "
usr/src/test/nvme-tests/tests/libnvme/device-empty.c
40
nvme_t *nvme;
usr/src/test/nvme-tests/tests/libnvme/device-empty.c
43
libnvme_test_init(&nvme, &ctrl);
usr/src/test/nvme-tests/tests/libnvme/device-empty.c
55
nvme_fini(nvme);
usr/src/test/nvme-tests/tests/libnvme/device-reset.c
102
libnvme_test_init(&nvme, &ctrl);
usr/src/test/nvme-tests/tests/libnvme/device-reset.c
116
nvme_fini(nvme);
usr/src/test/nvme-tests/tests/libnvme/device-reset.c
99
nvme_t *nvme;
usr/src/test/nvme-tests/tests/libnvme/format.c
372
nvme_t *nvme;
usr/src/test/nvme-tests/tests/libnvme/format.c
378
libnvme_test_init(&nvme, &ctrl);
usr/src/test/nvme-tests/tests/libnvme/format.c
508
nvme_fini(nvme);
usr/src/test/nvme-tests/tests/libnvme/info-roundtrip.c
323
nvme_t *nvme;
usr/src/test/nvme-tests/tests/libnvme/info-roundtrip.c
330
libnvme_test_init(&nvme, &ctrl);
usr/src/test/nvme-tests/tests/libnvme/info-roundtrip.c
340
if (!nvme_ctrl_info_restore(nvme, nvl, &rest_info)) {
usr/src/test/nvme-tests/tests/libnvme/info-roundtrip.c
341
libnvme_test_hdl_fatal(nvme, "failed to restore controller "
usr/src/test/nvme-tests/tests/libnvme/info-roundtrip.c
468
nvme_fini(nvme);
usr/src/test/nvme-tests/tests/libnvme/libnvme_test_common.c
103
libnvme_test_hdl_vwarn(nvme, fmt, ap);
usr/src/test/nvme-tests/tests/libnvme/libnvme_test_common.c
166
nvme_t *nvme;
usr/src/test/nvme-tests/tests/libnvme/libnvme_test_common.c
170
nvme = nvme_init();
usr/src/test/nvme-tests/tests/libnvme/libnvme_test_common.c
171
if (nvme == NULL) {
usr/src/test/nvme-tests/tests/libnvme/libnvme_test_common.c
182
if (!nvme_ctrl_ns_init(nvme, dev, &ctrl, NULL)) {
usr/src/test/nvme-tests/tests/libnvme/libnvme_test_common.c
183
libnvme_test_hdl_fatal(nvme, "failed to open %s", dev);
usr/src/test/nvme-tests/tests/libnvme/libnvme_test_common.c
186
*nvmep = nvme;
usr/src/test/nvme-tests/tests/libnvme/libnvme_test_common.c
44
libnvme_test_hdl_vwarn(nvme_t *nvme, const char *fmt, va_list ap)
usr/src/test/nvme-tests/tests/libnvme/libnvme_test_common.c
49
nvme_errmsg(nvme), nvme_errtostr(nvme, nvme_err(nvme)),
usr/src/test/nvme-tests/tests/libnvme/libnvme_test_common.c
50
nvme_err(nvme), nvme_syserr(nvme));
usr/src/test/nvme-tests/tests/libnvme/libnvme_test_common.c
88
libnvme_test_hdl_warn(nvme_t *nvme, const char *fmt, ...)
usr/src/test/nvme-tests/tests/libnvme/libnvme_test_common.c
93
libnvme_test_hdl_vwarn(nvme, fmt, ap);
usr/src/test/nvme-tests/tests/libnvme/libnvme_test_common.c
98
libnvme_test_hdl_fatal(nvme_t *nvme, const char *fmt, ...)
usr/src/test/nvme-tests/tests/libnvme/missing-field-destruct.c
172
nvme_fini(nvme);
usr/src/test/nvme-tests/tests/libnvme/missing-field-destruct.c
45
nvme_t *nvme;
usr/src/test/nvme-tests/tests/libnvme/missing-field-destruct.c
53
libnvme_test_init(&nvme, &ctrl);
usr/src/test/nvme-tests/tests/libnvme/missing-field.c
134
nvme_fini(nvme);
usr/src/test/nvme-tests/tests/libnvme/missing-field.c
46
nvme_t *nvme;
usr/src/test/nvme-tests/tests/libnvme/missing-field.c
53
libnvme_test_init(&nvme, &ctrl);
usr/src/test/nvme-tests/tests/libnvme/need-write-lock.c
371
nvme_t *nvme;
usr/src/test/nvme-tests/tests/libnvme/need-write-lock.c
376
libnvme_test_init(&nvme, &ctrl);
usr/src/test/nvme-tests/tests/libnvme/need-write-lock.c
400
nvme_fini(nvme);
usr/src/test/nvme-tests/tests/libnvme/ns-cap.c
104
nvme_fini(nvme);
usr/src/test/nvme-tests/tests/libnvme/ns-cap.c
36
nvme_t *nvme;
usr/src/test/nvme-tests/tests/libnvme/ns-cap.c
44
libnvme_test_init(&nvme, &ctrl);
usr/src/test/nvme-tests/tests/libnvme/ns-disc.c
329
nvme_t *nvme;
usr/src/test/nvme-tests/tests/libnvme/ns-disc.c
337
libnvme_test_init(&nvme, &ctrl);
usr/src/test/nvme-tests/tests/libnvme/ns-disc.c
474
nvme_fini(nvme);
usr/src/test/nvme-tests/tests/libnvme/ns-info.c
353
nvme_t *nvme;
usr/src/test/nvme-tests/tests/libnvme/ns-info.c
360
libnvme_test_init(&nvme, &ctrl);
usr/src/test/nvme-tests/tests/libnvme/ns-info.c
406
nvme_fini(nvme);
usr/src/test/nvme-tests/tests/libnvme/ns-lifecycle.c
232
nvme_t *nvme;
usr/src/test/nvme-tests/tests/libnvme/ns-lifecycle.c
237
libnvme_test_init(&nvme, &ctrl);
usr/src/test/nvme-tests/tests/libnvme/ns-lifecycle.c
259
nvme_fini(nvme);
usr/src/test/nvme-tests/tests/libnvme/ns-max.c
148
nvme_t *nvme;
usr/src/test/nvme-tests/tests/libnvme/ns-max.c
154
libnvme_test_init(&nvme, &ctrl);
usr/src/test/nvme-tests/tests/libnvme/ns-max.c
257
nvme_fini(nvme);
usr/src/test/nvme-tests/tests/libnvme/pev.c
131
nvme_t *nvme;
usr/src/test/nvme-tests/tests/libnvme/pev.c
134
libnvme_test_init(&nvme, &ctrl);
usr/src/test/nvme-tests/tests/libnvme/pev.c
135
if (!pev_action(nvme, ctrl, NVME_PEV_LSP_REL_CTX, true,
usr/src/test/nvme-tests/tests/libnvme/pev.c
140
if (!pev_action(nvme, ctrl, NVME_PEV_LSP_REL_CTX, true,
usr/src/test/nvme-tests/tests/libnvme/pev.c
145
if (!pev_action(nvme, ctrl, NVME_PEV_LSP_READ, false,
usr/src/test/nvme-tests/tests/libnvme/pev.c
150
if (!pev_action(nvme, ctrl, NVME_PEV_LSP_EST_CTX_READ, true,
usr/src/test/nvme-tests/tests/libnvme/pev.c
155
if (!pev_action(nvme, ctrl, NVME_PEV_LSP_EST_CTX_READ, false,
usr/src/test/nvme-tests/tests/libnvme/pev.c
160
if (!pev_action(nvme, ctrl, NVME_PEV_LSP_READ, true,
usr/src/test/nvme-tests/tests/libnvme/pev.c
165
if (!pev_action(nvme, ctrl, NVME_PEV_LSP_REL_CTX, true,
usr/src/test/nvme-tests/tests/libnvme/pev.c
35
pev_action(nvme_t *nvme, nvme_ctrl_t *ctrl, nvme_pev_log_lsp_t pev, bool exp,
usr/src/test/nvme-tests/tests/libnvme/pev.c
87
nvme_errtostr(nvme, err), err,
usr/src/test/nvme-tests/tests/libnvme/pev.c
88
nvme_errtostr(nvme, NVME_ERR_CONTROLLER),
usr/src/test/util-tests/tests/ctf/test-sou.c
186
struct csts nvme;
usr/src/uts/common/io/nvme/nvme.c
1269
nvme_put64(nvme_t *nvme, uintptr_t reg, uint64_t val)
usr/src/uts/common/io/nvme/nvme.c
1271
ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0);
usr/src/uts/common/io/nvme/nvme.c
1274
ddi_put64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg), val);
usr/src/uts/common/io/nvme/nvme.c
1278
nvme_put32(nvme_t *nvme, uintptr_t reg, uint32_t val)
usr/src/uts/common/io/nvme/nvme.c
1280
ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0);
usr/src/uts/common/io/nvme/nvme.c
1283
ddi_put32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg), val);
usr/src/uts/common/io/nvme/nvme.c
1287
nvme_get64(nvme_t *nvme, uintptr_t reg)
usr/src/uts/common/io/nvme/nvme.c
1291
ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x7) == 0);
usr/src/uts/common/io/nvme/nvme.c
1294
val = ddi_get64(nvme->n_regh, (uint64_t *)(nvme->n_regs + reg));
usr/src/uts/common/io/nvme/nvme.c
1300
nvme_get32(nvme_t *nvme, uintptr_t reg)
usr/src/uts/common/io/nvme/nvme.c
1304
ASSERT(((uintptr_t)(nvme->n_regs + reg) & 0x3) == 0);
usr/src/uts/common/io/nvme/nvme.c
1307
val = ddi_get32(nvme->n_regh, (uint32_t *)(nvme->n_regs + reg));
usr/src/uts/common/io/nvme/nvme.c
1329
nvme_mgmt_unlock(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme.c
1331
nvme_mgmt_lock_t *lock = &nvme->n_mgmt;
usr/src/uts/common/io/nvme/nvme.c
1338
nvme_mgmt_lock_held(const nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme.c
1340
return (MUTEX_HELD(&nvme->n_mgmt.nml_lock) != 0);
usr/src/uts/common/io/nvme/nvme.c
1344
nvme_mgmt_lock(nvme_t *nvme, nvme_mgmt_lock_level_t level)
usr/src/uts/common/io/nvme/nvme.c
1346
nvme_mgmt_lock_t *lock = &nvme->n_mgmt;
usr/src/uts/common/io/nvme/nvme.c
1363
nvme_mgmt_bd_start(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme.c
1365
nvme_mgmt_lock_t *lock = &nvme->n_mgmt;
usr/src/uts/common/io/nvme/nvme.c
1374
nvme_mgmt_bd_end(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme.c
1376
nvme_mgmt_lock_t *lock = &nvme->n_mgmt;
usr/src/uts/common/io/nvme/nvme.c
1405
nvme_ctrl_mark_dead(nvme_t *nvme, boolean_t removed)
usr/src/uts/common/io/nvme/nvme.c
1413
was_dead = atomic_cas_32((volatile uint32_t *)&nvme->n_dead, B_FALSE,
usr/src/uts/common/io/nvme/nvme.c
1423
nvme->n_dead_status = NVME_IOCTL_E_CTRL_GONE;
usr/src/uts/common/io/nvme/nvme.c
1436
ASSERT3U(nvme->n_dead_status, ==, NVME_IOCTL_E_CTRL_DEAD);
usr/src/uts/common/io/nvme/nvme.c
1437
ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST);
usr/src/uts/common/io/nvme/nvme.c
1440
taskq_dispatch_ent(nvme_dead_taskq, nvme_rwlock_ctrl_dead, nvme,
usr/src/uts/common/io/nvme/nvme.c
1441
TQ_NOSLEEP, &nvme->n_dead_tqent);
usr/src/uts/common/io/nvme/nvme.c
1445
nvme_ctrl_is_gone(const nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme.c
1447
if (nvme->n_dead && nvme->n_dead_status == NVME_IOCTL_E_CTRL_GONE)
usr/src/uts/common/io/nvme/nvme.c
1454
nvme_check_regs_hdl(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme.c
1458
ddi_fm_acc_err_get(nvme->n_regh, &error, DDI_FME_VERSION);
usr/src/uts/common/io/nvme/nvme.c
1509
nvme_alloc_dma_common(nvme_t *nvme, nvme_dma_t *dma,
usr/src/uts/common/io/nvme/nvme.c
1512
if (ddi_dma_alloc_handle(nvme->n_dip, dma_attr, DDI_DMA_SLEEP, NULL,
usr/src/uts/common/io/nvme/nvme.c
1519
dev_err(nvme->n_dip, CE_PANIC,
usr/src/uts/common/io/nvme/nvme.c
1528
(void) ddi_dma_mem_alloc(dma->nd_dmah, len, &nvme->n_reg_acc_attr,
usr/src/uts/common/io/nvme/nvme.c
1535
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
1537
NVME_BUMP_STAT(nvme, dma_bind_err);
usr/src/uts/common/io/nvme/nvme.c
1546
nvme_zalloc_dma(nvme_t *nvme, size_t len, uint_t flags,
usr/src/uts/common/io/nvme/nvme.c
1551
if (nvme_alloc_dma_common(nvme, dma, len, flags, dma_attr) !=
usr/src/uts/common/io/nvme/nvme.c
1568
nvme_t *nvme = (nvme_t *)private;
usr/src/uts/common/io/nvme/nvme.c
1573
if (nvme_alloc_dma_common(nvme, dma, nvme->n_pagesize,
usr/src/uts/common/io/nvme/nvme.c
1574
DDI_DMA_READ, &nvme->n_prp_dma_attr) != DDI_SUCCESS) {
usr/src/uts/common/io/nvme/nvme.c
1586
nvme_zalloc_queue_dma(nvme_t *nvme, uint32_t nentry, uint16_t qe_len,
usr/src/uts/common/io/nvme/nvme.c
1590
ddi_dma_attr_t q_dma_attr = nvme->n_queue_dma_attr;
usr/src/uts/common/io/nvme/nvme.c
1592
len = roundup(len, nvme->n_pagesize);
usr/src/uts/common/io/nvme/nvme.c
1594
if (nvme_zalloc_dma(nvme, len, flags, &q_dma_attr, dma)
usr/src/uts/common/io/nvme/nvme.c
1596
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
1602
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
1659
nvme_destroy_cq_array(nvme_t *nvme, uint_t start)
usr/src/uts/common/io/nvme/nvme.c
1663
for (i = start; i < nvme->n_cq_count; i++)
usr/src/uts/common/io/nvme/nvme.c
1664
if (nvme->n_cq[i] != NULL)
usr/src/uts/common/io/nvme/nvme.c
1665
nvme_free_cq(nvme->n_cq[i]);
usr/src/uts/common/io/nvme/nvme.c
1667
kmem_free(nvme->n_cq, sizeof (*nvme->n_cq) * nvme->n_cq_count);
usr/src/uts/common/io/nvme/nvme.c
1671
nvme_alloc_cq(nvme_t *nvme, uint32_t nentry, nvme_cq_t **cqp, uint16_t idx,
usr/src/uts/common/io/nvme/nvme.c
1678
DDI_INTR_PRI(nvme->n_intr_pri));
usr/src/uts/common/io/nvme/nvme.c
1680
if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_cqe_t),
usr/src/uts/common/io/nvme/nvme.c
1687
cq->ncq_hdbl = NVME_REG_CQHDBL(nvme, idx);
usr/src/uts/common/io/nvme/nvme.c
1693
ddi_driver_name(nvme->n_dip), ddi_get_instance(nvme->n_dip), idx);
usr/src/uts/common/io/nvme/nvme.c
1699
dev_err(nvme->n_dip, CE_WARN, "!failed to create cmd "
usr/src/uts/common/io/nvme/nvme.c
1721
nvme_create_cq_array(nvme_t *nvme, uint_t ncq, uint32_t nentry, uint_t nthr)
usr/src/uts/common/io/nvme/nvme.c
1726
ASSERT3U(ncq, >, nvme->n_cq_count);
usr/src/uts/common/io/nvme/nvme.c
1728
cq = nvme->n_cq;
usr/src/uts/common/io/nvme/nvme.c
1729
cq_count = nvme->n_cq_count;
usr/src/uts/common/io/nvme/nvme.c
1731
nvme->n_cq = kmem_zalloc(sizeof (*nvme->n_cq) * ncq, KM_SLEEP);
usr/src/uts/common/io/nvme/nvme.c
1732
nvme->n_cq_count = ncq;
usr/src/uts/common/io/nvme/nvme.c
1735
nvme->n_cq[i] = cq[i];
usr/src/uts/common/io/nvme/nvme.c
1737
for (; i < nvme->n_cq_count; i++)
usr/src/uts/common/io/nvme/nvme.c
1738
if (nvme_alloc_cq(nvme, nentry, &nvme->n_cq[i], i, nthr) !=
usr/src/uts/common/io/nvme/nvme.c
1748
nvme_destroy_cq_array(nvme, cq_count);
usr/src/uts/common/io/nvme/nvme.c
1752
nvme->n_cq_count = cq_count;
usr/src/uts/common/io/nvme/nvme.c
1753
nvme->n_cq = cq;
usr/src/uts/common/io/nvme/nvme.c
1759
nvme_alloc_qpair(nvme_t *nvme, uint32_t nentry, nvme_qpair_t **nqp,
usr/src/uts/common/io/nvme/nvme.c
1766
DDI_INTR_PRI(nvme->n_intr_pri));
usr/src/uts/common/io/nvme/nvme.c
1774
if (nvme_zalloc_queue_dma(nvme, nentry, sizeof (nvme_sqe_t),
usr/src/uts/common/io/nvme/nvme.c
1781
cq_idx = idx == 0 ? 0 : 1 + (idx - 1) % (nvme->n_cq_count - 1);
usr/src/uts/common/io/nvme/nvme.c
1782
qp->nq_cq = nvme->n_cq[cq_idx];
usr/src/uts/common/io/nvme/nvme.c
1786
qp->nq_sqtdbl = NVME_REG_SQTDBL(nvme, idx);
usr/src/uts/common/io/nvme/nvme.c
1815
nvme_alloc_cmd(nvme_t *nvme, int kmflag)
usr/src/uts/common/io/nvme/nvme.c
1821
cmd->nc_nvme = nvme;
usr/src/uts/common/io/nvme/nvme.c
1828
nvme_alloc_admin_cmd(nvme_t *nvme, int kmflag)
usr/src/uts/common/io/nvme/nvme.c
1830
nvme_cmd_t *cmd = nvme_alloc_cmd(nvme, kmflag);
usr/src/uts/common/io/nvme/nvme.c
1835
DDI_INTR_PRI(nvme->n_intr_pri));
usr/src/uts/common/io/nvme/nvme.c
1955
nvme_unqueue_cmd(nvme_t *nvme, nvme_qpair_t *qp, int cid)
usr/src/uts/common/io/nvme/nvme.c
1984
ASSERT3P(cmd->nc_nvme, ==, nvme);
usr/src/uts/common/io/nvme/nvme.c
1998
nvme_lost_cmd(nvme_t *nvme, nvme_cmd_t *cmd)
usr/src/uts/common/io/nvme/nvme.c
2004
nvme_qpair_t *qp = nvme->n_ioq[cmd->nc_sqid];
usr/src/uts/common/io/nvme/nvme.c
2014
(void) nvme_unqueue_cmd(nvme, qp, cmd->nc_sqe.sqe_cid);
usr/src/uts/common/io/nvme/nvme.c
2054
nvme_get_completed(nvme_t *nvme, nvme_cq_t *cq)
usr/src/uts/common/io/nvme/nvme.c
2069
qp = nvme->n_ioq[cqe->cqe_sqid];
usr/src/uts/common/io/nvme/nvme.c
2072
cmd = nvme_unqueue_cmd(nvme, qp, cqe->cqe_cid);
usr/src/uts/common/io/nvme/nvme.c
2083
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
2085
NVME_BUMP_STAT(nvme, unknown_cid);
usr/src/uts/common/io/nvme/nvme.c
2107
nvme_process_iocq(nvme_t *nvme, nvme_cq_t *cq)
usr/src/uts/common/io/nvme/nvme.c
2115
dev_err(nvme->n_dip, CE_WARN, "!ddi_dma_sync() failed in %s",
usr/src/uts/common/io/nvme/nvme.c
2120
while ((cmd = nvme_get_completed(nvme, cq)) != NULL) {
usr/src/uts/common/io/nvme/nvme.c
2156
nvme_put32(nvme, cq->ncq_hdbl, head.r);
usr/src/uts/common/io/nvme/nvme.c
2165
nvme_retrieve_cmd(nvme_t *nvme, nvme_qpair_t *qp)
usr/src/uts/common/io/nvme/nvme.c
2173
dev_err(nvme->n_dip, CE_WARN, "!ddi_dma_sync() failed in %s",
usr/src/uts/common/io/nvme/nvme.c
2178
if ((cmd = nvme_get_completed(nvme, cq)) != NULL) {
usr/src/uts/common/io/nvme/nvme.c
2180
nvme_put32(nvme, cq->ncq_hdbl, head.r);
usr/src/uts/common/io/nvme/nvme.c
2624
nvme_t *nvme = cmd->nc_nvme;
usr/src/uts/common/io/nvme/nvme.c
2626
if (nvme->n_dead) {
usr/src/uts/common/io/nvme/nvme.c
2627
return (nvme_ioctl_error(ioc, nvme->n_dead_status, 0, 0));
usr/src/uts/common/io/nvme/nvme.c
2653
nvme_t *nvme = cmd->nc_nvme;
usr/src/uts/common/io/nvme/nvme.c
2654
nvme_cmd_t *abort_cmd = nvme_alloc_admin_cmd(nvme, KM_SLEEP);
usr/src/uts/common/io/nvme/nvme.c
2658
sema_p(&nvme->n_abort_sema);
usr/src/uts/common/io/nvme/nvme.c
2680
sema_v(&nvme->n_abort_sema);
usr/src/uts/common/io/nvme/nvme.c
2700
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
2703
NVME_BUMP_STAT(nvme, abort_timeout);
usr/src/uts/common/io/nvme/nvme.c
2706
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
2712
NVME_BUMP_STAT(nvme, abort_failed);
usr/src/uts/common/io/nvme/nvme.c
2716
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
2722
NVME_BUMP_STAT(nvme, abort_successful);
usr/src/uts/common/io/nvme/nvme.c
2724
NVME_BUMP_STAT(nvme, abort_unsuccessful);
usr/src/uts/common/io/nvme/nvme.c
2746
nvme_t *nvme = cmd->nc_nvme;
usr/src/uts/common/io/nvme/nvme.c
2775
nvme_admin_stat_cmd(nvme, cmd);
usr/src/uts/common/io/nvme/nvme.c
2784
csts.r = nvme_get32(nvme, NVME_REG_CSTS);
usr/src/uts/common/io/nvme/nvme.c
2785
dev_err(nvme->n_dip, CE_WARN, "!command %d/%d timeout, "
usr/src/uts/common/io/nvme/nvme.c
2788
NVME_BUMP_STAT(nvme, cmd_timeout);
usr/src/uts/common/io/nvme/nvme.c
2796
nvme_check_regs_hdl(nvme) ||
usr/src/uts/common/io/nvme/nvme.c
2800
nvme_lost_cmd(nvme, cmd);
usr/src/uts/common/io/nvme/nvme.c
2835
ccnt = nvme_process_iocq(nvme, nvme->n_adminq->nq_cq);
usr/src/uts/common/io/nvme/nvme.c
2838
dev_err(nvme->n_dip, CE_WARN, "!possible missed interrupt "
usr/src/uts/common/io/nvme/nvme.c
2849
nvme_admin_stat_cmd(nvme, cmd);
usr/src/uts/common/io/nvme/nvme.c
2875
nvme_lost_cmd(nvme, cmd);
usr/src/uts/common/io/nvme/nvme.c
2895
nvme_t *nvme = cmd->nc_nvme;
usr/src/uts/common/io/nvme/nvme.c
2932
nvme->n_async_event_supported = B_FALSE;
usr/src/uts/common/io/nvme/nvme.c
2943
nvme_submit_admin_cmd(nvme->n_adminq, cmd, NULL);
usr/src/uts/common/io/nvme/nvme.c
2949
if (!nvme_get_logpage_int(nvme, B_FALSE,
usr/src/uts/common/io/nvme/nvme.c
2955
dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in "
usr/src/uts/common/io/nvme/nvme.c
2958
NVME_BUMP_STAT(nvme, wrong_logpage);
usr/src/uts/common/io/nvme/nvme.c
2964
dev_err(nvme->n_dip, CE_PANIC, "programming error: "
usr/src/uts/common/io/nvme/nvme.c
2969
dev_err(nvme->n_dip, CE_PANIC, "programming error: "
usr/src/uts/common/io/nvme/nvme.c
2974
dev_err(nvme->n_dip, CE_WARN, "!diagnostic failure");
usr/src/uts/common/io/nvme/nvme.c
2975
nvme_ctrl_mark_dead(nvme, B_FALSE);
usr/src/uts/common/io/nvme/nvme.c
2976
NVME_BUMP_STAT(nvme, diagfail_event);
usr/src/uts/common/io/nvme/nvme.c
2980
dev_err(nvme->n_dip, CE_WARN, "!persistent internal "
usr/src/uts/common/io/nvme/nvme.c
2982
nvme_ctrl_mark_dead(nvme, B_FALSE);
usr/src/uts/common/io/nvme/nvme.c
2983
NVME_BUMP_STAT(nvme, persistent_event);
usr/src/uts/common/io/nvme/nvme.c
2987
dev_err(nvme->n_dip, CE_WARN, "!transient internal "
usr/src/uts/common/io/nvme/nvme.c
2990
NVME_BUMP_STAT(nvme, transient_event);
usr/src/uts/common/io/nvme/nvme.c
2994
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
2996
NVME_BUMP_STAT(nvme, fw_load_event);
usr/src/uts/common/io/nvme/nvme.c
3003
if (!nvme_get_logpage_int(nvme, B_FALSE,
usr/src/uts/common/io/nvme/nvme.c
3009
dev_err(nvme->n_dip, CE_WARN, "!wrong logpage in "
usr/src/uts/common/io/nvme/nvme.c
3012
NVME_BUMP_STAT(nvme, wrong_logpage);
usr/src/uts/common/io/nvme/nvme.c
3018
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
3021
NVME_BUMP_STAT(nvme, reliability_event);
usr/src/uts/common/io/nvme/nvme.c
3025
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
3028
NVME_BUMP_STAT(nvme, temperature_event);
usr/src/uts/common/io/nvme/nvme.c
3032
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
3035
NVME_BUMP_STAT(nvme, spare_event);
usr/src/uts/common/io/nvme/nvme.c
3044
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
3048
NVME_BUMP_STAT(nvme, wrong_logpage);
usr/src/uts/common/io/nvme/nvme.c
3052
dev_err(nvme->n_dip, CE_NOTE,
usr/src/uts/common/io/nvme/nvme.c
3055
NVME_BUMP_STAT(nvme, notice_event);
usr/src/uts/common/io/nvme/nvme.c
3057
if (!nvme_get_logpage_int(nvme, B_FALSE,
usr/src/uts/common/io/nvme/nvme.c
3064
dev_err(nvme->n_dip, CE_CONT,
usr/src/uts/common/io/nvme/nvme.c
3070
nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME);
usr/src/uts/common/io/nvme/nvme.c
3078
dev_err(nvme->n_dip, CE_NOTE,
usr/src/uts/common/io/nvme/nvme.c
3080
ddi_get_instance(nvme->n_dip), nsid);
usr/src/uts/common/io/nvme/nvme.c
3082
if (nvme_init_ns(nvme, nsid) != DDI_SUCCESS)
usr/src/uts/common/io/nvme/nvme.c
3085
ns = nvme_nsid2ns(nvme, nsid);
usr/src/uts/common/io/nvme/nvme.c
3089
nvme_mgmt_bd_start(nvme);
usr/src/uts/common/io/nvme/nvme.c
3091
nvme_mgmt_bd_end(nvme);
usr/src/uts/common/io/nvme/nvme.c
3093
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
3098
dev_err(nvme->n_dip, CE_NOTE,
usr/src/uts/common/io/nvme/nvme.c
3101
NVME_BUMP_STAT(nvme, notice_event);
usr/src/uts/common/io/nvme/nvme.c
3105
dev_err(nvme->n_dip, CE_NOTE,
usr/src/uts/common/io/nvme/nvme.c
3108
NVME_BUMP_STAT(nvme, notice_event);
usr/src/uts/common/io/nvme/nvme.c
3112
dev_err(nvme->n_dip, CE_NOTE,
usr/src/uts/common/io/nvme/nvme.c
3115
NVME_BUMP_STAT(nvme, notice_event);
usr/src/uts/common/io/nvme/nvme.c
3119
dev_err(nvme->n_dip, CE_NOTE,
usr/src/uts/common/io/nvme/nvme.c
3122
NVME_BUMP_STAT(nvme, notice_event);
usr/src/uts/common/io/nvme/nvme.c
3126
dev_err(nvme->n_dip, CE_NOTE,
usr/src/uts/common/io/nvme/nvme.c
3129
NVME_BUMP_STAT(nvme, notice_event);
usr/src/uts/common/io/nvme/nvme.c
3133
dev_err(nvme->n_dip, CE_NOTE,
usr/src/uts/common/io/nvme/nvme.c
3136
NVME_BUMP_STAT(nvme, notice_event);
usr/src/uts/common/io/nvme/nvme.c
3140
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
3144
NVME_BUMP_STAT(nvme, unknown_event);
usr/src/uts/common/io/nvme/nvme.c
3150
dev_err(nvme->n_dip, CE_WARN, "!vendor specific async event "
usr/src/uts/common/io/nvme/nvme.c
3153
NVME_BUMP_STAT(nvme, vendor_event);
usr/src/uts/common/io/nvme/nvme.c
3157
dev_err(nvme->n_dip, CE_WARN, "!unknown async event received, "
usr/src/uts/common/io/nvme/nvme.c
3160
NVME_BUMP_STAT(nvme, unknown_event);
usr/src/uts/common/io/nvme/nvme.c
3197
nvme_async_event(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme.c
3201
cmd = nvme_alloc_admin_cmd(nvme, KM_SLEEP);
usr/src/uts/common/io/nvme/nvme.c
3207
nvme_submit_admin_cmd(nvme->n_adminq, cmd, NULL);
usr/src/uts/common/io/nvme/nvme.c
3216
nvme_no_blkdev_attached(nvme_t *nvme, uint32_t nsid)
usr/src/uts/common/io/nvme/nvme.c
3218
ASSERT(nvme_mgmt_lock_held(nvme));
usr/src/uts/common/io/nvme/nvme.c
3222
nvme_namespace_t *ns = nvme_nsid2ns(nvme, nsid);
usr/src/uts/common/io/nvme/nvme.c
3226
for (uint32_t i = 1; i <= nvme->n_namespace_count; i++) {
usr/src/uts/common/io/nvme/nvme.c
3227
nvme_namespace_t *ns = nvme_nsid2ns(nvme, i);
usr/src/uts/common/io/nvme/nvme.c
3238
nvme_format_nvm(nvme_t *nvme, nvme_ioctl_format_t *ioc)
usr/src/uts/common/io/nvme/nvme.c
3240
nvme_cmd_t *cmd = nvme_alloc_admin_cmd(nvme, KM_SLEEP);
usr/src/uts/common/io/nvme/nvme.c
3269
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
3287
nvme_get_logpage(nvme_t *nvme, boolean_t user, nvme_ioctl_get_logpage_t *log,
usr/src/uts/common/io/nvme/nvme.c
3290
nvme_cmd_t *cmd = nvme_alloc_admin_cmd(nvme, KM_SLEEP);
usr/src/uts/common/io/nvme/nvme.c
3337
if (nvme_zalloc_dma(nvme, log->nigl_len, DDI_DMA_READ,
usr/src/uts/common/io/nvme/nvme.c
3338
&nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) {
usr/src/uts/common/io/nvme/nvme.c
3339
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
3355
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
3381
nvme_get_logpage_int(nvme_t *nvme, boolean_t user, void **buf, size_t *bufsize,
usr/src/uts/common/io/nvme/nvme.c
3402
data.vcd_vers = &nvme->n_version;
usr/src/uts/common/io/nvme/nvme.c
3403
data.vcd_id = nvme->n_idctl;
usr/src/uts/common/io/nvme/nvme.c
3419
bret = nvme_get_logpage(nvme, user, &log, buf);
usr/src/uts/common/io/nvme/nvme.c
3429
nvme_identify(nvme_t *nvme, boolean_t user, nvme_ioctl_identify_t *ioc,
usr/src/uts/common/io/nvme/nvme.c
3432
nvme_cmd_t *cmd = nvme_alloc_admin_cmd(nvme, KM_SLEEP);
usr/src/uts/common/io/nvme/nvme.c
3450
if (nvme_zalloc_dma(nvme, NVME_IDENTIFY_BUFSIZE, DDI_DMA_READ,
usr/src/uts/common/io/nvme/nvme.c
3451
&nvme->n_prp_dma_attr, &cmd->nc_dma) != DDI_SUCCESS) {
usr/src/uts/common/io/nvme/nvme.c
3452
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
3460
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
3462
NVME_BUMP_STAT(nvme, too_many_cookies);
usr/src/uts/common/io/nvme/nvme.c
3482
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
3500
nvme_identify_int(nvme_t *nvme, uint32_t nsid, uint8_t cns, void **buf)
usr/src/uts/common/io/nvme/nvme.c
3508
return (nvme_identify(nvme, B_FALSE, &id, buf));
usr/src/uts/common/io/nvme/nvme.c
3512
nvme_get_current_nqueues(nvme_t *nvme, nvme_nqueues_t *nq)
usr/src/uts/common/io/nvme/nvme.c
3514
nvme_cmd_t *cmd = nvme_alloc_admin_cmd(nvme, KM_SLEEP);
usr/src/uts/common/io/nvme/nvme.c
3529
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
3544
nvme_set_features(nvme_t *nvme, boolean_t user, uint32_t nsid, uint8_t feature,
usr/src/uts/common/io/nvme/nvme.c
3548
nvme_cmd_t *cmd = nvme_alloc_admin_cmd(nvme, KM_SLEEP);
usr/src/uts/common/io/nvme/nvme.c
3564
if (!nvme->n_write_cache_present)
usr/src/uts/common/io/nvme/nvme.c
3578
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
3593
nvme_write_cache_set(nvme_t *nvme, boolean_t enable)
usr/src/uts/common/io/nvme/nvme.c
3608
return (nvme_set_features(nvme, B_TRUE, 0, NVME_FEAT_WRITE_CACHE,
usr/src/uts/common/io/nvme/nvme.c
3613
nvme_set_nqueues(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme.c
3621
if (nvme->n_completion_queues == -1)
usr/src/uts/common/io/nvme/nvme.c
3622
nvme->n_completion_queues = nvme->n_intr_cnt;
usr/src/uts/common/io/nvme/nvme.c
3628
nvme->n_completion_queues = MIN(nvme->n_completion_queues,
usr/src/uts/common/io/nvme/nvme.c
3629
nvme->n_intr_cnt);
usr/src/uts/common/io/nvme/nvme.c
3634
if (nvme->n_submission_queues == -1)
usr/src/uts/common/io/nvme/nvme.c
3635
nvme->n_submission_queues = nvme->n_completion_queues;
usr/src/uts/common/io/nvme/nvme.c
3641
nvme->n_completion_queues = MIN(nvme->n_completion_queues,
usr/src/uts/common/io/nvme/nvme.c
3642
nvme->n_submission_queues);
usr/src/uts/common/io/nvme/nvme.c
3644
ASSERT(nvme->n_submission_queues > 0);
usr/src/uts/common/io/nvme/nvme.c
3645
ASSERT(nvme->n_completion_queues > 0);
usr/src/uts/common/io/nvme/nvme.c
3647
nq.b.nq_nsq = nvme->n_submission_queues - 1;
usr/src/uts/common/io/nvme/nvme.c
3648
nq.b.nq_ncq = nvme->n_completion_queues - 1;
usr/src/uts/common/io/nvme/nvme.c
3650
ret = nvme_set_features(nvme, B_FALSE, 0, NVME_FEAT_NQUEUES, nq.r,
usr/src/uts/common/io/nvme/nvme.c
3657
nvme->n_submission_queues = MIN(nvme->n_submission_queues,
usr/src/uts/common/io/nvme/nvme.c
3659
nvme->n_completion_queues = MIN(nvme->n_completion_queues,
usr/src/uts/common/io/nvme/nvme.c
3667
nvme_create_completion_queue(nvme_t *nvme, nvme_cq_t *cq)
usr/src/uts/common/io/nvme/nvme.c
3669
nvme_cmd_t *cmd = nvme_alloc_admin_cmd(nvme, KM_SLEEP);
usr/src/uts/common/io/nvme/nvme.c
3679
c_dw11.b.cq_iv = cq->ncq_id % nvme->n_intr_cnt;
usr/src/uts/common/io/nvme/nvme.c
3691
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
3702
nvme_create_io_qpair(nvme_t *nvme, nvme_qpair_t *qp, uint16_t idx)
usr/src/uts/common/io/nvme/nvme.c
3716
nvme_create_completion_queue(nvme, cq) != DDI_SUCCESS)
usr/src/uts/common/io/nvme/nvme.c
3725
cmd = nvme_alloc_admin_cmd(nvme, KM_SLEEP);
usr/src/uts/common/io/nvme/nvme.c
3736
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
3747
nvme_reset(nvme_t *nvme, boolean_t quiesce)
usr/src/uts/common/io/nvme/nvme.c
3756
if (nvme_ctrl_is_gone(nvme)) {
usr/src/uts/common/io/nvme/nvme.c
3760
nvme_put32(nvme, NVME_REG_CC, 0);
usr/src/uts/common/io/nvme/nvme.c
3762
csts.r = nvme_get32(nvme, NVME_REG_CSTS);
usr/src/uts/common/io/nvme/nvme.c
3764
nvme_put32(nvme, NVME_REG_CC, 0);
usr/src/uts/common/io/nvme/nvme.c
3775
for (i = 0; i < nvme->n_timeout * 10; i++) {
usr/src/uts/common/io/nvme/nvme.c
3776
csts.r = nvme_get32(nvme, NVME_REG_CSTS);
usr/src/uts/common/io/nvme/nvme.c
3793
nvme_put32(nvme, NVME_REG_AQA, 0);
usr/src/uts/common/io/nvme/nvme.c
3794
nvme_put32(nvme, NVME_REG_ASQ, 0);
usr/src/uts/common/io/nvme/nvme.c
3795
nvme_put32(nvme, NVME_REG_ACQ, 0);
usr/src/uts/common/io/nvme/nvme.c
3797
csts.r = nvme_get32(nvme, NVME_REG_CSTS);
usr/src/uts/common/io/nvme/nvme.c
3802
nvme_shutdown(nvme_t *nvme, boolean_t quiesce)
usr/src/uts/common/io/nvme/nvme.c
3812
if (nvme_ctrl_is_gone(nvme)) {
usr/src/uts/common/io/nvme/nvme.c
3816
cc.r = nvme_get32(nvme, NVME_REG_CC);
usr/src/uts/common/io/nvme/nvme.c
3818
nvme_put32(nvme, NVME_REG_CC, cc.r);
usr/src/uts/common/io/nvme/nvme.c
3821
csts.r = nvme_get32(nvme, NVME_REG_CSTS);
usr/src/uts/common/io/nvme/nvme.c
3849
nvme_config_min_block_size(nvme_t *nvme, char *model, char *val)
usr/src/uts/common/io/nvme/nvme.c
3867
nvme->n_min_block_size = bsize;
usr/src/uts/common/io/nvme/nvme.c
3871
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
3875
nvme->n_min_block_size = NVME_DEFAULT_MIN_BLOCK_SIZE;
usr/src/uts/common/io/nvme/nvme.c
3879
nvme_config_boolean(nvme_t *nvme, char *model, char *name, char *val,
usr/src/uts/common/io/nvme/nvme.c
3889
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
3895
nvme_config_list(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme.c
3906
rv = ddi_prop_lookup_string_array(DDI_DEV_T_ANY, nvme->n_dip,
usr/src/uts/common/io/nvme/nvme.c
3911
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
3919
dev_err(nvme->n_dip, CE_WARN, "!nvme-config-list: must be "
usr/src/uts/common/io/nvme/nvme.c
3930
id_model_len = nvme_strlen(nvme->n_idctl->id_model,
usr/src/uts/common/io/nvme/nvme.c
3931
sizeof (nvme->n_idctl->id_model));
usr/src/uts/common/io/nvme/nvme.c
3936
if (strncmp(model, nvme->n_idctl->id_model, id_model_len) != 0)
usr/src/uts/common/io/nvme/nvme.c
3939
id_fwrev_len = nvme_strlen(nvme->n_idctl->id_fwrev,
usr/src/uts/common/io/nvme/nvme.c
3940
sizeof (nvme->n_idctl->id_fwrev));
usr/src/uts/common/io/nvme/nvme.c
3952
if (strncmp(fwr, nvme->n_idctl->id_fwrev,
usr/src/uts/common/io/nvme/nvme.c
3971
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
3986
nvme_config_boolean(nvme, model, name, val,
usr/src/uts/common/io/nvme/nvme.c
3987
&nvme->n_ignore_unknown_vendor_status);
usr/src/uts/common/io/nvme/nvme.c
3989
nvme_config_min_block_size(nvme, model, val);
usr/src/uts/common/io/nvme/nvme.c
3991
nvme_config_boolean(nvme, model, name, val,
usr/src/uts/common/io/nvme/nvme.c
3992
&nvme->n_write_cache_enabled);
usr/src/uts/common/io/nvme/nvme.c
3997
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
4009
nvme_prepare_devid(nvme_t *nvme, uint32_t nsid)
usr/src/uts/common/io/nvme/nvme.c
4019
char model[sizeof (nvme->n_idctl->id_model) + 1];
usr/src/uts/common/io/nvme/nvme.c
4020
char serial[sizeof (nvme->n_idctl->id_serial) + 1];
usr/src/uts/common/io/nvme/nvme.c
4022
bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model));
usr/src/uts/common/io/nvme/nvme.c
4023
bcopy(nvme->n_idctl->id_serial, serial,
usr/src/uts/common/io/nvme/nvme.c
4024
sizeof (nvme->n_idctl->id_serial));
usr/src/uts/common/io/nvme/nvme.c
4026
model[sizeof (nvme->n_idctl->id_model)] = '\0';
usr/src/uts/common/io/nvme/nvme.c
4027
serial[sizeof (nvme->n_idctl->id_serial)] = '\0';
usr/src/uts/common/io/nvme/nvme.c
4029
nvme_nsid2ns(nvme, nsid)->ns_devid = kmem_asprintf("%4X-%s-%s-%X",
usr/src/uts/common/io/nvme/nvme.c
4030
nvme->n_idctl->id_vid, model, serial, nsid);
usr/src/uts/common/io/nvme/nvme.c
4034
nvme_update_nsid_list(nvme_t *nvme, int cns)
usr/src/uts/common/io/nvme/nvme.c
4042
if (nvme_identify_int(nvme, 0, cns, (void **)&nslist))
usr/src/uts/common/io/nvme/nvme.c
4049
nvme_nsid2ns(nvme_t *nvme, uint32_t nsid)
usr/src/uts/common/io/nvme/nvme.c
4052
ASSERT3U(nsid, <=, nvme->n_namespace_count);
usr/src/uts/common/io/nvme/nvme.c
4053
return (&nvme->n_ns[nsid - 1]);
usr/src/uts/common/io/nvme/nvme.c
4059
nvme_t *nvme = ns->ns_nvme;
usr/src/uts/common/io/nvme/nvme.c
4062
ASSERT(nvme_mgmt_lock_held(nvme));
usr/src/uts/common/io/nvme/nvme.c
4067
if (NVME_VERSION_ATLEAST(&nvme->n_version, 1, 2) &&
usr/src/uts/common/io/nvme/nvme.c
4068
nvme->n_idctl->id_oacs.oa_nsmgmt != 0) {
usr/src/uts/common/io/nvme/nvme.c
4069
nvme_identify_nsid_list_t *nslist = nvme_update_nsid_list(nvme,
usr/src/uts/common/io/nvme/nvme.c
4102
nvme_t *nvme = ns->ns_nvme;
usr/src/uts/common/io/nvme/nvme.c
4106
ASSERT(nvme_mgmt_lock_held(nvme));
usr/src/uts/common/io/nvme/nvme.c
4111
if (NVME_VERSION_ATLEAST(&nvme->n_version, 1, 1)) {
usr/src/uts/common/io/nvme/nvme.c
4112
nvme_identify_nsid_list_t *nslist = nvme_update_nsid_list(nvme,
usr/src/uts/common/io/nvme/nvme.c
4151
nvme_init_ns(nvme_t *nvme, uint32_t nsid)
usr/src/uts/common/io/nvme/nvme.c
4153
nvme_namespace_t *ns = nvme_nsid2ns(nvme, nsid);
usr/src/uts/common/io/nvme/nvme.c
4157
ns->ns_nvme = nvme;
usr/src/uts/common/io/nvme/nvme.c
4159
ASSERT(nvme_mgmt_lock_held(nvme));
usr/src/uts/common/io/nvme/nvme.c
4167
if (!nvme_identify_int(nvme, nsid, NVME_IDENTIFY_NSID,
usr/src/uts/common/io/nvme/nvme.c
4169
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
4211
if (NVME_VERSION_ATLEAST(&nvme->n_version, 1, 1))
usr/src/uts/common/io/nvme/nvme.c
4217
if (NVME_VERSION_ATLEAST(&nvme->n_version, 1, 2))
usr/src/uts/common/io/nvme/nvme.c
4222
nvme_prepare_devid(nvme, ns->ns_id);
usr/src/uts/common/io/nvme/nvme.c
4242
if (ns->ns_best_block_size < nvme->n_min_block_size)
usr/src/uts/common/io/nvme/nvme.c
4243
ns->ns_best_block_size = nvme->n_min_block_size;
usr/src/uts/common/io/nvme/nvme.c
4253
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
4260
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
4274
dev_err(nvme->n_dip, CE_PANIC, "namespace %u state "
usr/src/uts/common/io/nvme/nvme.c
4288
nvme->n_namespaces_attachable--;
usr/src/uts/common/io/nvme/nvme.c
4293
nvme->n_namespaces_attachable++;
usr/src/uts/common/io/nvme/nvme.c
4300
nvme_bd_attach_ns(nvme_t *nvme, nvme_ioctl_common_t *com)
usr/src/uts/common/io/nvme/nvme.c
4302
nvme_namespace_t *ns = nvme_nsid2ns(nvme, com->nioc_nsid);
usr/src/uts/common/io/nvme/nvme.c
4305
ASSERT(nvme_mgmt_lock_held(nvme));
usr/src/uts/common/io/nvme/nvme.c
4314
if (!nvme->n_idctl->id_oncs.on_dset_mgmt)
usr/src/uts/common/io/nvme/nvme.c
4317
ns->ns_bd_hdl = bd_alloc_handle(ns, &ops, &nvme->n_prp_dma_attr,
usr/src/uts/common/io/nvme/nvme.c
4321
dev_err(nvme->n_dip, CE_WARN, "!Failed to get blkdev "
usr/src/uts/common/io/nvme/nvme.c
4328
nvme_mgmt_bd_start(nvme);
usr/src/uts/common/io/nvme/nvme.c
4329
ret = bd_attach_handle(nvme->n_dip, ns->ns_bd_hdl);
usr/src/uts/common/io/nvme/nvme.c
4330
nvme_mgmt_bd_end(nvme);
usr/src/uts/common/io/nvme/nvme.c
4342
nvme_bd_detach_ns(nvme_t *nvme, nvme_ioctl_common_t *com)
usr/src/uts/common/io/nvme/nvme.c
4344
nvme_namespace_t *ns = nvme_nsid2ns(nvme, com->nioc_nsid);
usr/src/uts/common/io/nvme/nvme.c
4347
ASSERT(nvme_mgmt_lock_held(nvme));
usr/src/uts/common/io/nvme/nvme.c
4353
nvme_mgmt_bd_start(nvme);
usr/src/uts/common/io/nvme/nvme.c
4356
nvme_mgmt_bd_end(nvme);
usr/src/uts/common/io/nvme/nvme.c
4373
nvme_rescan_ns(nvme_t *nvme, uint32_t nsid)
usr/src/uts/common/io/nvme/nvme.c
4375
ASSERT(nvme_mgmt_lock_held(nvme));
usr/src/uts/common/io/nvme/nvme.c
4379
nvme_namespace_t *ns = nvme_nsid2ns(nvme, nsid);
usr/src/uts/common/io/nvme/nvme.c
4382
(void) nvme_init_ns(nvme, nsid);
usr/src/uts/common/io/nvme/nvme.c
4386
for (uint32_t i = 1; i <= nvme->n_namespace_count; i++) {
usr/src/uts/common/io/nvme/nvme.c
4387
nvme_namespace_t *ns = nvme_nsid2ns(nvme, i);
usr/src/uts/common/io/nvme/nvme.c
4390
(void) nvme_init_ns(nvme, i);
usr/src/uts/common/io/nvme/nvme.c
4405
nvme_detect_quirks(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme.c
4410
if (nqt->nq_vendor_id == nvme->n_vendor_id &&
usr/src/uts/common/io/nvme/nvme.c
4411
nqt->nq_device_id == nvme->n_device_id) {
usr/src/uts/common/io/nvme/nvme.c
4412
nvme->n_quirks = nqt->nq_quirks;
usr/src/uts/common/io/nvme/nvme.c
4432
nvme_enable_host_behavior(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme.c
4443
if (nvme->n_idctl->id_lpa.lp_da4s == 0)
usr/src/uts/common/io/nvme/nvme.c
4456
if (!nvme_ioc_cmd(nvme, &err, &args)) {
usr/src/uts/common/io/nvme/nvme.c
4457
dev_err(nvme->n_dip, CE_WARN, "failed to enable host behavior "
usr/src/uts/common/io/nvme/nvme.c
4466
nvme_init(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme.c
4478
char model[sizeof (nvme->n_idctl->id_model) + 1];
usr/src/uts/common/io/nvme/nvme.c
4483
vs.r = nvme_get32(nvme, NVME_REG_VS);
usr/src/uts/common/io/nvme/nvme.c
4484
nvme->n_version.v_major = vs.b.vs_mjr;
usr/src/uts/common/io/nvme/nvme.c
4485
nvme->n_version.v_minor = vs.b.vs_mnr;
usr/src/uts/common/io/nvme/nvme.c
4486
dev_err(nvme->n_dip, CE_CONT, "?NVMe spec version %d.%d\n",
usr/src/uts/common/io/nvme/nvme.c
4487
nvme->n_version.v_major, nvme->n_version.v_minor);
usr/src/uts/common/io/nvme/nvme.c
4489
if (nvme->n_version.v_major > nvme_version_major) {
usr/src/uts/common/io/nvme/nvme.c
4490
dev_err(nvme->n_dip, CE_WARN, "!no support for version > %d.x",
usr/src/uts/common/io/nvme/nvme.c
4492
if (nvme->n_strict_version)
usr/src/uts/common/io/nvme/nvme.c
4497
cap.r = nvme_get64(nvme, NVME_REG_CAP);
usr/src/uts/common/io/nvme/nvme.c
4500
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
4505
nvme->n_nssr_supported = cap.b.cap_nssrs;
usr/src/uts/common/io/nvme/nvme.c
4506
nvme->n_doorbell_stride = 4 << cap.b.cap_dstrd;
usr/src/uts/common/io/nvme/nvme.c
4507
nvme->n_timeout = cap.b.cap_to;
usr/src/uts/common/io/nvme/nvme.c
4508
nvme->n_arbitration_mechanisms = cap.b.cap_ams;
usr/src/uts/common/io/nvme/nvme.c
4509
nvme->n_cont_queues_reqd = cap.b.cap_cqr;
usr/src/uts/common/io/nvme/nvme.c
4510
nvme->n_max_queue_entries = cap.b.cap_mqes + 1;
usr/src/uts/common/io/nvme/nvme.c
4517
nvme->n_pageshift = MIN(MAX(cap.b.cap_mpsmin + 12, PAGESHIFT),
usr/src/uts/common/io/nvme/nvme.c
4519
nvme->n_pagesize = 1UL << (nvme->n_pageshift);
usr/src/uts/common/io/nvme/nvme.c
4524
nvme->n_queue_dma_attr.dma_attr_align = nvme->n_pagesize;
usr/src/uts/common/io/nvme/nvme.c
4525
nvme->n_queue_dma_attr.dma_attr_minxfer = nvme->n_pagesize;
usr/src/uts/common/io/nvme/nvme.c
4531
nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_pagesize;
usr/src/uts/common/io/nvme/nvme.c
4532
nvme->n_prp_dma_attr.dma_attr_minxfer = nvme->n_pagesize;
usr/src/uts/common/io/nvme/nvme.c
4533
nvme->n_prp_dma_attr.dma_attr_align = nvme->n_pagesize;
usr/src/uts/common/io/nvme/nvme.c
4534
nvme->n_prp_dma_attr.dma_attr_seg = nvme->n_pagesize - 1;
usr/src/uts/common/io/nvme/nvme.c
4539
if (nvme_reset(nvme, B_FALSE) == B_FALSE) {
usr/src/uts/common/io/nvme/nvme.c
4540
dev_err(nvme->n_dip, CE_WARN, "!unable to reset controller");
usr/src/uts/common/io/nvme/nvme.c
4541
ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST);
usr/src/uts/common/io/nvme/nvme.c
4542
nvme->n_dead = B_TRUE;
usr/src/uts/common/io/nvme/nvme.c
4550
if (nvme_create_cq_array(nvme, 1, nvme->n_admin_queue_len, 4) !=
usr/src/uts/common/io/nvme/nvme.c
4552
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
4559
if (nvme_alloc_qpair(nvme, nvme->n_admin_queue_len, &nvme->n_adminq, 0)
usr/src/uts/common/io/nvme/nvme.c
4561
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
4565
nvme->n_ioq = kmem_alloc(sizeof (nvme_qpair_t *), KM_SLEEP);
usr/src/uts/common/io/nvme/nvme.c
4566
nvme->n_ioq[0] = nvme->n_adminq;
usr/src/uts/common/io/nvme/nvme.c
4568
if (nvme->n_quirks & NVME_QUIRK_START_CID)
usr/src/uts/common/io/nvme/nvme.c
4569
nvme->n_adminq->nq_next_cmd++;
usr/src/uts/common/io/nvme/nvme.c
4571
nvme->n_progress |= NVME_ADMIN_QUEUE;
usr/src/uts/common/io/nvme/nvme.c
4573
(void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip,
usr/src/uts/common/io/nvme/nvme.c
4574
"admin-queue-len", nvme->n_admin_queue_len);
usr/src/uts/common/io/nvme/nvme.c
4576
aqa.b.aqa_asqs = aqa.b.aqa_acqs = nvme->n_admin_queue_len - 1;
usr/src/uts/common/io/nvme/nvme.c
4577
asq = nvme->n_adminq->nq_sqdma->nd_cookie.dmac_laddress;
usr/src/uts/common/io/nvme/nvme.c
4578
acq = nvme->n_adminq->nq_cq->ncq_dma->nd_cookie.dmac_laddress;
usr/src/uts/common/io/nvme/nvme.c
4580
ASSERT((asq & (nvme->n_pagesize - 1)) == 0);
usr/src/uts/common/io/nvme/nvme.c
4581
ASSERT((acq & (nvme->n_pagesize - 1)) == 0);
usr/src/uts/common/io/nvme/nvme.c
4583
nvme_put32(nvme, NVME_REG_AQA, aqa.r);
usr/src/uts/common/io/nvme/nvme.c
4584
nvme_put64(nvme, NVME_REG_ASQ, asq);
usr/src/uts/common/io/nvme/nvme.c
4585
nvme_put64(nvme, NVME_REG_ACQ, acq);
usr/src/uts/common/io/nvme/nvme.c
4589
cc.b.cc_mps = nvme->n_pageshift - 12;
usr/src/uts/common/io/nvme/nvme.c
4595
nvme_put32(nvme, NVME_REG_CC, cc.r);
usr/src/uts/common/io/nvme/nvme.c
4600
csts.r = nvme_get32(nvme, NVME_REG_CSTS);
usr/src/uts/common/io/nvme/nvme.c
4602
for (i = 0; i != nvme->n_timeout * 10; i++) {
usr/src/uts/common/io/nvme/nvme.c
4604
csts.r = nvme_get32(nvme, NVME_REG_CSTS);
usr/src/uts/common/io/nvme/nvme.c
4607
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
4609
ddi_fm_service_impact(nvme->n_dip,
usr/src/uts/common/io/nvme/nvme.c
4611
nvme->n_dead = B_TRUE;
usr/src/uts/common/io/nvme/nvme.c
4621
dev_err(nvme->n_dip, CE_WARN, "!controller not ready");
usr/src/uts/common/io/nvme/nvme.c
4622
ddi_fm_service_impact(nvme->n_dip, DDI_SERVICE_LOST);
usr/src/uts/common/io/nvme/nvme.c
4623
nvme->n_dead = B_TRUE;
usr/src/uts/common/io/nvme/nvme.c
4631
sema_init(&nvme->n_abort_sema, 1, NULL, SEMA_DRIVER, NULL);
usr/src/uts/common/io/nvme/nvme.c
4636
if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX, 1)
usr/src/uts/common/io/nvme/nvme.c
4638
(nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI, 1)
usr/src/uts/common/io/nvme/nvme.c
4640
(nvme_setup_interrupts(nvme, DDI_INTR_TYPE_FIXED, 1)
usr/src/uts/common/io/nvme/nvme.c
4642
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
4651
nvme->n_dead_status = NVME_IOCTL_E_CTRL_DEAD;
usr/src/uts/common/io/nvme/nvme.c
4656
if (!nvme_identify_int(nvme, 0, NVME_IDENTIFY_CTRL,
usr/src/uts/common/io/nvme/nvme.c
4657
(void **)&nvme->n_idctl)) {
usr/src/uts/common/io/nvme/nvme.c
4658
dev_err(nvme->n_dip, CE_WARN, "!failed to identify controller");
usr/src/uts/common/io/nvme/nvme.c
4665
nvme_config_list(nvme);
usr/src/uts/common/io/nvme/nvme.c
4670
bcopy(nvme->n_idctl->id_model, model, sizeof (nvme->n_idctl->id_model));
usr/src/uts/common/io/nvme/nvme.c
4671
model[sizeof (nvme->n_idctl->id_model)] = '\0';
usr/src/uts/common/io/nvme/nvme.c
4675
nvme->n_vendor = strdup("NVMe");
usr/src/uts/common/io/nvme/nvme.c
4677
nvme->n_vendor = strdup(vendor);
usr/src/uts/common/io/nvme/nvme.c
4679
nvme->n_product = strdup(product);
usr/src/uts/common/io/nvme/nvme.c
4684
nvme->n_async_event_limit = MAX(NVME_MIN_ASYNC_EVENT_LIMIT,
usr/src/uts/common/io/nvme/nvme.c
4685
MIN(nvme->n_admin_queue_len / 10,
usr/src/uts/common/io/nvme/nvme.c
4686
MIN(nvme->n_idctl->id_aerl + 1, nvme->n_async_event_limit)));
usr/src/uts/common/io/nvme/nvme.c
4688
(void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip,
usr/src/uts/common/io/nvme/nvme.c
4689
"async-event-limit", nvme->n_async_event_limit);
usr/src/uts/common/io/nvme/nvme.c
4691
nvme->n_abort_command_limit = nvme->n_idctl->id_acl + 1;
usr/src/uts/common/io/nvme/nvme.c
4699
sema_destroy(&nvme->n_abort_sema);
usr/src/uts/common/io/nvme/nvme.c
4700
sema_init(&nvme->n_abort_sema, nvme->n_abort_command_limit - 1, NULL,
usr/src/uts/common/io/nvme/nvme.c
4703
nvme->n_progress |= NVME_CTRL_LIMITS;
usr/src/uts/common/io/nvme/nvme.c
4705
if (nvme->n_idctl->id_mdts == 0)
usr/src/uts/common/io/nvme/nvme.c
4706
nvme->n_max_data_transfer_size = nvme->n_pagesize * 65536;
usr/src/uts/common/io/nvme/nvme.c
4708
nvme->n_max_data_transfer_size =
usr/src/uts/common/io/nvme/nvme.c
4709
1ull << (nvme->n_pageshift + nvme->n_idctl->id_mdts);
usr/src/uts/common/io/nvme/nvme.c
4711
nvme->n_error_log_len = nvme->n_idctl->id_elpe + 1;
usr/src/uts/common/io/nvme/nvme.c
4720
nvme->n_max_data_transfer_size = MIN(nvme->n_max_data_transfer_size,
usr/src/uts/common/io/nvme/nvme.c
4721
(nvme->n_pagesize / sizeof (uint64_t) * nvme->n_pagesize));
usr/src/uts/common/io/nvme/nvme.c
4723
nvme->n_prp_dma_attr.dma_attr_maxxfer = nvme->n_max_data_transfer_size;
usr/src/uts/common/io/nvme/nvme.c
4730
if (((1 << nvme->n_idctl->id_sqes.qes_min) > sizeof (nvme_sqe_t)) ||
usr/src/uts/common/io/nvme/nvme.c
4731
((1 << nvme->n_idctl->id_sqes.qes_max) < sizeof (nvme_sqe_t)) ||
usr/src/uts/common/io/nvme/nvme.c
4732
((1 << nvme->n_idctl->id_cqes.qes_min) > sizeof (nvme_cqe_t)) ||
usr/src/uts/common/io/nvme/nvme.c
4733
((1 << nvme->n_idctl->id_cqes.qes_max) < sizeof (nvme_cqe_t)))
usr/src/uts/common/io/nvme/nvme.c
4741
nvme->n_write_cache_present =
usr/src/uts/common/io/nvme/nvme.c
4742
nvme->n_idctl->id_vwc.vwc_present == 0 ? B_FALSE : B_TRUE;
usr/src/uts/common/io/nvme/nvme.c
4744
(void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip,
usr/src/uts/common/io/nvme/nvme.c
4746
nvme->n_write_cache_present ? 1 : 0);
usr/src/uts/common/io/nvme/nvme.c
4748
if (!nvme->n_write_cache_present) {
usr/src/uts/common/io/nvme/nvme.c
4749
nvme->n_write_cache_enabled = B_FALSE;
usr/src/uts/common/io/nvme/nvme.c
4750
} else if (nvme_write_cache_set(nvme, nvme->n_write_cache_enabled)
usr/src/uts/common/io/nvme/nvme.c
4752
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
4754
nvme->n_write_cache_enabled ? "en" : "dis");
usr/src/uts/common/io/nvme/nvme.c
4758
nvme->n_write_cache_enabled = B_TRUE;
usr/src/uts/common/io/nvme/nvme.c
4761
(void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip,
usr/src/uts/common/io/nvme/nvme.c
4763
nvme->n_write_cache_enabled ? 1 : 0);
usr/src/uts/common/io/nvme/nvme.c
4768
nvme->n_namespace_count = nvme->n_idctl->id_nn;
usr/src/uts/common/io/nvme/nvme.c
4770
if (nvme->n_namespace_count == 0) {
usr/src/uts/common/io/nvme/nvme.c
4771
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
4776
nvme->n_ns = kmem_zalloc(sizeof (nvme_namespace_t) *
usr/src/uts/common/io/nvme/nvme.c
4777
nvme->n_namespace_count, KM_SLEEP);
usr/src/uts/common/io/nvme/nvme.c
4783
if (nvme_ctrl_atleast(nvme, &nvme_vers_1v2) &&
usr/src/uts/common/io/nvme/nvme.c
4784
nvme->n_idctl->id_oacs.oa_nsmgmt != 0) {
usr/src/uts/common/io/nvme/nvme.c
4790
if (!nvme_identify_int(nvme, nsid, NVME_IDENTIFY_NSID,
usr/src/uts/common/io/nvme/nvme.c
4791
(void **)&nvme->n_idcomns)) {
usr/src/uts/common/io/nvme/nvme.c
4792
dev_err(nvme->n_dip, CE_WARN, "!failed to identify common "
usr/src/uts/common/io/nvme/nvme.c
4797
if (nvme_get_current_nqueues(nvme, &nq)) {
usr/src/uts/common/io/nvme/nvme.c
4798
nvme->n_submission_queues_supported = nq.b.nq_nsq + 1;
usr/src/uts/common/io/nvme/nvme.c
4799
nvme->n_completion_queues_supported = nq.b.nq_ncq + 1;
usr/src/uts/common/io/nvme/nvme.c
4801
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
4809
if ((nvme->n_intr_types & (DDI_INTR_TYPE_MSI | DDI_INTR_TYPE_MSIX))
usr/src/uts/common/io/nvme/nvme.c
4812
nvme->n_submission_queues_supported,
usr/src/uts/common/io/nvme/nvme.c
4813
nvme->n_completion_queues_supported);
usr/src/uts/common/io/nvme/nvme.c
4815
nvme_release_interrupts(nvme);
usr/src/uts/common/io/nvme/nvme.c
4816
if ((nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSIX,
usr/src/uts/common/io/nvme/nvme.c
4818
(nvme_setup_interrupts(nvme, DDI_INTR_TYPE_MSI,
usr/src/uts/common/io/nvme/nvme.c
4820
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
4830
if (nvme_set_nqueues(nvme) != 0) {
usr/src/uts/common/io/nvme/nvme.c
4831
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
4833
nvme->n_intr_cnt);
usr/src/uts/common/io/nvme/nvme.c
4840
kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *));
usr/src/uts/common/io/nvme/nvme.c
4841
nvme->n_ioq = kmem_zalloc(sizeof (nvme_qpair_t *) *
usr/src/uts/common/io/nvme/nvme.c
4842
(nvme->n_submission_queues + 1), KM_SLEEP);
usr/src/uts/common/io/nvme/nvme.c
4843
nvme->n_ioq[0] = nvme->n_adminq;
usr/src/uts/common/io/nvme/nvme.c
4849
ASSERT(nvme->n_submission_queues >= nvme->n_completion_queues);
usr/src/uts/common/io/nvme/nvme.c
4851
nvme->n_ioq_count = nvme->n_submission_queues;
usr/src/uts/common/io/nvme/nvme.c
4853
nvme->n_io_squeue_len =
usr/src/uts/common/io/nvme/nvme.c
4854
MIN(nvme->n_io_squeue_len, nvme->n_max_queue_entries);
usr/src/uts/common/io/nvme/nvme.c
4856
(void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, "io-squeue-len",
usr/src/uts/common/io/nvme/nvme.c
4857
nvme->n_io_squeue_len);
usr/src/uts/common/io/nvme/nvme.c
4865
if (nvme->n_submission_queues == nvme->n_completion_queues)
usr/src/uts/common/io/nvme/nvme.c
4866
nvme->n_io_cqueue_len = MIN(nvme->n_io_cqueue_len,
usr/src/uts/common/io/nvme/nvme.c
4867
nvme->n_io_squeue_len);
usr/src/uts/common/io/nvme/nvme.c
4869
nvme->n_io_cqueue_len = MIN(nvme->n_io_cqueue_len,
usr/src/uts/common/io/nvme/nvme.c
4870
nvme->n_max_queue_entries);
usr/src/uts/common/io/nvme/nvme.c
4872
(void) ddi_prop_update_int(DDI_DEV_T_NONE, nvme->n_dip, "io-cqueue-len",
usr/src/uts/common/io/nvme/nvme.c
4873
nvme->n_io_cqueue_len);
usr/src/uts/common/io/nvme/nvme.c
4886
tq_threads = ncpus_eff / nvme->n_completion_queues;
usr/src/uts/common/io/nvme/nvme.c
4894
if (nvme_create_cq_array(nvme, nvme->n_completion_queues + 1,
usr/src/uts/common/io/nvme/nvme.c
4895
nvme->n_io_cqueue_len, tq_threads) != DDI_SUCCESS) {
usr/src/uts/common/io/nvme/nvme.c
4896
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
4905
if (nvme->n_completion_queues + 1 < nvme->n_intr_cnt) {
usr/src/uts/common/io/nvme/nvme.c
4906
nvme_release_interrupts(nvme);
usr/src/uts/common/io/nvme/nvme.c
4908
if (nvme_setup_interrupts(nvme, nvme->n_intr_type,
usr/src/uts/common/io/nvme/nvme.c
4909
nvme->n_completion_queues + 1) != DDI_SUCCESS) {
usr/src/uts/common/io/nvme/nvme.c
4910
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
4920
for (i = 1; i != nvme->n_ioq_count + 1; i++) {
usr/src/uts/common/io/nvme/nvme.c
4921
if (nvme_alloc_qpair(nvme, nvme->n_io_squeue_len,
usr/src/uts/common/io/nvme/nvme.c
4922
&nvme->n_ioq[i], i) != DDI_SUCCESS) {
usr/src/uts/common/io/nvme/nvme.c
4923
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
4928
if (nvme_create_io_qpair(nvme, nvme->n_ioq[i], i) != 0) {
usr/src/uts/common/io/nvme/nvme.c
4929
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
4938
nvme_enable_host_behavior(nvme);
usr/src/uts/common/io/nvme/nvme.c
4943
(void) nvme_reset(nvme, B_FALSE);
usr/src/uts/common/io/nvme/nvme.c
4950
nvme_t *nvme = (nvme_t *)arg1;
usr/src/uts/common/io/nvme/nvme.c
4955
if (inum >= nvme->n_intr_cnt)
usr/src/uts/common/io/nvme/nvme.c
4958
if (nvme->n_dead) {
usr/src/uts/common/io/nvme/nvme.c
4959
return (nvme->n_intr_type == DDI_INTR_TYPE_FIXED ?
usr/src/uts/common/io/nvme/nvme.c
4970
qnum < nvme->n_cq_count && nvme->n_cq[qnum] != NULL;
usr/src/uts/common/io/nvme/nvme.c
4971
qnum += nvme->n_intr_cnt) {
usr/src/uts/common/io/nvme/nvme.c
4972
ccnt += nvme_process_iocq(nvme, nvme->n_cq[qnum]);
usr/src/uts/common/io/nvme/nvme.c
4979
nvme_release_interrupts(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme.c
4983
for (i = 0; i < nvme->n_intr_cnt; i++) {
usr/src/uts/common/io/nvme/nvme.c
4984
if (nvme->n_inth[i] == NULL)
usr/src/uts/common/io/nvme/nvme.c
4987
if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK)
usr/src/uts/common/io/nvme/nvme.c
4988
(void) ddi_intr_block_disable(&nvme->n_inth[i], 1);
usr/src/uts/common/io/nvme/nvme.c
4990
(void) ddi_intr_disable(nvme->n_inth[i]);
usr/src/uts/common/io/nvme/nvme.c
4992
(void) ddi_intr_remove_handler(nvme->n_inth[i]);
usr/src/uts/common/io/nvme/nvme.c
4993
(void) ddi_intr_free(nvme->n_inth[i]);
usr/src/uts/common/io/nvme/nvme.c
4996
kmem_free(nvme->n_inth, nvme->n_inth_sz);
usr/src/uts/common/io/nvme/nvme.c
4997
nvme->n_inth = NULL;
usr/src/uts/common/io/nvme/nvme.c
4998
nvme->n_inth_sz = 0;
usr/src/uts/common/io/nvme/nvme.c
5000
nvme->n_progress &= ~NVME_INTERRUPTS;
usr/src/uts/common/io/nvme/nvme.c
5004
nvme_setup_interrupts(nvme_t *nvme, int intr_type, int nqpairs)
usr/src/uts/common/io/nvme/nvme.c
5010
if (nvme->n_intr_types == 0) {
usr/src/uts/common/io/nvme/nvme.c
5011
ret = ddi_intr_get_supported_types(nvme->n_dip,
usr/src/uts/common/io/nvme/nvme.c
5012
&nvme->n_intr_types);
usr/src/uts/common/io/nvme/nvme.c
5014
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
5021
nvme->n_intr_types &= ~DDI_INTR_TYPE_MSIX;
usr/src/uts/common/io/nvme/nvme.c
5025
if ((nvme->n_intr_types & intr_type) == 0)
usr/src/uts/common/io/nvme/nvme.c
5028
ret = ddi_intr_get_nintrs(nvme->n_dip, intr_type, &nintrs);
usr/src/uts/common/io/nvme/nvme.c
5030
dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_nintrs failed",
usr/src/uts/common/io/nvme/nvme.c
5035
ret = ddi_intr_get_navail(nvme->n_dip, intr_type, &navail);
usr/src/uts/common/io/nvme/nvme.c
5037
dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_navail failed",
usr/src/uts/common/io/nvme/nvme.c
5046
nvme->n_inth_sz = sizeof (ddi_intr_handle_t) * navail;
usr/src/uts/common/io/nvme/nvme.c
5047
nvme->n_inth = kmem_zalloc(nvme->n_inth_sz, KM_SLEEP);
usr/src/uts/common/io/nvme/nvme.c
5049
ret = ddi_intr_alloc(nvme->n_dip, nvme->n_inth, intr_type, 0, navail,
usr/src/uts/common/io/nvme/nvme.c
5052
dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_alloc failed",
usr/src/uts/common/io/nvme/nvme.c
5057
nvme->n_intr_cnt = count;
usr/src/uts/common/io/nvme/nvme.c
5059
ret = ddi_intr_get_pri(nvme->n_inth[0], &nvme->n_intr_pri);
usr/src/uts/common/io/nvme/nvme.c
5061
dev_err(nvme->n_dip, CE_WARN, "!%s: ddi_intr_get_pri failed",
usr/src/uts/common/io/nvme/nvme.c
5067
ret = ddi_intr_add_handler(nvme->n_inth[i], nvme_intr,
usr/src/uts/common/io/nvme/nvme.c
5068
(void *)nvme, (void *)(uintptr_t)i);
usr/src/uts/common/io/nvme/nvme.c
5070
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
5076
(void) ddi_intr_get_cap(nvme->n_inth[0], &nvme->n_intr_cap);
usr/src/uts/common/io/nvme/nvme.c
5079
if (nvme->n_intr_cap & DDI_INTR_FLAG_BLOCK)
usr/src/uts/common/io/nvme/nvme.c
5080
ret = ddi_intr_block_enable(&nvme->n_inth[i], 1);
usr/src/uts/common/io/nvme/nvme.c
5082
ret = ddi_intr_enable(nvme->n_inth[i]);
usr/src/uts/common/io/nvme/nvme.c
5085
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
5091
nvme->n_intr_type = intr_type;
usr/src/uts/common/io/nvme/nvme.c
5093
nvme->n_progress |= NVME_INTERRUPTS;
usr/src/uts/common/io/nvme/nvme.c
5098
nvme_release_interrupts(nvme);
usr/src/uts/common/io/nvme/nvme.c
5116
nvme_t *nvme = a;
usr/src/uts/common/io/nvme/nvme.c
5118
nvme_ctrl_mark_dead(nvme, B_TRUE);
usr/src/uts/common/io/nvme/nvme.c
5124
for (uint_t i = 0; i < nvme->n_ioq_count + 1; i++) {
usr/src/uts/common/io/nvme/nvme.c
5125
nvme_qpair_t *qp = nvme->n_ioq[i];
usr/src/uts/common/io/nvme/nvme.c
5143
u_cmd = nvme_unqueue_cmd(nvme, qp, cmd->nc_sqe.sqe_cid);
usr/src/uts/common/io/nvme/nvme.c
5202
nvme_t *nvme;
usr/src/uts/common/io/nvme/nvme.c
5216
nvme = ddi_get_soft_state(nvme_state, instance);
usr/src/uts/common/io/nvme/nvme.c
5217
ddi_set_driver_private(dip, nvme);
usr/src/uts/common/io/nvme/nvme.c
5218
nvme->n_dip = dip;
usr/src/uts/common/io/nvme/nvme.c
5223
if (pci_config_setup(dip, &nvme->n_pcicfg_handle) != DDI_SUCCESS) {
usr/src/uts/common/io/nvme/nvme.c
5227
nvme->n_progress |= NVME_PCI_CONFIG;
usr/src/uts/common/io/nvme/nvme.c
5232
nvme->n_vendor_id =
usr/src/uts/common/io/nvme/nvme.c
5233
pci_config_get16(nvme->n_pcicfg_handle, PCI_CONF_VENID);
usr/src/uts/common/io/nvme/nvme.c
5234
nvme->n_device_id =
usr/src/uts/common/io/nvme/nvme.c
5235
pci_config_get16(nvme->n_pcicfg_handle, PCI_CONF_DEVID);
usr/src/uts/common/io/nvme/nvme.c
5236
nvme->n_revision_id =
usr/src/uts/common/io/nvme/nvme.c
5237
pci_config_get8(nvme->n_pcicfg_handle, PCI_CONF_REVID);
usr/src/uts/common/io/nvme/nvme.c
5238
nvme->n_subsystem_device_id =
usr/src/uts/common/io/nvme/nvme.c
5239
pci_config_get16(nvme->n_pcicfg_handle, PCI_CONF_SUBSYSID);
usr/src/uts/common/io/nvme/nvme.c
5240
nvme->n_subsystem_vendor_id =
usr/src/uts/common/io/nvme/nvme.c
5241
pci_config_get16(nvme->n_pcicfg_handle, PCI_CONF_SUBVENID);
usr/src/uts/common/io/nvme/nvme.c
5243
nvme_detect_quirks(nvme);
usr/src/uts/common/io/nvme/nvme.c
5254
if (ddi_get_eventcookie(nvme->n_dip, DDI_DEVI_REMOVE_EVENT,
usr/src/uts/common/io/nvme/nvme.c
5255
&nvme->n_rm_cookie) == DDI_SUCCESS) {
usr/src/uts/common/io/nvme/nvme.c
5256
if (ddi_add_event_handler(nvme->n_dip, nvme->n_rm_cookie,
usr/src/uts/common/io/nvme/nvme.c
5257
nvme_remove_callback, nvme, &nvme->n_ev_rm_cb_id) !=
usr/src/uts/common/io/nvme/nvme.c
5262
nvme->n_ev_rm_cb_id = NULL;
usr/src/uts/common/io/nvme/nvme.c
5265
mutex_init(&nvme->n_minor_mutex, NULL, MUTEX_DRIVER, NULL);
usr/src/uts/common/io/nvme/nvme.c
5266
nvme->n_progress |= NVME_MUTEX_INIT;
usr/src/uts/common/io/nvme/nvme.c
5268
nvme->n_strict_version = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
usr/src/uts/common/io/nvme/nvme.c
5270
nvme->n_ignore_unknown_vendor_status = ddi_prop_get_int(DDI_DEV_T_ANY,
usr/src/uts/common/io/nvme/nvme.c
5273
nvme->n_admin_queue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
usr/src/uts/common/io/nvme/nvme.c
5275
nvme->n_io_squeue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
usr/src/uts/common/io/nvme/nvme.c
5281
nvme->n_io_cqueue_len = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
usr/src/uts/common/io/nvme/nvme.c
5283
nvme->n_async_event_limit = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
usr/src/uts/common/io/nvme/nvme.c
5286
nvme->n_write_cache_enabled = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
usr/src/uts/common/io/nvme/nvme.c
5289
nvme->n_min_block_size = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
usr/src/uts/common/io/nvme/nvme.c
5292
nvme->n_submission_queues = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
usr/src/uts/common/io/nvme/nvme.c
5294
nvme->n_completion_queues = ddi_prop_get_int(DDI_DEV_T_ANY, dip,
usr/src/uts/common/io/nvme/nvme.c
5297
if (!ISP2(nvme->n_min_block_size) ||
usr/src/uts/common/io/nvme/nvme.c
5298
(nvme->n_min_block_size < NVME_DEFAULT_MIN_BLOCK_SIZE)) {
usr/src/uts/common/io/nvme/nvme.c
5300
"using default %d", ISP2(nvme->n_min_block_size) ?
usr/src/uts/common/io/nvme/nvme.c
5303
nvme->n_min_block_size = NVME_DEFAULT_MIN_BLOCK_SIZE;
usr/src/uts/common/io/nvme/nvme.c
5306
if (nvme->n_submission_queues != -1 &&
usr/src/uts/common/io/nvme/nvme.c
5307
(nvme->n_submission_queues < 1 ||
usr/src/uts/common/io/nvme/nvme.c
5308
nvme->n_submission_queues > UINT16_MAX)) {
usr/src/uts/common/io/nvme/nvme.c
5310
"valid. Must be [1..%d]", nvme->n_submission_queues,
usr/src/uts/common/io/nvme/nvme.c
5312
nvme->n_submission_queues = -1;
usr/src/uts/common/io/nvme/nvme.c
5315
if (nvme->n_completion_queues != -1 &&
usr/src/uts/common/io/nvme/nvme.c
5316
(nvme->n_completion_queues < 1 ||
usr/src/uts/common/io/nvme/nvme.c
5317
nvme->n_completion_queues > UINT16_MAX)) {
usr/src/uts/common/io/nvme/nvme.c
5319
"valid. Must be [1..%d]", nvme->n_completion_queues,
usr/src/uts/common/io/nvme/nvme.c
5321
nvme->n_completion_queues = -1;
usr/src/uts/common/io/nvme/nvme.c
5324
if (nvme->n_admin_queue_len < NVME_MIN_ADMIN_QUEUE_LEN)
usr/src/uts/common/io/nvme/nvme.c
5325
nvme->n_admin_queue_len = NVME_MIN_ADMIN_QUEUE_LEN;
usr/src/uts/common/io/nvme/nvme.c
5326
else if (nvme->n_admin_queue_len > NVME_MAX_ADMIN_QUEUE_LEN)
usr/src/uts/common/io/nvme/nvme.c
5327
nvme->n_admin_queue_len = NVME_MAX_ADMIN_QUEUE_LEN;
usr/src/uts/common/io/nvme/nvme.c
5329
if (nvme->n_io_squeue_len < NVME_MIN_IO_QUEUE_LEN)
usr/src/uts/common/io/nvme/nvme.c
5330
nvme->n_io_squeue_len = NVME_MIN_IO_QUEUE_LEN;
usr/src/uts/common/io/nvme/nvme.c
5331
if (nvme->n_io_cqueue_len < NVME_MIN_IO_QUEUE_LEN)
usr/src/uts/common/io/nvme/nvme.c
5332
nvme->n_io_cqueue_len = NVME_MIN_IO_QUEUE_LEN;
usr/src/uts/common/io/nvme/nvme.c
5334
if (nvme->n_async_event_limit < 1)
usr/src/uts/common/io/nvme/nvme.c
5335
nvme->n_async_event_limit = NVME_DEFAULT_ASYNC_EVENT_LIMIT;
usr/src/uts/common/io/nvme/nvme.c
5337
nvme->n_reg_acc_attr = nvme_reg_acc_attr;
usr/src/uts/common/io/nvme/nvme.c
5338
nvme->n_queue_dma_attr = nvme_queue_dma_attr;
usr/src/uts/common/io/nvme/nvme.c
5339
nvme->n_prp_dma_attr = nvme_prp_dma_attr;
usr/src/uts/common/io/nvme/nvme.c
5340
nvme->n_sgl_dma_attr = nvme_sgl_dma_attr;
usr/src/uts/common/io/nvme/nvme.c
5345
nvme->n_fm_cap = ddi_getprop(DDI_DEV_T_ANY, dip,
usr/src/uts/common/io/nvme/nvme.c
5350
ddi_fm_init(dip, &nvme->n_fm_cap, &nvme->n_fm_ibc);
usr/src/uts/common/io/nvme/nvme.c
5352
if (nvme->n_fm_cap) {
usr/src/uts/common/io/nvme/nvme.c
5353
if (nvme->n_fm_cap & DDI_FM_ACCCHK_CAPABLE)
usr/src/uts/common/io/nvme/nvme.c
5354
nvme->n_reg_acc_attr.devacc_attr_access =
usr/src/uts/common/io/nvme/nvme.c
5357
if (nvme->n_fm_cap & DDI_FM_DMACHK_CAPABLE) {
usr/src/uts/common/io/nvme/nvme.c
5358
nvme->n_prp_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
usr/src/uts/common/io/nvme/nvme.c
5359
nvme->n_sgl_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
usr/src/uts/common/io/nvme/nvme.c
5362
if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) ||
usr/src/uts/common/io/nvme/nvme.c
5363
DDI_FM_ERRCB_CAP(nvme->n_fm_cap))
usr/src/uts/common/io/nvme/nvme.c
5366
if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap))
usr/src/uts/common/io/nvme/nvme.c
5368
(void *)nvme);
usr/src/uts/common/io/nvme/nvme.c
5371
nvme->n_progress |= NVME_FMA_INIT;
usr/src/uts/common/io/nvme/nvme.c
5382
if (ddi_regs_map_setup(dip, 1, &nvme->n_regs, 0, regsize,
usr/src/uts/common/io/nvme/nvme.c
5383
&nvme->n_reg_acc_attr, &nvme->n_regh) != DDI_SUCCESS) {
usr/src/uts/common/io/nvme/nvme.c
5388
nvme->n_progress |= NVME_REGS_MAPPED;
usr/src/uts/common/io/nvme/nvme.c
5393
if (!nvme_stat_init(nvme)) {
usr/src/uts/common/io/nvme/nvme.c
5397
nvme->n_progress |= NVME_STAT_INIT;
usr/src/uts/common/io/nvme/nvme.c
5404
nvme->n_prp_cache = kmem_cache_create(name, sizeof (nvme_dma_t),
usr/src/uts/common/io/nvme/nvme.c
5406
NULL, (void *)nvme, NULL, 0);
usr/src/uts/common/io/nvme/nvme.c
5408
if (nvme_init(nvme) != DDI_SUCCESS)
usr/src/uts/common/io/nvme/nvme.c
5415
&nvme->n_ufmh, nvme) != 0) {
usr/src/uts/common/io/nvme/nvme.c
5419
mutex_init(&nvme->n_fwslot_mutex, NULL, MUTEX_DRIVER, NULL);
usr/src/uts/common/io/nvme/nvme.c
5420
ddi_ufm_update(nvme->n_ufmh);
usr/src/uts/common/io/nvme/nvme.c
5421
nvme->n_progress |= NVME_UFM_INIT;
usr/src/uts/common/io/nvme/nvme.c
5423
nvme_mgmt_lock_init(&nvme->n_mgmt);
usr/src/uts/common/io/nvme/nvme.c
5424
nvme_lock_init(&nvme->n_lock);
usr/src/uts/common/io/nvme/nvme.c
5425
nvme->n_progress |= NVME_MGMT_INIT;
usr/src/uts/common/io/nvme/nvme.c
5430
nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME);
usr/src/uts/common/io/nvme/nvme.c
5433
for (uint32_t i = 1; i <= nvme->n_namespace_count; i++) {
usr/src/uts/common/io/nvme/nvme.c
5434
nvme_namespace_t *ns = nvme_nsid2ns(nvme, i);
usr/src/uts/common/io/nvme/nvme.c
5448
if (nvme_init_ns(nvme, i) != 0) {
usr/src/uts/common/io/nvme/nvme.c
5449
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
5466
NVME_MINOR_MAX, nvme->n_namespace_count);
usr/src/uts/common/io/nvme/nvme.c
5472
if (ddi_create_minor_node(nvme->n_dip, ns->ns_name, S_IFCHR,
usr/src/uts/common/io/nvme/nvme.c
5473
NVME_MINOR(ddi_get_instance(nvme->n_dip), i),
usr/src/uts/common/io/nvme/nvme.c
5475
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
5487
nvme->n_progress |= NVME_NS_INIT;
usr/src/uts/common/io/nvme/nvme.c
5492
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
5502
for (uint32_t i = 1; i <= nvme->n_namespace_count; i++) {
usr/src/uts/common/io/nvme/nvme.c
5503
nvme_namespace_t *ns = nvme_nsid2ns(nvme, i);
usr/src/uts/common/io/nvme/nvme.c
5509
if (!nvme_bd_attach_ns(nvme, &com) && com.nioc_drv_err !=
usr/src/uts/common/io/nvme/nvme.c
5511
dev_err(nvme->n_dip, CE_WARN, "!failed to attach "
usr/src/uts/common/io/nvme/nvme.c
5517
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
5536
nvme->n_async_event_supported = B_TRUE;
usr/src/uts/common/io/nvme/nvme.c
5537
for (uint16_t i = 0; i < nvme->n_async_event_limit; i++) {
usr/src/uts/common/io/nvme/nvme.c
5538
nvme_async_event(nvme);
usr/src/uts/common/io/nvme/nvme.c
5546
if (nvme->n_dead)
usr/src/uts/common/io/nvme/nvme.c
5558
nvme_t *nvme;
usr/src/uts/common/io/nvme/nvme.c
5565
nvme = ddi_get_soft_state(nvme_state, instance);
usr/src/uts/common/io/nvme/nvme.c
5567
if (nvme == NULL)
usr/src/uts/common/io/nvme/nvme.c
5583
if (nvme->n_ev_rm_cb_id != NULL) {
usr/src/uts/common/io/nvme/nvme.c
5584
(void) ddi_remove_event_handler(nvme->n_ev_rm_cb_id);
usr/src/uts/common/io/nvme/nvme.c
5586
nvme->n_ev_rm_cb_id = NULL;
usr/src/uts/common/io/nvme/nvme.c
5598
if (nvme->n_ns) {
usr/src/uts/common/io/nvme/nvme.c
5599
for (uint32_t i = 1; i <= nvme->n_namespace_count; i++) {
usr/src/uts/common/io/nvme/nvme.c
5600
nvme_namespace_t *ns = nvme_nsid2ns(nvme, i);
usr/src/uts/common/io/nvme/nvme.c
5617
kmem_free(nvme->n_ns, sizeof (nvme_namespace_t) *
usr/src/uts/common/io/nvme/nvme.c
5618
nvme->n_namespace_count);
usr/src/uts/common/io/nvme/nvme.c
5621
if (nvme->n_progress & NVME_MGMT_INIT) {
usr/src/uts/common/io/nvme/nvme.c
5622
nvme_lock_fini(&nvme->n_lock);
usr/src/uts/common/io/nvme/nvme.c
5623
nvme_mgmt_lock_fini(&nvme->n_mgmt);
usr/src/uts/common/io/nvme/nvme.c
5626
if (nvme->n_progress & NVME_UFM_INIT) {
usr/src/uts/common/io/nvme/nvme.c
5627
ddi_ufm_fini(nvme->n_ufmh);
usr/src/uts/common/io/nvme/nvme.c
5628
mutex_destroy(&nvme->n_fwslot_mutex);
usr/src/uts/common/io/nvme/nvme.c
5631
if (nvme->n_progress & NVME_INTERRUPTS)
usr/src/uts/common/io/nvme/nvme.c
5632
nvme_release_interrupts(nvme);
usr/src/uts/common/io/nvme/nvme.c
5634
for (uint_t i = 0; i < nvme->n_cq_count; i++) {
usr/src/uts/common/io/nvme/nvme.c
5635
if (nvme->n_cq[i]->ncq_cmd_taskq != NULL)
usr/src/uts/common/io/nvme/nvme.c
5636
taskq_wait(nvme->n_cq[i]->ncq_cmd_taskq);
usr/src/uts/common/io/nvme/nvme.c
5639
if (nvme->n_progress & NVME_MUTEX_INIT) {
usr/src/uts/common/io/nvme/nvme.c
5640
mutex_destroy(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
5643
if (nvme->n_ioq_count > 0) {
usr/src/uts/common/io/nvme/nvme.c
5644
for (uint_t i = 1; i != nvme->n_ioq_count + 1; i++) {
usr/src/uts/common/io/nvme/nvme.c
5645
if (nvme->n_ioq[i] != NULL) {
usr/src/uts/common/io/nvme/nvme.c
5647
nvme_free_qpair(nvme->n_ioq[i]);
usr/src/uts/common/io/nvme/nvme.c
5651
kmem_free(nvme->n_ioq, sizeof (nvme_qpair_t *) *
usr/src/uts/common/io/nvme/nvme.c
5652
(nvme->n_ioq_count + 1));
usr/src/uts/common/io/nvme/nvme.c
5655
if (nvme->n_prp_cache != NULL) {
usr/src/uts/common/io/nvme/nvme.c
5656
kmem_cache_destroy(nvme->n_prp_cache);
usr/src/uts/common/io/nvme/nvme.c
5659
if (nvme->n_progress & NVME_REGS_MAPPED) {
usr/src/uts/common/io/nvme/nvme.c
5660
nvme_shutdown(nvme, B_FALSE);
usr/src/uts/common/io/nvme/nvme.c
5661
(void) nvme_reset(nvme, B_FALSE);
usr/src/uts/common/io/nvme/nvme.c
5664
if (nvme->n_progress & NVME_CTRL_LIMITS)
usr/src/uts/common/io/nvme/nvme.c
5665
sema_destroy(&nvme->n_abort_sema);
usr/src/uts/common/io/nvme/nvme.c
5667
if (nvme->n_progress & NVME_ADMIN_QUEUE)
usr/src/uts/common/io/nvme/nvme.c
5668
nvme_free_qpair(nvme->n_adminq);
usr/src/uts/common/io/nvme/nvme.c
5670
if (nvme->n_cq_count > 0) {
usr/src/uts/common/io/nvme/nvme.c
5671
nvme_destroy_cq_array(nvme, 0);
usr/src/uts/common/io/nvme/nvme.c
5672
nvme->n_cq = NULL;
usr/src/uts/common/io/nvme/nvme.c
5673
nvme->n_cq_count = 0;
usr/src/uts/common/io/nvme/nvme.c
5676
if (nvme->n_idcomns)
usr/src/uts/common/io/nvme/nvme.c
5677
kmem_free(nvme->n_idcomns, NVME_IDENTIFY_BUFSIZE);
usr/src/uts/common/io/nvme/nvme.c
5679
if (nvme->n_idctl)
usr/src/uts/common/io/nvme/nvme.c
5680
kmem_free(nvme->n_idctl, NVME_IDENTIFY_BUFSIZE);
usr/src/uts/common/io/nvme/nvme.c
5682
if (nvme->n_progress & NVME_REGS_MAPPED)
usr/src/uts/common/io/nvme/nvme.c
5683
ddi_regs_map_free(&nvme->n_regh);
usr/src/uts/common/io/nvme/nvme.c
5685
if (nvme->n_progress & NVME_STAT_INIT)
usr/src/uts/common/io/nvme/nvme.c
5686
nvme_stat_cleanup(nvme);
usr/src/uts/common/io/nvme/nvme.c
5688
if (nvme->n_progress & NVME_FMA_INIT) {
usr/src/uts/common/io/nvme/nvme.c
5689
if (DDI_FM_ERRCB_CAP(nvme->n_fm_cap))
usr/src/uts/common/io/nvme/nvme.c
5690
ddi_fm_handler_unregister(nvme->n_dip);
usr/src/uts/common/io/nvme/nvme.c
5692
if (DDI_FM_EREPORT_CAP(nvme->n_fm_cap) ||
usr/src/uts/common/io/nvme/nvme.c
5693
DDI_FM_ERRCB_CAP(nvme->n_fm_cap))
usr/src/uts/common/io/nvme/nvme.c
5694
pci_ereport_teardown(nvme->n_dip);
usr/src/uts/common/io/nvme/nvme.c
5696
ddi_fm_fini(nvme->n_dip);
usr/src/uts/common/io/nvme/nvme.c
5699
if (nvme->n_progress & NVME_PCI_CONFIG)
usr/src/uts/common/io/nvme/nvme.c
5700
pci_config_teardown(&nvme->n_pcicfg_handle);
usr/src/uts/common/io/nvme/nvme.c
5702
if (nvme->n_vendor != NULL)
usr/src/uts/common/io/nvme/nvme.c
5703
strfree(nvme->n_vendor);
usr/src/uts/common/io/nvme/nvme.c
5705
if (nvme->n_product != NULL)
usr/src/uts/common/io/nvme/nvme.c
5706
strfree(nvme->n_product);
usr/src/uts/common/io/nvme/nvme.c
5717
nvme_t *nvme;
usr/src/uts/common/io/nvme/nvme.c
5721
nvme = ddi_get_soft_state(nvme_state, instance);
usr/src/uts/common/io/nvme/nvme.c
5723
if (nvme == NULL)
usr/src/uts/common/io/nvme/nvme.c
5726
nvme_shutdown(nvme, B_TRUE);
usr/src/uts/common/io/nvme/nvme.c
5728
(void) nvme_reset(nvme, B_TRUE);
usr/src/uts/common/io/nvme/nvme.c
5736
nvme_t *nvme = cmd->nc_nvme;
usr/src/uts/common/io/nvme/nvme.c
5766
nprp_per_page = nvme->n_pagesize / sizeof (uint64_t);
usr/src/uts/common/io/nvme/nvme.c
5782
cmd->nc_prp = kmem_cache_alloc(nvme->n_prp_cache, KM_SLEEP);
usr/src/uts/common/io/nvme/nvme.c
5814
nvme_t *nvme = cmd->nc_nvme;
usr/src/uts/common/io/nvme/nvme.c
5832
cmd->nc_prp = kmem_cache_alloc(nvme->n_prp_cache, allocflag);
usr/src/uts/common/io/nvme/nvme.c
5865
nvme_t *nvme = ns->ns_nvme;
usr/src/uts/common/io/nvme/nvme.c
5873
cmd = nvme_alloc_cmd(nvme, allocflag);
usr/src/uts/common/io/nvme/nvme.c
5937
nvme_t *nvme = ns->ns_nvme;
usr/src/uts/common/io/nvme/nvme.c
5938
uint_t ns_count = MAX(1, nvme->n_namespaces_attachable);
usr/src/uts/common/io/nvme/nvme.c
5940
nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_BDRO);
usr/src/uts/common/io/nvme/nvme.c
5947
drive->d_qcount = nvme->n_ioq_count;
usr/src/uts/common/io/nvme/nvme.c
5968
drive->d_qsize = nvme->n_io_squeue_len / ns_count;
usr/src/uts/common/io/nvme/nvme.c
5987
drive->d_model = nvme->n_idctl->id_model;
usr/src/uts/common/io/nvme/nvme.c
5988
drive->d_model_len = sizeof (nvme->n_idctl->id_model);
usr/src/uts/common/io/nvme/nvme.c
5989
drive->d_vendor = nvme->n_vendor;
usr/src/uts/common/io/nvme/nvme.c
5990
drive->d_vendor_len = strlen(nvme->n_vendor);
usr/src/uts/common/io/nvme/nvme.c
5991
drive->d_product = nvme->n_product;
usr/src/uts/common/io/nvme/nvme.c
5992
drive->d_product_len = strlen(nvme->n_product);
usr/src/uts/common/io/nvme/nvme.c
5993
drive->d_serial = nvme->n_idctl->id_serial;
usr/src/uts/common/io/nvme/nvme.c
5994
drive->d_serial_len = sizeof (nvme->n_idctl->id_serial);
usr/src/uts/common/io/nvme/nvme.c
5995
drive->d_revision = nvme->n_idctl->id_fwrev;
usr/src/uts/common/io/nvme/nvme.c
5996
drive->d_revision_len = sizeof (nvme->n_idctl->id_fwrev);
usr/src/uts/common/io/nvme/nvme.c
6003
if (nvme->n_idctl->id_oncs.on_dset_mgmt)
usr/src/uts/common/io/nvme/nvme.c
6006
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
6013
nvme_t *nvme = ns->ns_nvme;
usr/src/uts/common/io/nvme/nvme.c
6015
if (nvme->n_dead) {
usr/src/uts/common/io/nvme/nvme.c
6019
nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_BDRO);
usr/src/uts/common/io/nvme/nvme.c
6028
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
6036
nvme_t *nvme = ns->ns_nvme;
usr/src/uts/common/io/nvme/nvme.c
6042
if (nvme->n_dead) {
usr/src/uts/common/io/nvme/nvme.c
6051
ASSERT(cmd->nc_sqid <= nvme->n_ioq_count);
usr/src/uts/common/io/nvme/nvme.c
6052
ioq = nvme->n_ioq[cmd->nc_sqid];
usr/src/uts/common/io/nvme/nvme.c
6070
cmd = nvme_retrieve_cmd(nvme, ioq);
usr/src/uts/common/io/nvme/nvme.c
6127
nvme_t *nvme = ns->ns_nvme;
usr/src/uts/common/io/nvme/nvme.c
6129
if (nvme->n_dead) {
usr/src/uts/common/io/nvme/nvme.c
6166
nvme_t *nvme;
usr/src/uts/common/io/nvme/nvme.c
6178
nvme = ddi_get_soft_state(nvme_state, NVME_MINOR_INST(m));
usr/src/uts/common/io/nvme/nvme.c
6181
if (nvme == NULL)
usr/src/uts/common/io/nvme/nvme.c
6184
if (nsid > MIN(nvme->n_namespace_count, NVME_MINOR_MAX))
usr/src/uts/common/io/nvme/nvme.c
6187
if (nvme->n_dead)
usr/src/uts/common/io/nvme/nvme.c
6202
minor->nm_ctrl_lock.nli_nvme = nvme;
usr/src/uts/common/io/nvme/nvme.c
6205
minor->nm_ns_lock.nli_nvme = nvme;
usr/src/uts/common/io/nvme/nvme.c
6213
minor->nm_ctrl = nvme;
usr/src/uts/common/io/nvme/nvme.c
6215
minor->nm_ns = nvme_nsid2ns(nvme, nsid);
usr/src/uts/common/io/nvme/nvme.c
6232
mutex_enter(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
6247
mutex_exit(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
6257
mutex_exit(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
6268
nvme_t *nvme;
usr/src/uts/common/io/nvme/nvme.c
6287
nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
6288
mutex_enter(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
6302
mutex_exit(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
6357
nvme_t *const nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
6371
ns = nvme_nsid2ns(nvme, ioc->nioc_nsid);
usr/src/uts/common/io/nvme/nvme.c
6374
mutex_enter(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
6375
ctrl_is_excl = nvme->n_lock.nl_writer != NULL;
usr/src/uts/common/io/nvme/nvme.c
6376
have_ctrl = nvme->n_lock.nl_writer == &minor->nm_ctrl_lock;
usr/src/uts/common/io/nvme/nvme.c
6396
mutex_exit(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
6537
nvme_t *const nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
6570
if (!nvme_identify(nvme, B_TRUE, &id, &idbuf)) {
usr/src/uts/common/io/nvme/nvme.c
6580
bcopy(nvme->n_idcomns, &info->nci_common_ns,
usr/src/uts/common/io/nvme/nvme.c
6583
info->nci_vers = nvme->n_version;
usr/src/uts/common/io/nvme/nvme.c
6590
cap.r = nvme_get64(nvme, NVME_REG_CAP);
usr/src/uts/common/io/nvme/nvme.c
6594
info->nci_nintrs = (uint32_t)nvme->n_intr_cnt;
usr/src/uts/common/io/nvme/nvme.c
6610
nvme_t *const nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
6637
ns = nvme_nsid2ns(nvme, ns_info->nni_common.nioc_nsid);
usr/src/uts/common/io/nvme/nvme.c
6645
if (!nvme_identify(nvme, B_TRUE, &id, &idbuf)) {
usr/src/uts/common/io/nvme/nvme.c
6652
nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME);
usr/src/uts/common/io/nvme/nvme.c
6661
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
6667
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
6684
nvme_t *const nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
6726
if (!nvme_validate_identify(nvme, &id, ns_minor)) {
usr/src/uts/common/io/nvme/nvme.c
6730
if (nvme_identify(nvme, B_TRUE, &id, &idctl)) {
usr/src/uts/common/io/nvme/nvme.c
6775
nvme_ioc_cmd(nvme_t *nvme, nvme_ioctl_common_t *ioc, nvme_ioc_cmd_args_t *args)
usr/src/uts/common/io/nvme/nvme.c
6780
cmd = nvme_alloc_admin_cmd(nvme, KM_SLEEP);
usr/src/uts/common/io/nvme/nvme.c
6802
if (nvme_zalloc_dma(nvme, args->ica_data_len,
usr/src/uts/common/io/nvme/nvme.c
6803
args->ica_dma_flags, &nvme->n_prp_dma_attr, &cmd->nc_dma) !=
usr/src/uts/common/io/nvme/nvme.c
6805
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme.c
6856
nvme_t *const nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
6907
if (!nvme_validate_logpage(nvme, &log)) {
usr/src/uts/common/io/nvme/nvme.c
6911
if (nvme_get_logpage(nvme, B_TRUE, &log, &buf)) {
usr/src/uts/common/io/nvme/nvme.c
6962
nvme_t *const nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
7011
if (!nvme_validate_get_feature(nvme, &feat)) {
usr/src/uts/common/io/nvme/nvme.c
7030
if (!nvme_ioc_cmd(nvme, &feat.nigf_common, &args)) {
usr/src/uts/common/io/nvme/nvme.c
7071
nvme_t *const nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
7088
if (!nvme_validate_format(nvme, &ioc)) {
usr/src/uts/common/io/nvme/nvme.c
7098
nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME);
usr/src/uts/common/io/nvme/nvme.c
7100
if (!nvme_no_blkdev_attached(nvme, ioc.nif_common.nioc_nsid)) {
usr/src/uts/common/io/nvme/nvme.c
7101
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
7107
nvme_namespace_t *ns = nvme_nsid2ns(nvme,
usr/src/uts/common/io/nvme/nvme.c
7112
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
7117
if (nvme_format_nvm(nvme, &ioc)) {
usr/src/uts/common/io/nvme/nvme.c
7119
nvme_rescan_ns(nvme, ioc.nif_common.nioc_nsid);
usr/src/uts/common/io/nvme/nvme.c
7121
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
7136
nvme_t *const nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
7154
nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME);
usr/src/uts/common/io/nvme/nvme.c
7155
if (nvme_bd_detach_ns(nvme, &com)) {
usr/src/uts/common/io/nvme/nvme.c
7158
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
7173
nvme_t *const nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
7192
nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME);
usr/src/uts/common/io/nvme/nvme.c
7193
ns = nvme_nsid2ns(nvme, com.nioc_nsid);
usr/src/uts/common/io/nvme/nvme.c
7203
nvme_rescan_ns(nvme, com.nioc_nsid);
usr/src/uts/common/io/nvme/nvme.c
7206
if (nvme_bd_attach_ns(nvme, &com)) {
usr/src/uts/common/io/nvme/nvme.c
7209
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
7228
nvme_ctrl_attach_detach_ns(nvme_t *nvme, nvme_namespace_t *ns,
usr/src/uts/common/io/nvme/nvme.c
7236
ASSERT(nvme_mgmt_lock_held(nvme));
usr/src/uts/common/io/nvme/nvme.c
7253
ctrlids[1] = nvme->n_idctl->id_cntlid;
usr/src/uts/common/io/nvme/nvme.c
7262
return (nvme_ioc_cmd(nvme, ioc, &args));
usr/src/uts/common/io/nvme/nvme.c
7269
nvme_t *const nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
7288
if (!nvme_validate_ctrl_attach_detach_ns(nvme, &com)) {
usr/src/uts/common/io/nvme/nvme.c
7292
nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME);
usr/src/uts/common/io/nvme/nvme.c
7293
ns = nvme_nsid2ns(nvme, com.nioc_nsid);
usr/src/uts/common/io/nvme/nvme.c
7296
if (nvme_ctrl_attach_detach_ns(nvme, ns, &com, B_FALSE)) {
usr/src/uts/common/io/nvme/nvme.c
7297
nvme_rescan_ns(nvme, com.nioc_nsid);
usr/src/uts/common/io/nvme/nvme.c
7301
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
7316
nvme_t *const nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
7335
if (!nvme_validate_ns_create(nvme, &create)) {
usr/src/uts/common/io/nvme/nvme.c
7386
nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME);
usr/src/uts/common/io/nvme/nvme.c
7387
if (nvme_ioc_cmd(nvme, &create.nnc_common, &args)) {
usr/src/uts/common/io/nvme/nvme.c
7389
nvme_rescan_ns(nvme, create.nnc_nsid);
usr/src/uts/common/io/nvme/nvme.c
7392
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
7409
nvme_t *const nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
7427
if (!nvme_validate_ns_delete(nvme, &com)) {
usr/src/uts/common/io/nvme/nvme.c
7431
nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME);
usr/src/uts/common/io/nvme/nvme.c
7433
if (!nvme_no_blkdev_attached(nvme, com.nioc_nsid)) {
usr/src/uts/common/io/nvme/nvme.c
7434
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
7440
nvme_namespace_t *ns = nvme_nsid2ns(nvme, com.nioc_nsid);
usr/src/uts/common/io/nvme/nvme.c
7443
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
7467
if (nvme_ioc_cmd(nvme, &com, &args)) {
usr/src/uts/common/io/nvme/nvme.c
7468
nvme_rescan_ns(nvme, com.nioc_nsid);
usr/src/uts/common/io/nvme/nvme.c
7471
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
7486
nvme_t *const nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
7505
if (!nvme_validate_ctrl_attach_detach_ns(nvme, &com)) {
usr/src/uts/common/io/nvme/nvme.c
7509
nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME);
usr/src/uts/common/io/nvme/nvme.c
7510
ns = nvme_nsid2ns(nvme, com.nioc_nsid);
usr/src/uts/common/io/nvme/nvme.c
7513
if (nvme_ctrl_attach_detach_ns(nvme, ns, &com, B_TRUE)) {
usr/src/uts/common/io/nvme/nvme.c
7514
nvme_rescan_ns(nvme, com.nioc_nsid);
usr/src/uts/common/io/nvme/nvme.c
7518
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
7530
nvme_ufm_update(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme.c
7532
mutex_enter(&nvme->n_fwslot_mutex);
usr/src/uts/common/io/nvme/nvme.c
7533
ddi_ufm_update(nvme->n_ufmh);
usr/src/uts/common/io/nvme/nvme.c
7534
if (nvme->n_fwslot != NULL) {
usr/src/uts/common/io/nvme/nvme.c
7535
kmem_free(nvme->n_fwslot, sizeof (nvme_fwslot_log_t));
usr/src/uts/common/io/nvme/nvme.c
7536
nvme->n_fwslot = NULL;
usr/src/uts/common/io/nvme/nvme.c
7538
mutex_exit(&nvme->n_fwslot_mutex);
usr/src/uts/common/io/nvme/nvme.c
7552
nvme_t *const nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
7578
if (!nvme_validate_fw_load(nvme, &fw)) {
usr/src/uts/common/io/nvme/nvme.c
7599
data.vcd_vers = &nvme->n_version;
usr/src/uts/common/io/nvme/nvme.c
7600
data.vcd_id = nvme->n_idctl;
usr/src/uts/common/io/nvme/nvme.c
7603
if ((nvme->n_max_data_transfer_size % gran) == 0) {
usr/src/uts/common/io/nvme/nvme.c
7604
maxcopy = nvme->n_max_data_transfer_size;
usr/src/uts/common/io/nvme/nvme.c
7605
} else if (gran <= nvme->n_max_data_transfer_size) {
usr/src/uts/common/io/nvme/nvme.c
7627
if (!nvme_ioc_cmd(nvme, &fw.fwl_common, &args)) {
usr/src/uts/common/io/nvme/nvme.c
7649
nvme_t *const nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
7672
if (!nvme_validate_fw_commit(nvme, &fw)) {
usr/src/uts/common/io/nvme/nvme.c
7688
(void) nvme_ioc_cmd(nvme, &fw.fwc_common, &args);
usr/src/uts/common/io/nvme/nvme.c
7695
nvme_ufm_update(nvme);
usr/src/uts/common/io/nvme/nvme.c
7799
nvme_t *const nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
7823
if (!nvme_validate_vuc(nvme, &pass)) {
usr/src/uts/common/io/nvme/nvme.c
7827
nvme_mgmt_lock(nvme, NVME_MGMT_LOCK_NVME);
usr/src/uts/common/io/nvme/nvme.c
7834
if (!nvme_no_blkdev_attached(nvme, NVME_NSID_BCAST)) {
usr/src/uts/common/io/nvme/nvme.c
7835
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
7863
if (nvme_ioc_cmd(nvme, &pass.npc_common, &args)) {
usr/src/uts/common/io/nvme/nvme.c
7866
nvme_rescan_ns(nvme, NVME_NSID_BCAST);
usr/src/uts/common/io/nvme/nvme.c
7869
nvme_mgmt_unlock(nvme);
usr/src/uts/common/io/nvme/nvme.c
7884
nvme_t *nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
7957
mutex_enter(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
7962
mutex_exit(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
7973
mutex_exit(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
7981
mutex_exit(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
7990
mutex_exit(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
7998
mutex_exit(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
8041
mutex_exit(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
8057
nvme_t *const nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
8098
mutex_enter(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
8101
mutex_exit(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
8108
mutex_exit(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
8122
mutex_exit(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
8130
mutex_exit(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
8142
lock = &nvme->n_lock;
usr/src/uts/common/io/nvme/nvme.c
8148
ns = nvme_nsid2ns(nvme, nsid);
usr/src/uts/common/io/nvme/nvme.c
8154
mutex_exit(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme.c
8175
nvme_t *nvme;
usr/src/uts/common/io/nvme/nvme.c
8182
nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme.c
8183
if (nvme == NULL)
usr/src/uts/common/io/nvme/nvme.c
8187
return (ndi_devctl_ioctl(nvme->n_dip, cmd, arg, mode, 0));
usr/src/uts/common/io/nvme/nvme.c
8189
if (nvme->n_dead && (cmd != NVME_IOC_BD_DETACH && cmd !=
usr/src/uts/common/io/nvme/nvme.c
8195
return (nvme_ioctl_copyout_error(nvme->n_dead_status, arg,
usr/src/uts/common/io/nvme/nvme.c
8259
ASSERT(!nvme_mgmt_lock_held(nvme));
usr/src/uts/common/io/nvme/nvme.c
8270
nvme_t *nvme = arg;
usr/src/uts/common/io/nvme/nvme.c
8276
ddi_ufm_image_set_nslots(img, nvme->n_idctl->id_frmw.fw_nslot);
usr/src/uts/common/io/nvme/nvme.c
8295
nvme_t *nvme = arg;
usr/src/uts/common/io/nvme/nvme.c
8301
if (imgno > 0 || slotno > (nvme->n_idctl->id_frmw.fw_nslot - 1))
usr/src/uts/common/io/nvme/nvme.c
8304
mutex_enter(&nvme->n_fwslot_mutex);
usr/src/uts/common/io/nvme/nvme.c
8305
if (nvme->n_fwslot == NULL) {
usr/src/uts/common/io/nvme/nvme.c
8306
if (!nvme_get_logpage_int(nvme, B_TRUE, &log, &bufsize,
usr/src/uts/common/io/nvme/nvme.c
8311
mutex_exit(&nvme->n_fwslot_mutex);
usr/src/uts/common/io/nvme/nvme.c
8314
nvme->n_fwslot = (nvme_fwslot_log_t *)log;
usr/src/uts/common/io/nvme/nvme.c
8320
if (slotno == (nvme->n_fwslot->fw_afi - 1))
usr/src/uts/common/io/nvme/nvme.c
8323
if (slotno != 0 || nvme->n_idctl->id_frmw.fw_readonly == 0)
usr/src/uts/common/io/nvme/nvme.c
8326
if (nvme->n_fwslot->fw_frs[slotno][0] == '\0') {
usr/src/uts/common/io/nvme/nvme.c
8329
(void) strncpy(fw_ver, nvme->n_fwslot->fw_frs[slotno],
usr/src/uts/common/io/nvme/nvme.c
8334
mutex_exit(&nvme->n_fwslot_mutex);
usr/src/uts/common/io/nvme/nvme.c
8349
nvme_ctrl_atleast(nvme_t *nvme, const nvme_version_t *min)
usr/src/uts/common/io/nvme/nvme.c
8351
return (nvme_vers_atleast(&nvme->n_version, min) ? B_TRUE : B_FALSE);
usr/src/uts/common/io/nvme/nvme.c
842
#define NVME_BUMP_STAT(nvme, stat) \
usr/src/uts/common/io/nvme/nvme.c
843
atomic_inc_64(&nvme->n_device_stat.nds_ ## stat.value.ui64)
usr/src/uts/common/io/nvme/nvme_lock.c
102
for (uint32_t i = 1; i <= nvme->n_namespace_count; i++) {
usr/src/uts/common/io/nvme/nvme_lock.c
103
nvme_namespace_t *ns = nvme_nsid2ns(nvme, i);
usr/src/uts/common/io/nvme/nvme_lock.c
121
nvme_rwlock_handoff_ctrl_wrlock(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme_lock.c
124
ASSERT3P(nvme->n_lock.nl_writer, ==, NULL);
usr/src/uts/common/io/nvme/nvme_lock.c
126
if (list_is_empty(&nvme->n_lock.nl_readers) == 0) {
usr/src/uts/common/io/nvme/nvme_lock.c
130
for (uint32_t i = 1; i <= nvme->n_namespace_count; i++) {
usr/src/uts/common/io/nvme/nvme_lock.c
131
nvme_namespace_t *ns = nvme_nsid2ns(nvme, i);
usr/src/uts/common/io/nvme/nvme_lock.c
147
nvme_rwlock_handoff_ns_wrlock(nvme_t *nvme, nvme_namespace_t *ns)
usr/src/uts/common/io/nvme/nvme_lock.c
149
if (nvme_rwlock_wr_or_pend(&nvme->n_lock) ||
usr/src/uts/common/io/nvme/nvme_lock.c
150
list_is_empty(&nvme->n_lock.nl_readers) == 0) {
usr/src/uts/common/io/nvme/nvme_lock.c
275
nvme_rwlock_wakeup(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme_lock.c
277
nvme_lock_t *ctrl_lock = &nvme->n_lock;
usr/src/uts/common/io/nvme/nvme_lock.c
298
if (!nvme_rwlock_handoff_ctrl_wrlock(nvme))
usr/src/uts/common/io/nvme/nvme_lock.c
313
for (uint32_t i = 1; i <= nvme->n_namespace_count; i++) {
usr/src/uts/common/io/nvme/nvme_lock.c
314
nvme_namespace_t *ns = nvme_nsid2ns(nvme, i);
usr/src/uts/common/io/nvme/nvme_lock.c
320
if (!nvme_rwlock_handoff_ns_wrlock(nvme, ns))
usr/src/uts/common/io/nvme/nvme_lock.c
355
nvme_t *const nvme = info->nli_nvme;
usr/src/uts/common/io/nvme/nvme_lock.c
358
VERIFY(MUTEX_HELD(&nvme->n_minor_mutex));
usr/src/uts/common/io/nvme/nvme_lock.c
380
nvme_rwlock_wakeup(nvme);
usr/src/uts/common/io/nvme/nvme_lock.c
46
nvme_rwlock_block_ns_rdlock(nvme_t *nvme, nvme_namespace_t *ns)
usr/src/uts/common/io/nvme/nvme_lock.c
463
nvme_t *const nvme = minor->nm_ctrl;
usr/src/uts/common/io/nvme/nvme_lock.c
473
VERIFY(MUTEX_HELD(&nvme->n_minor_mutex));
usr/src/uts/common/io/nvme/nvme_lock.c
477
lock = &nvme->n_lock;
usr/src/uts/common/io/nvme/nvme_lock.c
48
return (nvme_rwlock_wr_or_pend(&nvme->n_lock) ||
usr/src/uts/common/io/nvme/nvme_lock.c
480
waiters = nvme_rwlock_block_ctrl_rdlock(nvme);
usr/src/uts/common/io/nvme/nvme_lock.c
482
waiters = nvme_rwlock_block_ctrl_wrlock(nvme);
usr/src/uts/common/io/nvme/nvme_lock.c
490
ns = nvme_nsid2ns(nvme, nsid);
usr/src/uts/common/io/nvme/nvme_lock.c
495
waiters = nvme_rwlock_block_ns_rdlock(nvme, ns);
usr/src/uts/common/io/nvme/nvme_lock.c
497
waiters = nvme_rwlock_block_ns_wrlock(nvme, ns);
usr/src/uts/common/io/nvme/nvme_lock.c
557
if (cv_wait_sig(&minor->nm_cv, &nvme->n_minor_mutex) == 0) {
usr/src/uts/common/io/nvme/nvme_lock.c
597
nvme_rwlock_ctrl_dead_cleanup_one(nvme_t *nvme, nvme_minor_lock_info_t *info)
usr/src/uts/common/io/nvme/nvme_lock.c
607
nvme_rwlock_signal_one(info, nvme->n_dead_status);
usr/src/uts/common/io/nvme/nvme_lock.c
63
nvme_rwlock_block_ns_wrlock(nvme_t *nvme, nvme_namespace_t *ns)
usr/src/uts/common/io/nvme/nvme_lock.c
630
nvme_t *nvme = arg;
usr/src/uts/common/io/nvme/nvme_lock.c
631
nvme_lock_t *ctrl_lock = &nvme->n_lock;
usr/src/uts/common/io/nvme/nvme_lock.c
634
mutex_enter(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme_lock.c
635
if ((nvme->n_progress & NVME_NS_INIT) == 0) {
usr/src/uts/common/io/nvme/nvme_lock.c
636
mutex_exit(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme_lock.c
640
for (uint32_t i = 1; i <= nvme->n_namespace_count; i++) {
usr/src/uts/common/io/nvme/nvme_lock.c
641
nvme_namespace_t *ns = nvme_nsid2ns(nvme, i);
usr/src/uts/common/io/nvme/nvme_lock.c
646
nvme_rwlock_ctrl_dead_cleanup_one(nvme, info);
usr/src/uts/common/io/nvme/nvme_lock.c
65
return (nvme_rwlock_wr_or_pend(&nvme->n_lock) ||
usr/src/uts/common/io/nvme/nvme_lock.c
651
nvme_rwlock_ctrl_dead_cleanup_one(nvme, info);
usr/src/uts/common/io/nvme/nvme_lock.c
656
nvme_rwlock_ctrl_dead_cleanup_one(nvme, info);
usr/src/uts/common/io/nvme/nvme_lock.c
661
nvme_rwlock_ctrl_dead_cleanup_one(nvme, info);
usr/src/uts/common/io/nvme/nvme_lock.c
663
mutex_exit(&nvme->n_minor_mutex);
usr/src/uts/common/io/nvme/nvme_lock.c
76
nvme_rwlock_block_ctrl_rdlock(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme_lock.c
78
return (nvme_rwlock_wr_or_pend(&nvme->n_lock));
usr/src/uts/common/io/nvme/nvme_lock.c
95
nvme_rwlock_block_ctrl_wrlock(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme_lock.c
97
if (nvme_rwlock_wr_or_pend(&nvme->n_lock) ||
usr/src/uts/common/io/nvme/nvme_lock.c
98
list_is_empty(&nvme->n_lock.nl_readers) == 0) {
usr/src/uts/common/io/nvme/nvme_reg.h
57
#define NVME_REG_SQTDBL(nvme, n) \
usr/src/uts/common/io/nvme/nvme_reg.h
58
(0x1000 + ((2 * (n)) * nvme->n_doorbell_stride))
usr/src/uts/common/io/nvme/nvme_reg.h
59
#define NVME_REG_CQHDBL(nvme, n) \
usr/src/uts/common/io/nvme/nvme_reg.h
60
(0x1000 + ((2 * (n) + 1) * nvme->n_doorbell_stride))
usr/src/uts/common/io/nvme/nvme_stat.c
114
kstat_install(nvme->n_device_kstat);
usr/src/uts/common/io/nvme/nvme_stat.c
119
nvme_stat_admin_cleanup(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme_stat.c
121
if (nvme->n_admin_kstat != NULL) {
usr/src/uts/common/io/nvme/nvme_stat.c
122
kstat_delete(nvme->n_admin_kstat);
usr/src/uts/common/io/nvme/nvme_stat.c
123
nvme->n_admin_kstat = NULL;
usr/src/uts/common/io/nvme/nvme_stat.c
124
mutex_destroy(&nvme->n_admin_stat_mutex);
usr/src/uts/common/io/nvme/nvme_stat.c
129
nvme_stat_admin_init(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme_stat.c
132
ddi_get_instance(nvme->n_dip), "admin", "controller",
usr/src/uts/common/io/nvme/nvme_stat.c
136
nvme_admin_stat_t *nas = &nvme->n_admin_stat;
usr/src/uts/common/io/nvme/nvme_stat.c
139
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme_stat.c
144
nvme->n_admin_kstat = ksp;
usr/src/uts/common/io/nvme/nvme_stat.c
146
mutex_init(&nvme->n_admin_stat_mutex, NULL, MUTEX_DRIVER, NULL);
usr/src/uts/common/io/nvme/nvme_stat.c
171
kstat_install(nvme->n_admin_kstat);
usr/src/uts/common/io/nvme/nvme_stat.c
176
nvme_stat_cleanup(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme_stat.c
178
nvme_stat_device_cleanup(nvme);
usr/src/uts/common/io/nvme/nvme_stat.c
179
nvme_stat_admin_cleanup(nvme);
usr/src/uts/common/io/nvme/nvme_stat.c
183
nvme_stat_init(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme_stat.c
185
if (!nvme_stat_device_init(nvme) || !nvme_stat_admin_init(nvme)) {
usr/src/uts/common/io/nvme/nvme_stat.c
186
nvme_stat_cleanup(nvme);
usr/src/uts/common/io/nvme/nvme_stat.c
193
nvme_admin_stat_cmd(nvme_t *nvme, nvme_cmd_t *cmd)
usr/src/uts/common/io/nvme/nvme_stat.c
20
nvme_stat_device_cleanup(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme_stat.c
207
data = nvme->n_admin_stat.nas_getlogpage;
usr/src/uts/common/io/nvme/nvme_stat.c
210
data = nvme->n_admin_stat.nas_identify;
usr/src/uts/common/io/nvme/nvme_stat.c
213
data = nvme->n_admin_stat.nas_abort;
usr/src/uts/common/io/nvme/nvme_stat.c
216
data = nvme->n_admin_stat.nas_fwactivate;
usr/src/uts/common/io/nvme/nvme_stat.c
219
data = nvme->n_admin_stat.nas_fwimgload;
usr/src/uts/common/io/nvme/nvme_stat.c
22
if (nvme->n_device_kstat != NULL) {
usr/src/uts/common/io/nvme/nvme_stat.c
222
data = nvme->n_admin_stat.nas_nsformat;
usr/src/uts/common/io/nvme/nvme_stat.c
225
data = nvme->n_admin_stat.nas_vendor;
usr/src/uts/common/io/nvme/nvme_stat.c
228
data = nvme->n_admin_stat.nas_other;
usr/src/uts/common/io/nvme/nvme_stat.c
23
kstat_delete(nvme->n_device_kstat);
usr/src/uts/common/io/nvme/nvme_stat.c
238
mutex_enter(&nvme->n_admin_stat_mutex);
usr/src/uts/common/io/nvme/nvme_stat.c
24
nvme->n_device_kstat = NULL;
usr/src/uts/common/io/nvme/nvme_stat.c
256
mutex_exit(&nvme->n_admin_stat_mutex);
usr/src/uts/common/io/nvme/nvme_stat.c
29
nvme_stat_device_init(nvme_t *nvme)
usr/src/uts/common/io/nvme/nvme_stat.c
32
ddi_get_instance(nvme->n_dip), "device", "controller",
usr/src/uts/common/io/nvme/nvme_stat.c
36
nvme_device_stat_t *nds = &nvme->n_device_stat;
usr/src/uts/common/io/nvme/nvme_stat.c
39
dev_err(nvme->n_dip, CE_WARN,
usr/src/uts/common/io/nvme/nvme_stat.c
44
nvme->n_device_kstat = ksp;
usr/src/uts/common/io/nvme/nvme_validate.c
1028
nvme_validate_ns_create(nvme_t *nvme, nvme_ioctl_ns_create_t *ioc)
usr/src/uts/common/io/nvme/nvme_validate.c
1030
const nvme_identify_nsid_t *idns = nvme->n_idcomns;
usr/src/uts/common/io/nvme/nvme_validate.c
1033
ctrl_data.vcd_vers = &nvme->n_version;
usr/src/uts/common/io/nvme/nvme_validate.c
1034
ctrl_data.vcd_id = nvme->n_idctl;
usr/src/uts/common/io/nvme/nvme_validate.c
373
nvme_validate_logpage(nvme_t *nvme, nvme_ioctl_get_logpage_t *log)
usr/src/uts/common/io/nvme/nvme_validate.c
379
ctrl_data.vcd_vers = &nvme->n_version;
usr/src/uts/common/io/nvme/nvme_validate.c
380
ctrl_data.vcd_id = nvme->n_idctl;
usr/src/uts/common/io/nvme/nvme_validate.c
442
nvme_validate_get_feature(nvme_t *nvme, nvme_ioctl_get_feature_t *get)
usr/src/uts/common/io/nvme/nvme_validate.c
449
ctrl_data.vcd_vers = &nvme->n_version;
usr/src/uts/common/io/nvme/nvme_validate.c
450
ctrl_data.vcd_id = nvme->n_idctl;
usr/src/uts/common/io/nvme/nvme_validate.c
576
nvme_validate_identify(nvme_t *nvme, nvme_ioctl_identify_t *id,
usr/src/uts/common/io/nvme/nvme_validate.c
582
ctrl_data.vcd_vers = &nvme->n_version;
usr/src/uts/common/io/nvme/nvme_validate.c
583
ctrl_data.vcd_id = nvme->n_idctl;
usr/src/uts/common/io/nvme/nvme_validate.c
634
if (nvme_ctrl_atleast(nvme, &nvme_vers_1v2) &&
usr/src/uts/common/io/nvme/nvme_validate.c
635
nvme->n_idctl->id_oacs.oa_nsmgmt != 0) {
usr/src/uts/common/io/nvme/nvme_validate.c
673
if (ns == 0 || ns > nvme->n_namespace_count) {
usr/src/uts/common/io/nvme/nvme_validate.c
708
nvme_validate_vuc(nvme_t *nvme, nvme_ioctl_passthru_t *pass)
usr/src/uts/common/io/nvme/nvme_validate.c
714
ctrl_data.vcd_vers = &nvme->n_version;
usr/src/uts/common/io/nvme/nvme_validate.c
715
ctrl_data.vcd_id = nvme->n_idctl;
usr/src/uts/common/io/nvme/nvme_validate.c
720
if (nvme->n_idctl->id_nvscc.nv_spec == 0) {
usr/src/uts/common/io/nvme/nvme_validate.c
817
nvme_validate_format(nvme_t *nvme, nvme_ioctl_format_t *ioc)
usr/src/uts/common/io/nvme/nvme_validate.c
822
ctrl_data.vcd_vers = &nvme->n_version;
usr/src/uts/common/io/nvme/nvme_validate.c
823
ctrl_data.vcd_id = nvme->n_idctl;
usr/src/uts/common/io/nvme/nvme_validate.c
848
idns = nvme->n_idcomns;
usr/src/uts/common/io/nvme/nvme_validate.c
860
nvme->n_idctl->id_fna.fn_crypt_erase == 0) {
usr/src/uts/common/io/nvme/nvme_validate.c
873
if (nvme->n_idctl->id_fna.fn_format != 0) {
usr/src/uts/common/io/nvme/nvme_validate.c
879
nvme->n_idctl->id_fna.fn_sec_erase != 0) {
usr/src/uts/common/io/nvme/nvme_validate.c
898
nvme_validate_fw_load(nvme_t *nvme, nvme_ioctl_fw_load_t *fw)
usr/src/uts/common/io/nvme/nvme_validate.c
902
ctrl_data.vcd_vers = &nvme->n_version;
usr/src/uts/common/io/nvme/nvme_validate.c
903
ctrl_data.vcd_id = nvme->n_idctl;
usr/src/uts/common/io/nvme/nvme_validate.c
934
nvme_validate_fw_commit(nvme_t *nvme, nvme_ioctl_fw_commit_t *fw)
usr/src/uts/common/io/nvme/nvme_validate.c
938
ctrl_data.vcd_vers = &nvme->n_version;
usr/src/uts/common/io/nvme/nvme_validate.c
939
ctrl_data.vcd_id = nvme->n_idctl;
usr/src/uts/common/io/nvme/nvme_validate.c
961
if (fw->fwc_slot == 1 && nvme->n_idctl->id_frmw.fw_readonly &&
usr/src/uts/common/io/nvme/nvme_validate.c
976
nvme_validate_ctrl_attach_detach_ns(nvme_t *nvme, nvme_ioctl_common_t *com)
usr/src/uts/common/io/nvme/nvme_validate.c
980
ctrl_data.vcd_vers = &nvme->n_version;
usr/src/uts/common/io/nvme/nvme_validate.c
981
ctrl_data.vcd_id = nvme->n_idctl;
usr/src/uts/common/io/nvme/nvme_validate.c
992
nvme_validate_ns_delete(nvme_t *nvme, nvme_ioctl_common_t *com)
usr/src/uts/common/io/nvme/nvme_validate.c
996
ctrl_data.vcd_vers = &nvme->n_version;
usr/src/uts/common/io/nvme/nvme_validate.c
997
ctrl_data.vcd_id = nvme->n_idctl;
usr/src/uts/common/io/nvme/nvme_var.h
87
typedef struct nvme nvme_t;
usr/src/uts/common/sys/fibre-channel/fca/emlxs/emlxs_mbox.h
3936
uint32_t nvme :1;
usr/src/uts/common/sys/fibre-channel/fca/emlxs/emlxs_mbox.h
4073
uint32_t nvme:1;
