diff options
| author | Hao Wu <hao.a.wu@intel.com> | 2016-04-27 13:15:24 +0800 |
|---|---|---|
| committer | Hao Wu <hao.a.wu@intel.com> | 2016-05-19 12:37:59 +0800 |
| commit | 758ea94651545896309725b53407e57e79477f28 (patch) | |
| tree | 66d94d13496fffd24fa05ec0e0cfdc514dc88471 | |
| parent | b6c8ee68652e90b0ef9b563b00b5d0f3119956e7 (diff) | |
| download | edk2-platforms-758ea94651545896309725b53407e57e79477f28.tar.xz | |
MdeModulePkg NvmExpressDxe: Add BlockIo2 support
Together with EFI_BLOCK_IO_PROTOCOL, EFI_BLOCK_IO2_PROTOCOL is also
produced on NVMe devices.
The following Block I/O 2 functions are implemented:
Reset
ReadBlocksEx
WriteBlocksEx
FlushBlocksEx
Contributed-under: TianoCore Contribution Agreement 1.0
Signed-off-by: Hao Wu <hao.a.wu@intel.com>
Reviewed-by: Feng Tian <feng.tian@intel.com>
| -rw-r--r-- | MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c | 265 | ||||
| -rw-r--r-- | MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.h | 113 | ||||
| -rw-r--r-- | MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressBlockIo.c | 976 | ||||
| -rw-r--r-- | MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressBlockIo.h | 150 | ||||
| -rw-r--r-- | MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressDxe.inf | 3 | ||||
| -rw-r--r-- | MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c | 155 | ||||
| -rw-r--r-- | MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c | 72 |
7 files changed, 1642 insertions, 92 deletions
diff --git a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c index cad061bdf7..cb25b3e08d 100644 --- a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c +++ b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.c @@ -158,6 +158,16 @@ EnumerateNvmeDevNamespace ( Device->BlockIo.FlushBlocks = NvmeBlockIoFlushBlocks;
//
+ // Create BlockIo2 Protocol instance
+ //
+ Device->BlockIo2.Media = &Device->Media;
+ Device->BlockIo2.Reset = NvmeBlockIoResetEx;
+ Device->BlockIo2.ReadBlocksEx = NvmeBlockIoReadBlocksEx;
+ Device->BlockIo2.WriteBlocksEx = NvmeBlockIoWriteBlocksEx;
+ Device->BlockIo2.FlushBlocksEx = NvmeBlockIoFlushBlocksEx;
+ InitializeListHead (&Device->AsyncQueue);
+
+ //
// Create StorageSecurityProtocol Instance
//
Device->StorageSecurity.ReceiveData = NvmeStorageSecurityReceiveData;
@@ -213,6 +223,8 @@ EnumerateNvmeDevNamespace ( Device->DevicePath,
&gEfiBlockIoProtocolGuid,
&Device->BlockIo,
+ &gEfiBlockIo2ProtocolGuid,
+ &Device->BlockIo2,
&gEfiDiskInfoProtocolGuid,
&Device->DiskInfo,
NULL
@@ -239,6 +251,8 @@ EnumerateNvmeDevNamespace ( Device->DevicePath,
&gEfiBlockIoProtocolGuid,
&Device->BlockIo,
+ &gEfiBlockIo2ProtocolGuid,
+ &Device->BlockIo2,
&gEfiDiskInfoProtocolGuid,
&Device->DiskInfo,
NULL
@@ -380,6 +394,8 @@ UnregisterNvmeNamespace ( NVME_DEVICE_PRIVATE_DATA *Device;
NVME_CONTROLLER_PRIVATE_DATA *Private;
EFI_STORAGE_SECURITY_COMMAND_PROTOCOL *StorageSecurity;
+ BOOLEAN IsEmpty;
+ EFI_TPL OldTpl;
BlockIo = NULL;
@@ -399,6 +415,21 @@ UnregisterNvmeNamespace ( Private = Device->Controller;
//
+ // Wait for the device's asynchronous I/O queue to become empty.
+ //
+ while (TRUE) {
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
+ IsEmpty = IsListEmpty (&Device->AsyncQueue);
+ gBS->RestoreTPL (OldTpl);
+
+ if (IsEmpty) {
+ break;
+ }
+
+ gBS->Stall (100);
+ }
+
+ //
// Close the child handle
//
gBS->CloseProtocol (
@@ -418,6 +449,8 @@ UnregisterNvmeNamespace ( Device->DevicePath,
&gEfiBlockIoProtocolGuid,
&Device->BlockIo,
+ &gEfiBlockIo2ProtocolGuid,
+ &Device->BlockIo2,
&gEfiDiskInfoProtocolGuid,
&Device->DiskInfo,
NULL
@@ -480,6 +513,170 @@ UnregisterNvmeNamespace ( }
/**
+ Call back function when the timer event is signaled.
+
+ @param[in] Event The Event this notify function registered to.
+ @param[in] Context Pointer to the context data registered to the
+ Event.
+
+**/
+VOID
+EFIAPI
+ProcessAsyncTaskList (
+ IN EFI_EVENT Event,
+ IN VOID* Context
+ )
+{
+ NVME_CONTROLLER_PRIVATE_DATA *Private;
+ EFI_PCI_IO_PROTOCOL *PciIo;
+ NVME_CQ *Cq;
+ UINT16 QueueId;
+ UINT32 Data;
+ LIST_ENTRY *Link;
+ LIST_ENTRY *NextLink;
+ NVME_PASS_THRU_ASYNC_REQ *AsyncRequest;
+ NVME_BLKIO2_SUBTASK *Subtask;
+ NVME_BLKIO2_REQUEST *BlkIo2Request;
+ EFI_BLOCK_IO2_TOKEN *Token;
+ BOOLEAN HasNewItem;
+ EFI_STATUS Status;
+
+ Private = (NVME_CONTROLLER_PRIVATE_DATA*)Context;
+ QueueId = 2;
+ Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
+ HasNewItem = FALSE;
+
+ //
+ // Submit asynchronous subtasks to the NVMe Submission Queue
+ //
+ for (Link = GetFirstNode (&Private->UnsubmittedSubtasks);
+ !IsNull (&Private->UnsubmittedSubtasks, Link);
+ Link = NextLink) {
+ NextLink = GetNextNode (&Private->UnsubmittedSubtasks, Link);
+ Subtask = NVME_BLKIO2_SUBTASK_FROM_LINK (Link);
+ BlkIo2Request = Subtask->BlockIo2Request;
+ Token = BlkIo2Request->Token;
+ RemoveEntryList (Link);
+ BlkIo2Request->UnsubmittedSubtaskNum--;
+
+ //
+ // If any previous subtask fails, do not process subsequent ones.
+ //
+ if (Token->TransactionStatus != EFI_SUCCESS) {
+ if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&
+ BlkIo2Request->LastSubtaskSubmitted &&
+ (BlkIo2Request->UnsubmittedSubtaskNum == 0)) {
+ //
+ // Remove the BlockIo2 request from the device asynchronous queue.
+ //
+ RemoveEntryList (&BlkIo2Request->Link);
+ FreePool (BlkIo2Request);
+ gBS->SignalEvent (Token->Event);
+ }
+
+ FreePool (Subtask->CommandPacket->NvmeCmd);
+ FreePool (Subtask->CommandPacket->NvmeCompletion);
+ FreePool (Subtask->CommandPacket);
+ FreePool (Subtask);
+
+ continue;
+ }
+
+ Status = Private->Passthru.PassThru (
+ &Private->Passthru,
+ Subtask->NamespaceId,
+ Subtask->CommandPacket,
+ Subtask->Event
+ );
+ if (Status == EFI_NOT_READY) {
+ InsertHeadList (&Private->UnsubmittedSubtasks, Link);
+ BlkIo2Request->UnsubmittedSubtaskNum++;
+ break;
+ } else if (EFI_ERROR (Status)) {
+ Token->TransactionStatus = EFI_DEVICE_ERROR;
+
+ if (IsListEmpty (&BlkIo2Request->SubtasksQueue) &&
+ Subtask->IsLast) {
+ //
+ // Remove the BlockIo2 request from the device asynchronous queue.
+ //
+ RemoveEntryList (&BlkIo2Request->Link);
+ FreePool (BlkIo2Request);
+ gBS->SignalEvent (Token->Event);
+ }
+
+ FreePool (Subtask->CommandPacket->NvmeCmd);
+ FreePool (Subtask->CommandPacket->NvmeCompletion);
+ FreePool (Subtask->CommandPacket);
+ FreePool (Subtask);
+ } else {
+ InsertTailList (&BlkIo2Request->SubtasksQueue, Link);
+ if (Subtask->IsLast) {
+ BlkIo2Request->LastSubtaskSubmitted = TRUE;
+ }
+ }
+ }
+
+ while (Cq->Pt != Private->Pt[QueueId]) {
+ ASSERT (Cq->Sqid == QueueId);
+
+ HasNewItem = TRUE;
+
+ //
+ // Find the command with given Command Id.
+ //
+ for (Link = GetFirstNode (&Private->AsyncPassThruQueue);
+ !IsNull (&Private->AsyncPassThruQueue, Link);
+ Link = NextLink) {
+ NextLink = GetNextNode (&Private->AsyncPassThruQueue, Link);
+ AsyncRequest = NVME_PASS_THRU_ASYNC_REQ_FROM_THIS (Link);
+ if (AsyncRequest->CommandId == Cq->Cid) {
+ //
+ // Copy the Respose Queue entry for this command to the callers
+ // response buffer.
+ //
+ CopyMem (
+ AsyncRequest->Packet->NvmeCompletion,
+ Cq,
+ sizeof(EFI_NVM_EXPRESS_COMPLETION)
+ );
+
+ RemoveEntryList (Link);
+ gBS->SignalEvent (AsyncRequest->CallerEvent);
+ FreePool (AsyncRequest);
+
+ //
+ // Update submission queue head.
+ //
+ Private->AsyncSqHead = Cq->Sqhd;
+ break;
+ }
+ }
+
+ Private->CqHdbl[QueueId].Cqh++;
+ if (Private->CqHdbl[QueueId].Cqh > NVME_ASYNC_CCQ_SIZE) {
+ Private->CqHdbl[QueueId].Cqh = 0;
+ Private->Pt[QueueId] ^= 1;
+ }
+
+ Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
+ }
+
+ if (HasNewItem) {
+ PciIo = Private->PciIo;
+ Data = ReadUnaligned32 ((UINT32*)&Private->CqHdbl[QueueId]);
+ PciIo->Mem.Write (
+ PciIo,
+ EfiPciIoWidthUint32,
+ NVME_BAR,
+ NVME_CQHDBL_OFFSET(QueueId, Private->Cap.Dstrd),
+ 1,
+ &Data
+ );
+ }
+}
+
+/**
Tests to see if this driver supports a given controller. If a child device is provided,
it further tests to see if this driver supports creating a handle for the specified child device.
@@ -736,19 +933,21 @@ NvmExpressDriverBindingStart ( }
//
- // 4 x 4kB aligned buffers will be carved out of this buffer.
+ // 6 x 4kB aligned buffers will be carved out of this buffer.
// 1st 4kB boundary is the start of the admin submission queue.
// 2nd 4kB boundary is the start of the admin completion queue.
// 3rd 4kB boundary is the start of I/O submission queue #1.
// 4th 4kB boundary is the start of I/O completion queue #1.
+ // 5th 4kB boundary is the start of I/O submission queue #2.
+ // 6th 4kB boundary is the start of I/O completion queue #2.
//
- // Allocate 4 pages of memory, then map it for bus master read and write.
+ // Allocate 6 pages of memory, then map it for bus master read and write.
//
Status = PciIo->AllocateBuffer (
PciIo,
AllocateAnyPages,
EfiBootServicesData,
- 4,
+ 6,
(VOID**)&Private->Buffer,
0
);
@@ -756,7 +955,7 @@ NvmExpressDriverBindingStart ( goto Exit;
}
- Bytes = EFI_PAGES_TO_SIZE (4);
+ Bytes = EFI_PAGES_TO_SIZE (6);
Status = PciIo->Map (
PciIo,
EfiPciIoOperationBusMasterCommonBuffer,
@@ -766,7 +965,7 @@ NvmExpressDriverBindingStart ( &Private->Mapping
);
- if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (4))) {
+ if (EFI_ERROR (Status) || (Bytes != EFI_PAGES_TO_SIZE (6))) {
goto Exit;
}
@@ -784,12 +983,37 @@ NvmExpressDriverBindingStart ( Private->Passthru.BuildDevicePath = NvmExpressBuildDevicePath;
Private->Passthru.GetNamespace = NvmExpressGetNamespace;
CopyMem (&Private->PassThruMode, &gEfiNvmExpressPassThruMode, sizeof (EFI_NVM_EXPRESS_PASS_THRU_MODE));
+ InitializeListHead (&Private->AsyncPassThruQueue);
+ InitializeListHead (&Private->UnsubmittedSubtasks);
Status = NvmeControllerInit (Private);
if (EFI_ERROR(Status)) {
goto Exit;
}
+ //
+ // Start the asynchronous I/O completion monitor
+ //
+ Status = gBS->CreateEvent (
+ EVT_TIMER | EVT_NOTIFY_SIGNAL,
+ TPL_NOTIFY,
+ ProcessAsyncTaskList,
+ Private,
+ &Private->TimerEvent
+ );
+ if (EFI_ERROR (Status)) {
+ goto Exit;
+ }
+
+ Status = gBS->SetTimer (
+ Private->TimerEvent,
+ TimerPeriodic,
+ NVME_HC_ASYNC_TIMER
+ );
+ if (EFI_ERROR (Status)) {
+ goto Exit;
+ }
+
Status = gBS->InstallMultipleProtocolInterfaces (
&Controller,
&gEfiNvmExpressPassThruProtocolGuid,
@@ -850,7 +1074,7 @@ Exit: }
if ((Private != NULL) && (Private->Buffer != NULL)) {
- PciIo->FreeBuffer (PciIo, 4, Private->Buffer);
+ PciIo->FreeBuffer (PciIo, 6, Private->Buffer);
}
if ((Private != NULL) && (Private->ControllerData != NULL)) {
@@ -858,6 +1082,10 @@ Exit: }
if (Private != NULL) {
+ if (Private->TimerEvent != NULL) {
+ gBS->CloseEvent (Private->TimerEvent);
+ }
+
FreePool (Private);
}
@@ -921,6 +1149,8 @@ NvmExpressDriverBindingStop ( UINTN Index;
NVME_CONTROLLER_PRIVATE_DATA *Private;
EFI_NVM_EXPRESS_PASS_THRU_PROTOCOL *PassThru;
+ BOOLEAN IsEmpty;
+ EFI_TPL OldTpl;
if (NumberOfChildren == 0) {
Status = gBS->OpenProtocol (
@@ -934,6 +1164,23 @@ NvmExpressDriverBindingStop ( if (!EFI_ERROR (Status)) {
Private = NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU (PassThru);
+
+ //
+ // Wait for the asynchronous PassThru queue to become empty.
+ //
+ while (TRUE) {
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
+ IsEmpty = IsListEmpty (&Private->AsyncPassThruQueue) &&
+ IsListEmpty (&Private->UnsubmittedSubtasks);
+ gBS->RestoreTPL (OldTpl);
+
+ if (IsEmpty) {
+ break;
+ }
+
+ gBS->Stall (100);
+ }
+
gBS->UninstallMultipleProtocolInterfaces (
Controller,
&gEfiNvmExpressPassThruProtocolGuid,
@@ -941,12 +1188,16 @@ NvmExpressDriverBindingStop ( NULL
);
+ if (Private->TimerEvent != NULL) {
+ gBS->CloseEvent (Private->TimerEvent);
+ }
+
if (Private->Mapping != NULL) {
Private->PciIo->Unmap (Private->PciIo, Private->Mapping);
}
if (Private->Buffer != NULL) {
- Private->PciIo->FreeBuffer (Private->PciIo, 4, Private->Buffer);
+ Private->PciIo->FreeBuffer (Private->PciIo, 6, Private->Buffer);
}
FreePool (Private->ControllerData);
diff --git a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.h b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.h index 6cbef3e380..519772b3c7 100644 --- a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.h +++ b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpress.h @@ -28,6 +28,7 @@ #include <Protocol/PciIo.h>
#include <Protocol/NvmExpressPassthru.h>
#include <Protocol/BlockIo.h>
+#include <Protocol/BlockIo2.h>
#include <Protocol/DiskInfo.h>
#include <Protocol/DriverSupportedEfiVersion.h>
#include <Protocol/StorageSecurityCommand.h>
@@ -63,7 +64,18 @@ extern EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL gNvmExpressDriverSupportedEfiV #define NVME_CSQ_SIZE 1 // Number of I/O submission queue entries, which is 0-based
#define NVME_CCQ_SIZE 1 // Number of I/O completion queue entries, which is 0-based
-#define NVME_MAX_QUEUES 2 // Number of queues supported by the driver
+//
+// Number of asynchronous I/O submission queue entries, which is 0-based.
+// The asynchronous I/O submission queue size is 4kB in total.
+//
+#define NVME_ASYNC_CSQ_SIZE 63
+//
+// Number of asynchronous I/O completion queue entries, which is 0-based.
+// The asynchronous I/O completion queue size is 4kB in total.
+//
+#define NVME_ASYNC_CCQ_SIZE 255
+
+#define NVME_MAX_QUEUES 3 // Number of queues supported by the driver
#define NVME_CONTROLLER_ID 0
@@ -73,6 +85,11 @@ extern EFI_DRIVER_SUPPORTED_EFI_VERSION_PROTOCOL gNvmExpressDriverSupportedEfiV #define NVME_GENERIC_TIMEOUT EFI_TIMER_PERIOD_SECONDS (5)
//
+// Nvme async transfer timer interval, set by experience.
+//
+#define NVME_HC_ASYNC_TIMER EFI_TIMER_PERIOD_MILLISECONDS (1)
+
+//
// Unique signature for private data structure.
//
#define NVME_CONTROLLER_PRIVATE_DATA_SIGNATURE SIGNATURE_32 ('N','V','M','E')
@@ -101,11 +118,13 @@ struct _NVME_CONTROLLER_PRIVATE_DATA { NVME_ADMIN_CONTROLLER_DATA *ControllerData;
//
- // 4 x 4kB aligned buffers will be carved out of this buffer.
+ // 6 x 4kB aligned buffers will be carved out of this buffer.
// 1st 4kB boundary is the start of the admin submission queue.
// 2nd 4kB boundary is the start of the admin completion queue.
// 3rd 4kB boundary is the start of I/O submission queue #1.
// 4th 4kB boundary is the start of I/O completion queue #1.
+ // 5th 4kB boundary is the start of I/O submission queue #2.
+ // 6th 4kB boundary is the start of I/O completion queue #2.
//
UINT8 *Buffer;
UINT8 *BufferPciAddr;
@@ -123,6 +142,7 @@ struct _NVME_CONTROLLER_PRIVATE_DATA { //
NVME_SQTDBL SqTdbl[NVME_MAX_QUEUES];
NVME_CQHDBL CqHdbl[NVME_MAX_QUEUES];
+ UINT16 AsyncSqHead;
UINT8 Pt[NVME_MAX_QUEUES];
UINT16 Cid[NVME_MAX_QUEUES];
@@ -133,6 +153,13 @@ struct _NVME_CONTROLLER_PRIVATE_DATA { NVME_CAP Cap;
VOID *Mapping;
+
+ //
+ // For Non-blocking operations.
+ //
+ EFI_EVENT TimerEvent;
+ LIST_ENTRY AsyncPassThruQueue;
+ LIST_ENTRY UnsubmittedSubtasks;
};
#define NVME_CONTROLLER_PRIVATE_DATA_FROM_PASS_THRU(a) \
@@ -166,9 +193,12 @@ struct _NVME_DEVICE_PRIVATE_DATA { EFI_BLOCK_IO_MEDIA Media;
EFI_BLOCK_IO_PROTOCOL BlockIo;
+ EFI_BLOCK_IO2_PROTOCOL BlockIo2;
EFI_DISK_INFO_PROTOCOL DiskInfo;
EFI_STORAGE_SECURITY_COMMAND_PROTOCOL StorageSecurity;
+ LIST_ENTRY AsyncQueue;
+
EFI_LBA NumBlocks;
CHAR16 ModelName[80];
@@ -188,6 +218,13 @@ struct _NVME_DEVICE_PRIVATE_DATA { NVME_DEVICE_PRIVATE_DATA_SIGNATURE \
)
+#define NVME_DEVICE_PRIVATE_DATA_FROM_BLOCK_IO2(a) \
+ CR (a, \
+ NVME_DEVICE_PRIVATE_DATA, \
+ BlockIo2, \
+ NVME_DEVICE_PRIVATE_DATA_SIGNATURE \
+ )
+
#define NVME_DEVICE_PRIVATE_DATA_FROM_DISK_INFO(a) \
CR (a, \
NVME_DEVICE_PRIVATE_DATA, \
@@ -202,6 +239,67 @@ struct _NVME_DEVICE_PRIVATE_DATA { NVME_DEVICE_PRIVATE_DATA_SIGNATURE \
)
+//
+// Nvme block I/O 2 request.
+//
+#define NVME_BLKIO2_REQUEST_SIGNATURE SIGNATURE_32 ('N', 'B', '2', 'R')
+
+typedef struct {
+ UINT32 Signature;
+ LIST_ENTRY Link;
+
+ EFI_BLOCK_IO2_TOKEN *Token;
+ UINTN UnsubmittedSubtaskNum;
+ BOOLEAN LastSubtaskSubmitted;
+ //
+ // The queue for Nvme read/write sub-tasks of a BlockIo2 request.
+ //
+ LIST_ENTRY SubtasksQueue;
+} NVME_BLKIO2_REQUEST;
+
+#define NVME_BLKIO2_REQUEST_FROM_LINK(a) \
+ CR (a, NVME_BLKIO2_REQUEST, Link, NVME_BLKIO2_REQUEST_SIGNATURE)
+
+#define NVME_BLKIO2_SUBTASK_SIGNATURE SIGNATURE_32 ('N', 'B', '2', 'S')
+
+typedef struct {
+ UINT32 Signature;
+ LIST_ENTRY Link;
+
+ BOOLEAN IsLast;
+ UINT32 NamespaceId;
+ EFI_EVENT Event;
+ EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET *CommandPacket;
+ //
+ // The BlockIo2 request this subtask belongs to
+ //
+ NVME_BLKIO2_REQUEST *BlockIo2Request;
+} NVME_BLKIO2_SUBTASK;
+
+#define NVME_BLKIO2_SUBTASK_FROM_LINK(a) \
+ CR (a, NVME_BLKIO2_SUBTASK, Link, NVME_BLKIO2_SUBTASK_SIGNATURE)
+
+//
+// Nvme asynchronous passthru request.
+//
+#define NVME_PASS_THRU_ASYNC_REQ_SIG SIGNATURE_32 ('N', 'P', 'A', 'R')
+
+typedef struct {
+ UINT32 Signature;
+ LIST_ENTRY Link;
+
+ EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET *Packet;
+ UINT16 CommandId;
+ EFI_EVENT CallerEvent;
+} NVME_PASS_THRU_ASYNC_REQ;
+
+#define NVME_PASS_THRU_ASYNC_REQ_FROM_THIS(a) \
+ CR (a, \
+ NVME_PASS_THRU_ASYNC_REQ, \
+ Link, \
+ NVME_PASS_THRU_ASYNC_REQ_SIG \
+ )
+
/**
Retrieves a Unicode string that is the user readable name of the driver.
@@ -605,4 +703,15 @@ NvmExpressBuildDevicePath ( IN OUT EFI_DEVICE_PATH_PROTOCOL **DevicePath
);
+/**
+ Dump the execution status from a given completion queue entry.
+
+ @param[in] Cq A pointer to the NVME_CQ item.
+
+**/
+VOID
+NvmeDumpStatus (
+ IN NVME_CQ *Cq
+ );
+
#endif
diff --git a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressBlockIo.c b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressBlockIo.c index 92d7d923df..734e1286c6 100644 --- a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressBlockIo.c +++ b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressBlockIo.c @@ -172,6 +172,23 @@ NvmeRead ( NVME_CONTROLLER_PRIVATE_DATA *Private;
UINT32 MaxTransferBlocks;
UINTN OrginalBlocks;
+ BOOLEAN IsEmpty;
+ EFI_TPL OldTpl;
+
+ //
+ // Wait for the device's asynchronous I/O queue to become empty.
+ //
+ while (TRUE) {
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
+ IsEmpty = IsListEmpty (&Device->AsyncQueue);
+ gBS->RestoreTPL (OldTpl);
+
+ if (IsEmpty) {
+ break;
+ }
+
+ gBS->Stall (100);
+ }
Status = EFI_SUCCESS;
Private = Device->Controller;
@@ -233,6 +250,23 @@ NvmeWrite ( NVME_CONTROLLER_PRIVATE_DATA *Private;
UINT32 MaxTransferBlocks;
UINTN OrginalBlocks;
+ BOOLEAN IsEmpty;
+ EFI_TPL OldTpl;
+
+ //
+ // Wait for the device's asynchronous I/O queue to become empty.
+ //
+ while (TRUE) {
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
+ IsEmpty = IsListEmpty (&Device->AsyncQueue);
+ gBS->RestoreTPL (OldTpl);
+
+ if (IsEmpty) {
+ break;
+ }
+
+ gBS->Stall (100);
+ }
Status = EFI_SUCCESS;
Private = Device->Controller;
@@ -313,6 +347,587 @@ NvmeFlush ( return Status;
}
+/**
+ Nonblocking I/O callback funtion when the event is signaled.
+
+ @param[in] Event The Event this notify function registered to.
+ @param[in] Context Pointer to the context data registered to the
+ Event.
+
+**/
+VOID
+EFIAPI
+AsyncIoCallback (
+ IN EFI_EVENT Event,
+ IN VOID *Context
+ )
+{
+ NVME_BLKIO2_SUBTASK *Subtask;
+ NVME_BLKIO2_REQUEST *Request;
+ NVME_CQ *Completion;
+ EFI_BLOCK_IO2_TOKEN *Token;
+
+ gBS->CloseEvent (Event);
+
+ Subtask = (NVME_BLKIO2_SUBTASK *) Context;
+ Completion = (NVME_CQ *) Subtask->CommandPacket->NvmeCompletion;
+ Request = Subtask->BlockIo2Request;
+ Token = Request->Token;
+
+ if (Token->TransactionStatus == EFI_SUCCESS) {
+ //
+ // If previous subtask already fails, do not check the result of
+ // subsequent subtasks.
+ //
+ if ((Completion->Sct != 0) || (Completion->Sc != 0)) {
+ Token->TransactionStatus = EFI_DEVICE_ERROR;
+
+ //
+ // Dump completion entry status for debugging.
+ //
+ DEBUG_CODE_BEGIN();
+ NvmeDumpStatus (Completion);
+ DEBUG_CODE_END();
+ }
+ }
+
+ //
+ // Remove the subtask from the BlockIo2 subtasks list.
+ //
+ RemoveEntryList (&Subtask->Link);
+
+ if (IsListEmpty (&Request->SubtasksQueue) && Request->LastSubtaskSubmitted) {
+ //
+ // Remove the BlockIo2 request from the device asynchronous queue.
+ //
+ RemoveEntryList (&Request->Link);
+ FreePool (Request);
+ gBS->SignalEvent (Token->Event);
+ }
+
+ FreePool (Subtask->CommandPacket->NvmeCmd);
+ FreePool (Subtask->CommandPacket->NvmeCompletion);
+ FreePool (Subtask->CommandPacket);
+ FreePool (Subtask);
+}
+
+/**
+ Read some sectors from the device in an asynchronous manner.
+
+ @param Device The pointer to the NVME_DEVICE_PRIVATE_DATA data
+ structure.
+ @param Request The pointer to the NVME_BLKIO2_REQUEST data structure.
+ @param Buffer The buffer used to store the data read from the device.
+ @param Lba The start block number.
+ @param Blocks Total block number to be read.
+ @param IsLast The last subtask of an asynchronous read request.
+
+ @retval EFI_SUCCESS Asynchronous read request has been queued.
+ @retval Others Fail to send the asynchronous request.
+
+**/
+EFI_STATUS
+AsyncReadSectors (
+ IN NVME_DEVICE_PRIVATE_DATA *Device,
+ IN NVME_BLKIO2_REQUEST *Request,
+ IN UINT64 Buffer,
+ IN UINT64 Lba,
+ IN UINT32 Blocks,
+ IN BOOLEAN IsLast
+ )
+{
+ NVME_CONTROLLER_PRIVATE_DATA *Private;
+ UINT32 Bytes;
+ NVME_BLKIO2_SUBTASK *Subtask;
+ EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET *CommandPacket;
+ EFI_NVM_EXPRESS_COMMAND *Command;
+ EFI_NVM_EXPRESS_COMPLETION *Completion;
+ EFI_STATUS Status;
+ UINT32 BlockSize;
+ EFI_TPL OldTpl;
+
+ Private = Device->Controller;
+ BlockSize = Device->Media.BlockSize;
+ Bytes = Blocks * BlockSize;
+ CommandPacket = NULL;
+ Command = NULL;
+ Completion = NULL;
+
+ Subtask = AllocateZeroPool (sizeof (NVME_BLKIO2_SUBTASK));
+ if (Subtask == NULL) {
+ Status = EFI_OUT_OF_RESOURCES;
+ goto ErrorExit;
+ }
+
+ Subtask->Signature = NVME_BLKIO2_SUBTASK_SIGNATURE;
+ Subtask->IsLast = IsLast;
+ Subtask->NamespaceId = Device->NamespaceId;
+ Subtask->BlockIo2Request = Request;
+
+ CommandPacket = AllocateZeroPool (sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
+ if (CommandPacket == NULL) {
+ Status = EFI_OUT_OF_RESOURCES;
+ goto ErrorExit;
+ } else {
+ Subtask->CommandPacket = CommandPacket;
+ }
+
+ Command = AllocateZeroPool (sizeof (EFI_NVM_EXPRESS_COMMAND));
+ if (Command == NULL) {
+ Status = EFI_OUT_OF_RESOURCES;
+ goto ErrorExit;
+ }
+
+ Completion = AllocateZeroPool (sizeof (EFI_NVM_EXPRESS_COMPLETION));
+ if (Completion == NULL) {
+ Status = EFI_OUT_OF_RESOURCES;
+ goto ErrorExit;
+ }
+
+ //
+ // Create Event
+ //
+ Status = gBS->CreateEvent (
+ EVT_NOTIFY_SIGNAL,
+ TPL_NOTIFY,
+ AsyncIoCallback,
+ Subtask,
+ &Subtask->Event
+ );
+ if (EFI_ERROR(Status)) {
+ goto ErrorExit;
+ }
+
+ CommandPacket->NvmeCmd = Command;
+ CommandPacket->NvmeCompletion = Completion;
+
+ CommandPacket->NvmeCmd->Cdw0.Opcode = NVME_IO_READ_OPC;
+ CommandPacket->NvmeCmd->Nsid = Device->NamespaceId;
+ CommandPacket->TransferBuffer = (VOID *)(UINTN)Buffer;
+
+ CommandPacket->TransferLength = Bytes;
+ CommandPacket->CommandTimeout = NVME_GENERIC_TIMEOUT;
+ CommandPacket->QueueType = NVME_IO_QUEUE;
+
+ CommandPacket->NvmeCmd->Cdw10 = (UINT32)Lba;
+ CommandPacket->NvmeCmd->Cdw11 = (UINT32)RShiftU64(Lba, 32);
+ CommandPacket->NvmeCmd->Cdw12 = (Blocks - 1) & 0xFFFF;
+
+ CommandPacket->NvmeCmd->Flags = CDW10_VALID | CDW11_VALID | CDW12_VALID;
+
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
+ InsertTailList (&Private->UnsubmittedSubtasks, &Subtask->Link);
+ Request->UnsubmittedSubtaskNum++;
+ gBS->RestoreTPL (OldTpl);
+
+ return EFI_SUCCESS;
+
+ErrorExit:
+ //
+ // Resource cleanup if asynchronous read request has not been queued.
+ //
+ if (Completion != NULL) {
+ FreePool (Completion);
+ }
+
+ if (Command != NULL) {
+ FreePool (Command);
+ }
+
+ if (CommandPacket != NULL) {
+ FreePool (CommandPacket);
+ }
+
+ if (Subtask != NULL) {
+ if (Subtask->Event != NULL) {
+ gBS->CloseEvent (Subtask->Event);
+ }
+
+ FreePool (Subtask);
+ }
+
+ return Status;
+}
+
+/**
+ Write some sectors from the device in an asynchronous manner.
+
+ @param Device The pointer to the NVME_DEVICE_PRIVATE_DATA data
+ structure.
+ @param Request The pointer to the NVME_BLKIO2_REQUEST data structure.
+ @param Buffer The buffer used to store the data written to the
+ device.
+ @param Lba The start block number.
+ @param Blocks Total block number to be written.
+ @param IsLast The last subtask of an asynchronous write request.
+
+ @retval EFI_SUCCESS Asynchronous write request has been queued.
+ @retval Others Fail to send the asynchronous request.
+
+**/
+EFI_STATUS
+AsyncWriteSectors (
+ IN NVME_DEVICE_PRIVATE_DATA *Device,
+ IN NVME_BLKIO2_REQUEST *Request,
+ IN UINT64 Buffer,
+ IN UINT64 Lba,
+ IN UINT32 Blocks,
+ IN BOOLEAN IsLast
+ )
+{
+ NVME_CONTROLLER_PRIVATE_DATA *Private;
+ UINT32 Bytes;
+ NVME_BLKIO2_SUBTASK *Subtask;
+ EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET *CommandPacket;
+ EFI_NVM_EXPRESS_COMMAND *Command;
+ EFI_NVM_EXPRESS_COMPLETION *Completion;
+ EFI_STATUS Status;
+ UINT32 BlockSize;
+ EFI_TPL OldTpl;
+
+ Private = Device->Controller;
+ BlockSize = Device->Media.BlockSize;
+ Bytes = Blocks * BlockSize;
+ CommandPacket = NULL;
+ Command = NULL;
+ Completion = NULL;
+
+ Subtask = AllocateZeroPool (sizeof (NVME_BLKIO2_SUBTASK));
+ if (Subtask == NULL) {
+ Status = EFI_OUT_OF_RESOURCES;
+ goto ErrorExit;
+ }
+
+ Subtask->Signature = NVME_BLKIO2_SUBTASK_SIGNATURE;
+ Subtask->IsLast = IsLast;
+ Subtask->NamespaceId = Device->NamespaceId;
+ Subtask->BlockIo2Request = Request;
+
+ CommandPacket = AllocateZeroPool (sizeof (EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
+ if (CommandPacket == NULL) {
+ Status = EFI_OUT_OF_RESOURCES;
+ goto ErrorExit;
+ } else {
+ Subtask->CommandPacket = CommandPacket;
+ }
+
+ Command = AllocateZeroPool (sizeof (EFI_NVM_EXPRESS_COMMAND));
+ if (Command == NULL) {
+ Status = EFI_OUT_OF_RESOURCES;
+ goto ErrorExit;
+ }
+
+ Completion = AllocateZeroPool (sizeof (EFI_NVM_EXPRESS_COMPLETION));
+ if (Completion == NULL) {
+ Status = EFI_OUT_OF_RESOURCES;
+ goto ErrorExit;
+ }
+
+ //
+ // Create Event
+ //
+ Status = gBS->CreateEvent (
+ EVT_NOTIFY_SIGNAL,
+ TPL_NOTIFY,
+ AsyncIoCallback,
+ Subtask,
+ &Subtask->Event
+ );
+ if (EFI_ERROR(Status)) {
+ goto ErrorExit;
+ }
+
+ CommandPacket->NvmeCmd = Command;
+ CommandPacket->NvmeCompletion = Completion;
+
+ CommandPacket->NvmeCmd->Cdw0.Opcode = NVME_IO_WRITE_OPC;
+ CommandPacket->NvmeCmd->Nsid = Device->NamespaceId;
+ CommandPacket->TransferBuffer = (VOID *)(UINTN)Buffer;
+
+ CommandPacket->TransferLength = Bytes;
+ CommandPacket->CommandTimeout = NVME_GENERIC_TIMEOUT;
+ CommandPacket->QueueType = NVME_IO_QUEUE;
+
+ CommandPacket->NvmeCmd->Cdw10 = (UINT32)Lba;
+ CommandPacket->NvmeCmd->Cdw11 = (UINT32)RShiftU64(Lba, 32);
+ //
+ // Set Force Unit Access bit (bit 30) to use write-through behaviour
+ //
+ CommandPacket->NvmeCmd->Cdw12 = ((Blocks - 1) & 0xFFFF) | BIT30;
+
+ CommandPacket->NvmeCmd->Flags = CDW10_VALID | CDW11_VALID | CDW12_VALID;
+
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
+ InsertTailList (&Private->UnsubmittedSubtasks, &Subtask->Link);
+ Request->UnsubmittedSubtaskNum++;
+ gBS->RestoreTPL (OldTpl);
+
+ return EFI_SUCCESS;
+
+ErrorExit:
+ //
+ // Resource cleanup if asynchronous read request has not been queued.
+ //
+ if (Completion != NULL) {
+ FreePool (Completion);
+ }
+
+ if (Command != NULL) {
+ FreePool (Command);
+ }
+
+ if (CommandPacket != NULL) {
+ FreePool (CommandPacket);
+ }
+
+ if (Subtask != NULL) {
+ if (Subtask->Event != NULL) {
+ gBS->CloseEvent (Subtask->Event);
+ }
+
+ FreePool (Subtask);
+ }
+
+ return Status;
+}
+
+/**
+ Read some blocks from the device in an asynchronous manner.
+
+ @param Device The pointer to the NVME_DEVICE_PRIVATE_DATA data
+ structure.
+ @param Buffer The buffer used to store the data read from the device.
+ @param Lba The start block number.
+ @param Blocks Total block number to be read.
+ @param Token A pointer to the token associated with the transaction.
+
+ @retval EFI_SUCCESS Data are read from the device.
+ @retval Others Fail to read all the data.
+
+**/
+EFI_STATUS
+NvmeAsyncRead (
+ IN NVME_DEVICE_PRIVATE_DATA *Device,
+ OUT VOID *Buffer,
+ IN UINT64 Lba,
+ IN UINTN Blocks,
+ IN EFI_BLOCK_IO2_TOKEN *Token
+ )
+{
+ EFI_STATUS Status;
+ UINT32 BlockSize;
+ NVME_CONTROLLER_PRIVATE_DATA *Private;
+ NVME_BLKIO2_REQUEST *BlkIo2Req;
+ UINT32 MaxTransferBlocks;
+ UINTN OrginalBlocks;
+ BOOLEAN IsEmpty;
+ EFI_TPL OldTpl;
+
+ Status = EFI_SUCCESS;
+ Private = Device->Controller;
+ BlockSize = Device->Media.BlockSize;
+ OrginalBlocks = Blocks;
+ BlkIo2Req = AllocateZeroPool (sizeof (NVME_BLKIO2_REQUEST));
+ if (BlkIo2Req == NULL) {
+ return EFI_OUT_OF_RESOURCES;
+ }
+
+ BlkIo2Req->Signature = NVME_BLKIO2_REQUEST_SIGNATURE;
+ BlkIo2Req->Token = Token;
+
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
+ InsertTailList (&Device->AsyncQueue, &BlkIo2Req->Link);
+ gBS->RestoreTPL (OldTpl);
+
+ InitializeListHead (&BlkIo2Req->SubtasksQueue);
+
+ if (Private->ControllerData->Mdts != 0) {
+ MaxTransferBlocks = (1 << (Private->ControllerData->Mdts)) * (1 << (Private->Cap.Mpsmin + 12)) / BlockSize;
+ } else {
+ MaxTransferBlocks = 1024;
+ }
+
+ while (Blocks > 0) {
+ if (Blocks > MaxTransferBlocks) {
+ Status = AsyncReadSectors (
+ Device,
+ BlkIo2Req, (UINT64)(UINTN)Buffer,
+ Lba,
+ MaxTransferBlocks,
+ FALSE
+ );
+
+ Blocks -= MaxTransferBlocks;
+ Buffer = (VOID *)(UINTN)((UINT64)(UINTN)Buffer + MaxTransferBlocks * BlockSize);
+ Lba += MaxTransferBlocks;
+ } else {
+ Status = AsyncReadSectors (
+ Device,
+ BlkIo2Req,
+ (UINT64)(UINTN)Buffer,
+ Lba,
+ (UINT32)Blocks,
+ TRUE
+ );
+
+ Blocks = 0;
+ }
+
+ if (EFI_ERROR(Status)) {
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
+ IsEmpty = IsListEmpty (&BlkIo2Req->SubtasksQueue) &&
+ (BlkIo2Req->UnsubmittedSubtaskNum == 0);
+
+ if (IsEmpty) {
+ //
+ // Remove the BlockIo2 request from the device asynchronous queue.
+ //
+ RemoveEntryList (&BlkIo2Req->Link);
+ FreePool (BlkIo2Req);
+ Status = EFI_DEVICE_ERROR;
+ } else {
+ //
+ // There are previous BlockIo2 subtasks still running, EFI_SUCCESS
+ // should be returned to make sure that the caller does not free
+ // resources still using by these requests.
+ //
+ Status = EFI_SUCCESS;
+ Token->TransactionStatus = EFI_DEVICE_ERROR;
+ BlkIo2Req->LastSubtaskSubmitted = TRUE;
+ }
+
+ gBS->RestoreTPL (OldTpl);
+
+ break;
+ }
+ }
+
+ DEBUG ((EFI_D_VERBOSE, "%a: Lba = 0x%08Lx, Original = 0x%08Lx, "
+ "Remaining = 0x%08Lx, BlockSize = 0x%x, Status = %r\n", __FUNCTION__, Lba,
+ (UINT64)OrginalBlocks, (UINT64)Blocks, BlockSize, Status));
+
+ return Status;
+}
+
+/**
+ Write some blocks from the device in an asynchronous manner.
+
+ @param Device The pointer to the NVME_DEVICE_PRIVATE_DATA data
+ structure.
+ @param Buffer The buffer used to store the data written to the
+ device.
+ @param Lba The start block number.
+ @param Blocks Total block number to be written.
+ @param Token A pointer to the token associated with the transaction.
+
+ @retval EFI_SUCCESS Data are written to the device.
+ @retval Others Fail to write all the data.
+
+**/
+EFI_STATUS
+NvmeAsyncWrite (
+ IN NVME_DEVICE_PRIVATE_DATA *Device,
+ IN VOID *Buffer,
+ IN UINT64 Lba,
+ IN UINTN Blocks,
+ IN EFI_BLOCK_IO2_TOKEN *Token
+ )
+{
+ EFI_STATUS Status;
+ UINT32 BlockSize;
+ NVME_CONTROLLER_PRIVATE_DATA *Private;
+ NVME_BLKIO2_REQUEST *BlkIo2Req;
+ UINT32 MaxTransferBlocks;
+ UINTN OrginalBlocks;
+ BOOLEAN IsEmpty;
+ EFI_TPL OldTpl;
+
+ Status = EFI_SUCCESS;
+ Private = Device->Controller;
+ BlockSize = Device->Media.BlockSize;
+ OrginalBlocks = Blocks;
+ BlkIo2Req = AllocateZeroPool (sizeof (NVME_BLKIO2_REQUEST));
+ if (BlkIo2Req == NULL) {
+ return EFI_OUT_OF_RESOURCES;
+ }
+
+ BlkIo2Req->Signature = NVME_BLKIO2_REQUEST_SIGNATURE;
+ BlkIo2Req->Token = Token;
+
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
+ InsertTailList (&Device->AsyncQueue, &BlkIo2Req->Link);
+ gBS->RestoreTPL (OldTpl);
+
+ InitializeListHead (&BlkIo2Req->SubtasksQueue);
+
+ if (Private->ControllerData->Mdts != 0) {
+ MaxTransferBlocks = (1 << (Private->ControllerData->Mdts)) * (1 << (Private->Cap.Mpsmin + 12)) / BlockSize;
+ } else {
+ MaxTransferBlocks = 1024;
+ }
+
+ while (Blocks > 0) {
+ if (Blocks > MaxTransferBlocks) {
+ Status = AsyncWriteSectors (
+ Device,
+ BlkIo2Req,
+ (UINT64)(UINTN)Buffer,
+ Lba,
+ MaxTransferBlocks,
+ FALSE
+ );
+
+ Blocks -= MaxTransferBlocks;
+ Buffer = (VOID *)(UINTN)((UINT64)(UINTN)Buffer + MaxTransferBlocks * BlockSize);
+ Lba += MaxTransferBlocks;
+ } else {
+ Status = AsyncWriteSectors (
+ Device,
+ BlkIo2Req,
+ (UINT64)(UINTN)Buffer,
+ Lba,
+ (UINT32)Blocks,
+ TRUE
+ );
+
+ Blocks = 0;
+ }
+
+ if (EFI_ERROR(Status)) {
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
+ IsEmpty = IsListEmpty (&BlkIo2Req->SubtasksQueue) &&
+ (BlkIo2Req->UnsubmittedSubtaskNum == 0);
+
+ if (IsEmpty) {
+ //
+ // Remove the BlockIo2 request from the device asynchronous queue.
+ //
+ RemoveEntryList (&BlkIo2Req->Link);
+ FreePool (BlkIo2Req);
+ Status = EFI_DEVICE_ERROR;
+ } else {
+ //
+ // There are previous BlockIo2 subtasks still running, EFI_SUCCESS
+ // should be returned to make sure that the caller does not free
+ // resources still using by these requests.
+ //
+ Status = EFI_SUCCESS;
+ Token->TransactionStatus = EFI_DEVICE_ERROR;
+ BlkIo2Req->LastSubtaskSubmitted = TRUE;
+ }
+
+ gBS->RestoreTPL (OldTpl);
+
+ break;
+ }
+ }
+
+ DEBUG ((EFI_D_VERBOSE, "%a: Lba = 0x%08Lx, Original = 0x%08Lx, "
+ "Remaining = 0x%08Lx, BlockSize = 0x%x, Status = %r\n", __FUNCTION__, Lba,
+ (UINT64)OrginalBlocks, (UINT64)Blocks, BlockSize, Status));
+
+ return Status;
+}
/**
Reset the Block Device.
@@ -568,6 +1183,361 @@ NvmeBlockIoFlushBlocks ( }
/**
+ Reset the block device hardware.
+
+ @param[in] This Indicates a pointer to the calling context.
+ @param[in] ExtendedVerification Indicates that the driver may perform a more
+ exhausive verfication operation of the
+ device during reset.
+
+ @retval EFI_SUCCESS The device was reset.
+ @retval EFI_DEVICE_ERROR The device is not functioning properly and could
+ not be reset.
+
+**/
+EFI_STATUS
+EFIAPI
+NvmeBlockIoResetEx (
+ IN EFI_BLOCK_IO2_PROTOCOL *This,
+ IN BOOLEAN ExtendedVerification
+ )
+{
+ EFI_STATUS Status;
+ NVME_DEVICE_PRIVATE_DATA *Device;
+ NVME_CONTROLLER_PRIVATE_DATA *Private;
+ BOOLEAN IsEmpty;
+ EFI_TPL OldTpl;
+
+ if (This == NULL) {
+ return EFI_INVALID_PARAMETER;
+ }
+
+ Device = NVME_DEVICE_PRIVATE_DATA_FROM_BLOCK_IO2 (This);
+ Private = Device->Controller;
+
+ //
+ // Wait for the asynchronous PassThru queue to become empty.
+ //
+ while (TRUE) {
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
+ IsEmpty = IsListEmpty (&Private->AsyncPassThruQueue) &&
+ IsListEmpty (&Private->UnsubmittedSubtasks);
+ gBS->RestoreTPL (OldTpl);
+
+ if (IsEmpty) {
+ break;
+ }
+
+ gBS->Stall (100);
+ }
+
+ OldTpl = gBS->RaiseTPL (TPL_CALLBACK);
+
+ Status = NvmeControllerInit (Private);
+
+ if (EFI_ERROR (Status)) {
+ Status = EFI_DEVICE_ERROR;
+ }
+
+ gBS->RestoreTPL (OldTpl);
+
+ return Status;
+}
+
+/**
+ Read BufferSize bytes from Lba into Buffer.
+
+ This function reads the requested number of blocks from the device. All the
+ blocks are read, or an error is returned.
+ If EFI_DEVICE_ERROR, EFI_NO_MEDIA,_or EFI_MEDIA_CHANGED is returned and
+ non-blocking I/O is being used, the Event associated with this request will
+ not be signaled.
+
+ @param[in] This Indicates a pointer to the calling context.
+ @param[in] MediaId Id of the media, changes every time the media is
+ replaced.
+ @param[in] Lba The starting Logical Block Address to read from.
+ @param[in, out] Token A pointer to the token associated with the
+ transaction.
+ @param[in] BufferSize Size of Buffer, must be a multiple of device
+ block size.
+ @param[out] Buffer A pointer to the destination buffer for the data.
+ The caller is responsible for either having
+ implicit or explicit ownership of the buffer.
+
+ @retval EFI_SUCCESS The read request was queued if Token->Event is
+ not NULL.The data was read correctly from the
+ device if the Token->Event is NULL.
+ @retval EFI_DEVICE_ERROR The device reported an error while performing
+ the read.
+ @retval EFI_NO_MEDIA There is no media in the device.
+ @retval EFI_MEDIA_CHANGED The MediaId is not for the current media.
+ @retval EFI_BAD_BUFFER_SIZE The BufferSize parameter is not a multiple of
+ the intrinsic block size of the device.
+ @retval EFI_INVALID_PARAMETER The read request contains LBAs that are not
+ valid, or the buffer is not on proper
+ alignment.
+ @retval EFI_OUT_OF_RESOURCES The request could not be completed due to a
+ lack of resources.
+
+**/
+EFI_STATUS
+EFIAPI
+NvmeBlockIoReadBlocksEx (
+ IN EFI_BLOCK_IO2_PROTOCOL *This,
+ IN UINT32 MediaId,
+ IN EFI_LBA Lba,
+ IN OUT EFI_BLOCK_IO2_TOKEN *Token,
+ IN UINTN BufferSize,
+ OUT VOID *Buffer
+ )
+{
+ NVME_DEVICE_PRIVATE_DATA *Device;
+ EFI_STATUS Status;
+ EFI_BLOCK_IO_MEDIA *Media;
+ UINTN BlockSize;
+ UINTN NumberOfBlocks;
+ UINTN IoAlign;
+ EFI_TPL OldTpl;
+
+ //
+ // Check parameters.
+ //
+ if (This == NULL) {
+ return EFI_INVALID_PARAMETER;
+ }
+
+ Media = This->Media;
+
+ if (MediaId != Media->MediaId) {
+ return EFI_MEDIA_CHANGED;
+ }
+
+ if (Buffer == NULL) {
+ return EFI_INVALID_PARAMETER;
+ }
+
+ if (BufferSize == 0) {
+ if ((Token != NULL) && (Token->Event != NULL)) {
+ Token->TransactionStatus = EFI_SUCCESS;
+ gBS->SignalEvent (Token->Event);
+ }
+ return EFI_SUCCESS;
+ }
+
+ BlockSize = Media->BlockSize;
+ if ((BufferSize % BlockSize) != 0) {
+ return EFI_BAD_BUFFER_SIZE;
+ }
+
+ NumberOfBlocks = BufferSize / BlockSize;
+ if ((Lba + NumberOfBlocks - 1) > Media->LastBlock) {
+ return EFI_INVALID_PARAMETER;
+ }
+
+ IoAlign = Media->IoAlign;
+ if (IoAlign > 0 && (((UINTN) Buffer & (IoAlign - 1)) != 0)) {
+ return EFI_INVALID_PARAMETER;
+ }
+
+ OldTpl = gBS->RaiseTPL (TPL_CALLBACK);
+
+ Device = NVME_DEVICE_PRIVATE_DATA_FROM_BLOCK_IO2 (This);
+
+ if ((Token != NULL) && (Token->Event != NULL)) {
+ Token->TransactionStatus = EFI_SUCCESS;
+ Status = NvmeAsyncRead (Device, Buffer, Lba, NumberOfBlocks, Token);
+ } else {
+ Status = NvmeRead (Device, Buffer, Lba, NumberOfBlocks);
+ }
+
+ gBS->RestoreTPL (OldTpl);
+ return Status;
+}
+
+/**
+ Write BufferSize bytes from Lba into Buffer.
+
+ This function writes the requested number of blocks to the device. All blocks
+ are written, or an error is returned.If EFI_DEVICE_ERROR, EFI_NO_MEDIA,
+ EFI_WRITE_PROTECTED or EFI_MEDIA_CHANGED is returned and non-blocking I/O is
+ being used, the Event associated with this request will not be signaled.
+
+ @param[in] This Indicates a pointer to the calling context.
+ @param[in] MediaId The media ID that the write request is for.
+ @param[in] Lba The starting logical block address to be written.
+ The caller is responsible for writing to only
+ legitimate locations.
+ @param[in, out] Token A pointer to the token associated with the
+ transaction.
+ @param[in] BufferSize Size of Buffer, must be a multiple of device
+ block size.
+ @param[in] Buffer A pointer to the source buffer for the data.
+
+ @retval EFI_SUCCESS The write request was queued if Event is not
+ NULL.
+ The data was written correctly to the device if
+ the Event is NULL.
+ @retval EFI_WRITE_PROTECTED The device can not be written to.
+ @retval EFI_NO_MEDIA There is no media in the device.
+ @retval EFI_MEDIA_CHNAGED The MediaId does not matched the current
+ device.
+ @retval EFI_DEVICE_ERROR The device reported an error while performing
+ the write.
+ @retval EFI_BAD_BUFFER_SIZE The Buffer was not a multiple of the block size
+ of the device.
+ @retval EFI_INVALID_PARAMETER The write request contains LBAs that are not
+ valid, or the buffer is not on proper
+ alignment.
+ @retval EFI_OUT_OF_RESOURCES The request could not be completed due to a
+ lack of resources.
+
+**/
+EFI_STATUS
+EFIAPI
+NvmeBlockIoWriteBlocksEx (
+ IN EFI_BLOCK_IO2_PROTOCOL *This,
+ IN UINT32 MediaId,
+ IN EFI_LBA Lba,
+ IN OUT EFI_BLOCK_IO2_TOKEN *Token,
+ IN UINTN BufferSize,
+ IN VOID *Buffer
+ )
+{
+ NVME_DEVICE_PRIVATE_DATA *Device;
+ EFI_STATUS Status;
+ EFI_BLOCK_IO_MEDIA *Media;
+ UINTN BlockSize;
+ UINTN NumberOfBlocks;
+ UINTN IoAlign;
+ EFI_TPL OldTpl;
+
+ //
+ // Check parameters.
+ //
+ if (This == NULL) {
+ return EFI_INVALID_PARAMETER;
+ }
+
+ Media = This->Media;
+
+ if (MediaId != Media->MediaId) {
+ return EFI_MEDIA_CHANGED;
+ }
+
+ if (Buffer == NULL) {
+ return EFI_INVALID_PARAMETER;
+ }
+
+ if (BufferSize == 0) {
+ if ((Token != NULL) && (Token->Event != NULL)) {
+ Token->TransactionStatus = EFI_SUCCESS;
+ gBS->SignalEvent (Token->Event);
+ }
+ return EFI_SUCCESS;
+ }
+
+ BlockSize = Media->BlockSize;
+ if ((BufferSize % BlockSize) != 0) {
+ return EFI_BAD_BUFFER_SIZE;
+ }
+
+ NumberOfBlocks = BufferSize / BlockSize;
+ if ((Lba + NumberOfBlocks - 1) > Media->LastBlock) {
+ return EFI_INVALID_PARAMETER;
+ }
+
+ IoAlign = Media->IoAlign;
+ if (IoAlign > 0 && (((UINTN) Buffer & (IoAlign - 1)) != 0)) {
+ return EFI_INVALID_PARAMETER;
+ }
+
+ OldTpl = gBS->RaiseTPL (TPL_CALLBACK);
+
+ Device = NVME_DEVICE_PRIVATE_DATA_FROM_BLOCK_IO2 (This);
+
+ if ((Token != NULL) && (Token->Event != NULL)) {
+ Token->TransactionStatus = EFI_SUCCESS;
+ Status = NvmeAsyncWrite (Device, Buffer, Lba, NumberOfBlocks, Token);
+ } else {
+ Status = NvmeWrite (Device, Buffer, Lba, NumberOfBlocks);
+ }
+
+ gBS->RestoreTPL (OldTpl);
+ return Status;
+}
+
+/**
+ Flush the Block Device.
+
+ If EFI_DEVICE_ERROR, EFI_NO_MEDIA,_EFI_WRITE_PROTECTED or EFI_MEDIA_CHANGED
+ is returned and non-blocking I/O is being used, the Event associated with
+ this request will not be signaled.
+
+ @param[in] This Indicates a pointer to the calling context.
+ @param[in,out] Token A pointer to the token associated with the
+ transaction.
+
+ @retval EFI_SUCCESS The flush request was queued if Event is not
+ NULL.
+ All outstanding data was written correctly to
+ the device if the Event is NULL.
+ @retval EFI_DEVICE_ERROR The device reported an error while writting back
+ the data.
+ @retval EFI_WRITE_PROTECTED The device cannot be written to.
+ @retval EFI_NO_MEDIA There is no media in the device.
+ @retval EFI_MEDIA_CHANGED The MediaId is not for the current media.
+ @retval EFI_OUT_OF_RESOURCES The request could not be completed due to a lack
+ of resources.
+
+**/
+EFI_STATUS
+EFIAPI
+NvmeBlockIoFlushBlocksEx (
+ IN EFI_BLOCK_IO2_PROTOCOL *This,
+ IN OUT EFI_BLOCK_IO2_TOKEN *Token
+ )
+{
+ NVME_DEVICE_PRIVATE_DATA *Device;
+ BOOLEAN IsEmpty;
+ EFI_TPL OldTpl;
+
+ //
+ // Check parameters.
+ //
+ if (This == NULL) {
+ return EFI_INVALID_PARAMETER;
+ }
+
+ Device = NVME_DEVICE_PRIVATE_DATA_FROM_BLOCK_IO2 (This);
+
+ //
+ // Wait for the asynchronous I/O queue to become empty.
+ //
+ while (TRUE) {
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
+ IsEmpty = IsListEmpty (&Device->AsyncQueue);
+ gBS->RestoreTPL (OldTpl);
+
+ if (IsEmpty) {
+ break;
+ }
+
+ gBS->Stall (100);
+ }
+
+ //
+ // Signal caller event
+ //
+ if ((Token != NULL) && (Token->Event != NULL)) {
+ Token->TransactionStatus = EFI_SUCCESS;
+ gBS->SignalEvent (Token->Event);
+ }
+
+ return EFI_SUCCESS;
+}
+
+/**
Trust transfer data from/to NVMe device.
This function performs one NVMe transaction to do a trust transfer from/to NVMe device.
@@ -619,7 +1589,7 @@ TrustTransferNvmeDevice ( CommandPacket.NvmeCmd = &Command;
CommandPacket.NvmeCompletion = &Completion;
- //
+ //
// Change Endianness of SecurityProtocolSpecificData
//
SpecificData = (((SecurityProtocolSpecificData << 8) & 0xFF00) | (SecurityProtocolSpecificData >> 8));
@@ -748,7 +1718,7 @@ NvmeStorageSecurityReceiveData ( {
EFI_STATUS Status;
NVME_DEVICE_PRIVATE_DATA *Device;
-
+
Status = EFI_SUCCESS;
if ((PayloadBuffer == NULL) || (PayloadTransferSize == NULL) || (PayloadBufferSize == 0)) {
@@ -852,7 +1822,7 @@ NvmeStorageSecuritySendData ( IN VOID *PayloadBuffer
)
{
- EFI_STATUS Status;
+ EFI_STATUS Status;
NVME_DEVICE_PRIVATE_DATA *Device;
Status = EFI_SUCCESS;
diff --git a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressBlockIo.h b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressBlockIo.h index 1c71a81ec5..9199f284d8 100644 --- a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressBlockIo.h +++ b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressBlockIo.h @@ -1,7 +1,7 @@ /** @file
Header file for EFI_BLOCK_IO_PROTOCOL interface.
-Copyright (c) 2013 - 2015, Intel Corporation. All rights reserved.<BR>
+Copyright (c) 2013 - 2016, Intel Corporation. All rights reserved.<BR>
This program and the accompanying materials
are licensed and made available under the terms and conditions of the BSD License
which accompanies this distribution. The full text of the license may be found at
@@ -109,6 +109,154 @@ NvmeBlockIoFlushBlocks ( );
/**
+ Reset the block device hardware.
+
+ @param[in] This Indicates a pointer to the calling context.
+ @param[in] ExtendedVerification Indicates that the driver may perform a more
+ exhausive verfication operation of the
+ device during reset.
+
+ @retval EFI_SUCCESS The device was reset.
+ @retval EFI_DEVICE_ERROR The device is not functioning properly and could
+ not be reset.
+
+**/
+EFI_STATUS
+EFIAPI
+NvmeBlockIoResetEx (
+ IN EFI_BLOCK_IO2_PROTOCOL *This,
+ IN BOOLEAN ExtendedVerification
+ );
+
+/**
+ Read BufferSize bytes from Lba into Buffer.
+
+ This function reads the requested number of blocks from the device. All the
+ blocks are read, or an error is returned.
+ If EFI_DEVICE_ERROR, EFI_NO_MEDIA,_or EFI_MEDIA_CHANGED is returned and
+ non-blocking I/O is being used, the Event associated with this request will
+ not be signaled.
+
+ @param[in] This Indicates a pointer to the calling context.
+ @param[in] MediaId Id of the media, changes every time the media is
+ replaced.
+ @param[in] Lba The starting Logical Block Address to read from.
+ @param[in, out] Token A pointer to the token associated with the
+ transaction.
+ @param[in] BufferSize Size of Buffer, must be a multiple of device
+ block size.
+ @param[out] Buffer A pointer to the destination buffer for the data.
+ The caller is responsible for either having
+ implicit or explicit ownership of the buffer.
+
+ @retval EFI_SUCCESS The read request was queued if Token->Event is
+ not NULL.The data was read correctly from the
+ device if the Token->Event is NULL.
+ @retval EFI_DEVICE_ERROR The device reported an error while performing
+ the read.
+ @retval EFI_NO_MEDIA There is no media in the device.
+ @retval EFI_MEDIA_CHANGED The MediaId is not for the current media.
+ @retval EFI_BAD_BUFFER_SIZE The BufferSize parameter is not a multiple of
+ the intrinsic block size of the device.
+ @retval EFI_INVALID_PARAMETER The read request contains LBAs that are not
+ valid, or the buffer is not on proper
+ alignment.
+ @retval EFI_OUT_OF_RESOURCES The request could not be completed due to a
+ lack of resources.
+
+**/
+EFI_STATUS
+EFIAPI
+NvmeBlockIoReadBlocksEx (
+ IN EFI_BLOCK_IO2_PROTOCOL *This,
+ IN UINT32 MediaId,
+ IN EFI_LBA Lba,
+ IN OUT EFI_BLOCK_IO2_TOKEN *Token,
+ IN UINTN BufferSize,
+ OUT VOID *Buffer
+ );
+
+/**
+ Write BufferSize bytes from Lba into Buffer.
+
+ This function writes the requested number of blocks to the device. All blocks
+ are written, or an error is returned.If EFI_DEVICE_ERROR, EFI_NO_MEDIA,
+ EFI_WRITE_PROTECTED or EFI_MEDIA_CHANGED is returned and non-blocking I/O is
+ being used, the Event associated with this request will not be signaled.
+
+ @param[in] This Indicates a pointer to the calling context.
+ @param[in] MediaId The media ID that the write request is for.
+ @param[in] Lba The starting logical block address to be written.
+ The caller is responsible for writing to only
+ legitimate locations.
+ @param[in, out] Token A pointer to the token associated with the
+ transaction.
+ @param[in] BufferSize Size of Buffer, must be a multiple of device
+ block size.
+ @param[in] Buffer A pointer to the source buffer for the data.
+
+ @retval EFI_SUCCESS The write request was queued if Event is not
+ NULL.
+ The data was written correctly to the device if
+ the Event is NULL.
+ @retval EFI_WRITE_PROTECTED The device can not be written to.
+ @retval EFI_NO_MEDIA There is no media in the device.
+ @retval EFI_MEDIA_CHNAGED The MediaId does not matched the current
+ device.
+ @retval EFI_DEVICE_ERROR The device reported an error while performing
+ the write.
+ @retval EFI_BAD_BUFFER_SIZE The Buffer was not a multiple of the block size
+ of the device.
+ @retval EFI_INVALID_PARAMETER The write request contains LBAs that are not
+ valid, or the buffer is not on proper
+ alignment.
+ @retval EFI_OUT_OF_RESOURCES The request could not be completed due to a
+ lack of resources.
+
+**/
+EFI_STATUS
+EFIAPI
+NvmeBlockIoWriteBlocksEx (
+ IN EFI_BLOCK_IO2_PROTOCOL *This,
+ IN UINT32 MediaId,
+ IN EFI_LBA Lba,
+ IN OUT EFI_BLOCK_IO2_TOKEN *Token,
+ IN UINTN BufferSize,
+ IN VOID *Buffer
+ );
+
+/**
+ Flush the Block Device.
+
+ If EFI_DEVICE_ERROR, EFI_NO_MEDIA,_EFI_WRITE_PROTECTED or EFI_MEDIA_CHANGED
+ is returned and non-blocking I/O is being used, the Event associated with
+ this request will not be signaled.
+
+ @param[in] This Indicates a pointer to the calling context.
+ @param[in,out] Token A pointer to the token associated with the
+ transaction.
+
+ @retval EFI_SUCCESS The flush request was queued if Event is not
+ NULL.
+ All outstanding data was written correctly to
+ the device if the Event is NULL.
+ @retval EFI_DEVICE_ERROR The device reported an error while writting back
+ the data.
+ @retval EFI_WRITE_PROTECTED The device cannot be written to.
+ @retval EFI_NO_MEDIA There is no media in the device.
+ @retval EFI_MEDIA_CHANGED The MediaId is not for the current media.
+ @retval EFI_OUT_OF_RESOURCES The request could not be completed due to a lack
+ of resources.
+
+**/
+EFI_STATUS
+EFIAPI
+NvmeBlockIoFlushBlocksEx (
+ IN EFI_BLOCK_IO2_PROTOCOL *This,
+ IN OUT EFI_BLOCK_IO2_TOKEN *Token
+ );
+
+/**
Send a security protocol command to a device that receives data and/or the result
of one or more commands sent by SendData.
diff --git a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressDxe.inf b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressDxe.inf index 00acf2b5cf..3270042e02 100644 --- a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressDxe.inf +++ b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressDxe.inf @@ -4,7 +4,7 @@ # NvmExpressDxe driver is used to manage non-volatile memory subsystem which follows
# NVM Express specification.
#
-# Copyright (c) 2013 - 2015, Intel Corporation. All rights reserved.<BR>
+# Copyright (c) 2013 - 2016, Intel Corporation. All rights reserved.<BR>
#
# This program and the accompanying materials
# are licensed and made available under the terms and conditions of the BSD License
@@ -67,6 +67,7 @@ gEfiDevicePathProtocolGuid
gEfiNvmExpressPassThruProtocolGuid ## BY_START
gEfiBlockIoProtocolGuid ## BY_START
+ gEfiBlockIo2ProtocolGuid ## BY_START
gEfiDiskInfoProtocolGuid ## BY_START
gEfiStorageSecurityCommandProtocolGuid ## BY_START
gEfiDriverSupportedEfiVersionProtocolGuid ## PRODUCES
diff --git a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c index e2201b9cef..dcfe1e865d 100644 --- a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c +++ b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressHci.c @@ -682,33 +682,39 @@ NvmeCreateIoCompletionQueue ( EFI_NVM_EXPRESS_COMPLETION Completion;
EFI_STATUS Status;
NVME_ADMIN_CRIOCQ CrIoCq;
-
- ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
- ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));
- ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));
- ZeroMem (&CrIoCq, sizeof(NVME_ADMIN_CRIOCQ));
-
- CommandPacket.NvmeCmd = &Command;
- CommandPacket.NvmeCompletion = &Completion;
-
- Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_CMD;
- CommandPacket.TransferBuffer = Private->CqBufferPciAddr[1];
- CommandPacket.TransferLength = EFI_PAGE_SIZE;
- CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
- CommandPacket.QueueType = NVME_ADMIN_QUEUE;
-
- CrIoCq.Qid = NVME_IO_QUEUE;
- CrIoCq.Qsize = NVME_CCQ_SIZE;
- CrIoCq.Pc = 1;
- CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));
- CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
-
- Status = Private->Passthru.PassThru (
- &Private->Passthru,
- 0,
- &CommandPacket,
- NULL
- );
+ UINT32 Index;
+
+ for (Index = 1; Index < NVME_MAX_QUEUES; Index++) {
+ ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
+ ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));
+ ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));
+ ZeroMem (&CrIoCq, sizeof(NVME_ADMIN_CRIOCQ));
+
+ CommandPacket.NvmeCmd = &Command;
+ CommandPacket.NvmeCompletion = &Completion;
+
+ Command.Cdw0.Opcode = NVME_ADMIN_CRIOCQ_CMD;
+ CommandPacket.TransferBuffer = Private->CqBufferPciAddr[Index];
+ CommandPacket.TransferLength = EFI_PAGE_SIZE;
+ CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
+ CommandPacket.QueueType = NVME_ADMIN_QUEUE;
+
+ CrIoCq.Qid = Index;
+ CrIoCq.Qsize = (Index == 1) ? NVME_CCQ_SIZE : NVME_ASYNC_CCQ_SIZE;
+ CrIoCq.Pc = 1;
+ CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoCq, sizeof (NVME_ADMIN_CRIOCQ));
+ CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
+
+ Status = Private->Passthru.PassThru (
+ &Private->Passthru,
+ 0,
+ &CommandPacket,
+ NULL
+ );
+ if (EFI_ERROR (Status)) {
+ break;
+ }
+ }
return Status;
}
@@ -732,35 +738,41 @@ NvmeCreateIoSubmissionQueue ( EFI_NVM_EXPRESS_COMPLETION Completion;
EFI_STATUS Status;
NVME_ADMIN_CRIOSQ CrIoSq;
-
- ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
- ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));
- ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));
- ZeroMem (&CrIoSq, sizeof(NVME_ADMIN_CRIOSQ));
-
- CommandPacket.NvmeCmd = &Command;
- CommandPacket.NvmeCompletion = &Completion;
-
- Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_CMD;
- CommandPacket.TransferBuffer = Private->SqBufferPciAddr[1];
- CommandPacket.TransferLength = EFI_PAGE_SIZE;
- CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
- CommandPacket.QueueType = NVME_ADMIN_QUEUE;
-
- CrIoSq.Qid = NVME_IO_QUEUE;
- CrIoSq.Qsize = NVME_CSQ_SIZE;
- CrIoSq.Pc = 1;
- CrIoSq.Cqid = NVME_IO_QUEUE;
- CrIoSq.Qprio = 0;
- CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));
- CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
-
- Status = Private->Passthru.PassThru (
- &Private->Passthru,
- 0,
- &CommandPacket,
- NULL
- );
+ UINT32 Index;
+
+ for (Index = 1; Index < NVME_MAX_QUEUES; Index++) {
+ ZeroMem (&CommandPacket, sizeof(EFI_NVM_EXPRESS_PASS_THRU_COMMAND_PACKET));
+ ZeroMem (&Command, sizeof(EFI_NVM_EXPRESS_COMMAND));
+ ZeroMem (&Completion, sizeof(EFI_NVM_EXPRESS_COMPLETION));
+ ZeroMem (&CrIoSq, sizeof(NVME_ADMIN_CRIOSQ));
+
+ CommandPacket.NvmeCmd = &Command;
+ CommandPacket.NvmeCompletion = &Completion;
+
+ Command.Cdw0.Opcode = NVME_ADMIN_CRIOSQ_CMD;
+ CommandPacket.TransferBuffer = Private->SqBufferPciAddr[Index];
+ CommandPacket.TransferLength = EFI_PAGE_SIZE;
+ CommandPacket.CommandTimeout = NVME_GENERIC_TIMEOUT;
+ CommandPacket.QueueType = NVME_ADMIN_QUEUE;
+
+ CrIoSq.Qid = Index;
+ CrIoSq.Qsize = (Index == 1) ? NVME_CSQ_SIZE : NVME_ASYNC_CSQ_SIZE;
+ CrIoSq.Pc = 1;
+ CrIoSq.Cqid = Index;
+ CrIoSq.Qprio = 0;
+ CopyMem (&CommandPacket.NvmeCmd->Cdw10, &CrIoSq, sizeof (NVME_ADMIN_CRIOSQ));
+ CommandPacket.NvmeCmd->Flags = CDW10_VALID | CDW11_VALID;
+
+ Status = Private->Passthru.PassThru (
+ &Private->Passthru,
+ 0,
+ &CommandPacket,
+ NULL
+ );
+ if (EFI_ERROR (Status)) {
+ break;
+ }
+ }
return Status;
}
@@ -844,12 +856,17 @@ NvmeControllerInit ( Private->Cid[0] = 0;
Private->Cid[1] = 0;
+ Private->Cid[2] = 0;
Private->Pt[0] = 0;
Private->Pt[1] = 0;
+ Private->Pt[2] = 0;
Private->SqTdbl[0].Sqt = 0;
Private->SqTdbl[1].Sqt = 0;
+ Private->SqTdbl[2].Sqt = 0;
Private->CqHdbl[0].Cqh = 0;
Private->CqHdbl[1].Cqh = 0;
+ Private->CqHdbl[2].Cqh = 0;
+ Private->AsyncSqHead = 0;
Status = NvmeDisableController (Private);
@@ -878,7 +895,7 @@ NvmeControllerInit ( //
// Address of I/O submission & completion queue.
//
- ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (4));
+ ZeroMem (Private->Buffer, EFI_PAGES_TO_SIZE (6));
Private->SqBuffer[0] = (NVME_SQ *)(UINTN)(Private->Buffer);
Private->SqBufferPciAddr[0] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr);
Private->CqBuffer[0] = (NVME_CQ *)(UINTN)(Private->Buffer + 1 * EFI_PAGE_SIZE);
@@ -887,14 +904,20 @@ NvmeControllerInit ( Private->SqBufferPciAddr[1] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 2 * EFI_PAGE_SIZE);
Private->CqBuffer[1] = (NVME_CQ *)(UINTN)(Private->Buffer + 3 * EFI_PAGE_SIZE);
Private->CqBufferPciAddr[1] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 3 * EFI_PAGE_SIZE);
+ Private->SqBuffer[2] = (NVME_SQ *)(UINTN)(Private->Buffer + 4 * EFI_PAGE_SIZE);
+ Private->SqBufferPciAddr[2] = (NVME_SQ *)(UINTN)(Private->BufferPciAddr + 4 * EFI_PAGE_SIZE);
+ Private->CqBuffer[2] = (NVME_CQ *)(UINTN)(Private->Buffer + 5 * EFI_PAGE_SIZE);
+ Private->CqBufferPciAddr[2] = (NVME_CQ *)(UINTN)(Private->BufferPciAddr + 5 * EFI_PAGE_SIZE);
DEBUG ((EFI_D_INFO, "Private->Buffer = [%016X]\n", (UINT64)(UINTN)Private->Buffer));
- DEBUG ((EFI_D_INFO, "Admin Submission Queue size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));
- DEBUG ((EFI_D_INFO, "Admin Completion Queue size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs));
- DEBUG ((EFI_D_INFO, "Admin Submission Queue (SqBuffer[0]) = [%016X]\n", Private->SqBuffer[0]));
- DEBUG ((EFI_D_INFO, "Admin Completion Queue (CqBuffer[0]) = [%016X]\n", Private->CqBuffer[0]));
- DEBUG ((EFI_D_INFO, "I/O Submission Queue (SqBuffer[1]) = [%016X]\n", Private->SqBuffer[1]));
- DEBUG ((EFI_D_INFO, "I/O Completion Queue (CqBuffer[1]) = [%016X]\n", Private->CqBuffer[1]));
+ DEBUG ((EFI_D_INFO, "Admin Submission Queue size (Aqa.Asqs) = [%08X]\n", Aqa.Asqs));
+ DEBUG ((EFI_D_INFO, "Admin Completion Queue size (Aqa.Acqs) = [%08X]\n", Aqa.Acqs));
+ DEBUG ((EFI_D_INFO, "Admin Submission Queue (SqBuffer[0]) = [%016X]\n", Private->SqBuffer[0]));
+ DEBUG ((EFI_D_INFO, "Admin Completion Queue (CqBuffer[0]) = [%016X]\n", Private->CqBuffer[0]));
+ DEBUG ((EFI_D_INFO, "Sync I/O Submission Queue (SqBuffer[1]) = [%016X]\n", Private->SqBuffer[1]));
+ DEBUG ((EFI_D_INFO, "Sync I/O Completion Queue (CqBuffer[1]) = [%016X]\n", Private->CqBuffer[1]));
+ DEBUG ((EFI_D_INFO, "Async I/O Submission Queue (SqBuffer[2]) = [%016X]\n", Private->SqBuffer[2]));
+ DEBUG ((EFI_D_INFO, "Async I/O Completion Queue (CqBuffer[2]) = [%016X]\n", Private->CqBuffer[2]));
//
// Program admin queue attributes.
@@ -971,7 +994,8 @@ NvmeControllerInit ( DEBUG ((EFI_D_INFO, " NN : 0x%x\n", Private->ControllerData->Nn));
//
- // Create one I/O completion queue.
+ // Create two I/O completion queues.
+ // One for blocking I/O, one for non-blocking I/O.
//
Status = NvmeCreateIoCompletionQueue (Private);
if (EFI_ERROR(Status)) {
@@ -979,7 +1003,8 @@ NvmeControllerInit ( }
//
- // Create one I/O Submission queue.
+ // Create two I/O Submission queues.
+ // One for blocking I/O, one for non-blocking I/O.
//
Status = NvmeCreateIoSubmissionQueue (Private);
if (EFI_ERROR(Status)) {
diff --git a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c index f9871527dd..02219219b6 100644 --- a/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c +++ b/MdeModulePkg/Bus/Pci/NvmExpressDxe/NvmExpressPassthru.c @@ -362,7 +362,7 @@ NvmExpressPassThru ( EFI_PCI_IO_PROTOCOL *PciIo;
NVME_SQ *Sq;
NVME_CQ *Cq;
- UINT8 QueueType;
+ UINT16 QueueId;
UINT32 Bytes;
UINT16 Offset;
EFI_EVENT TimerEvent;
@@ -376,6 +376,8 @@ NvmExpressPassThru ( VOID *PrpListHost;
UINTN PrpListNo;
UINT32 Data;
+ NVME_PASS_THRU_ASYNC_REQ *AsyncRequest;
+ EFI_TPL OldTpl;
//
// check the data fields in Packet parameter.
@@ -403,9 +405,25 @@ NvmExpressPassThru ( TimerEvent = NULL;
Status = EFI_SUCCESS;
- QueueType = Packet->QueueType;
- Sq = Private->SqBuffer[QueueType] + Private->SqTdbl[QueueType].Sqt;
- Cq = Private->CqBuffer[QueueType] + Private->CqHdbl[QueueType].Cqh;
+ if (Packet->QueueType == NVME_ADMIN_QUEUE) {
+ QueueId = 0;
+ } else {
+ if (Event == NULL) {
+ QueueId = 1;
+ } else {
+ QueueId = 2;
+
+ //
+ // Submission queue full check.
+ //
+ if ((Private->SqTdbl[QueueId].Sqt + 1) % (NVME_ASYNC_CSQ_SIZE + 1) ==
+ Private->AsyncSqHead) {
+ return EFI_NOT_READY;
+ }
+ }
+ }
+ Sq = Private->SqBuffer[QueueId] + Private->SqTdbl[QueueId].Sqt;
+ Cq = Private->CqBuffer[QueueId] + Private->CqHdbl[QueueId].Cqh;
if (Packet->NvmeCmd->Nsid != NamespaceId) {
return EFI_INVALID_PARAMETER;
@@ -414,7 +432,7 @@ NvmExpressPassThru ( ZeroMem (Sq, sizeof (NVME_SQ));
Sq->Opc = (UINT8)Packet->NvmeCmd->Cdw0.Opcode;
Sq->Fuse = (UINT8)Packet->NvmeCmd->Cdw0.FusedOperation;
- Sq->Cid = Private->Cid[QueueType]++;
+ Sq->Cid = Private->Cid[QueueId]++;
Sq->Nsid = Packet->NvmeCmd->Nsid;
//
@@ -528,17 +546,45 @@ NvmExpressPassThru ( //
// Ring the submission queue doorbell.
//
- Private->SqTdbl[QueueType].Sqt ^= 1;
- Data = ReadUnaligned32 ((UINT32*)&Private->SqTdbl[QueueType]);
+ if (Event != NULL) {
+ Private->SqTdbl[QueueId].Sqt =
+ (Private->SqTdbl[QueueId].Sqt + 1) % (NVME_ASYNC_CSQ_SIZE + 1);
+ } else {
+ Private->SqTdbl[QueueId].Sqt ^= 1;
+ }
+ Data = ReadUnaligned32 ((UINT32*)&Private->SqTdbl[QueueId]);
PciIo->Mem.Write (
PciIo,
EfiPciIoWidthUint32,
NVME_BAR,
- NVME_SQTDBL_OFFSET(QueueType, Private->Cap.Dstrd),
+ NVME_SQTDBL_OFFSET(QueueId, Private->Cap.Dstrd),
1,
&Data
);
+ //
+ // For non-blocking requests, return directly if the command is placed
+ // in the submission queue.
+ //
+ if (Event != NULL) {
+ AsyncRequest = AllocateZeroPool (sizeof (NVME_PASS_THRU_ASYNC_REQ));
+ if (AsyncRequest == NULL) {
+ Status = EFI_DEVICE_ERROR;
+ goto EXIT;
+ }
+
+ AsyncRequest->Signature = NVME_PASS_THRU_ASYNC_REQ_SIG;
+ AsyncRequest->Packet = Packet;
+ AsyncRequest->CommandId = Sq->Cid;
+ AsyncRequest->CallerEvent = Event;
+
+ OldTpl = gBS->RaiseTPL (TPL_NOTIFY);
+ InsertTailList (&Private->AsyncPassThruQueue, &AsyncRequest->Link);
+ gBS->RestoreTPL (OldTpl);
+
+ return EFI_SUCCESS;
+ }
+
Status = gBS->CreateEvent (
EVT_TIMER,
TPL_CALLBACK,
@@ -561,7 +607,7 @@ NvmExpressPassThru ( //
Status = EFI_TIMEOUT;
while (EFI_ERROR (gBS->CheckEvent (TimerEvent))) {
- if (Cq->Pt != Private->Pt[QueueType]) {
+ if (Cq->Pt != Private->Pt[QueueId]) {
Status = EFI_SUCCESS;
break;
}
@@ -589,16 +635,16 @@ NvmExpressPassThru ( }
}
- if ((Private->CqHdbl[QueueType].Cqh ^= 1) == 0) {
- Private->Pt[QueueType] ^= 1;
+ if ((Private->CqHdbl[QueueId].Cqh ^= 1) == 0) {
+ Private->Pt[QueueId] ^= 1;
}
- Data = ReadUnaligned32 ((UINT32*)&Private->CqHdbl[QueueType]);
+ Data = ReadUnaligned32 ((UINT32*)&Private->CqHdbl[QueueId]);
PciIo->Mem.Write (
PciIo,
EfiPciIoWidthUint32,
NVME_BAR,
- NVME_CQHDBL_OFFSET(QueueType, Private->Cap.Dstrd),
+ NVME_CQHDBL_OFFSET(QueueId, Private->Cap.Dstrd),
1,
&Data
);
|