New dev_top and dev_fault ref manuals, dev_fifo updates

1. Added an important chunk of comments to dev_fifo.ref.txt,
thanks to work by Prateek Srivastaval, as requested by Ben Skeggs.

2. Two new manuals are now published:
    dev_top.ref.txt
    dev_fault.ref.txt

3. Minor updates to pri_mmu_hub.ref.txt

Reviewed by: Maneet Singh

5 files changed, 543 insertions, 2 deletions

diff --git a/manuals/volta/gv100/dev_fault.ref.txt b/manuals/volta/gv100/dev_fault.ref.txt
new file mode 100644
index 0000000..3ae81be
--- /dev/null
+++ b/manuals/volta/gv100/dev_fault.ref.txt
@@ -0,0 +1,129 @@
+Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+--------------------------------------------------------------------------------
+
+#define NV_PFAULT_FAULT_TYPE_PDE                  0x00000000 /*       */
+#define NV_PFAULT_FAULT_TYPE_PDE_SIZE             0x00000001 /*       */
+#define NV_PFAULT_FAULT_TYPE_PTE                  0x00000002 /*       */
+#define NV_PFAULT_FAULT_TYPE_VA_LIMIT_VIOLATION   0x00000003 /*       */
+#define NV_PFAULT_FAULT_TYPE_UNBOUND_INST_BLOCK   0x00000004 /*       */
+#define NV_PFAULT_FAULT_TYPE_PRIV_VIOLATION       0x00000005 /*       */
+#define NV_PFAULT_FAULT_TYPE_RO_VIOLATION         0x00000006 /*       */
+#define NV_PFAULT_FAULT_TYPE_WO_VIOLATION         0x00000007 /*       */
+#define NV_PFAULT_FAULT_TYPE_PITCH_MASK_VIOLATION 0x00000008 /*       */
+#define NV_PFAULT_FAULT_TYPE_WORK_CREATION        0x00000009 /*       */
+#define NV_PFAULT_FAULT_TYPE_UNSUPPORTED_APERTURE 0x0000000a /*       */
+#define NV_PFAULT_FAULT_TYPE_COMPRESSION_FAILURE  0x0000000b /*       */
+#define NV_PFAULT_FAULT_TYPE_UNSUPPORTED_KIND     0x0000000c /*       */
+#define NV_PFAULT_FAULT_TYPE_REGION_VIOLATION     0x0000000d /*       */
+#define NV_PFAULT_FAULT_TYPE_POISONED             0x0000000e /*       */
+#define NV_PFAULT_FAULT_TYPE_ATOMIC_VIOLATION     0x0000000f /*       */
+#define NV_PFAULT_ACCESS_TYPE_READ       0x00000000 /*       */
+#define NV_PFAULT_ACCESS_TYPE_WRITE      0x00000001 /*       */
+#define NV_PFAULT_ACCESS_TYPE_ATOMIC     0x00000002 /*       */
+#define NV_PFAULT_ACCESS_TYPE_PREFETCH   0x00000003 /*       */
+#define NV_PFAULT_ACCESS_TYPE_VIRT_READ          0x00000000 /*       */
+#define NV_PFAULT_ACCESS_TYPE_VIRT_WRITE         0x00000001 /*       */
+#define NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC        0x00000002 /*       */
+#define NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC_STRONG 0x00000002 /*       */
+#define NV_PFAULT_ACCESS_TYPE_VIRT_PREFETCH      0x00000003 /*       */
+#define NV_PFAULT_ACCESS_TYPE_VIRT_ATOMIC_WEAK   0x00000004 /*       */
+#define NV_PFAULT_ACCESS_TYPE_PHYS_READ          0x00000008 /*       */
+#define NV_PFAULT_ACCESS_TYPE_PHYS_WRITE         0x00000009 /*       */
+#define NV_PFAULT_ACCESS_TYPE_PHYS_ATOMIC        0x0000000a /*       */
+#define NV_PFAULT_ACCESS_TYPE_PHYS_PREFETCH      0x0000000b /*       */
+#define NV_PFAULT_MMU_CLIENT_TYPE_GPC    0x00000000 /*       */
+#define NV_PFAULT_MMU_CLIENT_TYPE_HUB    0x00000001 /*       */
+
+--------------------------------------------------------------------------------
+                         KEY LEGEND
+--------------------------------------------------------------------------------
+
+Each define in the .ref file has a 5 field code to say what kind of define it is: i.e. /* RW--R */
+The following legend shows accepted values for each of the 5 fields:
+  Read, Write, Internal State, Declaration/Size, and Define Indicator.
+
+  Read
+    ' ' = Other Information
+    '-' = Field is part of a write-only register
+    'C' = Value read is always the same, constant value line follows (C)
+    'R' = Value is read
+
+
+  Write
+    ' ' = Other Information
+    '-' = Must not be written (D), value ignored when written (R,A,F)
+    'W' = Can be written
+
+
+  Internal State
+    ' ' = Other Information
+    '-' = No internal state
+    'X' = Internal state, initial value is unknown
+    'I' = Internal state, initial value is known and follows (I), see "Reset Signal" section for signal.
+    'E' = Internal state, initial value is known and follows (E), see "Reset Signal" section for signal.
+    'B' = Internal state, initial value is known and follows (B), see "Reset Signal" section for signal.
+    'C' = Internal state, initial value is known and follows (C), see "Reset Signal" section for signal.
+
+    'V' = (legacy) Internal state, initialize at volatile reset
+    'D' = (legacy) Internal state, default initial value at object creation (legacy: Only used in dev_ram.ref)
+    'C' = (legacy) Internal state, initial value at object creation
+    'C' = (legacy) Internal state, class-based initial value at object creation (legacy: Only used in dev_ram.ref)
+
+
+  Declaration/Size
+    ' ' = Other Information
+    '-' = Does Not Apply
+    'V' = Type is void
+    'U' = Type is unsigned integer
+    'S' = Type is signed integer
+    'F' = Type is IEEE floating point
+    '1' = Byte size (008)
+    '2' = Short size (016)
+    '3' = Three byte size (024)
+    '4' = Word size (032)
+    '8' = Double size (064)
+
+
+  Define Indicator
+    ' ' = Other Information
+    'C' = Clear value
+    'D' = Device
+    'L' = Logical device.
+    'M' = Memory
+    'R' = Register
+    'A' = Array of Registers
+    'F' = Field
+    'V' = Value
+    'T' = Task
+    'P' = Phantom Register
+
+    'B' = (legacy) Bundle address
+    'G' = (legacy) General purpose configuration register
+    'C' = (legacy) Class
+
+  Reset signal defaults for graphics engine registers.
+    All graphics engine registers use the following defaults for reset signals:
+     'E' = initialized with engine_reset_
+     'I' = initialized with context_reset_
+     'B' = initialized with reset_IB_dly_
+
+  Reset signal
+    For units that differ from the graphics engine defaults, the reset signals should be defined here:
diff --git a/manuals/volta/gv100/dev_fifo.ref.txt b/manuals/volta/gv100/dev_fifo.ref.txt
index 8b590cb..fe0bea2 100644
--- a/manuals/volta/gv100/dev_fifo.ref.txt
+++ b/manuals/volta/gv100/dev_fifo.ref.txt
@@ -640,6 +640,103 @@ DEALINGS IN THE SOFTWARE.
 #define NV_PFIFO_PBDMA_STATUS_INST_VALID_FALSE                0x00000000 /* R-E-V */
 #define NV_PFIFO_PBDMA_STATUS_INST_VALID_TRUE                 0x00000001 /* R---V */
 
+Channel Teardown Sequence
+===============================================================================
+
+     This section describes the sequence software (specifically RM) can use to
+tear down a channel for robust channels (RC) recovery or in the case of a fault.
+
+     In the case of a fault, Host does not guarantee that a PBDMA has saved out
+prior to RM receiving notification of the fault.  RM must determine which
+context has faulted by processing the fault buffer as described in the
+NV_PFB_PRI_MMU_FAULT_BUFFER_* register documentation in pri_mmu_hub.ref and in
+the fault buffer NV_MMU_FAULT_BUF_ENTRY documentation in dev_mmu_fault.ref.
+This context can then be torn down using the following procedure.
+     Note when a PBDMA fault or CE fault occurs, the PBDMA will save out
+automatically.  The TSG related to the context in which the fault occurred will
+not be scheduled again until the fault is handled.
+     In the case of some other issue requiring the engine to be reset, the TSG
+will need to be manually preempted.
+     In all cases, a PBDMA interrupt may occur prior to the PBDMA being able to
+switch out.  SW must handle these interrupts according to the relevant handling
+procedure before the PBDMA preempt can complete.
+
+Context TSG tear-down procedure:
+
+  1. Disable scheduling for the engine's runlist via NV_PFIFO_SCHED_DISABLE.
+     This enables SW to determine whether a context has hung later in the
+     process: otherwise, ongoing work on the runlist may keep ENG_STATUS from
+     reaching a steady state.
+
+  2. Disable all channels in the TSG being torn down or submit a new runlist
+     that does not contain the TSG.  This is to prevent the TSG from being
+     rescheduled once scheduling is re-enabled in step 6.
+
+  3. Initiate a preempt of the engine by writing the bit associated with its
+     runlist to NV_PFIFO_RUNLIST_PREEMPT.  This allows us to begin the preempt
+     process prior to doing the slow register reads needed to determine whether
+     the context has hit any interrupts or is hung.  Do not poll
+     NV_PFIFO_RUNLIST_PREEMPT for the preempt to complete.
+
+  4. Check for interrupts or hangs while waiting for the preempt to complete.
+     During the below polling, any stalling interrupts relating to the runlist
+     must be detected and handled in order for the preemption to complete.  SW
+     may opt to simply reset the engine immediately, or perform the following
+     sub-steps to more cleanly tear down the context:
+
+    a. Wait for PBDMA preempt completion: For each PBDMA which serves the
+       runlist, poll NV_PFIFO_PBDMA_STATUS(pbdma_id) to reach CHAN_STATUS
+       INVALID, indicating the no further work will run on the PBDMA during the
+       tear-down sequence.  Interleaved with the polling, PBDMA interrupts must
+       be serviced as they arise: such an interrupt can prevent the PBDMA from
+       completing its channel save.
+
+    b. Wait for engine context preempt completion: For each engine served by the
+       runlist, read NV_PFIFO_ENGINE_STATUS(engine_id) to verify the channel/TSG
+       has saved off the engine, or tell if the CTXSW is hung, via the
+       CTX_STATUS, ID, and NEXT_ID fields.  Take action based on the following
+       values for the CTX_STATUS field:
+
+      i. CTX_STATUS_SWITCH: Engine save hasn't started yet, continue to poll
+         (repeat step 4b).
+
+      ii. CTX_STATUS_INVALID: The engine context has switched off.  The
+         preemption step for this engine is complete.
+
+      iii. CTX_STATUS_VALID or CTX_STATUS_CTXSW_SAVE: check the ID field:
+        * If ID matches the TSG for the context being torn down, the engine
+          reset procedure can be performed (see step 5), or SW can continue
+          waiting by repeating step 4b.
+        * If ID does NOT match, then skip engine reset (skip step 5) for this
+          engine.  The context isn't running on the engine.
+
+      iv. CTX_STATUS_LOAD: check the NEXT_ID field:
+        * If NEXT_ID matches the TSG of the context being torn down, the engine
+          is loading the context and reset (see step 5) can be performed
+          immediately or after a delay to allow the context a chance to load and
+          be saved off.
+        * If NEXT_ID does not match the TSG ID or CHID then the context is no
+          longer on the engine.  Skip engine reset (skip step 5) for this
+          engine.
+
+       SW may alternatively wait for the CTX_STATUS to reach INVALID, but this
+       may take longer if an unrelated context is currently on the engine or
+       being switched to.
+
+  5. If a reset is needed as determined by step 4:
+
+    a. Halt the memory interface for the engine (as per the relevant engine
+       procedure).
+
+    b. Reset the engine via NV_PMC_ENABLE.
+
+    c. Take the engine out of reset and re-init the engine (as per the relevant
+       engine procedure)
+
+  6. Re-enable scheduling for the engine's runlist via NV_PFIFO_SCHED_ENABLE.
+
+After this sequence, resources for the channels in the TSG may be reclaimed.
+
 --------------------------------------------------------------------------------
                          KEY LEGEND
 --------------------------------------------------------------------------------
diff --git a/manuals/volta/gv100/dev_top.ref.txt b/manuals/volta/gv100/dev_top.ref.txt
new file mode 100644
index 0000000..e314c50
--- /dev/null
+++ b/manuals/volta/gv100/dev_top.ref.txt
@@ -0,0 +1,315 @@
+Copyright (c) 2019, NVIDIA CORPORATION. All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining a
+copy of this software and associated documentation files (the "Software"),
+to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense,
+and/or sell copies of the Software, and to permit persons to whom the
+Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
+THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+DEALINGS IN THE SOFTWARE.
+--------------------------------------------------------------------------------
+
+#define NV_PTOP                               0x000227FF:0x00022400 /* RW--D */
+#define NV_PTOP_SCAL_NUM_GPCS                                                0x00022430 /* R--4R */
+#define NV_PTOP_SCAL_NUM_GPCS_VALUE                                                 4:0 /* R-IVF */
+#define NV_PTOP_SCAL_NUM_GPCS_VALUE_DEFAULT                                           0 /* R-I-V */
+#define NV_PTOP_SCAL_NUM_TPC_PER_GPC                                         0x00022434 /* R--4R */
+#define NV_PTOP_SCAL_NUM_TPC_PER_GPC_VALUE                                          4:0 /* R-IVF */
+#define NV_PTOP_SCAL_NUM_TPC_PER_GPC_VALUE_DEFAULT                                    0 /* R-I-V */
+#define NV_PTOP_SCAL_NUM_FBPS                                                0x00022438 /* R--4R */
+#define NV_PTOP_SCAL_NUM_FBPS_VALUE                                                 4:0 /* R-IVF */
+#define NV_PTOP_SCAL_NUM_FBPS_VALUE_DEFAULT                                           0 /* R-I-V */
+#define NV_PTOP_SCAL_NUM_FBPAS                                               0x0002243C /* R--4R */
+#define NV_PTOP_SCAL_NUM_FBPAS_VALUE                                                4:0 /* R-IVF */
+#define NV_PTOP_SCAL_NUM_FBPAS_VALUE_DEFAULT                                          0 /* R-I-V */
+#define NV_PTOP_SCAL_NUM_LTC_PER_FBP                                        0x00022450 /* R--4R */
+#define NV_PTOP_SCAL_NUM_LTC_PER_FBP_VALUE                                         4:0 /* R-IVF */
+#define NV_PTOP_SCAL_NUM_LTC_PER_FBP_VALUE_DEFAULT                                   0 /* R-I-V */
+#define NV_PTOP_SCAL_NUM_LTCS                                               0x00022454 /* R--4R */
+#define NV_PTOP_SCAL_NUM_LTCS_VALUE                                                4:0 /* R-IVF */
+#define NV_PTOP_SCAL_NUM_LTCS_VALUE_DEFAULT                                          0 /* R-I-V */
+#define NV_PTOP_SCAL_NUM_FBPA_PER_FBP                                       0x00022458 /* R--4R */
+#define NV_PTOP_SCAL_NUM_FBPA_PER_FBP_VALUE                                        4:0 /* R-IVF */
+#define NV_PTOP_SCAL_NUM_FBPA_PER_FBP_VALUE_DEFAULT                                  0 /* R-I-V */
+#define NV_PTOP_SCAL_NUM_SLICES_PER_LTC                                     0x0002245c /* R--4R */
+#define NV_PTOP_SCAL_NUM_SLICES_PER_LTC_VALUE                                      4:0 /* R-IVF */
+#define NV_PTOP_SCAL_NUM_SLICES_PER_LTC_VALUE_DEFAULT                                0 /* R-I-V */
+
+
+DEVICE_INFO REGISTERS
+===============================================================================
+
+The device info mechanism provides a PRI readable array containing identifying
+information for the devices and engines available on the chip.  Information on a
+device's reset, interrupts, basic type, version registers, engine enumeration,
+and runlist enumeration is included, along with the ability to add additional
+enumerations.
+
+Currently, only Host-driven engines and NVLINK (via IOCTRL) are present in the
+table.
+
+
+DEVICE_INFO is an array of registers, here referred to as entries.  Each device
+in the table is described using one or more entries.  The entries are grouped
+according to the CHAIN field in each entry, which occupies the top bit.  The
+interpretation of the data in an entry is determined by the ENTRY field, which
+is present in the bottom bits of each entry.  Within each entry, valid bits
+control whether each piece of information is applicable to the current device.
+
+CHAIN, when set to ENABLE, indicates the next entry is part of the same device
+as the current entry.  DISABLE means the next entry is either invalid or part of
+a different device.
+
+ENTRY specifies the type of the entry, and defines how that entry is to be
+interpreted:
+
+     * NOT_VALID - the entry is to be ignored
+     * DATA - the entry contains the instance number, PRI base, and fault id for
+              the device
+     * ENUM - the entry provides the engine number, runlist number,
+              interrupt id, and reset id for the device
+     * ENGINE_TYPE - the entry lists the engine type (e.g. "graphics")
+
+Note the device info table does not contain engine versions or class id information.
+
+
+
+NV_PTOP_DEVICE_INFO_ENTRY == ENUM definitions:
+
+     Define prefix: NV_PTOP_DEVICE_INFO
+
+ENGINE_ENUM, when ENGINE is VALID, contains the Host engine ID for the current
+device if it is a Host engine, meaning Host can send methods to the engine.
+This id is used to index into any register array whose __SIZE_1 is equal to
+NV_HOST_NUM_ENGINES.  A given ENGINE_ENUM can be present for at most one device
+in the table.  Devices corresponding to all ENGINE_ENUM ids 0 through
+NV_HOST_NUM_ENGINES - 1 must be present in the device info table.
+
+ENGINE controls whether ENGINE_ENUM applies to the current device.  ENGINE_ENUM
+is valid only when the ENGINE field equals VALID.  If the device is not a Host
+engine, ENGINE will be NOT_VALID and ENGINE_ENUM will be 0.
+
+RUNLIST_ENUM, when RUNLIST is VALID, contains the Host runlist ID on which
+methods for the current device should be submitted if the device is a Host
+engine.  The runlist id is used to index into any register array whose __SIZE_1
+is equal to NV_HOST_NUM_RUNLISTS.  Devices corresponding to all RUNLIST_ENUM ids
+0 through NV_HOST_NUM_RUNLISTS - 1 must be present in the device info table.
+
+RUNLIST controls whether RUNLIST_ENUM applies to the current device.
+RUNLIST_ENUM is valid only when the RUNLIST field equals VALID.  If the device
+is not a Host engine, RUNLIST will be NOT_VALID and RUNLIST_ENUM will be 0.
+
+INTR_ENUM, when INTR is VALID, contains the INTR id for the current device.
+The INTR id is used to index into the NV_PMC_INTR_*_DEVICE register bitfields:
+NV_PMC_INTR_DEVICE, NV_PMC_INTR_EN_DEVICE, NV_PMC_INTR_EN_SET_DEVICE,
+NV_PMC_INTR_EN_CLEAR_DEVICE.
+
+INTR controls whether INTR_ENUM applies to the current device.  INTR_ENUM is
+valid only when the INTR field equals VALID.  If the device has no interrupt,
+INTR will be NOT_VALID and INTR_ENUM will be 0.
+
+RESET_ENUM, when RESET is VALID, contains the reset id for the current device.
+It is used to index into the NV_PMC_ENABLE_DEVICE(i),
+NV_PMC_ELPG_ENABLE_DEVICE(i) register bitfields.
+
+RESET controls whether RESET_ENUM applies to the current device.  RESET_ENUM is
+valid only when the RESET field equals VALID.  If there is no reset specific to
+the device, RESET will be NOT_VALID and RESET_ENUM will be 0.
+
+
+
+NV_PTOP_DEVICE_INFO_ENTRY == ENGINE_TYPE definitions:
+
+     Define prefix: NV_PTOP_DEVICE_INFO
+
+TYPE_ENUM specifies the engine type for the current device when that device is a
+Host engine.  The enumeration is defined by software for software.  Hardware
+does not use these values in any way.  When new engine types are defined, software
+must provide a new enumeration value for the engine.  This data is not used for
+class codes, engine versions, etc.  It is used for generic descriptions of the
+engine, like graphics, or copy engine.  A Host engine can be uniquely identified
+by its TYPE_ENUM:INST_ID pair; see the DATA entry interpretation for INST_ID.
+
+TYPE_ENUM values are expected to stay constant across GPUs and architectures.
+
+
+
+NV_PTOP_DEVICE_INFO_ENTRY == DATA definitions:
+
+     Define prefix: NV_PTOP_DEVICE_INFO_DATA
+
+TYPE defines how to interpret the fields in the DATA group.  However
+today, there is only one available interpretation which is ENUM2.  In
+the future there may be additional interpretations.
+
+INST_ID specifies the instance of a device, allowing software to distinguish
+between multiple copies of a device present on the same chip.  Devices that do
+not support instancing will report as the zeroth instance, so there is no
+associated VALID field.
+
+PRI_BASE allows SW to determine the BAR0 offset for registers in the device:
+
+  BAR0 base = (PRI_BASE << NV_PTOP_DEVICE_INFO_DATA_PRI_BASE_ALIGN)
+
+The field is designed such that the low order bit of PRI_BASE is already aligned
+appropriately within the entry register at bit position
+NV_PTOP_DEVICE_INFO_DATA_PRI_BASE_ALIGN.  The size of the range is at a minimum
+4kB but should be implied by NV_PTOP_DEVICE_INFO_TYPE_ENUM or can be defined in
+the first 4kB of the device itself.  Note that some instanced devices (such as
+logical copy engines aka LCE) share a PRI_BASE across all devices of the same
+engine type; such devices require an additional offset:
+
+  instanced base = BAR0 base + stride * INST_ID
+
+Every device must have a valid PRI_BASE defined.
+
+FAULT_ID_ENUM, when FAULT_ID is VALID, contains the MMU fault id used by this
+device.  These ids correspond to the NV_PFAULT_MMU_ENG_ID define list.
+
+FAULT_ID controls whether FAULT_ID_ENUM applies to the current device.
+FAULT_ID_ENUM is valid only when the FAULT_ID field equals VALID.  If the device
+does not have its own bind point to the MMU, FAULT_ID will be NOT_VALID and
+FAULT_ID_ENUM will be 0.
+
+
+#define NV_PTOP_DEVICE_INFO(i)                         (0x00022700+(i)*4) /* R--4A */
+#define NV_PTOP_DEVICE_INFO__SIZE_1                                    64 /*       */
+#define NV_PTOP_DEVICE_INFO_VALUE                                    31:0 /* R--VF */
+#define NV_PTOP_DEVICE_INFO_CHAIN                                   31:31 /*       */
+#define NV_PTOP_DEVICE_INFO_CHAIN_ENABLE                                1 /*       */
+#define NV_PTOP_DEVICE_INFO_CHAIN_DISABLE                               0 /*       */
+#define NV_PTOP_DEVICE_INFO_ENGINE_ENUM                             29:26 /*       */
+#define NV_PTOP_DEVICE_INFO_RUNLIST_ENUM                            24:21 /*       */
+#define NV_PTOP_DEVICE_INFO_INTR_ENUM                               19:15 /*       */
+#define NV_PTOP_DEVICE_INFO_RESET_ENUM                               13:9 /*       */
+#define NV_PTOP_DEVICE_INFO_ENGINE                                    5:5 /*       */
+#define NV_PTOP_DEVICE_INFO_ENGINE_VALID                              0x1 /*       */
+#define NV_PTOP_DEVICE_INFO_ENGINE_NOT_VALID                          0x0 /*       */
+#define NV_PTOP_DEVICE_INFO_RUNLIST                                   4:4 /*       */
+#define NV_PTOP_DEVICE_INFO_RUNLIST_VALID                             0x1 /*       */
+#define NV_PTOP_DEVICE_INFO_RUNLIST_NOT_VALID                         0x0 /*       */
+#define NV_PTOP_DEVICE_INFO_INTR                                      3:3 /*       */
+#define NV_PTOP_DEVICE_INFO_INTR_VALID                                0x1 /*       */
+#define NV_PTOP_DEVICE_INFO_INTR_NOT_VALID                            0x0 /*       */
+#define NV_PTOP_DEVICE_INFO_RESET                                     2:2 /*       */
+#define NV_PTOP_DEVICE_INFO_RESET_VALID                               0x1 /*       */
+#define NV_PTOP_DEVICE_INFO_RESET_NOT_VALID                           0x0 /*       */
+#define NV_PTOP_DEVICE_INFO_TYPE_ENUM                          30:2 /*       */
+#define NV_PTOP_DEVICE_INFO_TYPE_ENUM_GRAPHICS                 0 /*       */
+#define NV_PTOP_DEVICE_INFO_TYPE_ENUM_NVDEC                    16 /*       */
+#define NV_PTOP_DEVICE_INFO_TYPE_ENUM_NVENC                    14 /*       */
+#define NV_PTOP_DEVICE_INFO_TYPE_ENUM_SEC                      13 /*       */
+#define NV_PTOP_DEVICE_INFO_TYPE_ENUM_LCE                      19 /*       */
+#define NV_PTOP_DEVICE_INFO_TYPE_ENUM_GSP                      20 /*       */
+#define NV_PTOP_DEVICE_INFO_TYPE_ENUM_IOCTRL                   18 /*       */
+#define NV_PTOP_DEVICE_INFO_TYPE_ENUM_MSPDEC                   8 /*       */
+#define NV_PTOP_DEVICE_INFO_TYPE_ENUM_MSPPP                    9 /*       */
+#define NV_PTOP_DEVICE_INFO_TYPE_ENUM_MSVLD                    10 /*       */
+#define NV_PTOP_DEVICE_INFO_TYPE_ENUM_MSENC                    11 /*       */
+#define NV_PTOP_DEVICE_INFO_TYPE_ENUM_VIC                      12 /*       */
+#define NV_PTOP_DEVICE_INFO_TYPE_ENUM_COPY0                    1 /*       */
+#define NV_PTOP_DEVICE_INFO_TYPE_ENUM_COPY1                    2 /*       */
+#define NV_PTOP_DEVICE_INFO_TYPE_ENUM_COPY2                    3 /*       */
+#define NV_PTOP_DEVICE_INFO_TYPE_ENUM_NVENC0                   14 /*       */
+#define NV_PTOP_DEVICE_INFO_TYPE_ENUM_NVENC1                   15 /*       */
+#define NV_PTOP_DEVICE_INFO_DATA                                     30:2 /*       */
+#define NV_PTOP_DEVICE_INFO_DATA_TYPE                                30:30 /*       */
+#define NV_PTOP_DEVICE_INFO_DATA_TYPE_ENUM2                          0 /*       */
+#define NV_PTOP_DEVICE_INFO_DATA_INST_ID                             29:26 /*       */
+#define NV_PTOP_DEVICE_INFO_DATA_PRI_BASE                            23:12      /*       */
+#define NV_PTOP_DEVICE_INFO_DATA_PRI_BASE_ALIGN                      12      /*       */
+#define NV_PTOP_DEVICE_INFO_DATA_FAULT_ID_ENUM                       9:3 /*       */
+#define NV_PTOP_DEVICE_INFO_DATA_FAULT_ID                            2:2      /*       */
+#define NV_PTOP_DEVICE_INFO_DATA_FAULT_ID_VALID                               0x1 /*       */
+#define NV_PTOP_DEVICE_INFO_DATA_FAULT_ID_NOT_VALID                           0x0 /*       */
+#define NV_PTOP_DEVICE_INFO_ENTRY                                     1:0 /*       */
+#define NV_PTOP_DEVICE_INFO_ENTRY_NOT_VALID                             0 /*       */
+#define NV_PTOP_DEVICE_INFO_ENTRY_DATA                                  1 /*       */
+#define NV_PTOP_DEVICE_INFO_ENTRY_ENUM                                  2 /*       */
+#define NV_PTOP_DEVICE_INFO_ENTRY_ENGINE_TYPE                           3 /*       */
+
+--------------------------------------------------------------------------------
+                         KEY LEGEND
+--------------------------------------------------------------------------------
+
+Each define in the .ref file has a 5 field code to say what kind of define it is: i.e. /* RW--R */
+The following legend shows accepted values for each of the 5 fields:
+  Read, Write, Internal State, Declaration/Size, and Define Indicator.
+
+  Read
+    ' ' = Other Information
+    '-' = Field is part of a write-only register
+    'C' = Value read is always the same, constant value line follows (C)
+    'R' = Value is read
+
+
+  Write
+    ' ' = Other Information
+    '-' = Must not be written (D), value ignored when written (R,A,F)
+    'W' = Can be written
+
+
+  Internal State
+    ' ' = Other Information
+    '-' = No internal state
+    'X' = Internal state, initial value is unknown
+    'I' = Internal state, initial value is known and follows (I), see "Reset Signal" section for signal.
+    'E' = Internal state, initial value is known and follows (E), see "Reset Signal" section for signal.
+    'B' = Internal state, initial value is known and follows (B), see "Reset Signal" section for signal.
+    'C' = Internal state, initial value is known and follows (C), see "Reset Signal" section for signal.
+
+    'V' = (legacy) Internal state, initialize at volatile reset
+    'D' = (legacy) Internal state, default initial value at object creation (legacy: Only used in dev_ram.ref)
+    'C' = (legacy) Internal state, initial value at object creation
+    'C' = (legacy) Internal state, class-based initial value at object creation (legacy: Only used in dev_ram.ref)
+
+
+  Declaration/Size
+    ' ' = Other Information
+    '-' = Does Not Apply
+    'V' = Type is void
+    'U' = Type is unsigned integer
+    'S' = Type is signed integer
+    'F' = Type is IEEE floating point
+    '1' = Byte size (008)
+    '2' = Short size (016)
+    '3' = Three byte size (024)
+    '4' = Word size (032)
+    '8' = Double size (064)
+
+
+  Define Indicator
+    ' ' = Other Information
+    'C' = Clear value
+    'D' = Device
+    'L' = Logical device.
+    'M' = Memory
+    'R' = Register
+    'A' = Array of Registers
+    'F' = Field
+    'V' = Value
+    'T' = Task
+    'P' = Phantom Register
+
+    'B' = (legacy) Bundle address
+    'G' = (legacy) General purpose configuration register
+    'C' = (legacy) Class
+
+  Reset signal defaults for graphics engine registers.
+    All graphics engine registers use the following defaults for reset signals:
+     'E' = initialized with engine_reset_
+     'I' = initialized with context_reset_
+     'B' = initialized with reset_IB_dly_
+
+  Reset signal
+    For units that differ from the graphics engine defaults, the reset signals should be defined here:
diff --git a/manuals/volta/gv100/index.html b/manuals/volta/gv100/index.html
index fcada35..07a1dac 100644
--- a/manuals/volta/gv100/index.html
+++ b/manuals/volta/gv100/index.html
@@ -4,12 +4,14 @@
         <h1>gv100</h1>
         <a href="dev_bus.ref.txt">dev_bus.ref.txt</a><br/>
         <a href="dev_display_withoffset.ref.txt">dev_display_withoffset.ref.txt</a><br/>
+        <a href="dev_fault.ref.txt">dev_fault.ref.txt</a><br/>
         <a href="dev_fifo.ref.txt">dev_fifo.ref.txt</a><br/>
         <a href="dev_master.ref.txt">dev_master.ref.txt</a><br/>
         <a href="dev_mmu_fault.ref.txt">dev_mmu_fault.ref.txt</a><br/>
         <a href="dev_pbdma.ref.txt">dev_pbdma.ref.txt</a><br/>
         <a href="dev_ram.ref.txt">dev_ram.ref.txt</a><br/>
         <a href="dev_timer.ref.txt">dev_timer.ref.txt</a><br/>
+        <a href="dev_top.ref.txt">dev_top.ref.txt</a><br/>
         <a href="dev_usermode.ref.txt">dev_usermode.ref.txt</a><br/>
         <a href="pri_mmu_both.ref.txt">pri_mmu_both.ref.txt</a><br/>
         <a href="pri_mmu_gpc.ref.txt">pri_mmu_gpc.ref.txt</a><br/>
diff --git a/manuals/volta/gv100/pri_mmu_hub.ref.txt b/manuals/volta/gv100/pri_mmu_hub.ref.txt
index 3c47294..3c334f5 100644
--- a/manuals/volta/gv100/pri_mmu_hub.ref.txt
+++ b/manuals/volta/gv100/pri_mmu_hub.ref.txt
@@ -34,8 +34,6 @@ DEALINGS IN THE SOFTWARE.
 #define NV_PFB_PRI_MMU_BIND_IMB_ADDR                                      31:4 /* RWXVF */
 #define NV_PFB_PRI_MMU_BIND_IMB_ADDR_ALIGNMENT                      0x0000000c /*       */
 #define NV_PFB_PRI_MMU_BIND                                         0x00100CB0 /* RW-4R */
-#define NV_PFB_PRI_MMU_BIND_ENGINE_ID                                      7:0 /* RWEVF */
-#define NV_PFB_PRI_MMU_BIND_ENGINE_ID_INIT                          0x00000000 /* RWE-V */
 #define NV_PFB_PRI_MMU_BIND_UPPER_IMB_ADDR                                25:8 /* RWEVF */
 #define NV_PFB_PRI_MMU_BIND_UPPER_IMB_ADDR_INIT                     0x00000000 /* RWE-V */
 #define NV_PFB_PRI_MMU_BIND_OP                                           30:29 /* RWEVF */