x86: Rework how VEX prefixes are decoded.

Remove redundant information from the ExtMachInst, hash the vex
information to ensure the decode cache works properly, print the vex info
when printing an ExtMachInst, consider the vex info when comparing two
ExtMachInsts, fold the info from the vex prefixes into existing settings,
remove redundant decode code, handle vex prefixes one byte at a time and
don't bother building up the entire prefix, and let instructions that care
about vex use it in their implementation, instead of developing an entire
parallel decode tree.

This also eliminates the error prone vex immediate decode table which was
incomplete and would result in an out of bounds access for incorrectly
encoded instructions or when the CPU was mispeculating, as it was (as far
as I can tell) redundant with the tables that already existed for two and
three byte opcodes. There were differences, but I think those may have
been mistakes based on the documentation I found.

Also, in 32 bit mode, the VEX prefixes might actually be LDS or LES
instructions which are still legal in that mode. A valid VEX prefix would
look like an LDS/LES with an otherwise invalid modrm encoding, so use that
as a signal to abort processing the VEX and turn the instruction into an
LES/LDS as appropriate.

Change-Id: Icb367eaaa35590692df1c98862f315da4c139f5c
Reviewed-on: https://gem5-review.googlesource.com/3501
Reviewed-by: Joe Gross <joe.gross@amd.com>
Reviewed-by: Jason Lowe-Power <jason@lowepower.com>
Maintainer: Anthony Gutierrez <anthony.gutierrez@amd.com>

1 files changed, 46 insertions, 50 deletions

diff --git a/src/arch/x86/types.hh b/src/arch/x86/types.hh
index 8d47b7efb..6e1b1cf2f 100644
--- a/src/arch/x86/types.hh
+++ b/src/arch/x86/types.hh
@@ -106,47 +106,45 @@ namespace X86ISA
         Bitfield<0> b;
     EndBitUnion(Rex)
 
-    BitUnion(uint32_t, ThreeByteVex)
-        Bitfield<7,0> zero;
-        SubBitUnion(first, 15, 8)
-            // Inverted one-bit extension of ModRM reg field
-            Bitfield<15> r;
-            // Inverted one-bit extension of SIB index field
-            Bitfield<14> x;
-            // Inverted one-bit extension, r/m field or SIB base field
-            Bitfield<13> b;
-            // Opcode map select
-            Bitfield<12, 8> map_select;
-        EndSubBitUnion(first)
-        SubBitUnion(second, 23, 16)
-            // Default operand size override for a general purpose register to
-            // 64-bit size in 64-bit mode; operand configuration specifier for
-            // certain YMM/XMM-based operations.
-            Bitfield<23> w;
-            // Source or destination register selector, in ones' complement
-            // format
-            Bitfield<22, 19>  vvvv;
-            // Vector length specifier
-            Bitfield<18> l;
-            // Implied 66, F2, or F3 opcode extension
-            Bitfield<17, 16> pp;
-        EndSubBitUnion(second)
-    EndBitUnion(ThreeByteVex)
-
-    BitUnion16(TwoByteVex)
-        Bitfield<7,0> zero;
-        SubBitUnion(first, 15, 8)
-            // Inverted one-bit extension of ModRM reg field
-            Bitfield<15> r;
-            // Source or destination register selector, in ones' complement
-            // format
-            Bitfield<14, 11>  vvvv;
-            // Vector length specifier
-            Bitfield<10> l;
-            // Implied 66, F2, or F3 opcode extension
-            Bitfield<9, 8> pp;
-        EndSubBitUnion(first)
-    EndBitUnion(TwoByteVex)
+    BitUnion8(Vex2Of3)
+        // Inverted bits from the REX prefix.
+        Bitfield<7> r;
+        Bitfield<6> x;
+        Bitfield<5> b;
+        // Selector for what would be two or three byte opcode types.
+        Bitfield<4, 0> m;
+    EndBitUnion(Vex2Of3)
+
+    BitUnion8(Vex3Of3)
+        // Bit from the REX prefix.
+        Bitfield<7> w;
+        // Inverted extra register index.
+        Bitfield<6, 3>  v;
+        // Vector length specifier.
+        Bitfield<2> l;
+        // Implied 66, F2, or F3 opcode prefix.
+        Bitfield<1, 0> p;
+    EndBitUnion(Vex3Of3)
+
+    BitUnion8(Vex2Of2)
+        // Inverted bit from the REX prefix.
+        Bitfield<7> r;
+        // Inverted extra register index.
+        Bitfield<6, 3>  v;
+        // Vector length specifier
+        Bitfield<2> l;
+        // Implied 66, F2, or F3 opcode prefix.
+        Bitfield<1, 0> p;
+    EndBitUnion(Vex2Of2)
+
+    BitUnion8(VexInfo)
+        // Extra register index.
+        Bitfield<6, 3> v;
+        // Vector length specifier.
+        Bitfield<2> l;
+        // Whether the VEX prefix was used.
+        Bitfield<0> present;
+    EndBitUnion(VexInfo)
 
     enum OpcodeType {
         BadOpcode,
@@ -154,7 +152,6 @@ namespace X86ISA
         TwoByteOpcode,
         ThreeByte0F38Opcode,
         ThreeByte0F3AOpcode,
-        Vex,
     };
 
     static inline const char *
@@ -171,8 +168,6 @@ namespace X86ISA
             return "three byte 0f38";
           case ThreeByte0F3AOpcode:
             return "three byte 0f3a";
-          case Vex:
-            return "vex";
           default:
             return "unrecognized!";
         }
@@ -207,9 +202,7 @@ namespace X86ISA
         //Prefixes
         LegacyPrefixVector legacy;
         Rex rex;
-        // We use the following field for encoding both two byte and three byte
-        // escape sequences
-        ThreeByteVex vex;
+        VexInfo vex;
 
         //This holds all of the bytes of the opcode
         struct
@@ -248,7 +241,7 @@ namespace X86ISA
                      "immediate = %#x,\n\tdisplacement = %#x\n\t"
                      "dispSize = %d}\n",
                      (uint8_t)emi.legacy, (uint8_t)emi.rex,
-                     (uint32_t)emi.vex,
+                     (uint8_t)emi.vex,
                      opcodeTypeToStr(emi.opcode.type), (uint8_t)emi.opcode.op,
                      (uint8_t)emi.modRM, (uint8_t)emi.sib,
                      emi.immediate, emi.displacement, emi.dispSize);
@@ -262,6 +255,8 @@ namespace X86ISA
             return false;
         if (emi1.rex != emi2.rex)
             return false;
+        if (emi1.vex != emi2.vex)
+            return false;
         if (emi1.opcode.type != emi2.opcode.type)
             return false;
         if (emi1.opcode.op != emi2.opcode.op)
@@ -357,8 +352,9 @@ namespace std {
     template<>
     struct hash<X86ISA::ExtMachInst> {
         size_t operator()(const X86ISA::ExtMachInst &emi) const {
-            return (((uint64_t)emi.legacy << 40) |
-                    ((uint64_t)emi.rex  << 32) |
+            return (((uint64_t)emi.legacy << 48) |
+                    ((uint64_t)emi.rex << 40) |
+                    ((uint64_t)emi.vex << 32) |
                     ((uint64_t)emi.modRM << 24) |
                     ((uint64_t)emi.sib << 16) |
                     ((uint64_t)emi.opcode.type << 8) |