Fix DX12 Capture transition state handling for MSAA (#532)

ScreenGrab/ScreenGrab9.cpp

@@ -20,6 +20,7 @@
 #include <algorithm>
 #include <cassert>
 #include <cstddef>
+#include <cstdint>
 #include <cstring>
 #include <memory>
 #include <new>
@@ -63,9 +64,9 @@ using Microsoft::WRL::ComPtr;
 //--------------------------------------------------------------------------------------
 namespace
 {
-#pragma pack(push,1)
+    #pragma pack(push,1)
 
-#define DDS_MAGIC 0x20534444 // "DDS "
+    constexpr uint32_t DDS_MAGIC = 0x20534444; // "DDS "
 
     struct DDS_PIXELFORMAT
     {
@@ -79,22 +80,22 @@ namespace
         uint32_t    ABitMask;
     };
 
-#define DDS_FOURCC        0x00000004  // DDPF_FOURCC
-#define DDS_RGB           0x00000040  // DDPF_RGB
-#define DDS_RGBA          0x00000041  // DDPF_RGB | DDPF_ALPHAPIXELS
-#define DDS_LUMINANCE     0x00020000  // DDPF_LUMINANCE
-#define DDS_LUMINANCEA    0x00020001  // DDPF_LUMINANCE | DDPF_ALPHAPIXELS
-#define DDS_ALPHA         0x00000002  // DDPF_ALPHA
-#define DDS_BUMPLUMINANCE 0x00040000  // DDPF_BUMPLUMINANCE
-#define DDS_BUMPDUDV      0x00080000  // DDPF_BUMPDUDV
-#define DDS_BUMPDUDVA     0x00080001  // DDPF_BUMPDUDV | DDPF_ALPHAPIXELS
+    #define DDS_FOURCC        0x00000004  // DDPF_FOURCC
+    #define DDS_RGB           0x00000040  // DDPF_RGB
+    #define DDS_RGBA          0x00000041  // DDPF_RGB | DDPF_ALPHAPIXELS
+    #define DDS_LUMINANCE     0x00020000  // DDPF_LUMINANCE
+    #define DDS_LUMINANCEA    0x00020001  // DDPF_LUMINANCE | DDPF_ALPHAPIXELS
+    #define DDS_ALPHA         0x00000002  // DDPF_ALPHA
+    #define DDS_BUMPLUMINANCE 0x00040000  // DDPF_BUMPLUMINANCE
+    #define DDS_BUMPDUDV      0x00080000  // DDPF_BUMPDUDV
+    #define DDS_BUMPDUDVA     0x00080001  // DDPF_BUMPDUDV | DDPF_ALPHAPIXELS
 
-#define DDS_HEADER_FLAGS_TEXTURE        0x00001007  // DDSD_CAPS | DDSD_HEIGHT | DDSD_WIDTH | DDSD_PIXELFORMAT
-#define DDS_HEADER_FLAGS_MIPMAP         0x00020000  // DDSD_MIPMAPCOUNT
-#define DDS_HEADER_FLAGS_PITCH          0x00000008  // DDSD_PITCH
-#define DDS_HEADER_FLAGS_LINEARSIZE     0x00080000  // DDSD_LINEARSIZE
+    #define DDS_HEADER_FLAGS_TEXTURE        0x00001007  // DDSD_CAPS | DDSD_HEIGHT | DDSD_WIDTH | DDSD_PIXELFORMAT
+    #define DDS_HEADER_FLAGS_MIPMAP         0x00020000  // DDSD_MIPMAPCOUNT
+    #define DDS_HEADER_FLAGS_PITCH          0x00000008  // DDSD_PITCH
+    #define DDS_HEADER_FLAGS_LINEARSIZE     0x00080000  // DDSD_LINEARSIZE
 
-#define DDS_SURFACE_FLAGS_TEXTURE 0x00001000 // DDSCAPS_TEXTURE
+    #define DDS_SURFACE_FLAGS_TEXTURE 0x00001000 // DDSCAPS_TEXTURE
 
     struct DDS_HEADER
     {
@@ -114,7 +115,12 @@ namespace
         uint32_t        reserved2;
     };
 
-#pragma pack(pop)
+    #pragma pack(pop)
+
+    static_assert(sizeof(DDS_PIXELFORMAT) == 32, "DDS pixel format size mismatch");
+    static_assert(sizeof(DDS_HEADER) == 124, "DDS Header size mismatch");
+
+    constexpr size_t DDS_DX9_HEADER_SIZE = sizeof(uint32_t) + sizeof(DDS_HEADER);
 
     const DDS_PIXELFORMAT DDSPF_DXT1 =
     { sizeof(DDS_PIXELFORMAT), DDS_FOURCC, MAKEFOURCC('D','X','T','1'), 0, 0, 0, 0, 0 };
@@ -606,13 +612,11 @@ HRESULT DirectX::SaveDDSTextureToFile(
     auto_delete_file delonfail(hFile.get());
 
     // Setup header
-    constexpr size_t MAX_HEADER_SIZE = sizeof(uint32_t) + sizeof(DDS_HEADER);
-    uint8_t fileHeader[MAX_HEADER_SIZE] = {};
+    uint8_t fileHeader[DDS_DX9_HEADER_SIZE] = {};
 
     *reinterpret_cast<uint32_t*>(&fileHeader[0]) = DDS_MAGIC;
 
     auto header = reinterpret_cast<DDS_HEADER*>(&fileHeader[0] + sizeof(uint32_t));
-    constexpr size_t headerSize = sizeof(uint32_t) + sizeof(DDS_HEADER);
     header->size = sizeof(DDS_HEADER);
     header->flags = DDS_HEADER_FLAGS_TEXTURE | DDS_HEADER_FLAGS_MIPMAP;
     header->height = desc.Height;
@@ -726,10 +730,10 @@ HRESULT DirectX::SaveDDSTextureToFile(
 
     // Write header & pixels
     DWORD bytesWritten;
-    if (!WriteFile(hFile.get(), fileHeader, static_cast<DWORD>(headerSize), &bytesWritten, nullptr))
+    if (!WriteFile(hFile.get(), fileHeader, static_cast<DWORD>(DDS_DX9_HEADER_SIZE), &bytesWritten, nullptr))
         return HRESULT_FROM_WIN32(GetLastError());
 
-    if (bytesWritten != headerSize)
+    if (bytesWritten != DDS_DX9_HEADER_SIZE)
         return E_FAIL;
 
     if (!WriteFile(hFile.get(), pixels.get(), static_cast<DWORD>(slicePitch), &bytesWritten, nullptr))