Fix CrytoPkg issue in GCC X64 tip.

Signed-off-by: tye
Reviewed-by: gdong1
Reviewed-by: ftian


git-svn-id: https://edk2.svn.sourceforge.net/svnroot/edk2/trunk/edk2@12784 6f19259b-4bc3-4df7-8a09-765794883524

3 files changed, 91 insertions, 19 deletions

diff --git a/CryptoPkg/Library/BaseCryptLib/Pk/CryptPkcs7.c b/CryptoPkg/Library/BaseCryptLib/Pk/CryptPkcs7.c
index 47bab1004b..83f0dfcd5a 100644
--- a/CryptoPkg/Library/BaseCryptLib/Pk/CryptPkcs7.c
+++ b/CryptoPkg/Library/BaseCryptLib/Pk/CryptPkcs7.c
@@ -190,6 +190,7 @@ Pkcs7Sign (
   //

   Key = EVP_PKEY_new ();

   if (Key == NULL) {

+    Status = FALSE;

     goto _Exit;

   }

   Key->save_type = EVP_PKEY_RSA;

@@ -213,6 +214,7 @@ Pkcs7Sign (
             PKCS7_BINARY | PKCS7_NOATTR | PKCS7_DETACHED

             );

   if (Pkcs7 == NULL) {

+    Status = FALSE;

     goto _Exit;

   }

 

@@ -221,9 +223,16 @@ Pkcs7Sign (
   //

   P7DataSize = i2d_PKCS7 (Pkcs7, NULL);

   if (P7DataSize <= 19) {

+    Status = FALSE;

     goto _Exit;

   }

+

   P7Data     = OPENSSL_malloc (P7DataSize);

+  if (P7Data == NULL) {

+    Status = FALSE;

+    goto _Exit;

+  }

+

   Tmp        = P7Data;

   P7DataSize = i2d_PKCS7 (Pkcs7, (unsigned char **) &Tmp);

 

@@ -233,6 +242,12 @@ Pkcs7Sign (
   //

   *SignedDataSize = P7DataSize - 19;

   *SignedData     = OPENSSL_malloc (*SignedDataSize);

+  if (*SignedData == NULL) {

+    Status = FALSE;

+    OPENSSL_free (P7Data);

+    goto _Exit;

+  }

+

   CopyMem (*SignedData, P7Data + 19, *SignedDataSize);

   

   OPENSSL_free (P7Data);

diff --git a/CryptoPkg/Library/OpensslLib/EDKII_openssl-0.9.8l.patch b/CryptoPkg/Library/OpensslLib/EDKII_openssl-0.9.8l.patch
index 231ec6e23a..d14b08e770 100644
--- a/CryptoPkg/Library/OpensslLib/EDKII_openssl-0.9.8l.patch
+++ b/CryptoPkg/Library/OpensslLib/EDKII_openssl-0.9.8l.patch
@@ -24,6 +24,59 @@
  #endif /* OPENSSL_NO_STDIO */

  

  #endif /* HEADER_BSS_FILE_C */

+--- crypto/err/err.c

++++ crypto/err/err.c

+@@ -313,7 +313,12 @@

+ 	es->err_data_flags[i]=flags;

+ 	}

+ 

++/* Add EFIAPI for UEFI version. */

++#if defined(OPENSSL_SYS_UEFI)

++void EFIAPI ERR_add_error_data(int num, ...)

++#else

+ void ERR_add_error_data(int num, ...)

++#endif

+ 	{

+ 	va_list args;

+ 	int i,n,s;

+--- crypto/err/err.h

++++ crypto/err/err.h

+@@ -286,8 +286,14 @@

+ #endif

+ #ifndef OPENSSL_NO_BIO

+ void ERR_print_errors(BIO *bp);

++

++/* Add EFIAPI for UEFI version. */

++#if defined(OPENSSL_SYS_UEFI)

++void EFIAPI ERR_add_error_data(int num, ...);

++#else

+ void ERR_add_error_data(int num, ...);

+ #endif

++#endif

+ void ERR_load_strings(int lib,ERR_STRING_DATA str[]);

+ void ERR_unload_strings(int lib,ERR_STRING_DATA str[]);

+ void ERR_load_ERR_strings(void);

+--- crypto/opensslconf.h

++++ crypto/opensslconf.h

+@@ -162,6 +162,9 @@

+ /* The prime number generation stuff may not work when

+  * EIGHT_BIT but I don't care since I've only used this mode

+  * for debuging the bignum libraries */

++

++/* Bypass following definition for UEFI version. */

++#if !defined(OPENSSL_SYS_UEFI)

+ #undef SIXTY_FOUR_BIT_LONG

+ #undef SIXTY_FOUR_BIT

+ #define THIRTY_TWO_BIT

+@@ -169,6 +172,8 @@

+ #undef EIGHT_BIT

+ #endif

+ 

++#endif

++

+ #if defined(HEADER_RC4_LOCL_H) && !defined(CONFIG_HEADER_RC4_LOCL_H)

+ #define CONFIG_HEADER_RC4_LOCL_H

+ /* if this is defined data[i] is used instead of *data, this is a %20

 --- crypto/pkcs7/pk7_smime.c	2009-03-15 21:36:02.000000000 +0800

 +++ crypto/pkcs7/pk7_smime.c	2011-09-13 14:11:36.019454700 +0800

 @@ -88,7 +88,10 @@

diff --git a/CryptoPkg/Library/OpensslLib/OpensslLib.inf b/CryptoPkg/Library/OpensslLib/OpensslLib.inf
index a74a0c19ed..e85b4b7450 100644
--- a/CryptoPkg/Library/OpensslLib/OpensslLib.inf
+++ b/CryptoPkg/Library/OpensslLib/OpensslLib.inf
@@ -583,22 +583,26 @@
    #

    # Override MSFT build option to remove /W4 (to silence warning messages when building OpenSSL).

    #

-   MSFT:DEBUG_VS2003_IA32_CC_FLAGS        == /nologo /c /WX /Gs32768 /Gy /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /GX- /Zi /Gm -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w

-   MSFT:RELEASE_VS2003_IA32_CC_FLAGS      == /nologo /c /WX /Gs32768 /Gy /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /GX- -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w

-   MSFT:DEBUG_VS2003xASL_IA32_CC_FLAGS    == /nologo /c /WX /Gs32768 /Gy /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /GX- /Zi /Gm -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w

-   MSFT:RELEASE_VS2003xASL_IA32_CC_FLAGS  == /nologo /c /WX /Gs32768 /Gy /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /GX- -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w

-   MSFT:DEBUG_DDK3790_IA32_CC_FLAGS       == /nologo /c /WX /Gy /Gs32768 /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /Zi /Gm -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w

-   MSFT:RELEASE_DDK3790_IA32_CC_FLAGS     == /nologo /c /WX /Gy /Gs32768 /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w

-   MSFT:DEBUG_DDK3790xASL_IA32_CC_FLAGS   == /nologo /c /WX /Gy /Gs32768 /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /Zi /Gm -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w

-   MSFT:RELEASE_DDK3790xASL_IA32_CC_FLAGS == /nologo /c /WX /Gy /Gs32768 /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w

-   MSFT:DEBUG_*_IA32_CC_FLAGS             == /nologo /c /WX /GS- /Gs32768 /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /Gy /Zi /Gm -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w

-   MSFT:RELEASE_*_IA32_CC_FLAGS           == /nologo /c /WX /GS- /Gs32768 /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w

-   MSFT:DEBUG_*_X64_CC_FLAGS              == /nologo /c /WX /GS- /X /Gs32768 /D UNICODE /O1ib2s /GL /Gy /FIAutoGen.h /EHs-c- /GR- /GF /Zi /Gm -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w

-   MSFT:RELEASE_*_X64_CC_FLAGS            == /nologo /c /WX /GS- /X /Gs32768 /D UNICODE /O1ib2s /GL /Gy /FIAutoGen.h /EHs-c- /GR- /GF -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w

-   MSFT:DEBUG_*_IPF_CC_FLAGS              == /nologo /c /WX /GS- /X /EHs-c- /GR- /Gy /Os /GL /FIAutoGen.h /QIPF_fr32 /Zi -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w

-   MSFT:RELEASE_*_IPF_CC_FLAGS            == /nologo /c /WX /GS- /X /EHs-c- /GR- /Gy /Os /GL /FIAutoGen.h /QIPF_fr32 -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w

-   MSFT:NOOPT_*_IA32_CC_FLAGS             == /nologo /c /WX /GS- /Gs32768 /D UNICODE /Od /FIAutoGen.h /EHs-c- /GR- /GF /Gy /Zi /Gm -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w

-   INTEL:*_*_*_CC_FLAGS                   = -U_WIN32 -U_WIN64 -U_MSC_VER -U__ICC $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w

-   GCC:*_*_*_CC_FLAGS                     = -U_WIN32 -U_WIN64 $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) -w

-   RVCT:DEBUG_*_*_CC_FLAGS                = $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) --library_interface=aeabi_clib99 --fpu=vfpv3

-   RVCT:RELEASE_*_*_CC_FLAGS              = $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) --library_interface=aeabi_clib99 --fpu=vfpv3

+   MSFT:DEBUG_VS2003_IA32_CC_FLAGS        == /nologo /c /WX /Gs32768 /Gy /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /GX- /Zi /Gm -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w -DTHIRTY_TWO_BIT

+   MSFT:RELEASE_VS2003_IA32_CC_FLAGS      == /nologo /c /WX /Gs32768 /Gy /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /GX- -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w  -DTHIRTY_TWO_BIT

+   MSFT:DEBUG_VS2003xASL_IA32_CC_FLAGS    == /nologo /c /WX /Gs32768 /Gy /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /GX- /Zi /Gm -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w -DTHIRTY_TWO_BIT

+   MSFT:RELEASE_VS2003xASL_IA32_CC_FLAGS  == /nologo /c /WX /Gs32768 /Gy /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /GX- -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w -DTHIRTY_TWO_BIT

+   MSFT:DEBUG_DDK3790_IA32_CC_FLAGS       == /nologo /c /WX /Gy /Gs32768 /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /Zi /Gm -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w -DTHIRTY_TWO_BIT

+   MSFT:RELEASE_DDK3790_IA32_CC_FLAGS     == /nologo /c /WX /Gy /Gs32768 /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w -DTHIRTY_TWO_BIT

+   MSFT:DEBUG_DDK3790xASL_IA32_CC_FLAGS   == /nologo /c /WX /Gy /Gs32768 /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /Zi /Gm -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w -DTHIRTY_TWO_BIT

+   MSFT:RELEASE_DDK3790xASL_IA32_CC_FLAGS == /nologo /c /WX /Gy /Gs32768 /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w -DTHIRTY_TWO_BIT

+   MSFT:DEBUG_*_IA32_CC_FLAGS             == /nologo /c /WX /GS- /Gs32768 /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF /Gy /Zi /Gm -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w -DTHIRTY_TWO_BIT

+   MSFT:RELEASE_*_IA32_CC_FLAGS           == /nologo /c /WX /GS- /Gs32768 /D UNICODE /O1ib2 /GL /FIAutoGen.h /EHs-c- /GR- /GF -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w -DTHIRTY_TWO_BIT

+   MSFT:DEBUG_*_X64_CC_FLAGS              == /nologo /c /WX /GS- /X /Gs32768 /D UNICODE /O1ib2s /GL /Gy /FIAutoGen.h /EHs-c- /GR- /GF /Zi /Gm -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w -DSIXTY_FOUR_BIT

+   MSFT:RELEASE_*_X64_CC_FLAGS            == /nologo /c /WX /GS- /X /Gs32768 /D UNICODE /O1ib2s /GL /Gy /FIAutoGen.h /EHs-c- /GR- /GF -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w -DSIXTY_FOUR_BIT

+   MSFT:DEBUG_*_IPF_CC_FLAGS              == /nologo /c /WX /GS- /X /EHs-c- /GR- /Gy /Os /GL /FIAutoGen.h /QIPF_fr32 /Zi -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w -DSIXTY_FOUR_BIT

+   MSFT:RELEASE_*_IPF_CC_FLAGS            == /nologo /c /WX /GS- /X /EHs-c- /GR- /Gy /Os /GL /FIAutoGen.h /QIPF_fr32 -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w -DSIXTY_FOUR_BIT

+   MSFT:NOOPT_*_IA32_CC_FLAGS             == /nologo /c /WX /GS- /Gs32768 /D UNICODE /Od /FIAutoGen.h /EHs-c- /GR- /GF /Gy /Zi /Gm -U_WIN32 -U_WIN64 -U_MSC_VER $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w -DTHIRTY_TWO_BIT

+   INTEL:*_*_IA32_CC_FLAGS                = -U_WIN32 -U_WIN64 -U_MSC_VER -U__ICC $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w -DTHIRTY_TWO_BIT

+   INTEL:*_*_X64_CC_FLAGS                 = -U_WIN32 -U_WIN64 -U_MSC_VER -U__ICC $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) /w -DSIXTY_FOUR_BIT

+   GCC:*_*_IA32_CC_FLAGS                  = -U_WIN32 -U_WIN64 $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) -w -DTHIRTY_TWO_BIT

+   GCC:*_*_X64_CC_FLAGS                   = -U_WIN32 -U_WIN64 $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) -w -DSIXTY_FOUR_BIT

+   RVCT:DEBUG_*_IA32_CC_FLAGS             = $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) --library_interface=aeabi_clib99 --fpu=vfpv3 -DTHIRTY_TWO_BIT

+   RVCT:RELEASE_*_IA32_CC_FLAGS           = $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) --library_interface=aeabi_clib99 --fpu=vfpv3 -DTHIRTY_TWO_BIT

+   RVCT:DEBUG_*_X64_CC_FLAGS              = $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) --library_interface=aeabi_clib99 --fpu=vfpv3 -DSIXTY_FOUR_BIT

+   RVCT:RELEASE_*_X64_CC_FLAGS            = $(OPENSSL_FLAGS) $(OPENSSL_EXFLAGS) --library_interface=aeabi_clib99 --fpu=vfpv3 -DSIXTY_FOUR_BIT