summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJordan Justen <jordan.l.justen@intel.com>2015-06-23 23:34:22 +0000
committerjljusten <jljusten@Edk2>2015-06-23 23:34:22 +0000
commitdadfab5b23c3f02ab8396a5487122de797188845 (patch)
treef1c11df3a8df1e8da144a1071e054ffd44da8ff9
parentd80e451b187c9d33cbd771253fbd5119670f75c6 (diff)
downloadedk2-platforms-dadfab5b23c3f02ab8396a5487122de797188845.tar.xz
BaseTools/Tests: Verify unsupported UTF-16 are rejected
Supplementary Plane characters can exist in UTF-16 files, but they are not valid UCS-2 characters. For example, this python interpreter code: >>> import codecs >>> codecs.encode(u'\U00010300', 'utf-16') '\xff\xfe\x00\xd8\x00\xdf' Therefore the UCS-4 0x00010300 character is encoded as two 16-bit numbers (0xd800 0xdf00) in a little endian UTF-16 file. For more information, see: http://en.wikipedia.org/wiki/UTF-16#U.2B10000_to_U.2B10FFFF This test checks to make sure that BaseTools will reject these characters in UTF-16 files. The range of 0xd800 - 0xdfff should also be rejected as unicode code points because they are reserved for the surrogate pair usage in UTF-16 files. This test was fixed by the previous commit: "BaseTools/UniClassObject: Verify valid UCS-2 chars in UTF-16 .uni files" Contributed-under: TianoCore Contribution Agreement 1.0 Signed-off-by: Jordan Justen <jordan.l.justen@intel.com> Reviewed-by: Laszlo Ersek <lersek@redhat.com> Reviewed-by: Michael D Kinney <michael.d.kinney@intel.com> Reviewed-by: Yingke Liu <yingke.d.liu@intel.com> git-svn-id: https://svn.code.sf.net/p/edk2/code/trunk/edk2@17695 6f19259b-4bc3-4df7-8a09-765794883524
-rw-r--r--BaseTools/Tests/CheckUnicodeSourceFiles.py35
1 files changed, 34 insertions, 1 deletions
diff --git a/BaseTools/Tests/CheckUnicodeSourceFiles.py b/BaseTools/Tests/CheckUnicodeSourceFiles.py
index 0083ad8589..ad5fd18963 100644
--- a/BaseTools/Tests/CheckUnicodeSourceFiles.py
+++ b/BaseTools/Tests/CheckUnicodeSourceFiles.py
@@ -38,7 +38,10 @@ class Tests(TestTools.BaseToolsTest):
def EncodeToFile(self, encoding, string=None):
if string is None:
string = self.SampleData
- data = codecs.encode(string, encoding)
+ if encoding is not None:
+ data = codecs.encode(string, encoding)
+ else:
+ data = string
path = 'input.uni'
self.WriteTmpFile(path, data)
return PathClass(self.GetTmpFilePath(path))
@@ -81,6 +84,36 @@ class Tests(TestTools.BaseToolsTest):
def testUtf16InUniFile(self):
self.CheckFile('utf_16', shouldPass=True)
+ def testSupplementaryPlaneUnicodeCharInUtf16File(self):
+ #
+ # Supplementary Plane characters can exist in UTF-16 files,
+ # but they are not valid UCS-2 characters.
+ #
+ # This test makes sure that BaseTools rejects these characters
+ # if seen in a .uni file.
+ #
+ data = u'''
+ #langdef en-US "English"
+ #string STR_A #language en-US "CodePoint (\U00010300) > 0xFFFF"
+ '''
+
+ self.CheckFile('utf_16', shouldPass=False, string=data)
+
+ def testSurrogatePairUnicodeCharInUtf16File(self):
+ #
+ # Surrogate Pair code points are used in UTF-16 files to
+ # encode the Supplementary Plane characters. But, a Surrogate
+ # Pair code point which is not followed by another Surrogate
+ # Pair code point might be interpreted as a single code point
+ # with the Surrogate Pair code point.
+ #
+ # This test makes sure that BaseTools rejects these characters
+ # if seen in a .uni file.
+ #
+ data = codecs.BOM_UTF16_LE + '//\x01\xd8 '
+
+ self.CheckFile(encoding=None, shouldPass=False, string=data)
+
TheTestSuite = TestTools.MakeTheTestSuite(locals())
if __name__ == '__main__':