8000 TSI tables content as utf-8 and unicode instead of bytes by moyogo · Pull Request #1060 · fonttools/fonttools · GitHub
[go: up one dir, main page]
More Web Proxy on the site http://driver.im/
Skip to content

TSI tables content as utf-8 and unicode instead of bytes #1060

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Sep 21, 2017
Merged

TSI tables content as utf-8 and unicode instead of bytes #1060

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
fcce935
TSI1: update tests for utf-8
Sep 21, 2017
35625df
TSI tables content as unicode
Sep 20, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 4 additions & 3 deletions Lib/fontTools/ttLib/tables/T_S_I__1.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ def decompile(self, data, ttFont):
"%r textLength (%d) must not be > 32768" % (name, textLength))
text = data[textOffset:textOffset+textLength]
assert len(text) == textLength
text = tounicode(text, encoding='utf-8')
if text:
programs[name] = text
if isExtra:
Expand All @@ -90,7 +91,7 @@ def compile(self, ttFont):
data = data + b"\015" # align on 2-byte boundaries, fill with return chars. Yum.
name = glyphNames[i]
if name in self.glyphPrograms:
text = tobytes(self.glyphPrograms[name])
text = tobytes(self.glyphPrograms[name], encoding="utf-8")
else:
text = b""
textLength = len(text)
Expand Down Expand Up @@ -126,7 +127,7 @@ def toXML(self, writer, ttFont):
continue
writer.begintag("glyphProgram", name=name)
writer.newline()
writer.write_noindent(text.replace(b"\r", b"\n"))
writer.write_noindent(text.replace("\r", "\n"))
writer.newline()
writer.endtag("glyphProgram")
writer.newline()
Expand All @@ -138,7 +139,7 @@ def toXML(self, writer, ttFont):
continue
writer.begintag("extraProgram", name=name)
writer.newline()
writer.write_noindent(text.replace(b"\r", b"\n"))
writer.write_noindent(text.replace("\r", "\n"))
writer.newline()
writer.endtag("extraProgram")
writer.newline()
Expand Down
74 changes: 48 additions & 26 deletions Tests/ttLib/tables/T_S_I__1_test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
from __future__ import print_function, division, absolute_import
from fontTools.misc.py23 import unichr
from __future__ import (
print_function, division, absolute_import, unicode_literals
)
from fontTools.misc.py23 import unichr, tobytes
from fontTools.misc.loggingTools import CapturingLogHandler
from fontTools.ttLib import TTFont, TTLibError
from fontTools.ttLib.tables.T_S_I__0 import table_T_S_I__0
Expand All @@ -8,6 +10,7 @@


TSI1_DATA = b"""abcdefghijklmnopqrstuvxywz0123456789"""
TSI1_UTF8_DATA = b"""abcd\xc3\xa9ghijklmnopqrstuvxywz0123456789"""


@pytest.fixture
Expand Down Expand Up @@ -56,16 +59,33 @@ def test_decompile(font):
table.decompile(TSI1_DATA, font)

assert table.glyphPrograms == {
'a': b'a',
'b': b'bcdef',
# 'c': b'', # zero-length entries are skipped
# 'd': b'',
'e': b'ghijklmn'}
'a': 'a',
'b': 'bcdef',
# 'c': '', # zero-length entries are skipped
# 'd': '',
'e': 'ghijklmn'}
assert table.extraPrograms == {
'ppgm': b'op',
'cvt': b'qrst',
'reserved': b'uvxywz',
'fpgm': b'0123456789'}
'ppgm': 'op',
'cvt': 'qrst',
'reserved': 'uvxywz',
'fpgm': '0123456789'}


def test_decompile_utf8(font):
table = table_T_S_I__1()
table.decompile(TSI1_UTF8_DATA, font)

assert table.glyphPrograms == {
'a': 'a',
'b': 'bcd\u00e9',
# 'c': '', # zero-length entries are skipped
# 'd': '',
'e': 'ghijklmn'}
assert table.extraPrograms == {
'ppgm': 'op',
'cvt': 'qrst',
'reserved': 'uvxywz',
'fpgm': '0123456789'}


def test_decompile_empty(empty_font):
Expand All @@ -88,27 +108,29 @@ def test_decompile_invalid_length(empty_font):

def test_decompile_offset_past_end(empty_font):
empty_font.glyphOrder = ['foo', 'bar']
data = b'baz'
content = 'baz'
data = tobytes(content)
empty_font['TSI0'].indices = [(0, len(data), 0), (1, 1, len(data)+1)]

table = table_T_S_I__1()
with CapturingLogHandler(table.log, "WARNING") as captor:
table.decompile(data, empty_font)

# the 'bar' program is skipped because its offset > len(data)
assert table.glyphPrograms == {'foo': b'baz'}
assert table.glyphPrograms == {'foo': 'baz'}
assert any("textOffset > totalLength" in r.msg for r in captor.records)


def test_decompile_magic_length_last_extra(empty_font):
indextable = empty_font['TSI0']
indextable.extra_indices[-1] = (0xFFFD, 0x8000, 0)
data = b"0" * (0x8000 + 1)
content = "0" * (0x8000 + 1)
data = tobytes(content)

table = table_T_S_I__1()
table.decompile(data, empty_font)

assert table.extraPrograms['fpgm'] == data
assert table.extraPrograms['fpgm'] == content


def test_decompile_magic_length_last_glyph(empty_font):
Expand All @@ -122,15 +144,15 @@ def test_decompile_magic_length_last_glyph(empty_font):
(0xFFFB, 0, 0x8004),
(0xFFFC, 0, 0x8004),
(0xFFFD, 0, 0x8004)]
foo_data = b"0" * 3
bar_data = b"1" * (0x8000 + 1)
data = foo_data + bar_data
foo_content = "0" * 3
bar_content = "1" * (0x8000 + 1)
data = tobytes(foo_content + bar_content)

table = table_T_S_I__1()
table.decompile(data, empty_font)

assert table.glyphPrograms['foo'] == foo_data
assert table.glyphPrograms['bar'] == bar_data
assert table.glyphPrograms['foo'] == foo_content
assert table.glyphPrograms['bar'] == bar_content


def test_decompile_magic_length_non_last(empty_font):
Expand All @@ -140,21 +162,21 @@ def test_decompile_magic_length_non_last(empty_font):
(0xFFFB, 0x8000, 3), # the actual length of 'cvt' program is:
(0xFFFC, 0, 0x8004), # nextTextOffset - textOffset: 0x8004 - 3
(0xFFFD, 0, 0x8004)]
ppgm_data = b"0" * 3
cvt_data = b"1" * (0x8000 + 1)
data = ppgm_data + cvt_data
ppgm_content = "0" * 3
cvt_content = "1" * (0x8000 + 1)
data = tobytes(ppgm_content + cvt_content)

table = table_T_S_I__1()
table.decompile(data, empty_font)

assert table.extraPrograms['ppgm'] == ppgm_data
assert table.extraPrograms['cvt'] == cvt_data
assert table.extraPrograms['ppgm'] == ppgm_content
assert table.extraPrograms['cvt'] == cvt_content

table = table_T_S_I__1()
with CapturingLogHandler(table.log, "WARNING") as captor:
table.decompile(data[:-1], empty_font) # last entry is truncated
captor.assertRegex("nextTextOffset > totalLength")
assert table.extraPrograms['cvt'] == cvt_data[:-1]
assert table.extraPrograms['cvt'] == cvt_content[:-1]


if __name__ == "__main__":
Expand Down
0