111 lines
4.1 KiB
Python
111 lines
4.1 KiB
Python
|
import json
|
|||
|
|
import sys
|
|||
|
|
import unittest
|
|||
|
|
from pathlib import Path
|
|||
|
|
|
|||
|
|
|
|||
|
|
MODULE_DIR = Path(__file__).resolve().parents[1]
|
|||
|
|
sys.path.insert(0, str(MODULE_DIR))
|
|||
|
|
|
|||
|
|
import prose_extract
|
|||
|
|
|
|||
|
|
FIXTURE = (
|
|||
|
|
MODULE_DIR.parents[1] / "Tests/NucleicCoreTests/Fixtures/context-switch-prose.json"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class ProseParityTests(unittest.TestCase):
|
|||
|
|
"""The shared fixture is the contract between this port and the Swift runtime.
|
|||
|
|
|
|||
|
|
``ContextSwitchTests.replyProseMatchesTheSharedParityFixture`` asserts the same file
|
|||
|
|
against ``HeuristicSummary.contextSwitchReplyProse``. A change to either extraction
|
|||
|
|
that is not mirrored in the other fails one of the two suites.
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
def setUp(self):
|
|||
|
|
self.cases = json.loads(FIXTURE.read_text(encoding="utf-8"))
|
|||
|
|
self.assertTrue(self.cases, "shared prose parity fixture is empty")
|
|||
|
|
|
|||
|
|
def test_matches_the_shared_runtime_fixture(self):
|
|||
|
|
for case in self.cases:
|
|||
|
|
with self.subTest(case["name"]):
|
|||
|
|
prose = prose_extract.reply_prose(case["reply"], case["characterLimit"])
|
|||
|
|
self.assertEqual(prose, case["expectedProse"])
|
|||
|
|
self.assertEqual(
|
|||
|
|
prose_extract.ends_in_question(prose), case["expectedEndsInQuestion"]
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
def test_extracted_prose_never_exceeds_the_limit_in_graphemes(self):
|
|||
|
|
for case in self.cases:
|
|||
|
|
prose = prose_extract.reply_prose(case["reply"], case["characterLimit"])
|
|||
|
|
if prose is None:
|
|||
|
|
continue
|
|||
|
|
with self.subTest(case["name"]):
|
|||
|
|
self.assertLessEqual(
|
|||
|
|
len(prose_extract.graphemes(prose)), case["characterLimit"]
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class GraphemeSegmentationTests(unittest.TestCase):
|
|||
|
|
def test_models_the_clusters_swift_treats_as_one_character(self):
|
|||
|
|
for text, expected in [
|
|||
|
|
("á", 1), # combining acute
|
|||
|
|
("\U0001f468\U0001f469\U0001f467", 1), # family ZWJ sequence
|
|||
|
|
("\U0001f44d\U0001f3fd", 1), # thumbs up + skin tone
|
|||
|
|
("\U0001f1fa\U0001f1f8", 1), # regional indicator pair
|
|||
|
|
("\U0001f1fa\U0001f1f8\U0001f1e9\U0001f1ea", 2), # two flags, not one run
|
|||
|
|
("\r\n", 1),
|
|||
|
|
("\n\n", 2),
|
|||
|
|
("2️⃣", 1), # keycap
|
|||
|
|
("plain", 5),
|
|||
|
|
]:
|
|||
|
|
with self.subTest(repr(text)):
|
|||
|
|
self.assertEqual(len(prose_extract.graphemes(text)), expected)
|
|||
|
|
|
|||
|
|
def test_reassembling_clusters_is_lossless(self):
|
|||
|
|
for text in ["", "ábc", "\U0001f468\U0001f469 x", "a\r\n b"]:
|
|||
|
|
with self.subTest(repr(text)):
|
|||
|
|
self.assertEqual("".join(prose_extract.graphemes(text)), text)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class TrimTests(unittest.TestCase):
|
|||
|
|
def test_uses_the_swift_whitespace_set_rather_than_str_strip(self):
|
|||
|
|
# str.strip() would remove the information separators; Swift's
|
|||
|
|
# .whitespacesAndNewlines does not, so neither does the port.
|
|||
|
|
self.assertEqual(prose_extract.trim(" Prose. "), " Prose. ")
|
|||
|
|
self.assertEqual(prose_extract.trim(" Prose. "), "Prose.")
|
|||
|
|
self.assertEqual(prose_extract.trim(" \t\r\n "), "")
|
|||
|
|
|
|||
|
|
def test_whitespace_set_is_exactly_unicode_white_space(self):
|
|||
|
|
expected = {
|
|||
|
|
*range(0x0009, 0x000E),
|
|||
|
|
0x0020,
|
|||
|
|
0x0085,
|
|||
|
|
0x00A0,
|
|||
|
|
0x1680,
|
|||
|
|
*range(0x2000, 0x200B),
|
|||
|
|
0x2028,
|
|||
|
|
0x2029,
|
|||
|
|
0x202F,
|
|||
|
|
0x205F,
|
|||
|
|
0x3000,
|
|||
|
|
}
|
|||
|
|
self.assertEqual({ord(c) for c in prose_extract.WHITESPACE}, expected)
|
|||
|
|
|
|||
|
|
|
|||
|
|
class FenceStrippingTests(unittest.TestCase):
|
|||
|
|
def test_drops_fenced_blocks_and_keeps_outside_line_structure(self):
|
|||
|
|
self.assertEqual(
|
|||
|
|
prose_extract.strip_fenced_code("a\n```\ncode\n```\nb"), "a\nb"
|
|||
|
|
)
|
|||
|
|
# An unclosed fence takes the remainder: half-streamed code is still code.
|
|||
|
|
self.assertEqual(prose_extract.strip_fenced_code("a\n```\ncode"), "a")
|
|||
|
|
# A backtick fence does not close a tilde fence.
|
|||
|
|
self.assertEqual(
|
|||
|
|
prose_extract.strip_fenced_code("a\n~~~\n```\nx\n```\n~~~\nb"), "a\nb"
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
|
|||
|
|
if __name__ == "__main__":
|
|||
|
|
unittest.main()
|