241 rewiew encoding before sending (#246)

* fix: Fix UTF-8 encoding when surrogates in string
* doc: Update version

Author: angel-devo

CHANGELOG.md

@@ -4,6 +4,16 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [5.1.9] - 2024-02-06
+
+### Added
+
+- Tests for encoding of messages in Sender.
+
+### Changed
+
+- UFT-8 encoding with "replace" is now used by default in Sender.
+
 ## [5.1.8] - 2024-01-10
 
 ### Added

LICENSE

@@ -1,6 +1,6 @@
 The MIT License (MIT)
 
-Copyright (C) 2023 Devo, Inc.
+Copyright (C) 2024 Devo, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

README.md

@@ -203,7 +203,7 @@ You can contact with us at _support@devo.com_.
 
 MIT License
 
-(C) 2023 Devo, Inc.
+(C) 2024 Devo, Inc.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy of
 this software and associated documentation files (the 'Software'), to deal in

devo/__version__.py

@@ -1,7 +1,7 @@
 __description__ = "Devo Python Library."
 __url__ = "http://www.devo.com"
-__version__ = "5.1.8"
+__version__ = "5.1.9"
 __author__ = "Devo"
 __author_email__ = "support@devo.com"
 __license__ = "MIT"
-__copyright__ = "Copyright 2023 Devo"
+__copyright__ = "Copyright 2024 Devo"

devo/sender/data.py

@@ -625,7 +625,7 @@ def __encode_record(record):
         :return: record encoded for PY3
         """
         if not isinstance(record, bytes):
-            return record.encode("utf-8")
+            return record.encode("utf-8", "replace")
         return record
 
     def __send_oc(self, record):

run_tests.py

@@ -22,6 +22,7 @@
 from tests.sender.test_number_lookup import TestLookup as SENDER_NUMBER_LOOKUP
 from tests.sender.test_read_csv import TestCSVRFC as SENDER_CSV
 from tests.sender.test_send_data import TestSender as SENDER_SEND_DATA
+from tests.sender.test_send_encoding import TestEncoding as SENDER_SEND_ENCODING
 from tests.sender.test_send_lookup import TestLookup as SENDER_SEND_LOOKUP
 
 module_paths = {
@@ -38,6 +39,7 @@
     "SENDER_CSV": SENDER_CSV,
     "SENDER_NUMBER_LOOKUP": SENDER_NUMBER_LOOKUP,
     "SENDER_SEND_DATA": SENDER_SEND_DATA,
+    "SENDER_SEND_ENCODING": SENDER_SEND_ENCODING,
     "SENDER_SEND_LOOKUP": SENDER_SEND_LOOKUP,
 }
 

tests/sender/test_send_encoding.py

@@ -0,0 +1,32 @@
+import unittest
+from devo.sender.data import Sender
+
+# This test case uses the private method __encode_record to test the encoding
+# of the records and needs some name mangling to access it.
+
+
+class TestEncoding(unittest.TestCase):
+
+    def test_encode_record_ascii(self):
+        record = 'Hello'  # ASCII Normal sequence
+        encoded_record = Sender._Sender__encode_record(record)
+        self.assertEqual(encoded_record, b'Hello')
+
+    def test_encode_record_utf8(self):
+        record = 'Hello 🌍, こんにちは'  # UTF-8 sequence
+        encoded_record = Sender._Sender__encode_record(record)
+        self.assertEqual(encoded_record, b'Hello \xf0\x9f\x8c\x8d, \xe3\x81\x93\xe3\x82\x93\xe3\x81\xab\xe3\x81\xa1\xe3\x81\xaf')
+
+    def test_encode_record_utf8_with_byte_sequence(self):
+        record = '10 €'  # UTF-8 valid byte sequence
+        encoded_record = Sender._Sender__encode_record(record)
+        self.assertEqual(encoded_record, b'10 \xe2\x82\xac')
+
+    def test_encode_record_utf16_surrogate(self):
+        record = '\uD83D Hello'  # UTF-16 sequence
+        encoded_record = Sender._Sender__encode_record(record)
+        self.assertEqual(encoded_record, b'? Hello')
+
+
+if __name__ == "__main__":
+    unittest.main()