Merge branch 'L2SS-1582_fix_json' into 'main'

L2SS-1582: Fix incorrect payload JSON serialization

See merge request !7

lofar_stingray/record.py

@@ -13,6 +13,7 @@ from lofar_station_client.statistics import receivers
 from logfmter import Logfmter
 from minio import Minio
 
+from lofar_stingray.utils import StingrayJsonEncoder
 from lofar_stingray.writer import Storage
 
 logger = logging.getLogger()
@@ -69,7 +70,7 @@ def main():
     with storage:
         for packet in receivers.create(args.source):
             data = dict(packet)
-            storage.write_line(json.dumps(data, default=str))
+            storage.write_line(json.dumps(data, cls=StingrayJsonEncoder))
 
     logger.info("End of packet stream. Shutting down.")
 

lofar_stingray/utils/__init__.py

+#  Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+#  SPDX-License-Identifier: Apache-2.0
+
+""" Util classes """
+
+from ._stringray_json_encoder import StingrayJsonEncoder
+
+__all__ = ["StingrayJsonEncoder"]

lofar_stingray/utils/_stringray_json_encoder.py

+#  Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+#  SPDX-License-Identifier: Apache-2.0
+
+""" additional stingray JSON encoders """
+from datetime import datetime
+from json import JSONEncoder
+
+import numpy
+
+
+class StingrayJsonEncoder(JSONEncoder):
+    """Additional json encoders for numpy and datetime"""
+
+    def default(self, o):
+        if isinstance(o, numpy.ndarray):
+            return o.tolist()
+        if isinstance(o, datetime):
+            return o.isoformat()
+        return JSONEncoder.default(self, o)

tests/utils/__init__.py

+#  Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+#  SPDX-License-Identifier: Apache-2.0

tests/utils/test_stingray_json_encoder.py

+#  Copyright (C) 2024 ASTRON (Netherlands Institute for Radio Astronomy)
+#  SPDX-License-Identifier: Apache-2.0
+
+""" numpy JSON encoder tests """
+import json
+from datetime import datetime
+from unittest import TestCase
+
+import numpy
+
+from lofar_stingray.utils import StingrayJsonEncoder
+
+
+class TestStingrayJsonEncoder(TestCase):
+    """Test cases of the StingrayJsonEncoder class"""
+
+    def test_serialize_np_array_simple(self):
+        """Test if a numpy array is correctly serialized"""
+
+        data = numpy.full(2, 10)
+        json_data = json.dumps(data, cls=StingrayJsonEncoder)
+
+        deserialized = json.loads(json_data)
+        self.assertEqual([10, 10], deserialized)
+
+    def test_serialize_np_array_multi(self):
+        """Test if a numpy array is correctly serialized"""
+
+        data = numpy.full((2, 2), 5)
+        json_data = json.dumps(data, cls=StingrayJsonEncoder)
+
+        deserialized = json.loads(json_data)
+        self.assertEqual([[5, 5], [5, 5]], deserialized)
+
+    def test_date(self):
+        """Test if a datetime is correctly serialized"""
+        data = {"ts": datetime.fromisoformat("2024-06-02T03:51:41.110706+00:00")}
+        json_data = json.dumps(data, cls=StingrayJsonEncoder)
+
+        deserialized = json.loads(json_data)
+        self.assertEqual({"ts": "2024-06-02T03:51:41.110706+00:00"}, deserialized)