Merge branch 'L2SS-412-replace-opcua-by-asyncua' into 'master'

L2SS-412: Use asyncio for opcua and other clients

Closes L2SS-412

See merge request !142

devices/test/clients/test_attr_wrapper.py

@@ -18,6 +18,8 @@ from devices.hardware_device import *
 from tango.test_context import DeviceTestContext
 from test import base
 
+import asyncio
+
 scalar_dims = (1,)
 spectrum_dims = (4,)
 image_dims = (3,2)
@@ -31,7 +33,7 @@ def dev_init(device):
     device.set_state(DevState.INIT)
     device.test_client = test_client(device.Fault, device)
     for i in device.attr_list():
-        i.set_comm_client(device.test_client)
+        asyncio.run(i.async_set_comm_client(device.test_client))
     device.test_client.start()
 
 
@@ -361,6 +363,9 @@ class TestAttributeTypes(base.TestCase):
 
     def read_RW_test(self, dev, dtype, test_type):
         '''Test device'''
+        expected = None
+        val = None
+
         try:
             with DeviceTestContext(dev, process=True) as proxy:
 

devices/test/clients/test_client.py

@@ -91,7 +91,7 @@ class test_client(CommClient):
         self.streams.debug_stream("created and bound example_client read/write functions to attribute_wrapper object")
         return read_function, write_function
 
-    def setup_attribute(self, annotation=None, attribute=None):
+    async def setup_attribute(self, annotation=None, attribute=None):
         """
         MANDATORY function: is used by the attribute wrapper to get read/write functions.
         must return the read and write functions

devices/test/clients/test_opcua_client.py

@@ -2,13 +2,15 @@ import numpy
 from clients.opcua_client import OPCUAConnection
 from clients import opcua_client
 
-import opcua
+import asyncua
 import io
+import asyncio
 
 from unittest import mock
 import unittest
 
 from test import base
+import asynctest
 
 
 class attr_props:
@@ -37,36 +39,39 @@ image_shape = (2, 3)
 dimension_tests = [scalar_shape, spectrum_shape, image_shape]
 
 
-class TestOPCua(base.TestCase):
-    @mock.patch.object(OPCUAConnection, "check_nodes")
-    @mock.patch.object(OPCUAConnection, "connect")
-    @mock.patch.object(opcua_client, "Client")
-    def test_opcua_connection(self, m_opc_client, m_connect, m_check):
+class TestOPCua(base.AsyncTestCase):
+    @asynctest.patch.object(OPCUAConnection, "ping")
+    @asynctest.patch.object(opcua_client, "Client")
+    async def test_opcua_connection(self, m_opc_client, m_ping):
         """
         This tests verifies whether the correct connection steps happen. It checks whether we can init an OPCUAConnection object
         Whether we can set the namespace, and the OPCua client.
         """
 
-        m_get_namespace = mock.Mock()
-        m_get_namespace.get_namespace_index.return_value = 42
-        m_opc_client.return_value = m_get_namespace
+        m_opc_client_members = asynctest.asynctest.CoroutineMock()
+        m_opc_client_members.get_namespace_index = asynctest.asynctest.CoroutineMock(return_value=42)
+        m_opc_client_members.connect = asynctest.asynctest.CoroutineMock()
+        m_opc_client_members.disconnect = asynctest.asynctest.CoroutineMock()
+        m_opc_client_members.send_hello = asynctest.asynctest.CoroutineMock()
+        m_opc_client.return_value = m_opc_client_members
 
-        test_client = OPCUAConnection("opc.tcp://localhost:4874/freeopcua/server/", "http://lofar.eu", 5, mock.Mock(), mock.Mock())
+        test_client = OPCUAConnection("opc.tcp://localhost:4874/freeopcua/server/", "http://lofar.eu", 5, mock.Mock(), self.loop)
+        try:
+            await test_client.start()
 
-        """Verify that construction of OPCUAConnection calls self.connect"""
-        m_connect.assert_called_once()  # the connect function in the opcua client
-        m_check.assert_called_once()  # debug function that prints out all nodes
-        m_opc_client.assert_called_once()  # makes sure the actual freeOPCua client object is created only once
+            m_opc_client.assert_called_once()  # makes sure the actual freeOPCua client object is created only once
 
-        m_get_namespace.get_namespace_index.assert_called_once_with("http://lofar.eu")
-        self.assertEqual(42, test_client.name_space_index)
+            # this also implies test_client.connect() is called
+            m_opc_client_members.get_namespace_index.assert_called_once_with("http://lofar.eu")
+            self.assertEqual(42, test_client.name_space_index)
+        finally:
+            await test_client.stop()
 
 
-    @mock.patch.object(OPCUAConnection, "check_nodes")
-    @mock.patch.object(OPCUAConnection, "connect")
-    @mock.patch.object(opcua_client, "Client")
+    @asynctest.patch.object(OPCUAConnection, "ping")
+    @asynctest.patch.object(opcua_client, "Client")
     @mock.patch.object(opcua_client, 'ProtocolAttribute')
-    def test_opcua_attr_setup(self, m_protocol_attr, m_opc_client, m_connect, m_check):
+    async def test_opcua_attr_setup(self, m_protocol_attr, m_opc_client, m_ping):
         """
         This tests covers the correct creation of read/write functions.
         In normal circumstances called by he attribute wrapper.
@@ -75,6 +80,16 @@ class TestOPCua(base.TestCase):
         Test succeeds if there are no errors.
         """
 
+        m_opc_client_members = asynctest.asynctest.CoroutineMock()
+        m_opc_client_members.get_namespace_index = asynctest.asynctest.CoroutineMock(return_value=2)
+        m_opc_client_members.connect = asynctest.asynctest.CoroutineMock()
+        m_opc_client_members.disconnect = asynctest.asynctest.CoroutineMock()
+        m_opc_client_members.send_hello = asynctest.asynctest.CoroutineMock()
+        m_objects_node = asynctest.Mock()
+        m_objects_node.get_child = asynctest.asynctest.CoroutineMock()
+        m_opc_client_members.get_objects_node = asynctest.Mock(return_value=m_objects_node)
+        m_opc_client.return_value = m_opc_client_members
+
         for i in attr_test_types:
             class mock_attr:
                 def __init__(self, dtype, x, y):
@@ -96,13 +111,15 @@ class TestOPCua(base.TestCase):
                 # pretend like there is a running OPCua server with a node that has this name
                 m_annotation = ["2:PCC", f"2:testNode_{str(i.numpy_type)}_{str(dim_x)}_{str(dim_y)}"]
 
-                test = OPCUAConnection("opc.tcp://localhost:4874/freeopcua/server/", "http://lofar.eu", 5, mock.Mock(), mock.Mock())
-                test.setup_attribute(m_annotation, m_attribute)
+                test_client = OPCUAConnection("opc.tcp://localhost:4874/freeopcua/server/", "http://lofar.eu", 5, mock.Mock(), self.loop)
+                try:
+                    await test_client.start()
+                    await test_client.setup_attribute(m_annotation, m_attribute)
+                finally:
+                    await test_client.stop()
 
                 # success if there are no errors.
 
-
-
     def test_protocol_attr(self):
         """
         This tests finding an OPCua node and returning a valid object with read/write functions.
@@ -136,7 +153,7 @@ class TestOPCua(base.TestCase):
                 self.assertTrue(hasattr(test, "write_function"), f"No write function found")
                 self.assertTrue(hasattr(test, "read_function"), f"No read function found")
 
-    def test_read(self):
+    async def test_read(self):
         """
         This tests the read functions.
         """
@@ -146,17 +163,17 @@ class TestOPCua(base.TestCase):
                 def get_test_value():
                     return numpy.zeros(j, i.numpy_type)
 
-                def get_flat_value():
+                async def get_flat_value():
                     return get_test_value().flatten()
 
-                m_node = mock.Mock()
+                m_node = asynctest.asynctest.CoroutineMock()
 
                 if len(j) == 1:
                     test = opcua_client.ProtocolAttribute(m_node, j[0], 0, opcua_client.numpy_to_OPCua_dict[i.numpy_type])
                 else:
                     test = opcua_client.ProtocolAttribute(m_node, j[1], j[0], opcua_client.numpy_to_OPCua_dict[i.numpy_type])
                 m_node.get_value = get_flat_value
-                val = test.read_function()
+                val = await test.read_function()
 
                 comp = val == get_test_value()
                 self.assertTrue(comp.all(), "Read value unequal to expected value: \n\t{} \n\t{}".format(val, get_test_value()))
@@ -175,15 +192,15 @@ class TestOPCua(base.TestCase):
               default_value = 42.25
 
             # apply our mapping
-            v = opcua.ua.uatypes.Variant(value=numpy_type(default_value), varianttype=opcua_type)
+            v = asyncua.ua.uatypes.Variant(Value=numpy_type(default_value), VariantType=opcua_type)
 
             try:
                 # try to convert it to binary to force opcua to parse the value as the type
-                binary = opcua.ua.ua_binary.variant_to_binary(v)
+                binary = asyncua.ua.ua_binary.variant_to_binary(v)
 
                 # reinterpret the resulting binary to obtain what opcua made of our value
                 binary_stream = io.BytesIO(binary)
-                reparsed_v = opcua.ua.ua_binary.variant_from_binary(binary_stream)
+                reparsed_v = asyncua.ua.ua_binary.variant_from_binary(binary_stream)
             except Exception as e:
                 raise Exception(f"Conversion {numpy_type} -> {opcua_type} failed.") from e
 
@@ -192,11 +209,11 @@ class TestOPCua(base.TestCase):
 
             # does the OPC-UA type have the same datasize (and thus, precision?)
             if numpy_type not in [str, numpy.str]:
-                self.assertEqual(numpy_type().itemsize, getattr(opcua.ua.ua_binary.Primitives, opcua_type.name).size, msg=f"Conversion {numpy_type} -> {opcua_type} failed: precision mismatch")
+                self.assertEqual(numpy_type().itemsize, getattr(asyncua.ua.ua_binary.Primitives, opcua_type.name).size, msg=f"Conversion {numpy_type} -> {opcua_type} failed: precision mismatch")
 
 
 
-    def test_write(self):
+    async def test_write(self):
         """
         Test the writing of values by instantiating a ProtocolAttribute attribute, and calling the write function.
         but the opcua function that writes to the server has been changed to the compare_values function.
@@ -215,24 +232,22 @@ class TestOPCua(base.TestCase):
 
                 # get opcua Varianttype array of the test value
                 def get_mock_value(value):
-                    return opcua.ua.uatypes.Variant(value=value, varianttype=opcua_client.numpy_to_OPCua_dict[i.numpy_type])
+                    return asyncua.ua.uatypes.Variant(Value=value, VariantType=opcua_client.numpy_to_OPCua_dict[i.numpy_type])
 
-                m_node = mock.Mock()
+                m_node = asynctest.asynctest.CoroutineMock()
 
                 # create the protocolattribute
                 if len(j) == 1:
-                    test = opcua_client.ProtocolAttribute(m_node, j[0], 0, opcua_client.numpy_to_OPCua_dict[i.numpy_type])
+                    test = opcua_client.ProtocolAttribute(m_node, j[0], 0, opcua_client.numpy_to_OPCua_dict[i.numpy_type], self.loop)
                 else:
-                    test = opcua_client.ProtocolAttribute(m_node, j[1], j[0], opcua_client.numpy_to_OPCua_dict[i.numpy_type])
-
-                test.node.get_data_value = mock.Mock()
+                    test = opcua_client.ProtocolAttribute(m_node, j[1], j[0], opcua_client.numpy_to_OPCua_dict[i.numpy_type], self.loop)
 
                 # comparison function that replaces `set_data_value` inside the attributes write function
-                def compare_values(val):
-                    # test values
+                async def compare_values(val):
+                    # test valuest
                     val = val.tolist() if type(val) == numpy.ndarray else val
                     if j != dimension_tests[0]:
-                        comp = val._value == get_mock_value(get_test_value().flatten())._value
+                        comp = val.Value == get_mock_value(get_test_value().flatten()).Value
                         self.assertTrue(comp.all(),
                                         "Array attempting to write unequal to expected array: \n\t got: {} \n\texpected: {}".format(val,get_mock_value(get_test_value())))
                     else:
@@ -243,4 +258,4 @@ class TestOPCua(base.TestCase):
                 m_node.set_data_value = compare_values
 
                 # call the write function with the test values
-                test.write_function(get_test_value())
+                await test.write_function(get_test_value())

docker-compose/integration-test.yml

@@ -26,4 +26,4 @@ services:
       - --timeout=30
       - --strict
       - --
-      - tox -e integration
+      - tox --recreate -e integration

docker-compose/itango/lofar-requirements.txt

 parso == 0.7.1
 jedi == 0.17.2
-opcua >= 0.98.13
+asyncua
 astropy 
 python-logstash-async
 gitpython
 PyMySQL[rsa]
 sqlalchemy
-timeout-decorator

docker-compose/lofar-device-base/lofar-requirements.txt

-opcua >= 0.98.9
+asyncua
 astropy
 python-logstash-async
 gitpython