OSB-28: merge with master branch and various updates and fixes

LCU/Maintenance/MDB_tools/test/t_client.run

+#!/bin/bash
+source python-coverage.sh
+
+python_coverage_test client t_client.py

LCU/Maintenance/MDB_tools/test/t_client.sh

+#!/bin/sh
+./runctest.sh t_client

LCU/Maintenance/MDB_tools/test/t_mdb_loader.py

@@ -24,60 +24,6 @@ def load_station_test():
 
 
 class TESTMDBLoader(unittest.TestCase):
-    def test_is_station_test_right_content(self):
-        station_test_content = load_station_test()
-        self.assertTrue(is_station_test(station_test_content))
-
-    def test_is_station_test_wrong_content(self):
-        station_test_content = load_rtsm_test()
-        self.assertFalse(is_station_test(station_test_content))
-
-    def test_is_rtsm_test_right_content(self):
-        rtsm_test_content = load_rtsm_test()
-        self.assertTrue(is_rtsm_test(rtsm_test_content))
-
-    def test_is_rtsm_test_wrong_content(self):
-        rtsm_test_content = load_station_test()
-        self.assertFalse(is_rtsm_test(rtsm_test_content))
-
-    @patch('requests.post')
-    def test_send_stationtest_rtsm_content_to_address_success(self, mocked_post):
-        station_test_content = load_station_test()
-        address = 'http://testaddress'
-        response = Mock(status_code=200, reason='ALL GOOD', content='YEAH')
-        mocked_post.return_value = response
-
-        full_address = '/'.join([address, compose_api_url_for_given_test_type(station_test_content)])
-        result = send_stationtest_rtsm_content_to_address('http://testaddress', station_test_content)
-        self.assertTrue(mocked_post.called)
-        mocked_post.assert_called_with(full_address, data=station_test_content)
-        self.assertTrue(result)
-
-    @patch('requests.post')
-    def test_send_stationtest_rtsm_content_to_address_failure(self, mocked_post):
-        station_test_content = load_station_test()
-        address = 'http://testaddress'
-        response = Mock(status_code=400, reason='Bad Request', content='Sorry')
-        mocked_post.return_value = response
-
-        full_address = '/'.join([address, compose_api_url_for_given_test_type(station_test_content)])
-        result = send_stationtest_rtsm_content_to_address('http://testaddress', station_test_content)
-        self.assertTrue(mocked_post.called)
-        mocked_post.assert_called_with(full_address, data=station_test_content)
-        self.assertFalse(result)
-
-    def test_compose_api_url_for_given_test_type_is_rtsm(self):
-        rtsm_test_content = load_rtsm_test()
-        self.assertEqual(compose_api_url_for_given_test_type(rtsm_test_content), 'rtsm/insert_raw')
-
-    def test_compose_api_url_for_given_test_type_is_station_test(self):
-        station_test_content = load_station_test()
-        self.assertEqual(compose_api_url_for_given_test_type(station_test_content), 'stationtests/insert_raw')
-
-    def test_compose_api_url_for_given_test_type_raises_content_unrecognized(self):
-        with self.assertRaises(ValueError):
-            compose_api_url_for_given_test_type(['Neque porro quisquam est qui dolorem ipsum quia dolor sit amet',
-                                                 'consectetur, adipisci velit...'])
 
     @patch('lofar.maintenance.utils.cli.mdb_loader.path')
     @patch('lofar.maintenance.utils.cli.mdb_loader.glob')
@@ -90,48 +36,6 @@ class TESTMDBLoader(unittest.TestCase):
         self.assertEqual(path_mock.isfile.call_count, 3)
         path_mock.isfile.assert_called_with('test2')
 
-    @patch('lofar.maintenance.utils.cli.mdb_loader.open', create=True)
-    def test_read_stationtest_rtsm_output_to_dict_rtsm_success(self, open_mock):
-        file_name = 'CS001C_12345_20180305.dat'
-        file_content = 'testline1\ntestline2'
-        open_mock.return_value = MagicMock()
-        file_handle = open_mock.return_value.__enter__.return_value
-
-        file_handle.read.return_value = file_content
-
-        result = read_stationtest_rtsm_output_to_dict(file_name)
-        open_mock.assert_called_with(file_name, 'r')
-        file_handle.read.assert_called()
-        self.assertEqual(result, dict(content=file_content, station_name='CS001C'))
-
-    @patch('lofar.maintenance.utils.cli.mdb_loader.open', create=True)
-    def test_read_stationtest_rtsm_output_to_dict_rtsm_failure(self, open_mock):
-        file_name = 'CS001C_12345_20180305.dat'
-        open_mock.return_value = MagicMock()
-
-        file_handle = open_mock.return_value.__enter__.return_value
-
-        file_handle.read.side_effect = IOError('Cannot read file')
-
-        result = read_stationtest_rtsm_output_to_dict(file_name)
-        open_mock.assert_called_with(file_name, 'r')
-        file_handle.read.assert_called()
-        self.assertIsNone(result)
-
-    @patch('lofar.maintenance.utils.cli.mdb_loader.open', create=True)
-    def test_read_stationtest_rtsm_output_to_dict_station_test_success(self, open_mock):
-        file_name = 'CS031C_L2_StationTestHistory.csv'
-        file_content = 'testline1\ntestline2'
-        open_mock.return_value = MagicMock()
-        file_handle = open_mock.return_value.__enter__.return_value
-
-        file_handle.read.return_value = file_content
-
-        result = read_stationtest_rtsm_output_to_dict(file_name)
-        open_mock.assert_called_with(file_name, 'r')
-        file_handle.read.assert_called()
-        self.assertEqual(result, dict(content=file_content))
-
     @patch('lofar.maintenance.utils.cli.mdb_loader.logger')
     @patch('lofar.maintenance.utils.cli.mdb_loader.obtain_file_list')
     @patch('lofar.maintenance.utils.cli.mdb_loader.read_stationtest_rtsm_output_to_dict')

LCU/Maintenance/MDB_tools/test/t_station_tests_watchdog.py

+import unittest
+
+from lofar.maintenance.utils.cli.station_tests_watchdog import *
+from mock import patch, MagicMock, call
+
+logger = logging.getLogger(__name__)
+
+
+class TestStationTestsWatchdog(unittest.TestCase):
+    def setUp(self):
+        self.mock_path = 'mydirectory'
+        self.mock_address = 'http://fakeaddress.fk'
+        self.mock_filename = 'likename'
+        self.mock_content = 'fake_content'
+
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.mdb_client')
+    def test_on_created_event_read_test_file(self, mock_mdb_client):
+        event = ('UNUSED', 'IN_CLOSE_WRITE', self.mock_path, self.mock_filename)
+
+        handle_inotify_event_create_file(event, self.mock_filename, self.mock_address)
+
+        mock_mdb_client.read_stationtest_rtsm_output_to_dict.assert_called_with(self.mock_path +
+                                                                                '/' +
+                                                                                self.mock_filename)
+
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.mdb_client')
+    def test_on_created_event_send_test_file_content(self, mock_mdb_client):
+        event = ('UNUSED', 'IN_CLOSE_WRITE', self.mock_path, self.mock_filename)
+
+        mock_mdb_client.read_stationtest_rtsm_output_to_dict.return_value = self.mock_content
+
+        handle_inotify_event_create_file(event, self.mock_filename, self.mock_address)
+
+        mock_mdb_client.send_stationtest_rtsm_content_to_address.assert_called_with(
+            self.mock_address,
+            self.mock_content)
+
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.mdb_client')
+    def test_on_created_event_wrong_expected_name_not_send_test_file_content(self, mock_mdb_client):
+        event = ('UNUSED', 'IN_MODIFIED', self.mock_path, self.mock_filename)
+        mock_mdb_client.read_stationtest_rtsm_output_to_dict.return_value = self.mock_content
+
+        handle_inotify_event_create_file(event, 'wrong_name', self.mock_address)
+
+        mock_mdb_client.send_stationtest_rtsm_content_to_address.assert_not_called()
+
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.mdb_client')
+    def test_on_modified_event_not_send_test_file_content(self, mock_mdb_client):
+        event = ('UNUSED', 'IN_MODIFIED', self.mock_path, self.mock_filename)
+        mock_mdb_client.read_stationtest_rtsm_output_to_dict.return_value = self.mock_content
+
+        handle_inotify_event_create_file(event, self.mock_filename, self.mock_address)
+
+        mock_mdb_client.send_stationtest_rtsm_content_to_address.assert_not_called()
+
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.mdb_client')
+    def test_on_modified_event_not_read_test_file_content(self, mock_mdb_client):
+        mock_mdb_client.read_stationtest_rtsm_output_to_dict.return_value = self.mock_content
+        event = ('UNUSED', 'IN_MODIFIED', self.mock_path, self.mock_filename)
+
+        handle_inotify_event_create_file(event, self.mock_filename, self.mock_address)
+
+        mock_mdb_client.read_stationtest_rtsm_output_to_dict.assert_not_called()
+
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.mdb_client')
+    def test_on_created_event_not_send_if_file_not_test(self, mock_mdb_client):
+        event = ('UNUSED', 'IN_CREATED', self.mock_path, self.mock_filename)
+
+        mock_mdb_client.read_stationtest_rtsm_output_to_dict.return_value = self.mock_content
+        mock_mdb_client.read_stationtest_rtsm_output_to_dict.side_effect = ValueError()
+
+        handle_inotify_event_create_file(event, self.mock_filename, self.mock_address)
+
+        mock_mdb_client.send_stationtest_rtsm_content_to_address.assert_not_called()
+
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.mdb_client')
+    def test_on_created_event_not_send_if_file_error_reading_file(self, mock_mdb_client):
+        event = ('UNUSED', 'IN_CREATED', self.mock_path, self.mock_filename)
+
+        mock_mdb_client.read_stationtest_rtsm_output_to_dict.return_value = self.mock_content
+        mock_mdb_client.read_stationtest_rtsm_output_to_dict.side_effect = IOError()
+
+        handle_inotify_event_create_file(event, self.mock_filename, self.mock_address)
+
+        mock_mdb_client.send_stationtest_rtsm_content_to_address.assert_not_called()
+
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.'
+           'handle_inotify_event_create_file')
+    def test_main_wait_event_loop_unexpected_exception_keyboard_interrupt(self,
+                                                            handle_inotify_event_create_file_mock):
+        active_watch_time = 3
+        test_event = ('_', 'fake_type', 'fake_path', 'fake_file')
+        observer_mock = MagicMock()
+        observer_mock.event_gen.return_value = [test_event]
+        handle_inotify_event_create_file_mock.side_effect = [[], Exception(), KeyboardInterrupt()]
+
+        self.assertTrue(
+            main_wait_event_loop(observer_mock, 'fake_name', self.mock_address, active_watch_time))
+        handle_inotify_event_create_file_mock.assert_has_calls(
+            [call(test_event, 'fake_name', self.mock_address)] * 3)
+
+    @patch('inotify.adapters.Inotify')
+    def test_instantiate_file_creation_event_observer(self, inotify_mock):
+        mock_path = 'path_to_file'
+        inotify_object_partial_mock = MagicMock(inotify.adapters.Inotify())
+        inotify_mock.return_value = inotify_object_partial_mock
+        inotify_object_partial_mock.add_watch = MagicMock()
+
+        instantiate_file_creation_event_observer(mock_path)
+        inotify_object_partial_mock.add_watch.assert_called_with(mock_path)
+
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.'
+           'setup_command_argument_parser')
+    def test_get_settings_from_command_arguments(self, mock_setup_argument_parser):
+        argument_parser = MagicMock()
+        mock_setup_argument_parser.return_value = argument_parser
+        argument_parser.parse_args = MagicMock()
+
+        get_settings_from_command_arguments([])
+
+        mock_setup_argument_parser.assert_called()
+        argument_parser.parse_args.assert_called()
+
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.'
+           'setup_logging_framework')
+    def test_apply_global_settings_set_verbose_logging(self, mock_setup_logging_framework):
+        partial_mock_settings = MagicMock()
+        partial_mock_settings.v = True
+
+        apply_global_settings(partial_mock_settings)
+
+        mock_setup_logging_framework.assert_called_with(logging.DEBUG)
+
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.'
+           'setup_logging_framework')
+    def test_apply_global_settings_do_not_set_verbose_logging(self, mock_setup_logging_framework):
+        partial_mock_settings = MagicMock()
+        partial_mock_settings.v = False
+
+        apply_global_settings(partial_mock_settings)
+
+        mock_setup_logging_framework.assert_not_called()
+
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.'
+           'main_wait_event_loop')
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.'
+           'instantiate_file_creation_event_observer')
+    def test_station_tests_watchdog(self,
+                                    mock_instantiate_file_creation_event_observer,
+                                    mock_main_wait_event_loop):
+        mock_arguments = [self.mock_path, self.mock_filename, '--addr', self.mock_address]
+
+        mock_observer = MagicMock()
+        mock_instantiate_file_creation_event_observer.return_value = mock_observer
+
+        station_tests_watchdog(mock_arguments)
+
+        mock_instantiate_file_creation_event_observer.assert_called_with(self.mock_path)
+        mock_main_wait_event_loop.assert_called_with(mock_observer, self.mock_filename,
+                                                     self.mock_address, MAIN_LOOP_WAIT_TIME)
+
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.'
+           'main_wait_event_loop')
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.'
+           'instantiate_file_creation_event_observer')
+    def test_parsing_command_arguments_correctly(self,
+                                                 mock_instantiate_file_creation_event_observer,
+                                                 mock_main_wait_event_loop):
+        mock_arguments = [self.mock_path, self.mock_filename, '--addr', self.mock_address]
+
+        mock_observer = MagicMock()
+        mock_instantiate_file_creation_event_observer.return_value = mock_observer
+
+        station_tests_watchdog(mock_arguments)
+
+        mock_instantiate_file_creation_event_observer.assert_called_with(self.mock_path)
+        mock_main_wait_event_loop.assert_called_with(mock_observer, self.mock_filename,
+                                                     self.mock_address, MAIN_LOOP_WAIT_TIME)
+
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.'
+           'sys')
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.'
+           'logging')
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.'
+           'main_wait_event_loop')
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.'
+           'instantiate_file_creation_event_observer')
+    def test_default_logging_level_info(self,
+                                        mock_instantiate_file_creation_event_observer,
+                                        mock_main_wait_event_loop,
+                                        mock_logging,
+                                        partial_mock_sys):
+        mock_arguments = [self.mock_path, self.mock_filename, '--addr', self.mock_address]
+        partial_mock_sys.argv = mock_arguments
+        mock_observer = MagicMock()
+        mock_instantiate_file_creation_event_observer.return_value = mock_observer
+
+        main()
+
+        mock_instantiate_file_creation_event_observer.assert_called_with(self.mock_path)
+        mock_main_wait_event_loop.assert_called_with(mock_observer, self.mock_filename,
+                                                     self.mock_address, MAIN_LOOP_WAIT_TIME)
+
+        mock_logging.basicConfig.assert_called_once_with(
+            format="%(asctime)s %(levelname)s: %(message)s",
+            level=logging.INFO)
+
+
+if __name__ == '__main__':
+    setup_logging_framework(logging.DEBUG)
+    unittest.main()

LCU/Maintenance/MDB_tools/test/t_station_tests_watchdog.run

+#!/bin/bash
+source python-coverage.sh
+
+python_coverage_test station_tests_watchdog t_station_tests_watchdog.py

LCU/Maintenance/MDB_tools/test/t_station_tests_watchdog.sh

+#!/bin/sh
+./runctest.sh t_station_tests_watchdog

LCU/Maintenance/MDB_tools/test/t_station_tests_watchdog_integration.py

+import unittest
+from tempfile import mkdtemp, mkstemp
+import time
+from lofar.maintenance.utils.cli.station_tests_watchdog import *
+from mock import patch
+
+logger = logging.getLogger(__name__)
+
+
+class TestStationTestsWatchdog(unittest.TestCase):
+    def setUp(self):
+        self.temp_file_list = []
+        self.temp_dir_list = []
+
+    @patch('lofar.maintenance.utils.cli.station_tests_watchdog.'
+           'handle_event_file_created')
+    def test_observer_loop(self, handle_event_file_created_mock):
+        tmp_dir = mkdtemp()
+        self.temp_dir_list.append(tmp_dir)
+
+        logger.info('created temp dir %s', tmp_dir)
+        observer = instantiate_file_creation_event_observer(tmp_dir)
+
+        tmp_file = mkstemp(dir=tmp_dir)
+        self.temp_file_list.append(tmp_file[1])
+
+        with open(tmp_file[1], 'w') as f_out_stream:
+            f_out_stream.write('test')
+        logger.info('created temp file %s', tmp_file)
+        main_wait_event_loop(observer, tmp_file[1].split('/')[-1], 'fake_address', 1, run_once=True)
+        handle_event_file_created_mock.assert_called()
+
+    def attemptDelete(self, path):
+        """
+        Attempts the to remove a file. If an exception is thrown prints out
+        the exception message, and the stacktrace, and the path that it was supposed to remove
+        :param path: the path to the file/directory to delete
+        """
+        try:
+            if os.path.isfile(path):
+                os.remove(path)
+            else:
+                os.rmdir(path)
+        except Exception as e:
+            logger.exception('Cannot remove temporary file/directory %s : %s', path, e)
+
+    def tearDown(self):
+        try:
+            for file in self.temp_file_list:
+                self.attemptDelete(file)
+            for directory in self.temp_dir_list:
+                self.attemptDelete(directory)
+        except Exception as e:
+            logger.exception('Cannot remove temp file/directory: %s', e)
+
+            
+if __name__ == '__main__':
+    setup_logging_framework(logging.DEBUG)
+    unittest.main()

LCU/Maintenance/MDB_tools/test/t_station_tests_watchdog_integration.run

+#!/bin/bash
+source python-coverage.sh
+
+python_coverage_test station_tests_watchdog_integration t_station_tests_watchdog_integration.py

LCU/Maintenance/MDB_tools/test/t_station_tests_watchdog_integration.sh

+#!/bin/sh
+./runctest.sh t_station_tests_watchdog_integration