diff --git a/LCU/Maintenance/MDB_tools/cli/probe_mdb.py b/LCU/Maintenance/MDB_tools/cli/probe_mdb.py
index 91735b7893b61f98156fa40269d657ab1036e883..3a04271cebc66602b895ecd03c8dda631255031f 100755
--- a/LCU/Maintenance/MDB_tools/cli/probe_mdb.py
+++ b/LCU/Maintenance/MDB_tools/cli/probe_mdb.py
@@ -14,8 +14,8 @@ import requests
logger = logging.getLogger('probe_mdb')
"""
-This program is meant to probe from the command line the station tests and RTSM in a certain period of
-time and on a certain station name
+This program is meant to probe from the command line the station tests and RTSM in a certain period
+ of time and on a certain station name
"""
CACHING_SIZE = 100
@@ -26,24 +26,27 @@ def setup_argument_parser():
:return: an instance of the argument parser with all the probe_mdb options
:rtype: argparse.ArgumentParser
"""
- parser = argparse.ArgumentParser(prog='probe_mdb', description="probe_mdb is a tool which simplify the queries to" +
- " the maintenance database. Specifically, it helps"
- " to gather past test output performed on a "
- " specific station, in a specific time range"
- " or a specific class of stations")
+ parser = argparse.ArgumentParser(prog='probe_mdb',
+ description="probe_mdb is a tool which simplify the queries" +
+ " to the maintenance database. Specifically, it"
+ " helps to gather past test output performed on a "
+ " specific station, in a specific time range"
+ " or a specific class of stations")
parser.add_argument('--address', help='address of the server. default [localhost]:8000',
default='http://localhost:8000')
parser.add_argument('--from_date', help='selects tests from date [YYYY-MM-DD]', default=None)
parser.add_argument('--to_date', help='selects tests to date [YYYY-MM-DD]', default=None)
parser.add_argument('--station', help='selects a specific station. ex. CS001C', default=None)
- parser.add_argument('--station_type', help='selects a specific station type. ex. I', choices=['C', 'R', 'I'])
+ parser.add_argument('--station_type', help='selects a specific station type. ex. I',
+ choices=['C', 'R', 'I'])
parser.add_argument('--last-month', help='selects the last month results', action='store_true')
- parser.add_argument('--request-size', help='change the request size. Current default size is {}'.
+ parser.add_argument('--request-size',
+ help='change the request size. Current default size is {}'.
format(CACHING_SIZE))
parser.add_argument('-v', help='verbose output', action="store_false")
# TODO implement it
- #parser.add_argument('--to_csv', help='s results into a csv file', default=None)
+ # parser.add_argument('--to_csv', help='s results into a csv file', default=None)
return parser
@@ -69,7 +72,8 @@ def query_api_for_stationtest_rtsm_results(query_string, address, next_url=None)
extracted_content = parsed_content['results']
next_url = parsed_content['next']
if next_url is not None:
- return extracted_content + query_api_for_stationtest_rtsm_results(query_string, address, next_url)
+ return extracted_content + query_api_for_stationtest_rtsm_results(query_string, address,
+ next_url)
else:
return extracted_content
else:
@@ -88,34 +92,15 @@ def get_url_string_to_select_timespan(from_date, to_date):
"""
query = ''
if from_date:
- try:
- if isinstance(from_date, datetime):
- date = from_date
- else:
- date = datetime.strptime(from_date, '%Y-%m-%d')
- query += '&start_time__year__gte=%d' % (date.year,)
- query += '&start_time__month__gte=%d' % (date.month,)
- query += '&start_time__day__gte=%d' % (date.day,)
-
- except ValueError:
- logger.error('format of from date not valid; YYYY-MM-DD es. 2016-15-12')
- raise
+ query += '&start_time__year__gte=%d' % (from_date.year,)
+ query += '&start_time__month__gte=%d' % (from_date.month,)
+ query += '&start_time__day__gte=%d' % (from_date.day,)
if to_date:
- try:
+ query += '&start_time__year__lte=%d' % (to_date.year,)
+ query += '&start_time__month__lte=%d' % (to_date.month,)
+ query += '&start_time__day__lte=%d' % (to_date.day,)
- if not isinstance(to_date, datetime):
- date = datetime.strptime(to_date, '%Y-%m-%d')
- else:
- date = to_date
-
- query += '&start_time__year__lte=%d' % (date.year,)
- query += '&start_time__month__lte=%d' % (date.month,)
- query += '&start_time__day__lte=%d' % (date.day,)
-
- except ValueError:
- logger.error('format of to date not valid; YYYY-MM-DD es. 2016-15-12')
- raise
return query
@@ -176,7 +161,7 @@ def retrieve_station_tests(address, from_time, to_time, station_name=""):
if from_time or to_time:
query_string += get_url_string_to_select_timespan(from_time, to_time)
- if station_name and station_name is not None:
+ if station_name:
query_string += get_url_string_to_select_station_name(station_name)
return query_api_for_stationtest_rtsm_results(query_string, address)
@@ -274,7 +259,8 @@ def print_out_component_errors(station_test):
Prints out the output of the component errors
:param station_test: the dict that contains the station_test
"""
- excluded_fields = {'parent', 'resourcetype', 'polymorphic_ctype', 'id', 'station_test_id', 'tile_errors',
+ excluded_fields = {'parent', 'resourcetype', 'polymorphic_ctype', 'id', 'station_test_id',
+ 'tile_errors',
'parent_component_error'}
summary_fields = {'component_type', 'component_id', 'error_type'}
tile_summary_fields = {'tile_id'}
@@ -374,7 +360,8 @@ def print_out_rtsm_summary(rtsm_results):
error_type = error['error_type']
rtsm_errors[(rcu_id, error_type, start_frequency, stop_frequency, mode)] += 1
for key, value in rtsm_errors.items():
- table.append_row([obs_id, station_name, start_time, end_time]+list(key) + [value*100/samples])
+ table.append_row(
+ [obs_id, station_name, start_time, end_time] + list(key) + [value * 100 / samples])
print(table)
print()
@@ -409,7 +396,8 @@ def print_out_rtsm(rtsm_results):
time = error['time']
mode = error['mode']
error_type = error['error_type']
- table.append_row([obs_id, station_name, time, rcu_id, mode, error_type, start_frequency, stop_frequency])
+ table.append_row([obs_id, station_name, time, rcu_id, mode, error_type, start_frequency,
+ stop_frequency])
print(table)
@@ -431,6 +419,24 @@ def print_out_rtsm_and_stationtests(station_test_results, rtsm_results):
print_out_rtsm_summary(rtsm_results)
+def parse_specified_datetime(specified_datetime):
+ """
+ Parses the input datetime and coverts it to datetime object
+ the expected datetime format '%Y-%m-%d'
+ :param specified_datetime: specified datetime from command line
+ :return: a datetime object
+ """
+ try:
+ if isinstance(specified_datetime, datetime):
+ date_str = specified_datetime
+ else:
+ date_str = datetime.strptime(specified_datetime, '%Y-%m-%d')
+ except ValueError:
+ logger.error('format of to date not valid; YYYY-MM-DD es. 2016-15-12')
+ raise
+ return date_str
+
+
def probe_mdb():
"""
Main entry for the script
@@ -442,11 +448,11 @@ def probe_mdb():
if args.v:
logging.basicConfig(format="%(asctime)s %(levelname)s: %(message)s", level=logging.DEBUG)
- from_date = args.from_date
- to_date = args.to_date
+ from_date = parse_specified_datetime(args.from_date)
+ to_date = parse_specified_datetime(args.to_date)
if args.last_month:
- from_date = datetime.now()-timedelta(days=30)
+ from_date = datetime.now() - timedelta(days=30)
station_test_results = retrieve_station_tests(args.address, from_date, to_date, args.station)
@@ -457,4 +463,3 @@ def probe_mdb():
def main():
probe_mdb()
-
diff --git a/LCU/Maintenance/MDB_tools/test/t_probe_mdb.py b/LCU/Maintenance/MDB_tools/test/t_probe_mdb.py
index 9a32350743fa840446727e0762c4140d120dde82..c0f590879ec5b6cc5c1d52e321f7c401f0b5b950 100644
--- a/LCU/Maintenance/MDB_tools/test/t_probe_mdb.py
+++ b/LCU/Maintenance/MDB_tools/test/t_probe_mdb.py
@@ -1,16 +1,19 @@
+import json
import logging
import unittest
from datetime import datetime
from mock import patch, Mock
-from output_testing import compare_output_with_file_content
-import json
from lofar.maintenance.utils.cli.probe_mdb import get_url_string_to_select_timespan, TestType, \
- get_url_string_to_select_test_type, CACHING_SIZE, get_url_string_to_select_station_name, retrieve_station_tests,\
- retrieve_rtsm, reformat_datetime, sort_component_errors, rtsm_sort_errors, query_api_for_stationtest_rtsm_results,\
- print_out_station_test_summary, print_out_component_errors_summary, print_out_component_errors,\
- print_out_rtsm_summary, print_out_rtsm, print_out_rtsm_and_stationtests
+ get_url_string_to_select_test_type, CACHING_SIZE, get_url_string_to_select_station_name, \
+ retrieve_station_tests, retrieve_rtsm, reformat_datetime, sort_component_errors, \
+ rtsm_sort_errors, query_api_for_stationtest_rtsm_results, \
+ print_out_station_test_summary, print_out_component_errors_summary, \
+ print_out_component_errors, \
+ print_out_rtsm_summary, print_out_rtsm, print_out_rtsm_and_stationtests, \
+ parse_specified_datetime
+from output_testing import compare_output_with_file_content
logger = logging.getLogger(__name__)
@@ -87,39 +90,47 @@ class TESTProbeMDB(unittest.TestCase):
self.assertEqual(response, [])
- def test_get_url_string_to_select_timespan_from_date_success(self):
- from_date = "2018-12-13"
- response = get_url_string_to_select_timespan(from_date, None)
- self.assertEqual(response, '&start_time__year__gte=2018&start_time__month__gte=12&start_time__day__gte=13')
-
+ def test_parse_specified_datetime(self):
+ from_date_str = "2018-12-13"
from_date = datetime(2018, 12, 13)
- response = get_url_string_to_select_timespan(from_date, None)
- self.assertEqual(response, '&start_time__year__gte=2018&start_time__month__gte=12&start_time__day__gte=13')
+ result = parse_specified_datetime(from_date_str)
+ self.assertEqual(result, from_date)
- def test_get_url_string_to_select_timespan_wrong_string_format(self):
- date = "2018-12-13-21"
- with self.assertRaises(ValueError):
- get_url_string_to_select_timespan(date, None)
+ result = parse_specified_datetime(from_date)
+
+ self.assertEqual(result, from_date)
+
+ from_date_str = "2018-12-13-M1"
with self.assertRaises(ValueError):
- get_url_string_to_select_timespan(None, date)
+ result = parse_specified_datetime(from_date_str)
- def test_get_url_string_to_select_timespan_to_date_success(self):
- to_date = "2018-12-13"
- response = get_url_string_to_select_timespan(None, to_date)
- self.assertEqual(response, '&start_time__year__lte=2018&start_time__month__lte=12&start_time__day__lte=13')
+ def test_get_url_string_to_select_timespan_from_date_success(self):
+ from_date = datetime(2018, 12, 13)
+ response = get_url_string_to_select_timespan(from_date, None)
+ self.assertEqual(response,
+ '&start_time__year__gte=2018&'
+ 'start_time__month__gte=12&'
+ 'start_time__day__gte=13')
+ def test_get_url_string_to_select_timespan_to_date_success(self):
to_date = datetime(2018, 12, 13)
response = get_url_string_to_select_timespan(None, to_date)
- self.assertEqual(response, '&start_time__year__lte=2018&start_time__month__lte=12&start_time__day__lte=13')
-
+ self.assertEqual(response,
+ '&start_time__year__lte=2018&'
+ 'start_time__month__lte=12&'
+ 'start_time__day__lte=13')
def test_get_query_string_none_specified(self):
self.assertEqual(get_url_string_to_select_timespan(None, None), '')
def test_get_query_string_both_specified(self):
- date = "2018-12-13"
- expected_result = '&start_time__year__gte=2018&start_time__month__gte=12&start_time__day__gte=13' + \
- '&start_time__year__lte=2018&start_time__month__lte=12&start_time__day__lte=13'
+ date = datetime(2018, 12, 13)
+ expected_result = '&start_time__year__gte=2018&' \
+ 'start_time__month__gte=12&' \
+ 'start_time__day__gte=13&' \
+ 'start_time__year__lte=2018&' \
+ 'start_time__month__lte=12&s' \
+ 'tart_time__day__lte=13'
self.assertEqual(get_url_string_to_select_timespan(date, date), expected_result)
def test_get_query_string_per_type_rtsm(self):
@@ -162,10 +173,11 @@ class TESTProbeMDB(unittest.TestCase):
@patch('lofar.maintenance.utils.cli.probe_mdb.get_url_string_to_select_timespan')
@patch('lofar.maintenance.utils.cli.probe_mdb.get_url_string_to_select_station_name')
@patch('lofar.maintenance.utils.cli.probe_mdb.query_api_for_stationtest_rtsm_results')
- def test_retrieve_station_tests_time_not_set(self, query_api_for_stationtest_rtsm_results_mock,
- get_url_string_to_select_station_name_mock,
- get_url_string_to_select_timespan_mock,
- get_url_string_to_select_test_type_mock):
+ def test_retrieve_station_tests_time_not_set(self,
+ query_api_for_stationtest_rtsm_results_mock,
+ get_url_string_to_select_station_name_mock,
+ get_url_string_to_select_timespan_mock,
+ get_url_string_to_select_test_type_mock):
from_time = None
to_time = None
address = "test_address"
@@ -181,10 +193,11 @@ class TESTProbeMDB(unittest.TestCase):
@patch('lofar.maintenance.utils.cli.probe_mdb.get_url_string_to_select_timespan')
@patch('lofar.maintenance.utils.cli.probe_mdb.get_url_string_to_select_station_name')
@patch('lofar.maintenance.utils.cli.probe_mdb.query_api_for_stationtest_rtsm_results')
- def test_retrieve_station_tests_station_name_not_set(self, query_api_for_stationtest_rtsm_results_mock,
- get_url_string_to_select_station_name_mock,
- get_url_string_to_select_timespan_mock,
- get_url_string_to_select_test_type_mock):
+ def test_retrieve_station_tests_station_name_not_set(self,
+ query_api_for_stationtest_rtsm_results_mock,
+ get_url_string_to_select_station_name_mock,
+ get_url_string_to_select_timespan_mock,
+ get_url_string_to_select_test_type_mock):
from_time = "2013-05-01"
to_time = "2013-05-01"
address = "test_address"
@@ -215,9 +228,9 @@ class TESTProbeMDB(unittest.TestCase):
@patch('lofar.maintenance.utils.cli.probe_mdb.get_url_string_to_select_station_name')
@patch('lofar.maintenance.utils.cli.probe_mdb.query_api_for_stationtest_rtsm_results')
def test_retrieve_rtsm_all_set(self, query_api_for_stationtest_rtsm_results_mock,
- get_url_string_to_select_station_name_mock,
- get_url_string_to_select_timespan_mock,
- get_url_string_to_select_test_type_mock):
+ get_url_string_to_select_station_name_mock,
+ get_url_string_to_select_timespan_mock,
+ get_url_string_to_select_test_type_mock):
from_time = "2013-12-12"
to_time = "2013-12-11"
address = "test_address"
@@ -234,9 +247,9 @@ class TESTProbeMDB(unittest.TestCase):
@patch('lofar.maintenance.utils.cli.probe_mdb.get_url_string_to_select_station_name')
@patch('lofar.maintenance.utils.cli.probe_mdb.query_api_for_stationtest_rtsm_results')
def test_retrieve_rtsm_time_not_set(self, query_api_for_stationtest_rtsm_results_mock,
- get_url_string_to_select_station_name_mock,
- get_url_string_to_select_timespan_mock,
- get_url_string_to_select_test_type_mock):
+ get_url_string_to_select_station_name_mock,
+ get_url_string_to_select_timespan_mock,
+ get_url_string_to_select_test_type_mock):
from_time = None
to_time = None
address = "test_address"
@@ -326,33 +339,45 @@ class TESTProbeMDB(unittest.TestCase):
def test_print_out_station_test_summary(self):
test_station_test = read_api_station_test_output()[0]
- with compare_output_with_file_content('t_probe_mdb.in.print_out_station_test_summary_expected_output.txt'):
+ with compare_output_with_file_content('t_probe_mdb.in.'
+ 'print_out_station_test_summary'
+ '_expected_output.txt'):
print_out_station_test_summary(test_station_test, 1)
def test_print_out_component_error_summary(self):
test_station_test = read_api_station_test_output()[0]
- with compare_output_with_file_content('t_probe_mdb.in.print_out_component_errors_summary_expected_output.txt'):
+ with compare_output_with_file_content('t_probe_mdb.in.'
+ 'print_out_component_errors_summary'
+ '_expected_output.txt'):
print_out_component_errors_summary(test_station_test)
def test_print_out_component_errors(self):
test_station_test = read_api_station_test_output()[0]
- with compare_output_with_file_content('t_probe_mdb.in.print_out_component_errors_expected_output.txt'):
+ with compare_output_with_file_content('t_probe_mdb.in.'
+ 'print_out_component_errors'
+ '_expected_output.txt'):
print_out_component_errors(test_station_test)
def test_print_out_rtsm_summary(self):
test_rtsm = read_api_rtsm_test_output()
- with compare_output_with_file_content('t_probe_mdb.in.print_out_rtsm_summary_expected_output.txt'):
+ with compare_output_with_file_content('t_probe_mdb.in.'
+ 'print_out_rtsm_summary'
+ '_expected_output.txt'):
print_out_rtsm_summary(test_rtsm)
def test_print_out_rtsm(self):
test_rtsm = read_api_rtsm_test_output()
- with compare_output_with_file_content('t_probe_mdb.in.print_out_rtsm_expected_output.txt'):
+ with compare_output_with_file_content('t_probe_mdb.in.'
+ 'print_out_rtsm_'
+ 'expected_output.txt'):
print_out_rtsm(test_rtsm)
def test_print_out_rtsm_and_stationtests(self):
test_rtsm = read_api_rtsm_test_output()
test_station_test = read_api_station_test_output()
- with compare_output_with_file_content('t_probe_mdb.in.print_out_rtsm_and_stationtests_expected_output.txt'):
+ with compare_output_with_file_content('t_probe_mdb.in.'
+ 'print_out_rtsm_and_stationtests_'
+ 'expected_output.txt'):
print_out_rtsm_and_stationtests(test_station_test, test_rtsm)