Task #8531: Almost working unit-test

.gitattributes

@@ -4821,9 +4821,13 @@ SAS/OTDB/src/setStatus.conf -text
 SAS/OTDB/test/tBrokenHardware.cc -text
 SAS/OTDB/test/tMetadata.cc -text
 SAS/OTDB/test/tQueryPIC.cc -text
+SAS/OTDB_Services/CMakeLists.txt -text
 SAS/OTDB_Services/TreeService.py -text
 SAS/OTDB_Services/TreeStatusEvents.py -text
+SAS/OTDB_Services/test/CMakeLists.txt -text
 SAS/OTDB_Services/test/t_TreeService.py -text
+SAS/OTDB_Services/test/t_TreeService.run -text svneol=unset#application/x-shellscript
+SAS/OTDB_Services/test/t_TreeService.sh -text svneol=unset#application/x-shellscript
 SAS/Scheduler/src/.default_settings.set -text
 SAS/Scheduler/src/LOFAR_libScheduler.pro -text
 SAS/Scheduler/src/conflictdialog.ui -text

LCS/Messaging/python/messaging/RPC.py

@@ -25,7 +25,7 @@ from lofar.messaging.messages import ServiceMessage, ReplyMessage
 import uuid
 
 class RPCException(Exception):
-    "Passing exception of the messahe handler function."
+    "Exception occured in the RPC code itself, like time-out, invalid message received, etc."
     pass
 
 class RPC():
@@ -110,7 +110,7 @@ class RPC():
             status["state"] = "TIMEOUT"
             status["errmsg"] = "RPC Timed out"
             status["backtrace"] = ""
-            return (None, status)
+            raise RPCException(status)
 
         # Check for illegal message type
         if isinstance(answer, ReplyMessage) is False:
@@ -118,7 +118,7 @@ class RPC():
             status["state"] = "ERROR"
             status["errmsg"] = "Incorrect messagetype (" + str(type(answer)) + ") received."
             status["backtrace"] = ""
-            return (None, status)
+            raise RPCException(status)
 
         # return content and status if status is 'OK'
         if (answer.status == "OK"):
@@ -133,7 +133,7 @@ class RPC():
             status["state"] = "ERROR"
             status["errmsg"] = "Return state in message not found"
             status["backtrace"] = ""
-            return (Null, status)
+            raise RPCException(status)
 
         # Does the client expect us to throw the exception?
         if self.ForwardExceptions is True:
@@ -143,7 +143,7 @@ class RPC():
                 instance = excep_class_(answer.backtrace)
                 raise (instance)
             else:
-                raise (RPCException(answer.errmsg))
-        return (None,status)
+                raise RPCException(answer.errmsg)
+        return (None, status)
 
 __all__ = ["RPC", "RPCException"]

SAS/OTDB/sql/exportTree_func.sql

@@ -228,30 +228,27 @@ CREATE OR REPLACE FUNCTION exportPICSubTree(INT4, INT4, INT4)
 	DECLARE
 	  vResult		TEXT := '';
 	  vRow			RECORD;
+      vName         PIChierarchy.name%TYPE;
 
 	BEGIN
-	  -- first dump own parameters
-	  FOR vRow IN
-	    SELECT	name     --, value
-	    FROM	PIChierarchy
-	    WHERE	treeID = $1
-	    AND	 	parentID = $2
-		AND		leaf = true
-		ORDER BY name
-	  LOOP
-		vResult := vResult || substr(vRow.name,$3) || chr(10); 
-	  END LOOP;
+	  -- first get name of top node
+      SELECT	name
+      INTO      vName
+	  FROM	PIChierarchy
+	  WHERE	treeID = $1
+	  AND	nodeID = $2;
+      vName := vName || '%';
 
 	  -- call myself for all the children
 	  FOR vRow IN
-	    SELECT	nodeID, name
+	    SELECT	name
 	    FROM	PIChierarchy
 	    WHERE	treeID = $1
-	    AND	 	parentID = $2
-		AND		leaf = false
-		ORDER BY name
+	    AND	 	name like vName
+		AND		leaf = true
+		ORDER BY nodeid
 	  LOOP
-		vResult := vResult || exportPICSubTree($1, vRow.nodeID, $3);
+		vResult := vResult || substr(vRow.name,$3) || chr(10); 
 	  END LOOP;
 
 	  RETURN vResult;

SAS/OTDB/sql/upgradeOTDB.sql

@@ -6,3 +6,6 @@ CREATE INDEX statehist_creation_idx ON statehistory(creation);
 
 -- Load new functions
 \i getStateChanges.sql
+
+-- Load modified/improved functions
+\i exportTree_func.sql

SAS/OTDB_Services/CMakeLists.txt

+# $Id$
+
+#lofar_package(OTDB_Services 1.0 DEPENDS PyMessaging OTDB)
+
+include(PythonInstall)
+
+set(_py_files
+  __init__.py
+  TreeService.py
+  TreeStatusEvents.py
+)
+
+python_install(${_py_files} DESTINATION bin)
+
+add_subdirectory(test)
+

SAS/OTDB_Services/TreeService.py

@@ -32,7 +32,6 @@ import os,sys,time,pg
 import logging
 from optparse import OptionParser
 from lofar.messaging.Service import *
-from lofar.messaging.RPC import *
 
 QUERY_EXCEPTIONS = (TypeError, ValueError, MemoryError, pg.ProgrammingError, pg.InternalError)
 
@@ -76,14 +75,20 @@ def TaskSpecificationRequest(input_dict, db_connection):
     except QUERY_EXCEPTIONS, exc_info:
         raise FunctionError("Error while requesting specs of tree %d: %s"% (tree_id, exc_info))
     # convert the single string into the dict
-    answer = {}
+    # Note: a PIC tree is a list of keys while a Template tree and a VIC tree is a list of key-values.
+    #       Since we don't know what kind of tree was requested we assume a Template/VIC tree (most likely)
+    #       but if this ends in exceptions we fall back to a PIC tree.
+    answer_dict = {}
+    answer_list = []
     for line in treeinfo.split('\n'):
         try:
             (key, value) = line.split("=", 1)
-            answer[key] = value
+            answer_dict[key] = value
         except ValueError:
-            pass
-    return answer
+            answer_list.append(line)
+    if len(answer_list)>1:		# there is always one empty line, ignore that one...
+        answer_dict["tree"] = answer_list
+    return answer_dict
 
 # Status Update Command
 def StatusUpdateCommand(input_dict, db_connection):
@@ -136,7 +141,7 @@ def StatusUpdateCommand(input_dict, db_connection):
             (tree_id, allowed_states[new_status], str(update_times))).getresult()[0][0] == 't')
     except QUERY_EXCEPTIONS, exc_info:
         raise FunctionError("Error while setting the status of tree %d: %s" % (tree_id, exc_info))
-    return success
+    return str(success)
 
 
 # Key Update Command
@@ -188,7 +193,8 @@ def KeyUpdateCommand(input_dict, db_connection):
             print "%s: %s ==> %s" % (key, record_list[0][2], value)
         except QUERY_EXCEPTIONS, exc:
             errors[key] = str(exc)
-
+    if len(errors):
+        raise FunctionError(("Not all key were updated:", errors))
     return errors
 
 class PostgressMessageHandlerInterface(MessageHandlerInterface):

SAS/OTDB_Services/test/CMakeLists.txt

+# $Id: CMakeLists.txt 1576 2015-09-29 15:22:28Z loose $
+
+include(LofarCTest)
+
+lofar_add_test(t_TreeService)
+

SAS/OTDB_Services/test/t_TreeService.py

@@ -32,34 +32,82 @@ import sys
 import logging
 from lofar.messaging.RPC import *
 
-logging.basicConfig(stream=sys.stdout, level=logging.INFO)
+logging.basicConfig(stream=sys.stdout, level=logging.WARNING)
 logger = logging.getLogger(__name__)
 
 def do_rpc(rpc_instance, arg_dict):
-    try:
-        (data,status) = (rpc_instance)(arg_dict)
-        if (status != "OK"):
-            raise Exception("Status returned is %s" % status)
-        print type(data)
-        for (key,value) in data.iteritems():
-            print "%s ==> %s" % (key, value)
-    except OverflowError as e:
-        pass
+#    try:
+    (data,status) = (rpc_instance)(arg_dict)
+    if (status != "OK"):
+        raise Exception("Status returned is %s" % status)
+    for key in sorted(data):
+        print "%s ==> %s" % (key, data[key])
+#    except OverflowError as e:
+#        pass
     print "======"
 
 if __name__ == "__main__":
     busname = sys.argv[1] if len(sys.argv) > 1 else "simpletest"
 
-#    with RPC("TaskSpecification", ForwardExceptions=True, busname=busname, timeout=2) as task_spec_request:
-#        do_rpc(task_spec_request, {'OtdbID':63370})
-#        do_rpc(task_spec_request, {'OtdbID':146300})
-#        do_rpc(task_spec_request, {'OtdbID':82111})
+    with RPC("TaskSpecification", ForwardExceptions=True, busname=busname, timeout=10) as task_spec_request:
+        do_rpc(task_spec_request, {'OtdbID':1099269})	# PIC
+        do_rpc(task_spec_request, {'OtdbID':1099238})	# Template
+        do_rpc(task_spec_request, {'OtdbID':1099266})	# VIC
 
-#    print StatusUpdateCmd({'OtdbID':146300, 'NewStatus':'finished', 'UpdateTimestamps':True})
+    with RPC("StatusUpdateCmd", ForwardExceptions=True, busname=busname, timeout=5) as status_update_command:
+        # PIC
+        (data, status) = status_update_command({'OtdbID':1099269, 'NewStatus':'finished', 'UpdateTimestamps':True})
+        print status, data
+        # Template
+        (data, status) = status_update_command({'OtdbID':1099238, 'NewStatus':'finished', 'UpdateTimestamps':True})
+        print status, data
+        # VIC
+        (data, status) = status_update_command({'OtdbID':1099266, 'NewStatus':'finished', 'UpdateTimestamps':True})
+        print status, data
 
-    with RPC("KeyUpdateCmd", ForwardExceptions=True, busname=busname, timeout=2) as key_update:
-        print key_update({'OtdbID':63370, 
-                          'Updates':{'LOFAR.ObsSW.Observation.ObservationControl.OnlineControl._hostname':'CCU099ABC'}})
+        # Nonexisting tree
+        try:
+            (data, status) = status_update_command({'OtdbID':10, 'NewStatus':'finished', 'UpdateTimestamps':True})
+        except RPCException as e:
+            print "Caught expected exception on invalid treeID in status update"
 
-   
+        # VIC tree: invalid status
+        try:
+            (data, status) = status_update_command({'OtdbID':1099266, 'NewStatus':'what_happend', 'UpdateTimestamps':True})
+        except RPCException as e:
+            print "Caught expected exception on invalid status in status update"
+
+
+    with RPC("KeyUpdateCmd", ForwardExceptions=True, busname=busname, timeout=5) as key_update:
+        # VIC tree: valid
+        (data, status) = key_update({'OtdbID':1099266, 
+                          'Updates':{'LOFAR.ObsSW.Observation.ObservationControl.PythonControl.pythonHost':'NameOfTestHost'}})
+        print status, data
+
+        # Template tree: not supported yet
+        try: 
+            (data, status) = key_update({'OtdbID':1099238, 
+                          'Updates':{'LOFAR.ObsSW.Observation.ObservationControl.PythonControl.pythonHost':'NameOfTestHost'}})
+        except RPCException as e:
+            print "Caught expected exception on invalid treetype in key update"
 
+        # PIC tree: not supported yet
+        try:
+            (data, status) = key_update({'OtdbID':1099269, 'Updates':{'LOFAR.PIC.Core.CS001.status_state':'50'}})
+        except RPCException as e:
+            print "Caught expected exception on invalid treetype (PIC) in key update"
+   
+        # Non exsisting tree
+        try:
+            (data, status) = key_update({'OtdbID':10, 
+                          'Updates':{'LOFAR.ObsSW.Observation.ObservationControl.PythonControl.pythonHost':'NameOfTestHost'}})
+        except RPCException as e:
+            print "Caught expected exception on invalid treeID in key update"
+   
+        # VIC tree: wrong key
+        try:
+            (data, status) = key_update({'OtdbID':1099266, 
+                          'Updates':{'LOFAR.ObsSW.Observation.ObservationControl.PythonControl.NoSuchKey':'NameOfTestHost'}})
+        except RPCException as e:
+            print "Caught expected exception on invalid key in key update"
+   

SAS/OTDB_Services/test/t_TreeService.run

+#!/bin/sh -ex
+# constants
+DBHOST=sas099.control.lofar
+
+#cleanup on normal exit and on SIGHUP, SIGINT, SIGQUIT, and SIGTERM
+trap 'qpid-config del exchange --force $queue ; kill ${SERVER_PID}' 0 1 2 3 15
+
+# Generate randome queue name
+queue=$(< /dev/urandom tr -dc [:alnum:] | head -c16)
+
+# Create the queue
+qpid-config add exchange topic $queue
+
+# Setup a clean database with predefined content
+dropdb -U postgres -h ${DBHOST} unittest_db
+createdb -U postgres -h ${DBHOST} unittest_db
+psql -U postgres -h ${DBHOST} -f $src_dir/unittest_db.dump
+python TreeService.py $queue -D unittest_db -H ${DBHOST} &
+SERVER_PID=$!
+# Starting up takes a while
+sleep 3
+
+# Run the unit test
+# either with or without code coverage measurements,
+# depending wheter coverage has been installed
+
+if type "coverage" > /dev/null; then
+    #run test using python coverage tool
+
+    #erase previous results
+    coverage erase
+
+    #setup coverage config file
+    printf "[report]\nexclude_lines = \n  if __name__ == .__main__.\n  def main\n" > .coveragerc
+
+    coverage run --branch --include=*Messaging/python* t_TreeService.py $queue
+    RESULT=$?
+    if [ $RESULT -eq 0 ]; then
+        echo " *** Code coverage results *** "
+        coverage report -m
+        echo " *** End coverage results *** "
+    fi
+    exit $RESULT
+else
+    #coverage not available
+    echo "Please run: 'pip install coverage' to enable code coverage reporting of the unit tests"
+    #run plain test script
+    python t_TreeService.py $queue 
+fi
+

SAS/OTDB_Services/test/t_TreeService.sh

+#!/bin/sh
+./runctest.sh t_TreeService