adding ingest errors

atdb_statistics/atdb_plot.py

@@ -223,7 +223,7 @@ def do_speed_plot(title, y_axis_title, subtitle, annotate, datapoints):
     :return:
     """
 
-    print('do_speed_plot()')
+    #print('do_speed_plot()')
 
     fig = plt.figure(figsize=(12,6))
     #fig, ax = plt.subplots()
@@ -243,7 +243,8 @@ def do_speed_plot(title, y_axis_title, subtitle, annotate, datapoints):
     observing_y = []
     ingesting_x = []
     ingesting_y = []
-
+    ingest_error_x = []
+    ingest_error_y = []
     i = 0
     for datapoint in datapoints:
 
@@ -254,8 +255,8 @@ def do_speed_plot(title, y_axis_title, subtitle, annotate, datapoints):
             observing_y.append(datapoint['speed_bps'])
 
             # plot start and end points
-            plt.plot(datapoint['timestamp'], datapoint['speed_bps'], 'r.',
-                     datapoint['timestamp_end'], datapoint['speed_bps'], 'r.')
+            plt.plot(datapoint['timestamp'], datapoint['speed_bps'], 'b.',
+                     datapoint['timestamp_end'], datapoint['speed_bps'], 'b.')
             if annotate is not None:
                 plt.text(datapoint['timestamp'], datapoint['speed_bps'], datapoint[annotate]+'...',
                          rotation='vertical', fontsize=8)
@@ -271,17 +272,28 @@ def do_speed_plot(title, y_axis_title, subtitle, annotate, datapoints):
             if annotate is not None:
                 plt.text(datapoint['timestamp'], datapoint['speed_bps'], datapoint[annotate]+'...',
                           rotation='vertical', fontsize=8)
+
+        if datapoint['type'] == 'ingest_error':
+            ingest_error_x.append(datapoint['timestamp'])
+            ingest_error_x.append(datapoint['speed_bps'])
+
+            plt.plot(datapoint['timestamp'], datapoint['speed_bps'], 'r.')
+            if annotate is not None:
+                plt.text(datapoint['timestamp'], datapoint['speed_bps'], datapoint[annotate] + '...',
+                         rotation='vertical', fontsize=8)
+
     # plot lines
     # connect a line
     for i in range(0,len(observing_x),2):
-        plt.plot(observing_x[i:i + 2], observing_y[i:i + 2], 'r:')
+        plt.plot(observing_x[i:i + 2], observing_y[i:i + 2], 'b:')
 
     for i in range(0,len(ingesting_x),2):
         plt.plot(ingesting_x[i:i + 2], ingesting_y[i:i + 2], 'g-')
 
     # legend
-    plt.plot(observing_x[i:i + 2], observing_y[i:i + 2], 'r:',label='Observing')
+    plt.plot(observing_x[i:i + 2], observing_y[i:i + 2], 'b:',label='Observing')
     plt.plot(ingesting_x[i:i + 2], ingesting_y[i:i + 2], 'g-',label='Ingesting')
+    #plt.plot(ingest_error_x[i:i], 'r.',label='ingest error')
     plt.legend(loc='upper right')
 
     plt.show()

atdb_stats.py

@@ -21,6 +21,7 @@
 import os
 import sys
 import datetime
+import time
 
 import plotly
 import requests
@@ -47,6 +48,21 @@ ALTA_API_PROD = "https://alta.astron.nl/altapi"
 TIME_FORMAT = "%Y-%m-%d %H:%M"
 
 #--- common functions ---
+# this is a decorator that can be put in front (around) a function all to measure its execution time
+def timeit(method):
+    def timed(*args, **kw):
+        ts = time.time()
+        result = method(*args, **kw)
+        te = time.time()
+        if 'log_time' in kw:
+            name = kw.get('log_name', method.__name__.upper())
+            kw['log_time'][name] = int((te - ts) * 1000)
+        else:
+            print('execution time: %r  %2.2f ms' % \
+                  (method.__name__, (te - ts) * 1000))
+        return result
+    return timed
+
 def isCalibrator(name):
     """
     check if the provided name is in the (hardcoded) list of defined Apertif Calibrators
@@ -244,8 +260,8 @@ def do_sky(args, starttime, endtime):
             connection.close()
             print('Database connection closed.')
 
-
-def do_ingest_speeds(args, starttime, endtime, plot_engine='plotly'):
+@timeit
+def do_ingest_speeds(args):
 
     # The request header
     ATDB_HEADER = {
@@ -255,9 +271,6 @@ def do_ingest_speeds(args, starttime, endtime, plot_engine='plotly'):
     }
 
     # input parameters
-    starttime = datetime.datetime.strptime(args.starttime, '%Y-%m-%d %H:%M')
-    endtime = datetime.datetime.strptime(args.endtime, '%Y-%m-%d %H:%M')
-
 
     url = args.atdb_host + "/times?" + str(args.query)
 
@@ -289,7 +302,7 @@ def do_ingest_speeds(args, starttime, endtime, plot_engine='plotly'):
             datapoint['timestamp_end'] = timestamp + datetime.timedelta(seconds=result['duration'])
             datapoint['speed_bps'] = result['write_speed'] * 8 / 1000
             datapoints.append(datapoint)
-            print(datapoint)
+            #print(datapoint)
 
         if result['ingest_speed'] is not None:
             datapoint = {}
@@ -302,8 +315,20 @@ def do_ingest_speeds(args, starttime, endtime, plot_engine='plotly'):
             datapoint['timestamp_end'] = timestamp + datetime.timedelta(seconds=result['ingest_duration'])
             datapoint['speed_bps'] = result['ingest_speed'] * 8 / 1000
             datapoints.append(datapoint)
+
+            prev_ingest_speed = datapoint['speed_bps']
             # print(datapoint)
 
+        if result['timestamp_ingest_error'] is not None:
+            datapoint = {}
+            datapoint['taskid'] = result['taskID']
+            nofrag,frag = result['timestamp_ingest_error'].split('.')
+            timestamp = datetime.datetime.strptime(nofrag, '%Y-%m-%dT%H:%M:%S')
+            datapoint['timestamp'] = timestamp
+            datapoint['type'] = 'ingest_error'
+            datapoint['speed_bps'] = prev_ingest_speed
+            datapoints.append(datapoint)
+
     sorted_datapoints = sorted(datapoints, key=lambda k: k['timestamp'])
 
     # plot the results
@@ -496,7 +521,7 @@ def main():
        do_sky(args, starttime, endtime)
 
     elif presentation=="ingest_speed":
-       do_ingest_speeds(args, starttime, endtime)
+       do_ingest_speeds(args)
 
     if args.remote_post_command != None:
         execute_remote_command(args.atdb_host, args.remote_post_command)

data/ingest_speed_imaging.args

 --presentation=ingest_speed
---atdb_host=http://192.168.22.22/atdb
+--atdb_host=http://atdb.astron.nl/atdb
 --title=I/O speeds on wcudata1 from ATDB
 --observing_mode=imaging
 --y_axis_title=I/O Speed in Gbps
 --starttime=2019-06-08 00:00
 --endtime=2019-06-09 00:00
 --query=taskID__contains=190608
---query=starttime__gt=2019-06-07T00:00:00Z&starttime__lt=2019-06-10T00:00:00Z
---annotate=taskid
+--query=starttime__gt=2019-06-06T00:00:00Z&starttime__lt=2019-06-11T00:00:00Z
 --plot_engine=mathplotlib
\ No newline at end of file

data/ingest_speed_test_system.args

+--presentation=ingest_speed
+--atdb_host=http://atdb-test/atdb
+--title=I/O speeds on wcudata1 from ATDB
+--observing_mode=imaging
+--y_axis_title=I/O Speed in Gbps
+--starttime=2019-06-08 00:00
+--endtime=2019-06-09 00:00
+--query=taskID__contains=190608
+--query=starttime__gt=2019-05-01T00:00:00Z&starttime__lt=2019-05-02T00:00:00Z
+--plot_engine=mathplotlib
\ No newline at end of file

data/ingest_speed_today_prod.args

+--presentation=ingest_speed
+--atdb_host=http://atdb.astron.nl/atdb
+--title=I/O speeds on wcudata1 from ATDB
+--y_axis_title=I/O Speed in Gbps
+--query=taskID__contains=190608
+--query=starttime__gt=2019-06-09T00:00:00Z&starttime__lt=2019-06-13T00:00:00Z
+--plot_engine=mathplotlib

data/ingest_speed_today_vm.args

+--presentation=ingest_speed
+--atdb_host=http://192.168.22.22/atdb
+--title=I/O speeds on wcudata1 from ATDB
+--y_axis_title=I/O Speed in Gbps
+--query=taskID__contains=190607
+--plot_engine=mathplotlib