Commit df292509 authored by Nico Vermaas's avatar Nico Vermaas

adding ingest errors

parent a361256e
......@@ -223,7 +223,7 @@ def do_speed_plot(title, y_axis_title, subtitle, annotate, datapoints):
:return:
"""
print('do_speed_plot()')
#print('do_speed_plot()')
fig = plt.figure(figsize=(12,6))
#fig, ax = plt.subplots()
......@@ -243,7 +243,8 @@ def do_speed_plot(title, y_axis_title, subtitle, annotate, datapoints):
observing_y = []
ingesting_x = []
ingesting_y = []
ingest_error_x = []
ingest_error_y = []
i = 0
for datapoint in datapoints:
......@@ -254,8 +255,8 @@ def do_speed_plot(title, y_axis_title, subtitle, annotate, datapoints):
observing_y.append(datapoint['speed_bps'])
# plot start and end points
plt.plot(datapoint['timestamp'], datapoint['speed_bps'], 'r.',
datapoint['timestamp_end'], datapoint['speed_bps'], 'r.')
plt.plot(datapoint['timestamp'], datapoint['speed_bps'], 'b.',
datapoint['timestamp_end'], datapoint['speed_bps'], 'b.')
if annotate is not None:
plt.text(datapoint['timestamp'], datapoint['speed_bps'], datapoint[annotate]+'...',
rotation='vertical', fontsize=8)
......@@ -271,17 +272,28 @@ def do_speed_plot(title, y_axis_title, subtitle, annotate, datapoints):
if annotate is not None:
plt.text(datapoint['timestamp'], datapoint['speed_bps'], datapoint[annotate]+'...',
rotation='vertical', fontsize=8)
if datapoint['type'] == 'ingest_error':
ingest_error_x.append(datapoint['timestamp'])
ingest_error_x.append(datapoint['speed_bps'])
plt.plot(datapoint['timestamp'], datapoint['speed_bps'], 'r.')
if annotate is not None:
plt.text(datapoint['timestamp'], datapoint['speed_bps'], datapoint[annotate] + '...',
rotation='vertical', fontsize=8)
# plot lines
# connect a line
for i in range(0,len(observing_x),2):
plt.plot(observing_x[i:i + 2], observing_y[i:i + 2], 'r:')
plt.plot(observing_x[i:i + 2], observing_y[i:i + 2], 'b:')
for i in range(0,len(ingesting_x),2):
plt.plot(ingesting_x[i:i + 2], ingesting_y[i:i + 2], 'g-')
# legend
plt.plot(observing_x[i:i + 2], observing_y[i:i + 2], 'r:',label='Observing')
plt.plot(observing_x[i:i + 2], observing_y[i:i + 2], 'b:',label='Observing')
plt.plot(ingesting_x[i:i + 2], ingesting_y[i:i + 2], 'g-',label='Ingesting')
#plt.plot(ingest_error_x[i:i], 'r.',label='ingest error')
plt.legend(loc='upper right')
plt.show()
......
......@@ -21,6 +21,7 @@
import os
import sys
import datetime
import time
import plotly
import requests
......@@ -47,6 +48,21 @@ ALTA_API_PROD = "https://alta.astron.nl/altapi"
TIME_FORMAT = "%Y-%m-%d %H:%M"
#--- common functions ---
# this is a decorator that can be put in front (around) a function all to measure its execution time
def timeit(method):
def timed(*args, **kw):
ts = time.time()
result = method(*args, **kw)
te = time.time()
if 'log_time' in kw:
name = kw.get('log_name', method.__name__.upper())
kw['log_time'][name] = int((te - ts) * 1000)
else:
print('execution time: %r %2.2f ms' % \
(method.__name__, (te - ts) * 1000))
return result
return timed
def isCalibrator(name):
"""
check if the provided name is in the (hardcoded) list of defined Apertif Calibrators
......@@ -244,8 +260,8 @@ def do_sky(args, starttime, endtime):
connection.close()
print('Database connection closed.')
def do_ingest_speeds(args, starttime, endtime, plot_engine='plotly'):
@timeit
def do_ingest_speeds(args):
# The request header
ATDB_HEADER = {
......@@ -255,9 +271,6 @@ def do_ingest_speeds(args, starttime, endtime, plot_engine='plotly'):
}
# input parameters
starttime = datetime.datetime.strptime(args.starttime, '%Y-%m-%d %H:%M')
endtime = datetime.datetime.strptime(args.endtime, '%Y-%m-%d %H:%M')
url = args.atdb_host + "/times?" + str(args.query)
......@@ -289,7 +302,7 @@ def do_ingest_speeds(args, starttime, endtime, plot_engine='plotly'):
datapoint['timestamp_end'] = timestamp + datetime.timedelta(seconds=result['duration'])
datapoint['speed_bps'] = result['write_speed'] * 8 / 1000
datapoints.append(datapoint)
print(datapoint)
#print(datapoint)
if result['ingest_speed'] is not None:
datapoint = {}
......@@ -302,8 +315,20 @@ def do_ingest_speeds(args, starttime, endtime, plot_engine='plotly'):
datapoint['timestamp_end'] = timestamp + datetime.timedelta(seconds=result['ingest_duration'])
datapoint['speed_bps'] = result['ingest_speed'] * 8 / 1000
datapoints.append(datapoint)
prev_ingest_speed = datapoint['speed_bps']
# print(datapoint)
if result['timestamp_ingest_error'] is not None:
datapoint = {}
datapoint['taskid'] = result['taskID']
nofrag,frag = result['timestamp_ingest_error'].split('.')
timestamp = datetime.datetime.strptime(nofrag, '%Y-%m-%dT%H:%M:%S')
datapoint['timestamp'] = timestamp
datapoint['type'] = 'ingest_error'
datapoint['speed_bps'] = prev_ingest_speed
datapoints.append(datapoint)
sorted_datapoints = sorted(datapoints, key=lambda k: k['timestamp'])
# plot the results
......@@ -496,7 +521,7 @@ def main():
do_sky(args, starttime, endtime)
elif presentation=="ingest_speed":
do_ingest_speeds(args, starttime, endtime)
do_ingest_speeds(args)
if args.remote_post_command != None:
execute_remote_command(args.atdb_host, args.remote_post_command)
......
--presentation=ingest_speed
--atdb_host=http://192.168.22.22/atdb
--atdb_host=http://atdb.astron.nl/atdb
--title=I/O speeds on wcudata1 from ATDB
--observing_mode=imaging
--y_axis_title=I/O Speed in Gbps
--starttime=2019-06-08 00:00
--endtime=2019-06-09 00:00
--query=taskID__contains=190608
--query=starttime__gt=2019-06-07T00:00:00Z&starttime__lt=2019-06-10T00:00:00Z
--annotate=taskid
--query=starttime__gt=2019-06-06T00:00:00Z&starttime__lt=2019-06-11T00:00:00Z
--plot_engine=mathplotlib
\ No newline at end of file
--presentation=ingest_speed
--atdb_host=http://atdb-test/atdb
--title=I/O speeds on wcudata1 from ATDB
--observing_mode=imaging
--y_axis_title=I/O Speed in Gbps
--starttime=2019-06-08 00:00
--endtime=2019-06-09 00:00
--query=taskID__contains=190608
--query=starttime__gt=2019-05-01T00:00:00Z&starttime__lt=2019-05-02T00:00:00Z
--plot_engine=mathplotlib
\ No newline at end of file
--presentation=ingest_speed
--atdb_host=http://atdb.astron.nl/atdb
--title=I/O speeds on wcudata1 from ATDB
--y_axis_title=I/O Speed in Gbps
--query=taskID__contains=190608
--query=starttime__gt=2019-06-09T00:00:00Z&starttime__lt=2019-06-13T00:00:00Z
--plot_engine=mathplotlib
--presentation=ingest_speed
--atdb_host=http://192.168.22.22/atdb
--title=I/O speeds on wcudata1 from ATDB
--y_axis_title=I/O Speed in Gbps
--query=taskID__contains=190607
--plot_engine=mathplotlib
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment