Commit 3b16656d authored by Nico Vermaas's avatar Nico Vermaas

initial gitlab commit

parent d5647fde
"""
File name: atdb_plot.py
version: 1.0.0 (28 mar 2019)
version: 1.0.0 (14 jun 2019)
Author: Copyright (C) 2019 - Nico Vermaas - ASTRON
Description: atdb plot module
"""
import datetime
import plotly
import plotly.graph_objs as go
......@@ -215,7 +216,7 @@ def do_sky_plot(plot_engine, title, x,y, duration, sizes, output_html,y_axis_tit
plt.show()
def do_speed_plot(title, y_axis_title, subtitle, annotate, datapoints):
def do_times_plot(title, y_axis_title, subtitle, annotate, datapoints):
"""
:param title: Title of Plot
:param x: dict with data for x-axis (time)
......@@ -229,7 +230,7 @@ def do_speed_plot(title, y_axis_title, subtitle, annotate, datapoints):
#fig, ax = plt.subplots()
plt.text(x=0.5, y=0.94, s=title, fontsize=14, ha="center", transform=fig.transFigure)
plt.text(x=0.5, y=0.90, s='query: '+subtitle, fontsize=10, ha="center", transform=fig.transFigure)
plt.text(x=0.5, y=0.90, s=subtitle, fontsize=10, ha="center", transform=fig.transFigure)
#plt.title(title)
#plt.suptitle(subtitle)
......@@ -298,3 +299,101 @@ def do_speed_plot(title, y_axis_title, subtitle, annotate, datapoints):
plt.show()
def do_speeds_plot(title, y_axis_title, subtitle, annotate, datapoints):
"""
:param title: Title of Plot
:param x: dict with data for x-axis (time)
:param y: dict with data for y_axix (usage)
:return:
"""
#print('do_speed_plot()')
fig = plt.figure(figsize=(12,6))
#fig, ax = plt.subplots()
plt.text(x=0.5, y=0.94, s=title, fontsize=14, ha="center", transform=fig.transFigure)
plt.text(x=0.5, y=0.90, s=subtitle, fontsize=10, ha="center", transform=fig.transFigure)
#plt.title(title)
#plt.suptitle(subtitle)
plt.xlabel('Timestamp')
plt.ylabel(y_axis_title)
plt.grid(True,alpha=0.3)
observing_x = []
observing_y = []
ingesting_x = []
ingesting_y = []
ingest_error_x = []
ingest_error_y = []
i = 0
for datapoint in datapoints:
# convert the time strings to timestamps for a better grid.
try:
timestamp_start = datetime.datetime.strptime(datapoint['timestamp'], '%Y-%m-%dT%H:%M:%SZ')
timestamp_end = datetime.datetime.strptime(datapoint['timestamp_end'], '%Y-%m-%dT%H:%M:%SZ')
except:
# probably ISO-8601 format, cut off the fraction
try:
nofrag, frag = datapoint['timestamp'].split('.')
timestamp_start = datetime.datetime.strptime(nofrag, '%Y-%m-%dT%H:%M:%S')
nofrag, frag = datapoint['timestamp_end'].split('.')
timestamp_end = datetime.datetime.strptime(nofrag, '%Y-%m-%dT%H:%M:%S')
except:
# okay, probably not then, whatever.. continue
pass
if datapoint['type'] == 'observing':
observing_x.append(timestamp_start)
observing_y.append(datapoint['speed_bps'])
observing_x.append(timestamp_end)
observing_y.append(datapoint['speed_bps'])
# plot start and end points
plt.plot(timestamp_start, datapoint['speed_bps'], 'b.',
timestamp_end, datapoint['speed_bps'], 'b.')
if annotate is not None:
plt.text(timestamp_start, datapoint['speed_bps'], datapoint[annotate]+'...',
rotation='vertical', fontsize=8)
if datapoint['type']=='ingesting':
ingesting_x.append(timestamp_start)
ingesting_y.append(datapoint['speed_bps'])
ingesting_x.append(timestamp_end)
ingesting_y.append(datapoint['speed_bps'])
plt.plot(timestamp_start, datapoint['speed_bps'], 'g.',
timestamp_end, datapoint['speed_bps'], 'g.')
if annotate is not None:
plt.text(timestamp_start, datapoint['speed_bps'], datapoint[annotate]+'...',
rotation='vertical', fontsize=8)
if datapoint['type'] == 'ingest_error':
ingest_error_x.append(timestamp_start)
ingest_error_x.append(datapoint['speed_bps'])
plt.plot(timestamp_start, datapoint['speed_bps'], 'r.')
if annotate is not None:
plt.text(timestamp_start, datapoint['speed_bps'], datapoint[annotate] + '...',
rotation='vertical', fontsize=8)
# plot lines
# connect a line
for i in range(0,len(observing_x),2):
plt.plot(observing_x[i:i + 2], observing_y[i:i + 2], 'b:')
for i in range(0,len(ingesting_x),2):
plt.plot(ingesting_x[i:i + 2], ingesting_y[i:i + 2], 'g-')
# legend
plt.plot(observing_x[i:i + 2], observing_y[i:i + 2], 'b:',label='Observing')
plt.plot(ingesting_x[i:i + 2], ingesting_y[i:i + 2], 'g-',label='Ingesting')
#plt.plot(ingest_error_x[i:i], 'r.',label='ingest error')
plt.legend(loc='upper right')
plt.show()
\ No newline at end of file
"""
File name: atdb_plot.py
version: 1.0.0 (28 mar 2019)
version: 1.0.0 (14 jun 2019)
Author: Copyright (C) 2019 - Nico Vermaas - ASTRON
Description: Plot data from ATDB
......@@ -261,7 +261,7 @@ def do_sky(args, starttime, endtime):
print('Database connection closed.')
@timeit
def do_ingest_speeds(args):
def do_times(args):
# The request header
ATDB_HEADER = {
......@@ -292,6 +292,9 @@ def do_ingest_speeds(args):
print('analyse the results.')
datapoints = []
for result in results:
if result['taskID'] == "190613088":
print(result['taskID'])
if result['write_speed'] > 0:
datapoint = {}
datapoint['taskid'] = result['taskID']
......@@ -304,7 +307,7 @@ def do_ingest_speeds(args):
datapoints.append(datapoint)
#print(datapoint)
if result['ingest_speed'] is not None:
if result['ingest_speed'] is not None and result['ingest_speed'] > 0:
datapoint = {}
datapoint['taskid'] = result['taskID']
nofrag,frag = result['timestamp_ingesting'].split('.')
......@@ -332,9 +335,40 @@ def do_ingest_speeds(args):
sorted_datapoints = sorted(datapoints, key=lambda k: k['timestamp'])
# plot the results
atdb_plot.do_speed_plot(args.title, args.y_axis_title, args.query, args.annotate, sorted_datapoints)
atdb_plot.do_times_plot(args.title, args.y_axis_title, args.subtitle, args.annotate, sorted_datapoints)
@timeit
def do_speeds(args):
# The request header
ATDB_HEADER = {
'content-type': "application/json",
'cache-control': "no-cache",
'authorization': "Basic YWRtaW46YWRtaW4="
}
# input parameters
url = args.atdb_host + "/speeds?" + str(args.query)
# do the request to the ATDB backend
print('request to '+url)
response = requests.request("GET", url, headers=ATDB_HEADER)
# parse the response
try:
json_response = json.loads(response.text)
datapoints = json_response["datapoints"]
except Exception as err:
print("Exception : " + str(err))
raise (Exception(
"ERROR: " + str(response.status_code) + ", " + str(response.reason) + ', ' + str(response.content)))
# plot the results
atdb_plot.do_speeds_plot(args.title, args.y_axis_title, args.subtitle, args.annotate, datapoints)
def get_arguments(parser):
"""
......@@ -385,7 +419,7 @@ def main():
default="192.168.22.25/atdb",
help="ATDB ReST API")
parser.add_argument("--plot_engine",
default="plotly",
default="mathplotlib",
help="options are: 'plotly' (for webpage)or 'mathplotlib")
parser.add_argument("--filename",
default=None,
......@@ -447,6 +481,9 @@ def main():
parser.add_argument("--title",
default="Title",
help="Title of the Plot")
parser.add_argument("--subtitle",
default="Subtitle",
help="Subtitle of the Plot")
parser.add_argument("--y_axis_title",
default="y-axis",
help="Title on the Y axis")
......@@ -520,8 +557,11 @@ def main():
elif presentation=="sky":
do_sky(args, starttime, endtime)
elif presentation=="ingest_speed":
do_ingest_speeds(args)
elif presentation=="times":
do_times(args)
elif presentation=="speeds":
do_speeds(args)
if args.remote_post_command != None:
execute_remote_command(args.atdb_host, args.remote_post_command)
......
......@@ -9,6 +9,6 @@
--y_axis_title=Size in TB
--interval=day
--plot_type=bar
--starttime=2019-03-11 00:00
--endtime=2019-05-11 00:00
--starttime=2019-06-01 00:00
--endtime=2019-06-12 00:00
--plot_engine=mathplotlib
\ No newline at end of file
......@@ -10,5 +10,5 @@
--y_axis_title=Size in TB
--interval=day
--plot_type=bar
--starttime=2019-03-11 00:00
--endtime=2019-05-21 00:00
--starttime=2019-06-01 00:00
--endtime=2019-06-13 00:00
......@@ -10,5 +10,5 @@
--y_axis_title=Size in TB
--interval=day
--plot_type=bar
--starttime=2019-03-11 00:00
--endtime=2019-05-11 00:00
--starttime=2019-06-01 00:00
--endtime=2019-06-12 00:00
......@@ -7,4 +7,4 @@
--plot_type=scatter
--colormap=viridis
--starttime=2018-01-01 00:00
--endtime=2019-04-14 00:00
--endtime=2019-06-17 00:00
--presentation=ingest_speed
--presentation=speeds
--atdb_host=http://192.168.22.22/atdb
--title=I/O speeds on wcudata1 from ATDB
--subtitle=1 - 8 jun 2019
--y_axis_title=I/O Speed in Gbps
--query=taskID__contains=190607
--query=taskID__gte=190610000&taskID__lte=190613000
--query=starttime__gt=2019-06-01T00:00:00Z&starttime__lt=2019-06-08T00:00:00Z
--query=taskID__gte=190607000&taskID__lte=190613000
--plot_engine=mathplotlib
--presentation=ingest_speed
--presentation=times
--atdb_host=http://192.168.22.22/atdb
--title=I/O speeds on wcudata1 from ATDB
--observing_mode=imaging
......
--presentation=ingest_speed
--presentation=times
--atdb_host=http://atdb.astron.nl/atdb
--title=I/O speeds on wcudata1 from ATDB
--observing_mode=imaging
......
--presentation=ingest_speed
--atdb_host=http://atdb-test/atdb
--presentation=times
--atdb_host=http://atdb-test.astron.nl/atdb
--title=I/O speeds on wcudata1 from ATDB
--observing_mode=imaging
--y_axis_title=I/O Speed in Gbps
......
--presentation=ingest_speed
--presentation=times
--atdb_host=http://atdb.astron.nl/atdb
--title=I/O speeds on wcudata1 from ATDB
--title=I/O speeds from ATDB
--subtitle=12 to 15 july 2019
--y_axis_title=I/O Speed in Gbps
--query=taskID__contains=190608
--query=starttime__gt=2019-06-08T00:00:00Z&starttime__lt=2019-06-13T00:00:00Z
--annotate=taskid
--plot_engine=mathplotlib
--query=starttime__gt=2019-07-12T00:00:00Z&starttime__lt=2019-07-15T00:00:00Z
--presentation=times
--atdb_host=http://192.168.22.22/atdb
--title=I/O speeds on wcudata1 from ATDB
--y_axis_title=I/O Speed in Gbps
--query=taskID__contains=190607
--query=taskID__gte=190610000&taskID__lte=190613000
--query=starttime__gt=2019-06-01T00:00:00Z&starttime__lt=2019-06-08T00:00:00Z
--plot_engine=mathplotlib
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment