Skip to content
Snippets Groups Projects
Commit 3b16656d authored by Nico Vermaas's avatar Nico Vermaas
Browse files

initial gitlab commit

parent d5647fde
No related branches found
No related tags found
No related merge requests found
"""
File name: atdb_plot.py
version: 1.0.0 (28 mar 2019)
version: 1.0.0 (14 jun 2019)
Author: Copyright (C) 2019 - Nico Vermaas - ASTRON
Description: atdb plot module
"""
import datetime
import plotly
import plotly.graph_objs as go
......@@ -215,7 +216,7 @@ def do_sky_plot(plot_engine, title, x,y, duration, sizes, output_html,y_axis_tit
plt.show()
def do_speed_plot(title, y_axis_title, subtitle, annotate, datapoints):
def do_times_plot(title, y_axis_title, subtitle, annotate, datapoints):
"""
:param title: Title of Plot
:param x: dict with data for x-axis (time)
......@@ -229,7 +230,7 @@ def do_speed_plot(title, y_axis_title, subtitle, annotate, datapoints):
#fig, ax = plt.subplots()
plt.text(x=0.5, y=0.94, s=title, fontsize=14, ha="center", transform=fig.transFigure)
plt.text(x=0.5, y=0.90, s='query: '+subtitle, fontsize=10, ha="center", transform=fig.transFigure)
plt.text(x=0.5, y=0.90, s=subtitle, fontsize=10, ha="center", transform=fig.transFigure)
#plt.title(title)
#plt.suptitle(subtitle)
......@@ -298,3 +299,101 @@ def do_speed_plot(title, y_axis_title, subtitle, annotate, datapoints):
plt.show()
def do_speeds_plot(title, y_axis_title, subtitle, annotate, datapoints):
"""
:param title: Title of Plot
:param x: dict with data for x-axis (time)
:param y: dict with data for y_axix (usage)
:return:
"""
#print('do_speed_plot()')
fig = plt.figure(figsize=(12,6))
#fig, ax = plt.subplots()
plt.text(x=0.5, y=0.94, s=title, fontsize=14, ha="center", transform=fig.transFigure)
plt.text(x=0.5, y=0.90, s=subtitle, fontsize=10, ha="center", transform=fig.transFigure)
#plt.title(title)
#plt.suptitle(subtitle)
plt.xlabel('Timestamp')
plt.ylabel(y_axis_title)
plt.grid(True,alpha=0.3)
observing_x = []
observing_y = []
ingesting_x = []
ingesting_y = []
ingest_error_x = []
ingest_error_y = []
i = 0
for datapoint in datapoints:
# convert the time strings to timestamps for a better grid.
try:
timestamp_start = datetime.datetime.strptime(datapoint['timestamp'], '%Y-%m-%dT%H:%M:%SZ')
timestamp_end = datetime.datetime.strptime(datapoint['timestamp_end'], '%Y-%m-%dT%H:%M:%SZ')
except:
# probably ISO-8601 format, cut off the fraction
try:
nofrag, frag = datapoint['timestamp'].split('.')
timestamp_start = datetime.datetime.strptime(nofrag, '%Y-%m-%dT%H:%M:%S')
nofrag, frag = datapoint['timestamp_end'].split('.')
timestamp_end = datetime.datetime.strptime(nofrag, '%Y-%m-%dT%H:%M:%S')
except:
# okay, probably not then, whatever.. continue
pass
if datapoint['type'] == 'observing':
observing_x.append(timestamp_start)
observing_y.append(datapoint['speed_bps'])
observing_x.append(timestamp_end)
observing_y.append(datapoint['speed_bps'])
# plot start and end points
plt.plot(timestamp_start, datapoint['speed_bps'], 'b.',
timestamp_end, datapoint['speed_bps'], 'b.')
if annotate is not None:
plt.text(timestamp_start, datapoint['speed_bps'], datapoint[annotate]+'...',
rotation='vertical', fontsize=8)
if datapoint['type']=='ingesting':
ingesting_x.append(timestamp_start)
ingesting_y.append(datapoint['speed_bps'])
ingesting_x.append(timestamp_end)
ingesting_y.append(datapoint['speed_bps'])
plt.plot(timestamp_start, datapoint['speed_bps'], 'g.',
timestamp_end, datapoint['speed_bps'], 'g.')
if annotate is not None:
plt.text(timestamp_start, datapoint['speed_bps'], datapoint[annotate]+'...',
rotation='vertical', fontsize=8)
if datapoint['type'] == 'ingest_error':
ingest_error_x.append(timestamp_start)
ingest_error_x.append(datapoint['speed_bps'])
plt.plot(timestamp_start, datapoint['speed_bps'], 'r.')
if annotate is not None:
plt.text(timestamp_start, datapoint['speed_bps'], datapoint[annotate] + '...',
rotation='vertical', fontsize=8)
# plot lines
# connect a line
for i in range(0,len(observing_x),2):
plt.plot(observing_x[i:i + 2], observing_y[i:i + 2], 'b:')
for i in range(0,len(ingesting_x),2):
plt.plot(ingesting_x[i:i + 2], ingesting_y[i:i + 2], 'g-')
# legend
plt.plot(observing_x[i:i + 2], observing_y[i:i + 2], 'b:',label='Observing')
plt.plot(ingesting_x[i:i + 2], ingesting_y[i:i + 2], 'g-',label='Ingesting')
#plt.plot(ingest_error_x[i:i], 'r.',label='ingest error')
plt.legend(loc='upper right')
plt.show()
\ No newline at end of file
"""
File name: atdb_plot.py
version: 1.0.0 (28 mar 2019)
version: 1.0.0 (14 jun 2019)
Author: Copyright (C) 2019 - Nico Vermaas - ASTRON
Description: Plot data from ATDB
......@@ -261,7 +261,7 @@ def do_sky(args, starttime, endtime):
print('Database connection closed.')
@timeit
def do_ingest_speeds(args):
def do_times(args):
# The request header
ATDB_HEADER = {
......@@ -292,6 +292,9 @@ def do_ingest_speeds(args):
print('analyse the results.')
datapoints = []
for result in results:
if result['taskID'] == "190613088":
print(result['taskID'])
if result['write_speed'] > 0:
datapoint = {}
datapoint['taskid'] = result['taskID']
......@@ -304,7 +307,7 @@ def do_ingest_speeds(args):
datapoints.append(datapoint)
#print(datapoint)
if result['ingest_speed'] is not None:
if result['ingest_speed'] is not None and result['ingest_speed'] > 0:
datapoint = {}
datapoint['taskid'] = result['taskID']
nofrag,frag = result['timestamp_ingesting'].split('.')
......@@ -332,9 +335,40 @@ def do_ingest_speeds(args):
sorted_datapoints = sorted(datapoints, key=lambda k: k['timestamp'])
# plot the results
atdb_plot.do_speed_plot(args.title, args.y_axis_title, args.query, args.annotate, sorted_datapoints)
atdb_plot.do_times_plot(args.title, args.y_axis_title, args.subtitle, args.annotate, sorted_datapoints)
@timeit
def do_speeds(args):
# The request header
ATDB_HEADER = {
'content-type': "application/json",
'cache-control': "no-cache",
'authorization': "Basic YWRtaW46YWRtaW4="
}
# input parameters
url = args.atdb_host + "/speeds?" + str(args.query)
# do the request to the ATDB backend
print('request to '+url)
response = requests.request("GET", url, headers=ATDB_HEADER)
# parse the response
try:
json_response = json.loads(response.text)
datapoints = json_response["datapoints"]
except Exception as err:
print("Exception : " + str(err))
raise (Exception(
"ERROR: " + str(response.status_code) + ", " + str(response.reason) + ', ' + str(response.content)))
# plot the results
atdb_plot.do_speeds_plot(args.title, args.y_axis_title, args.subtitle, args.annotate, datapoints)
def get_arguments(parser):
"""
......@@ -385,7 +419,7 @@ def main():
default="192.168.22.25/atdb",
help="ATDB ReST API")
parser.add_argument("--plot_engine",
default="plotly",
default="mathplotlib",
help="options are: 'plotly' (for webpage)or 'mathplotlib")
parser.add_argument("--filename",
default=None,
......@@ -447,6 +481,9 @@ def main():
parser.add_argument("--title",
default="Title",
help="Title of the Plot")
parser.add_argument("--subtitle",
default="Subtitle",
help="Subtitle of the Plot")
parser.add_argument("--y_axis_title",
default="y-axis",
help="Title on the Y axis")
......@@ -520,8 +557,11 @@ def main():
elif presentation=="sky":
do_sky(args, starttime, endtime)
elif presentation=="ingest_speed":
do_ingest_speeds(args)
elif presentation=="times":
do_times(args)
elif presentation=="speeds":
do_speeds(args)
if args.remote_post_command != None:
execute_remote_command(args.atdb_host, args.remote_post_command)
......
......@@ -9,6 +9,6 @@
--y_axis_title=Size in TB
--interval=day
--plot_type=bar
--starttime=2019-03-11 00:00
--endtime=2019-05-11 00:00
--starttime=2019-06-01 00:00
--endtime=2019-06-12 00:00
--plot_engine=mathplotlib
\ No newline at end of file
......@@ -10,5 +10,5 @@
--y_axis_title=Size in TB
--interval=day
--plot_type=bar
--starttime=2019-03-11 00:00
--endtime=2019-05-21 00:00
--starttime=2019-06-01 00:00
--endtime=2019-06-13 00:00
......@@ -10,5 +10,5 @@
--y_axis_title=Size in TB
--interval=day
--plot_type=bar
--starttime=2019-03-11 00:00
--endtime=2019-05-11 00:00
--starttime=2019-06-01 00:00
--endtime=2019-06-12 00:00
......@@ -7,4 +7,4 @@
--plot_type=scatter
--colormap=viridis
--starttime=2018-01-01 00:00
--endtime=2019-04-14 00:00
--endtime=2019-06-17 00:00
--presentation=ingest_speed
--presentation=speeds
--atdb_host=http://192.168.22.22/atdb
--title=I/O speeds on wcudata1 from ATDB
--subtitle=1 - 8 jun 2019
--y_axis_title=I/O Speed in Gbps
--query=taskID__contains=190607
--query=taskID__gte=190610000&taskID__lte=190613000
--query=starttime__gt=2019-06-01T00:00:00Z&starttime__lt=2019-06-08T00:00:00Z
--query=taskID__gte=190607000&taskID__lte=190613000
--plot_engine=mathplotlib
--presentation=ingest_speed
--presentation=times
--atdb_host=http://192.168.22.22/atdb
--title=I/O speeds on wcudata1 from ATDB
--observing_mode=imaging
......
--presentation=ingest_speed
--presentation=times
--atdb_host=http://atdb.astron.nl/atdb
--title=I/O speeds on wcudata1 from ATDB
--observing_mode=imaging
......
--presentation=ingest_speed
--atdb_host=http://atdb-test/atdb
--presentation=times
--atdb_host=http://atdb-test.astron.nl/atdb
--title=I/O speeds on wcudata1 from ATDB
--observing_mode=imaging
--y_axis_title=I/O Speed in Gbps
......
--presentation=ingest_speed
--presentation=times
--atdb_host=http://atdb.astron.nl/atdb
--title=I/O speeds on wcudata1 from ATDB
--title=I/O speeds from ATDB
--subtitle=12 to 15 july 2019
--y_axis_title=I/O Speed in Gbps
--query=taskID__contains=190608
--query=starttime__gt=2019-06-08T00:00:00Z&starttime__lt=2019-06-13T00:00:00Z
--annotate=taskid
--plot_engine=mathplotlib
--query=starttime__gt=2019-07-12T00:00:00Z&starttime__lt=2019-07-15T00:00:00Z
--presentation=times
--atdb_host=http://192.168.22.22/atdb
--title=I/O speeds on wcudata1 from ATDB
--y_axis_title=I/O Speed in Gbps
--query=taskID__contains=190607
--query=taskID__gte=190610000&taskID__lte=190613000
--query=starttime__gt=2019-06-01T00:00:00Z&starttime__lt=2019-06-08T00:00:00Z
--plot_engine=mathplotlib
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment