From 3b16656d4edddf63e3a891b0f62688d52a9ee556 Mon Sep 17 00:00:00 2001
From: Nico Vermaas <vermaas@astron.nl>
Date: Tue, 16 Jul 2019 10:25:49 +0200
Subject: [PATCH] initial gitlab commit
---
atdb_statistics/atdb_plot.py | 105 +++++++++++++++++-
atdb_stats.py | 54 +++++++--
data/ingest_sizes_arts_cumulative.args | 4 +-
data/ingest_sizes_imaging_cumulative.args | 4 +-
..._sizes_imaging_cumulative_mathplotlib.args | 4 +-
data/ingest_speed_today_prod.args | 8 --
data/sky.args | 2 +-
...eed_today_vm.args => speeds_today_vm.args} | 4 +-
...ed_annotated.args => times_annotated.args} | 2 +-
...peed_imaging.args => times_pentacost.args} | 2 +-
...est_system.args => times_test_system.args} | 4 +-
data/times_today_prod.args | 7 ++
data/times_today_vm.args | 8 ++
13 files changed, 177 insertions(+), 31 deletions(-)
delete mode 100644 data/ingest_speed_today_prod.args
rename data/{ingest_speed_today_vm.args => speeds_today_vm.args} (78%)
rename data/{ingest_speed_annotated.args => times_annotated.args} (92%)
rename data/{ingest_speed_imaging.args => times_pentacost.args} (92%)
rename data/{ingest_speed_test_system.args => times_test_system.args} (81%)
create mode 100644 data/times_today_prod.args
create mode 100644 data/times_today_vm.args
diff --git a/atdb_statistics/atdb_plot.py b/atdb_statistics/atdb_plot.py
index 45b3ca6..0123cda 100644
--- a/atdb_statistics/atdb_plot.py
+++ b/atdb_statistics/atdb_plot.py
@@ -1,10 +1,11 @@
"""
File name: atdb_plot.py
- version: 1.0.0 (28 mar 2019)
+ version: 1.0.0 (14 jun 2019)
Author: Copyright (C) 2019 - Nico Vermaas - ASTRON
Description: atdb plot module
"""
+import datetime
import plotly
import plotly.graph_objs as go
@@ -215,7 +216,7 @@ def do_sky_plot(plot_engine, title, x,y, duration, sizes, output_html,y_axis_tit
plt.show()
-def do_speed_plot(title, y_axis_title, subtitle, annotate, datapoints):
+def do_times_plot(title, y_axis_title, subtitle, annotate, datapoints):
"""
:param title: Title of Plot
:param x: dict with data for x-axis (time)
@@ -229,7 +230,7 @@ def do_speed_plot(title, y_axis_title, subtitle, annotate, datapoints):
#fig, ax = plt.subplots()
plt.text(x=0.5, y=0.94, s=title, fontsize=14, ha="center", transform=fig.transFigure)
- plt.text(x=0.5, y=0.90, s='query: '+subtitle, fontsize=10, ha="center", transform=fig.transFigure)
+ plt.text(x=0.5, y=0.90, s=subtitle, fontsize=10, ha="center", transform=fig.transFigure)
#plt.title(title)
#plt.suptitle(subtitle)
@@ -298,3 +299,101 @@ def do_speed_plot(title, y_axis_title, subtitle, annotate, datapoints):
plt.show()
+
+def do_speeds_plot(title, y_axis_title, subtitle, annotate, datapoints):
+ """
+ :param title: Title of Plot
+ :param x: dict with data for x-axis (time)
+ :param y: dict with data for y_axix (usage)
+ :return:
+ """
+
+ #print('do_speed_plot()')
+
+ fig = plt.figure(figsize=(12,6))
+ #fig, ax = plt.subplots()
+
+ plt.text(x=0.5, y=0.94, s=title, fontsize=14, ha="center", transform=fig.transFigure)
+ plt.text(x=0.5, y=0.90, s=subtitle, fontsize=10, ha="center", transform=fig.transFigure)
+
+ #plt.title(title)
+ #plt.suptitle(subtitle)
+
+ plt.xlabel('Timestamp')
+ plt.ylabel(y_axis_title)
+
+ plt.grid(True,alpha=0.3)
+
+ observing_x = []
+ observing_y = []
+ ingesting_x = []
+ ingesting_y = []
+ ingest_error_x = []
+ ingest_error_y = []
+ i = 0
+
+ for datapoint in datapoints:
+ # convert the time strings to timestamps for a better grid.
+ try:
+ timestamp_start = datetime.datetime.strptime(datapoint['timestamp'], '%Y-%m-%dT%H:%M:%SZ')
+ timestamp_end = datetime.datetime.strptime(datapoint['timestamp_end'], '%Y-%m-%dT%H:%M:%SZ')
+ except:
+ # probably ISO-8601 format, cut off the fraction
+ try:
+ nofrag, frag = datapoint['timestamp'].split('.')
+ timestamp_start = datetime.datetime.strptime(nofrag, '%Y-%m-%dT%H:%M:%S')
+ nofrag, frag = datapoint['timestamp_end'].split('.')
+ timestamp_end = datetime.datetime.strptime(nofrag, '%Y-%m-%dT%H:%M:%S')
+ except:
+ # okay, probably not then, whatever.. continue
+ pass
+
+ if datapoint['type'] == 'observing':
+ observing_x.append(timestamp_start)
+ observing_y.append(datapoint['speed_bps'])
+ observing_x.append(timestamp_end)
+ observing_y.append(datapoint['speed_bps'])
+
+ # plot start and end points
+ plt.plot(timestamp_start, datapoint['speed_bps'], 'b.',
+ timestamp_end, datapoint['speed_bps'], 'b.')
+ if annotate is not None:
+ plt.text(timestamp_start, datapoint['speed_bps'], datapoint[annotate]+'...',
+ rotation='vertical', fontsize=8)
+
+ if datapoint['type']=='ingesting':
+ ingesting_x.append(timestamp_start)
+ ingesting_y.append(datapoint['speed_bps'])
+ ingesting_x.append(timestamp_end)
+ ingesting_y.append(datapoint['speed_bps'])
+
+ plt.plot(timestamp_start, datapoint['speed_bps'], 'g.',
+ timestamp_end, datapoint['speed_bps'], 'g.')
+ if annotate is not None:
+ plt.text(timestamp_start, datapoint['speed_bps'], datapoint[annotate]+'...',
+ rotation='vertical', fontsize=8)
+
+ if datapoint['type'] == 'ingest_error':
+ ingest_error_x.append(timestamp_start)
+ ingest_error_x.append(datapoint['speed_bps'])
+
+ plt.plot(timestamp_start, datapoint['speed_bps'], 'r.')
+ if annotate is not None:
+ plt.text(timestamp_start, datapoint['speed_bps'], datapoint[annotate] + '...',
+ rotation='vertical', fontsize=8)
+
+ # plot lines
+ # connect a line
+ for i in range(0,len(observing_x),2):
+ plt.plot(observing_x[i:i + 2], observing_y[i:i + 2], 'b:')
+
+ for i in range(0,len(ingesting_x),2):
+ plt.plot(ingesting_x[i:i + 2], ingesting_y[i:i + 2], 'g-')
+
+ # legend
+ plt.plot(observing_x[i:i + 2], observing_y[i:i + 2], 'b:',label='Observing')
+ plt.plot(ingesting_x[i:i + 2], ingesting_y[i:i + 2], 'g-',label='Ingesting')
+ #plt.plot(ingest_error_x[i:i], 'r.',label='ingest error')
+ plt.legend(loc='upper right')
+
+ plt.show()
\ No newline at end of file
diff --git a/atdb_stats.py b/atdb_stats.py
index 82b2af0..ff0ab41 100644
--- a/atdb_stats.py
+++ b/atdb_stats.py
@@ -1,6 +1,6 @@
"""
File name: atdb_plot.py
- version: 1.0.0 (28 mar 2019)
+ version: 1.0.0 (14 jun 2019)
Author: Copyright (C) 2019 - Nico Vermaas - ASTRON
Description: Plot data from ATDB
@@ -261,7 +261,7 @@ def do_sky(args, starttime, endtime):
print('Database connection closed.')
@timeit
-def do_ingest_speeds(args):
+def do_times(args):
# The request header
ATDB_HEADER = {
@@ -292,6 +292,9 @@ def do_ingest_speeds(args):
print('analyse the results.')
datapoints = []
for result in results:
+ if result['taskID'] == "190613088":
+ print(result['taskID'])
+
if result['write_speed'] > 0:
datapoint = {}
datapoint['taskid'] = result['taskID']
@@ -304,7 +307,7 @@ def do_ingest_speeds(args):
datapoints.append(datapoint)
#print(datapoint)
- if result['ingest_speed'] is not None:
+ if result['ingest_speed'] is not None and result['ingest_speed'] > 0:
datapoint = {}
datapoint['taskid'] = result['taskID']
nofrag,frag = result['timestamp_ingesting'].split('.')
@@ -332,9 +335,40 @@ def do_ingest_speeds(args):
sorted_datapoints = sorted(datapoints, key=lambda k: k['timestamp'])
# plot the results
- atdb_plot.do_speed_plot(args.title, args.y_axis_title, args.query, args.annotate, sorted_datapoints)
+ atdb_plot.do_times_plot(args.title, args.y_axis_title, args.subtitle, args.annotate, sorted_datapoints)
+
+@timeit
+def do_speeds(args):
+
+ # The request header
+ ATDB_HEADER = {
+ 'content-type': "application/json",
+ 'cache-control': "no-cache",
+ 'authorization': "Basic YWRtaW46YWRtaW4="
+ }
+
+ # input parameters
+ url = args.atdb_host + "/speeds?" + str(args.query)
+
+ # do the request to the ATDB backend
+ print('request to '+url)
+ response = requests.request("GET", url, headers=ATDB_HEADER)
+
+ # parse the response
+ try:
+ json_response = json.loads(response.text)
+ datapoints = json_response["datapoints"]
+
+ except Exception as err:
+ print("Exception : " + str(err))
+ raise (Exception(
+ "ERROR: " + str(response.status_code) + ", " + str(response.reason) + ', ' + str(response.content)))
+
+ # plot the results
+ atdb_plot.do_speeds_plot(args.title, args.y_axis_title, args.subtitle, args.annotate, datapoints)
+
def get_arguments(parser):
"""
@@ -385,7 +419,7 @@ def main():
default="192.168.22.25/atdb",
help="ATDB ReST API")
parser.add_argument("--plot_engine",
- default="plotly",
+ default="mathplotlib",
help="options are: 'plotly' (for webpage)or 'mathplotlib")
parser.add_argument("--filename",
default=None,
@@ -447,6 +481,9 @@ def main():
parser.add_argument("--title",
default="Title",
help="Title of the Plot")
+ parser.add_argument("--subtitle",
+ default="Subtitle",
+ help="Subtitle of the Plot")
parser.add_argument("--y_axis_title",
default="y-axis",
help="Title on the Y axis")
@@ -520,8 +557,11 @@ def main():
elif presentation=="sky":
do_sky(args, starttime, endtime)
- elif presentation=="ingest_speed":
- do_ingest_speeds(args)
+ elif presentation=="times":
+ do_times(args)
+
+ elif presentation=="speeds":
+ do_speeds(args)
if args.remote_post_command != None:
execute_remote_command(args.atdb_host, args.remote_post_command)
diff --git a/data/ingest_sizes_arts_cumulative.args b/data/ingest_sizes_arts_cumulative.args
index 3dc7b35..b67c513 100644
--- a/data/ingest_sizes_arts_cumulative.args
+++ b/data/ingest_sizes_arts_cumulative.args
@@ -9,6 +9,6 @@
--y_axis_title=Size in TB
--interval=day
--plot_type=bar
---starttime=2019-03-11 00:00
---endtime=2019-05-11 00:00
+--starttime=2019-06-01 00:00
+--endtime=2019-06-12 00:00
--plot_engine=mathplotlib
\ No newline at end of file
diff --git a/data/ingest_sizes_imaging_cumulative.args b/data/ingest_sizes_imaging_cumulative.args
index a0bab7a..8d6ef78 100644
--- a/data/ingest_sizes_imaging_cumulative.args
+++ b/data/ingest_sizes_imaging_cumulative.args
@@ -10,5 +10,5 @@
--y_axis_title=Size in TB
--interval=day
--plot_type=bar
---starttime=2019-03-11 00:00
---endtime=2019-05-21 00:00
+--starttime=2019-06-01 00:00
+--endtime=2019-06-13 00:00
diff --git a/data/ingest_sizes_imaging_cumulative_mathplotlib.args b/data/ingest_sizes_imaging_cumulative_mathplotlib.args
index 282e987..b229074 100644
--- a/data/ingest_sizes_imaging_cumulative_mathplotlib.args
+++ b/data/ingest_sizes_imaging_cumulative_mathplotlib.args
@@ -10,5 +10,5 @@
--y_axis_title=Size in TB
--interval=day
--plot_type=bar
---starttime=2019-03-11 00:00
---endtime=2019-05-11 00:00
+--starttime=2019-06-01 00:00
+--endtime=2019-06-12 00:00
diff --git a/data/ingest_speed_today_prod.args b/data/ingest_speed_today_prod.args
deleted file mode 100644
index 77dc8de..0000000
--- a/data/ingest_speed_today_prod.args
+++ /dev/null
@@ -1,8 +0,0 @@
---presentation=ingest_speed
---atdb_host=http://atdb.astron.nl/atdb
---title=I/O speeds on wcudata1 from ATDB
---y_axis_title=I/O Speed in Gbps
---query=taskID__contains=190608
---query=starttime__gt=2019-06-08T00:00:00Z&starttime__lt=2019-06-13T00:00:00Z
---annotate=taskid
---plot_engine=mathplotlib
diff --git a/data/sky.args b/data/sky.args
index ae6ab3d..f5437f2 100644
--- a/data/sky.args
+++ b/data/sky.args
@@ -7,4 +7,4 @@
--plot_type=scatter
--colormap=viridis
--starttime=2018-01-01 00:00
---endtime=2019-04-14 00:00
+--endtime=2019-06-17 00:00
diff --git a/data/ingest_speed_today_vm.args b/data/speeds_today_vm.args
similarity index 78%
rename from data/ingest_speed_today_vm.args
rename to data/speeds_today_vm.args
index d65422e..5e6fb36 100644
--- a/data/ingest_speed_today_vm.args
+++ b/data/speeds_today_vm.args
@@ -1,9 +1,9 @@
---presentation=ingest_speed
+--presentation=speeds
--atdb_host=http://192.168.22.22/atdb
--title=I/O speeds on wcudata1 from ATDB
+--subtitle=1 - 8 jun 2019
--y_axis_title=I/O Speed in Gbps
--query=taskID__contains=190607
--query=taskID__gte=190610000&taskID__lte=190613000
--query=starttime__gt=2019-06-01T00:00:00Z&starttime__lt=2019-06-08T00:00:00Z
---query=taskID__gte=190607000&taskID__lte=190613000
--plot_engine=mathplotlib
diff --git a/data/ingest_speed_annotated.args b/data/times_annotated.args
similarity index 92%
rename from data/ingest_speed_annotated.args
rename to data/times_annotated.args
index e94892c..02838a4 100644
--- a/data/ingest_speed_annotated.args
+++ b/data/times_annotated.args
@@ -1,4 +1,4 @@
---presentation=ingest_speed
+--presentation=times
--atdb_host=http://192.168.22.22/atdb
--title=I/O speeds on wcudata1 from ATDB
--observing_mode=imaging
diff --git a/data/ingest_speed_imaging.args b/data/times_pentacost.args
similarity index 92%
rename from data/ingest_speed_imaging.args
rename to data/times_pentacost.args
index b65bcd4..5eb85e7 100644
--- a/data/ingest_speed_imaging.args
+++ b/data/times_pentacost.args
@@ -1,4 +1,4 @@
---presentation=ingest_speed
+--presentation=times
--atdb_host=http://atdb.astron.nl/atdb
--title=I/O speeds on wcudata1 from ATDB
--observing_mode=imaging
diff --git a/data/ingest_speed_test_system.args b/data/times_test_system.args
similarity index 81%
rename from data/ingest_speed_test_system.args
rename to data/times_test_system.args
index f8adc65..dbd3d3b 100644
--- a/data/ingest_speed_test_system.args
+++ b/data/times_test_system.args
@@ -1,5 +1,5 @@
---presentation=ingest_speed
---atdb_host=http://atdb-test/atdb
+--presentation=times
+--atdb_host=http://atdb-test.astron.nl/atdb
--title=I/O speeds on wcudata1 from ATDB
--observing_mode=imaging
--y_axis_title=I/O Speed in Gbps
diff --git a/data/times_today_prod.args b/data/times_today_prod.args
new file mode 100644
index 0000000..9334680
--- /dev/null
+++ b/data/times_today_prod.args
@@ -0,0 +1,7 @@
+--presentation=times
+--atdb_host=http://atdb.astron.nl/atdb
+--title=I/O speeds from ATDB
+--subtitle=12 to 15 july 2019
+--y_axis_title=I/O Speed in Gbps
+--query=taskID__contains=190608
+--query=starttime__gt=2019-07-12T00:00:00Z&starttime__lt=2019-07-15T00:00:00Z
diff --git a/data/times_today_vm.args b/data/times_today_vm.args
new file mode 100644
index 0000000..f205ccf
--- /dev/null
+++ b/data/times_today_vm.args
@@ -0,0 +1,8 @@
+--presentation=times
+--atdb_host=http://192.168.22.22/atdb
+--title=I/O speeds on wcudata1 from ATDB
+--y_axis_title=I/O Speed in Gbps
+--query=taskID__contains=190607
+--query=taskID__gte=190610000&taskID__lte=190613000
+--query=starttime__gt=2019-06-01T00:00:00Z&starttime__lt=2019-06-08T00:00:00Z
+--plot_engine=mathplotlib
--
GitLab