diff --git a/.gitignore b/.gitignore
index 5f4486cc2ee7c0296e89b72d0a2f13dccdcc40a7..e31bafe3dfffdc83de19c5982deebdefe4be79e3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1 +1 @@
-/atdb/logs/
+/atdb/logs/
\ No newline at end of file
diff --git a/atdb/atdb/settings/base.py b/atdb/atdb/settings/base.py
index 44ee30e981054689d74b66d045b2be17330fdd98..fc212fbe894e98bdaec3511d8669c44fe2fa5cfb 100644
--- a/atdb/atdb/settings/base.py
+++ b/atdb/atdb/settings/base.py
@@ -165,8 +165,8 @@ LOGGING = {
},
'loggers': {
'taskdatabase': {
- 'handlers': ['my_handler','mail_admins'],
- # 'handlers': ['my_handler', 'my_file_handler', 'mail_admins'],
+ #'handlers': ['my_handler','mail_admins'],
+ 'handlers': ['my_handler', 'my_file_handler'],
'level': 'INFO',
},
'django': {
diff --git a/atdb/logs/atdb.log b/atdb/logs/atdb.log
index 0622ae9937cc012e3f5bea6a98b43f8b55ff42df..cc8e195e25d1cd7df6c5d028613c41468a3bcdfa 100644
--- a/atdb/logs/atdb.log
+++ b/atdb/logs/atdb.log
@@ -2,3 +2,23 @@ logging[14/Apr/2023 14:17:24,827] unsupported operand type(s) for +=: 'NoneType'
[14/Apr/2023 14:17:24,828] task 25485 - (regular) - 146233 has no plots, skipped.
[14/Apr/2023 14:17:26,041] unsupported operand type(s) for +=: 'NoneType' and 'str'
[14/Apr/2023 14:17:26,042] task 25485 - (regular) - 146233 has no plots, skipped.
+[01/May/2023 17:07:12,510] unsupported operand type(s) for &: 'bool' and 'str'
+[01/May/2023 17:07:12,512] task 25485 - (regular) - 146233 has no plots, skipped.
+[01/May/2023 17:07:17,635] unsupported operand type(s) for &: 'bool' and 'str'
+[01/May/2023 17:07:17,635] task 25485 - (regular) - 146233 has no plots, skipped.
+[01/May/2023 17:07:28,708] unsupported operand type(s) for &: 'bool' and 'str'
+[01/May/2023 17:07:28,709] unsupported operand type(s) for &: 'bool' and 'str'
+[01/May/2023 17:07:28,709] unsupported operand type(s) for &: 'bool' and 'str'
+[01/May/2023 17:07:28,718] task 25485 - (regular) - 146233 has no plots, skipped.
+[01/May/2023 17:07:28,718] task 25485 - (regular) - 146233 has no plots, skipped.
+[01/May/2023 17:07:28,718] task 25484 - (regular) - 146232 has no plots, skipped.
+[01/May/2023 17:07:29,725] unsupported operand type(s) for &: 'bool' and 'str'
+[01/May/2023 17:07:29,725] task 25484 - (regular) - 146232 has no plots, skipped.
+[01/May/2023 17:07:30,279] unsupported operand type(s) for &: 'bool' and 'str'
+[01/May/2023 17:07:30,279] task 25484 - (regular) - 146232 has no plots, skipped.
+[01/May/2023 17:07:30,718] unsupported operand type(s) for &: 'bool' and 'str'
+[01/May/2023 17:07:30,718] task 25484 - (regular) - 146232 has no plots, skipped.
+[01/May/2023 17:07:47,382] unsupported operand type(s) for &: 'bool' and 'str'
+[01/May/2023 17:07:47,382] task 25485 - (regular) - 146233 has no plots, skipped.
+[01/May/2023 17:07:48,568] unsupported operand type(s) for &: 'bool' and 'str'
+[01/May/2023 17:07:48,569] task 25485 - (regular) - 146233 has no plots, skipped.
diff --git a/atdb/taskdatabase/services/algorithms.py b/atdb/taskdatabase/services/algorithms.py
index 2c1ff41f5759acd0db5c4923199fdd1a9cf17978..61c31d77251d5d4b16ba236c6405b6770e7782cb 100644
--- a/atdb/taskdatabase/services/algorithms.py
+++ b/atdb/taskdatabase/services/algorithms.py
@@ -3,7 +3,9 @@
Author: Nico Vermaas - Astron
Description: Business logic for ATDB. These functions are called from the views (views.py).
"""
-import json
+import os
+import requests
+import base64
from datetime import datetime, timedelta
from django.db.models import Q, Sum
import logging
@@ -678,8 +680,62 @@ def unique_values_for_aggregation_key(queryset, aggregation_key):
return list(map(lambda x: x[aggregation_key], queryset.values(aggregation_key).distinct()))
-def construct_inspectionplots(task,source='task_id'):
- #results = "<h4>Inspection Plots (for SAS_ID " + task.sas_id + ")</h4>"
+def add_plots(task, token, translation, results, expand_image="False"):
+ # keep a temporary list of filenames to check uniqueness
+ plot_files = []
+
+ plots = task.quality_json["plots"]
+ count = 0
+ for plot in plots:
+ basename = plot['basename']
+ checksum = plot['checksum']
+ # plot_file = basename + str(plot['size'])
+ plot_file = checksum
+
+ # only add unique files
+ if not plot_file in plot_files:
+ count = count + 1
+ surl = plot['surl'] + "?action=show&authz=" + str(token)
+
+ url = surl.replace(translation[0], translation[1])
+
+ if basename.endswith('png') and expand_image=="True":
+
+ # retrieve the url and add the binary data to the html
+ response = requests.get(url)
+
+ if response.status_code == 200:
+ content_as_string = base64.b64encode(response.content).decode("utf-8")
+ img_html = '<img width="800" src = "data:image/png;base64,' + content_as_string + '">'
+
+ collapseable = """
+ <p>
+ <button class="btn btn-primary" type="button" data-toggle="collapse" data-target="#replace_with_id" aria-expanded="false" aria-controls="replace_with_id">
+ <i class="fas fa-image"></i> replace_with_basename
+ </button>
+ </p>
+ <div class="collapse" id="replace_with_id">
+ <div class="card card-body">
+ <a href="replace_with_url" target='_blank'>replace_with_image</a>
+ <hr>
+ <i>(click on image for orignal)</i>
+ </div>
+ </div>
+ """
+ collapseable = collapseable.replace('replace_with_basename', basename)
+ collapseable = collapseable.replace('replace_with_image', img_html)
+ collapseable = collapseable.replace('replace_with_url', url)
+ collapseable = collapseable.replace('replace_with_id', "plot" + str(task.id)+'_'+str(count))
+ results += '<tr><td>' + collapseable + '</td></tr>'
+
+ else:
+ results += '<tr><td><a href="' + url + '" target="_blank">' + basename + '</a></td></tr>'
+ plot_files.append(plot_file)
+
+ return results
+
+
+def construct_inspectionplots(task, expand_image="False", source='task_id'):
# translate the path to a url
try:
@@ -687,8 +743,6 @@ def construct_inspectionplots(task,source='task_id'):
except:
srm_to_url = "srm://srm.grid.sara.nl/pnfs/grid.sara.nl/data/lofar/ops/disk/ldv/::https://webdav.grid.surfsara.nl/"
- # https://webdav.grid.surfsara.nl/pnfs/grid.sara.nl/data/lofar/ops/disk/ldv::https://webdav.grid.surfsara.nl
-
token = str(Configuration.objects.get(key='dcache:token').value)
translation = srm_to_url.split("::")
@@ -696,31 +750,13 @@ def construct_inspectionplots(task,source='task_id'):
if source == 'task_id':
results = "<h4>Inspection Plots and Summary Logs</h4>"
results += "<p>Clicking a link will redirect to SURF SARA in a new browser window. </p>"
-
- plots = task.quality_json["plots"]
-
- # keep a temporary list of filenames to check uniqueness
- plot_files = []
- for plot in plots:
- basename = plot['basename']
- if not basename in plot_files:
- surl = plot['surl'] + "?action=show&authz=" + str(token)
-
- url = surl.replace(translation[0],translation[1])
- results += '<tr><td><a href="' + url + '" target="_blank">'+ basename + '</a></td></tr>'
-# if basename.endswith('png'):
-# results += '<tr><td><a href="' + url + '" target="_blank"><img width="800" src="' + url + '" alt="' + basename + '"/></a></td></tr>'
-
- plot_files.append(basename)
+ results = add_plots(task, token, translation, results, expand_image)
elif source == 'sas_id':
sas_id = task.sas_id
results = "<h4>(Unique) Inspection Plots and Summary Logs for SAS_ID" + str(sas_id) + "</h4>"
results += "<p>Clicking a link will redirect to SURF SARA in a new browser window. </p>"
- # keep a temporary list of filenames to check uniqueness
- plot_files = []
-
tasks = Task.objects.filter(sas_id=sas_id)
for task in tasks:
@@ -731,27 +767,10 @@ def construct_inspectionplots(task,source='task_id'):
try:
results += '<tr style="background-color:#7EB1C4"><td colspan="3"><b>Task ' + str(task.id) + '</b></td></tr>'
- plots = task.quality_json["plots"]
-
- for plot in plots:
- basename = plot['basename']
- checksum = plot['checksum']
- #plot_file = basename + str(plot['size'])
- plot_file = checksum
-
- # only add unique files
- if not plot_file in plot_files:
- surl = plot['surl'] + "?action=show&authz=" + str(token)
-
- url = surl.replace(translation[0], translation[1])
- results += '<tr><td><a href="' + url + '" target="_blank">' + basename + '</a></td></tr>'
-
-# if basename.endswith('png'):
-# results += '<tr><td><a href="' + url + '" target="_blank"><img width="800" src="' + url + '" alt="'+basename+'"/></a></td></tr>'
-
- plot_files.append(plot_file)
+ results = add_plots(task, token, translation, results, expand_image)
except Exception as error:
+ logger.error(error)
logger.info('task ' + str(task) + ' has no plots, skipped.')
return results
diff --git a/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html b/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html
index d5e1eaabeb1ff39db5676c07b7b8567e21d50ae4..d89f7fa7d330a4da131128cca864fcccabe19161 100644
--- a/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html
+++ b/atdb/taskdatabase/templates/taskdatabase/validation/tasks.html
@@ -18,10 +18,15 @@
{% if task.has_plots %}
<td>
<a class="open-modal btn btn-primary btn-sm"
- href="{% url 'inspection-plots' task.id my_tasks.number %}"
- data-popup-url="{% url 'inspection-plots-sasid' task.id my_tasks.number %}">
+ data-popup-url="{% url 'inspection-plots-sasid' task.id False %}">
<img src="{% static 'taskdatabase/surfsara.jpg' %}" height="20" alt="inspection plots">
</a>
+
+ <a class="open-modal btn btn-warning btn-sm"
+ data-popup-url="{% url 'inspection-plots-sasid' task.id True %}">
+ <i class="fas fa-images"></i>
+ </a>
+
</td>
{% else %}
<td>-</td>
diff --git a/atdb/taskdatabase/urls.py b/atdb/taskdatabase/urls.py
index 16c5316e7a4ff703462af4f5e9f5c2feeddf82fc..221d39ac911e24a19cd698493a7564de432c4c8a 100644
--- a/atdb/taskdatabase/urls.py
+++ b/atdb/taskdatabase/urls.py
@@ -30,7 +30,7 @@ urlpatterns = [
path('annotate_quality_sasid/<int:id>', views.AnnotateQualitySasId, name='annotate-quality-sasid'),
path('annotate_quality_sasid/<int:id>/<page>', views.AnnotateQualitySasId, name='annotate-quality-sasid'),
path('show_inspectionplots/<int:id>/<page>', views.ShowInspectionPlots, name='inspection-plots'),
- path('show_inspectionplots_sasid/<int:id>/<page>', views.ShowInspectionPlotsSasId, name='inspection-plots-sasid'),
+ path('show_inspectionplots_sasid/<int:id>/<expand_image>', views.ShowInspectionPlotsSasId, name='inspection-plots-sasid'),
path('show_summary/<int:id>/<page>', views.ShowSummarySasId, name='summary'),
diff --git a/atdb/taskdatabase/views.py b/atdb/taskdatabase/views.py
index fc8ce497125b8d91ac3085c3f66dd1606e0e857d..a64eea07a5c490aebc7b722c507c45225de3e2dc 100644
--- a/atdb/taskdatabase/views.py
+++ b/atdb/taskdatabase/views.py
@@ -643,17 +643,17 @@ def ShowInspectionPlots(request, id=0, page=0):
task = Task.objects.get(id=id)
# convert the path to a url
- plots_html = algorithms.construct_inspectionplots(task, source='task_id')
+ plots_html = algorithms.construct_inspectionplots(task, source='task_id', expand_image=True)
return render(request, "taskdatabase/validation/inspection_plots.html", {'task': task, 'my_plots': plots_html})
-def ShowInspectionPlotsSasId(request, id=0, page=0):
+def ShowInspectionPlotsSasId(request, id=0, expand_image="False"):
# a GET means that the form should be presented to be filled in
task = Task.objects.get(id=id)
# convert the path to a url
- plots_html = algorithms.construct_inspectionplots(task, source='sas_id')
+ plots_html = algorithms.construct_inspectionplots(task, expand_image, source='sas_id')
return render(request, "taskdatabase/validation/inspection_plots.html", {'task': task, 'my_plots': plots_html})