Merge branch 'SDC-437-adapt-dashboard' into 'master'

https://support.astron.nl/jira/browse/SDC-437

See merge request !165

atdb/atdb/settings/base.py

@@ -188,8 +188,9 @@ LOGOUT_REDIRECT_URL = '/atdb'
 STATIC_URL = '/atdb/static/'
 STATIC_ROOT = os.path.join(BASE_DIR, 'static')
 
-ALL_STATUSSES = ['defining','defined','staging','staged','fetching','fetched','processing','processed','storing','stored','validated', 'scrubbing','scrubbed','archiving','archived','finished']
-ACTIVE_STATUSSES = ['staging','staged','fetching','fetched','processing','processed','validated','storing','stored','scrubbing','scrubbed','archiving','archived']
+ALL_STATUSSES = ['defining','defined','staging','staged','processing','processed','storing','stored','validated', 'archiving','archived','finished']
+ACTIVE_STATUSSES = ['staging','staged','processing','processed','validated','storing','stored','archiving','archived']
 STATUSSES_WITH_DATA = ['staged','fetching','fetched','processing','processed','validated','storing','stored','scrubbing','scrubbed','archiving','archived']
+AGGREGATES = ['failed','active','total']
 
 QUERY_LIMIT_MULTI_CHANGE = 1000
\ No newline at end of file

atdb/atdb/static/taskdatabase/style.css

@@ -17,7 +17,20 @@ TD {
   background-color: lemonchiffon;
 }
 
-.error,.failed,.staging_failed,.processed_failed,.scrubbed_failed,.stored_failed {
+.max {
+   font-weight: bold;
+   color: blue;
+   background-color: lightgreen;
+}
+
+.aggregate {
+  font-weight: bold;
+  color: blue;
+  background-color: lightgreen;
+}
+
+
+.error,.failed,.staging_failed,.processed_failed,.scrubbed_failed,.stored_failed,.archived_failed {
   color: red;
   font-weight: bold;
 }
@@ -62,11 +75,6 @@ p.title {
 .info {
    background-color: #E0F8F8;
 }
-.max {
-   font-weight: bold;
-   color: green;
-   background-color: lightgreen;
-}
 
 .form-signin {
   width: 100%;

atdb/taskdatabase/services/algorithms.py

@@ -247,13 +247,15 @@ def aggregate_resources_tasks(selection):
         # split per status, to see the progress
         nr_per_status = {}
 
+        nr_per_status['failed'] = tasks_per_workflow.filter(status__icontains='failed').count()
+        nr_per_status['active'] = nr_of_active_tasks_per_workflow
+        nr_per_status['total'] = nr_of_tasks_per_workflow
+
         for status in settings.ALL_STATUSSES:
             nr_for_this_status = tasks_per_workflow.filter(status=status).count()
             nr_per_status[status] = nr_for_this_status
 
-        nr_per_status['failed'] = tasks_per_workflow.filter(status__icontains='failed').count()
-        nr_per_status['active'] = nr_of_active_tasks_per_workflow
-        nr_per_status['total'] = nr_of_tasks_per_workflow
+
 
         # store the results in a dict
         workflow_result['id'] = workflow.id
@@ -319,47 +321,6 @@ def aggregate_resources_logs(selection):
     return workflow_results
 
 
-# aggregate information from the logentries table per workflow per status
-def aggregate_resources_logs_version1():
-    records = []
-
-    # get all active tasks
-    active_tasks = Task.objects.filter(status__in=settings.ACTIVE_STATUSSES).filter(task_type='regular')
-    active_tasks_count = active_tasks.count()
-
-    # retrieve all unique workflows
-    active_workflows = active_tasks.values('workflow').distinct()
-
-    # iterate through the filters and accumulate logentries
-    for w in active_workflows:
-        workflow_result = {}
-
-        # extract the workflow object (cheap)
-        workflow = Workflow.objects.get(id=w['workflow'])
-
-        # aggregate logentries per step for all active statusses
-        for status in settings.ACTIVE_STATUSSES:
-            record = {}
-            record['name'] = str(workflow.id) + ' - ' + workflow.workflow_uri
-            # record['name'] = str(workflow.id)
-            record['status'] = status
-
-            # aggregate logentries per step for all active statusses (expensive)
-            logs = LogEntry.objects.filter(status=status) \
-                .filter(task__status__in=settings.ACTIVE_STATUSSES) \
-                .filter(task__workflow=workflow)
-
-            sum_cpu_cycles = logs.aggregate(Sum('cpu_cycles'))
-            record['cpu_cycles'] = sum_cpu_cycles['cpu_cycles__sum']
-
-            wall_clock_time = logs.aggregate(Sum('wall_clock_time'))
-            record['wall_clock_time'] = wall_clock_time['wall_clock_time__sum']
-
-            records.append(record)
-
-    return records
-
-
 def construct_link_to_tasks_api(request, status, workflow_id, count):
     link = str(count)
     try:
@@ -439,13 +400,13 @@ def construct_tasks_per_workflow_html(request, workflow_results):
     results_tasks = "<p>Progress of tasks per workflow</p>"
 
     # construct the header
-    ##header = "<th>Workflow</th>"
-    header = ""
+    ###header = ""
+    header = '<th class="aggregate_failed">failed</th><th class="aggregate">active</th><th class="aggregate">total</th>'
     for status in settings.ALL_STATUSSES:
         header += "<th>" + status + "</th>"
 
-    header += '<th class="failed">failed</th><th class="active">active</th><th>total</th>'
-    ## header += '<th>to process</th><th>processed</th><th>processing time</th>'
+    ###header += '<th class="failed">failed</th><th class="active">active</th><th>total</th>'
+
     results_tasks += header
 
     for workflow_result in workflow_results:
@@ -484,6 +445,9 @@ def construct_tasks_per_workflow_html(request, workflow_results):
             if key in settings.ACTIVE_STATUSSES or key == 'active':
                 style = "active"
 
+            if key in settings.AGGREGATES:
+                style = "aggregate"
+
             # bonus: add a query link
             link = construct_link_to_tasks_api(request, key, workflow_result['id'], d[key])
             values += "<td class=" + style + ">" + str(percentage) + "% (" + link + ")</td>"

atdb/taskdatabase/static/taskdatabase/style.css

@@ -17,6 +17,22 @@ TD {
   background-color: lemonchiffon;
 }
 
+.max {
+   font-weight: bold;
+   background-color: lightgreen;
+}
+
+.aggregate {
+  font-weight: bold;
+  background-color: lightgreen;
+}
+
+.aggregate_failed {
+  font-weight: bold;
+  color: red;
+  background-color: lightgreen;
+}
+
 .error,.failed,.staging_failed,.processed_failed,.scrubbed_failed,.stored_failed,.archived_failed {
   color: red;
   font-weight: bold;
@@ -62,11 +78,6 @@ p.title {
 .info {
    background-color: #E0F8F8;
 }
-.max {
-   font-weight: bold;
-   color: green;
-   background-color: lightgreen;
-}
 
 .form-signin {
   width: 100%;

atdb/taskdatabase/templates/taskdatabase/index.html

@@ -80,7 +80,7 @@
         {% include 'taskdatabase/pagination.html' %}
        </div>
     </div>
-    <p class="footer"> Version 1.0.0 (24 jan 2021 - 10:00)
+    <p class="footer"> Version 1.0.0 (25 jan 2021 - 10:00)
 
 </div>