diff --git a/.gitignore b/.gitignore
index 835d123e55024f7ef741087f7c198d1f936a4dae..6d6cef476da10824272a8c80600560dd9256dd54 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,8 +3,8 @@
__pycache__/
# Robot framework reports
-*.html
-*xml
+integration/*.xml
+integration/*.html
# Integration logs
integration/*.log
diff --git a/README.md b/README.md
index fa311be715d08f420a747c8b1b674290b38c3c18..6fe28edcff6598fb58f66b3500a71c1fdbd5e0ca 100644
--- a/README.md
+++ b/README.md
@@ -35,39 +35,19 @@ docker exec -ti ldv-specification python manage.py migrate --settings ldvspec.se
## Local Development Environment
-### Postgres Database in Docker
-
-Run `docker-compose -f docker-compose-postgres-dev.yml up -d` with the following compose file to spin up a new Postgres container.
-See the `docker-compose-postgres-dev.yml` file in the `docker` directory.
-(the shown configuration is based on the `dev.py` file. You can change, but then also change it in `dev.py`)
-```yaml
-
-version: "3.7"
-services:
-
- ldv-spec-db:
- image: postgres:14
- container_name: ldv-spec-postgres
- ports:
- - "5432:5432"
- environment:
- POSTGRES_PASSWORD: "atdb123"
- POSTGRES_USER: "atdb_admin"
- POSTGRES_DB: "ldv-spec-db"
- volumes:
- - ldv-spec-db:/var/lib/postgresql/data
- restart: always
-
- rabbitmq:
- container_name: ldv-spec-rabbit
- image: rabbitmq:3-management
- ports:
- - "5672:5672"
-
-volumes:
- ldv-spec-db:
-
-```
+### Start developing
+
+- Copy the `ldvspec.example.env`, rename it to `ldvspec.env` and fill in the variables. The variables should match the `local.py` settings which are coherent with the `docker-compose-local.yml` setup.
+- Run `docker-compose -f docker-compose-local.yml up -d` with the following compose file to spin up a new Postgres container, celery worker and rabbitMQ.
+- Run the following python command to start developing
+ > python manage.py migrate --settings=ldvspec.settings.local
+ >
+ > python manage.py createsuperuser --settings=ldvspec.settings.local
+ >
+ > python manage.py runserver --settings=ldvspec.settings.local
+ >
+
+-[ ] TODO: load fixture to have sample data
### Django Application
* clone the repo
diff --git a/ldvspec/Dockerfile b/ldvspec/Dockerfile
index 8233ad763713b4ec57644ebb7f441a7861247d9e..c2c972e4f26c4be1dc030333880b9780e4846d9c 100644
--- a/ldvspec/Dockerfile
+++ b/ldvspec/Dockerfile
@@ -5,9 +5,8 @@ FROM ${BASE_IMAGE}
# Make sure we are in the source directory
WORKDIR /src
COPY requirements /src/requirements
-# install dependencies and clean up apk build libs
-RUN pip install --no-cache-dir -r requirements/prod.txt && \
- apk --purge del .build-deps
+# install dependencies and clean up apt build libs
+RUN pip install --no-cache-dir -r requirements/prod.txt
# Copy the rest of the files
COPY . .
diff --git a/ldvspec/Dockerfile.base b/ldvspec/Dockerfile.base
index 71aa3d9334c3d7920bf5e80e72a6de6b15b92aac..7c36318086ed9f6a07f2753c3be6f78ac83f8cfa 100644
--- a/ldvspec/Dockerfile.base
+++ b/ldvspec/Dockerfile.base
@@ -1,9 +1,4 @@
-FROM python:3.10-alpine
+FROM python:3.10
ENV PYTHONUNBUFFERED 1
# Main dependencies
-RUN apk update && apk add --no-cache \
- bash nano mc postgresql-libs
-
-# Cache the build dependencies
-WORKDIR /src
-RUN apk add --no-cache --virtual .build-deps gcc python3-dev musl-dev postgresql-dev
+RUN apt-get update && apt-get install -y postgresql-client build-essential
\ No newline at end of file
diff --git a/ldvspec/docker/docker-compose-local.yml b/ldvspec/docker/docker-compose-local.yml
new file mode 100644
index 0000000000000000000000000000000000000000..4eb901f4d35736429178947c82a6c5c8dc2b8006
--- /dev/null
+++ b/ldvspec/docker/docker-compose-local.yml
@@ -0,0 +1,49 @@
+version: '3.4'
+networks:
+ ldv_network:
+ traefik_proxy:
+ external:
+ name: traefik_proxy
+ default:
+ driver: bridge
+
+services:
+ ldv-spec-db:
+ image: postgres:14
+ container_name: ldv-spec-postgres
+ ports:
+ - "5433:5432"
+ networks:
+ - traefik_proxy
+ - ldv_network
+ env_file:
+ - ../ldvspec.env
+ volumes:
+ - ldv-spec-db:/var/lib/postgresql/data
+ restart: always
+
+ rabbitmq:
+ image: rabbitmq:3-management
+ ports:
+ - "5672:5672"
+ networks:
+ - ldv_network
+ container_name: ldv-spec-rabbit
+
+ ldv-specification-background:
+ container_name: ldv-specification-background
+ image: git.astron.nl:5000/astron-sdc/ldv-specification:${LDVSPEC_VERSION:-latest}
+ networks:
+ - ldv_network
+ depends_on:
+ - ldv-spec-db
+ environment:
+ CELERY_BROKER_URL: amqp://guest@rabbitmq:5672
+ DJANGO_SETTINGS_MODULE: 'ldvspec.settings.local'
+ env_file:
+ - ../ldvspec.env
+ command: celery -A ldvspec worker -l INFO
+ restart: always
+
+volumes:
+ ldv-spec-db:
diff --git a/ldvspec/docker/docker-compose-postgres-dev.yml b/ldvspec/docker/docker-compose-postgres-dev.yml
deleted file mode 100644
index 2769910429ab49605891edf1002756efcc062bf2..0000000000000000000000000000000000000000
--- a/ldvspec/docker/docker-compose-postgres-dev.yml
+++ /dev/null
@@ -1,25 +0,0 @@
-version: "3.7"
-services:
-
- ldv-spec-db:
- image: postgres:14
- container_name: ldv-spec-postgres
- expose:
- - "5433"
- ports:
- - "5433:5432"
- environment:
- POSTGRES_PASSWORD: "secret"
- POSTGRES_USER: "postgres"
- POSTGRES_DB: "ldv-spec-db"
- volumes:
- - ldv-spec-db:/var/lib/postgresql/data
- restart: always
-
- rabbitmq:
- image: rabbitmq:3-management
- ports:
- - "5672:5672"
-
-volumes:
- ldv-spec-db:
diff --git a/ldvspec/ldvspec.env b/ldvspec/ldvspec.env
new file mode 100755
index 0000000000000000000000000000000000000000..356d0d57e452b1bb69cba5dfb7715242ab5b25dd
--- /dev/null
+++ b/ldvspec/ldvspec.env
@@ -0,0 +1,11 @@
+DEBUG=True
+POSTGRES_PASSWORD=secret
+POSTGRES_USER=postgres
+POSTGRES_DB=ldv-spec-db
+DATABASE_HOST=ldv-spec-db
+DATABASE_PORT=5433
+DATABASE_NAME=ldv-spec-db
+DATABASE_USER=postgres
+DATABASE_PASSWORD=secret
+ATDB_HOST=https://sdc-dev.astron.nl:5554/atdb/
+CELERY_BROKER_URL=amqp://guest@localhost:5672
diff --git a/ldvspec/ldvspec.example.env b/ldvspec/ldvspec.example.env
new file mode 100755
index 0000000000000000000000000000000000000000..ececa1f9d28990cba63495bfae3de0eb2bc85e8b
--- /dev/null
+++ b/ldvspec/ldvspec.example.env
@@ -0,0 +1,11 @@
+DEBUG=True
+POSTGRES_PASSWORD=
+POSTGRES_USER=
+POSTGRES_DB=
+DATABASE_HOST=
+DATABASE_PORT=
+DATABASE_NAME=
+DATABASE_USER=
+DATABASE_PASSWORD=
+ATDB_HOST=
+CELERY_BROKER_URL=
\ No newline at end of file
diff --git a/ldvspec/ldvspec/settings/local.py b/ldvspec/ldvspec/settings/local.py
new file mode 100644
index 0000000000000000000000000000000000000000..97827532c7bf836be7e16ddf63bec56fd461c1dd
--- /dev/null
+++ b/ldvspec/ldvspec/settings/local.py
@@ -0,0 +1,18 @@
+from ldvspec.settings.base import *
+
+DEV = True
+DEBUG = True
+
+ALLOWED_HOSTS = ["*"]
+CORS_ORIGIN_ALLOW_ALL = True
+
+DATABASES = {
+ 'default': {
+ 'ENGINE': 'django.db.backends.postgresql_psycopg2',
+ 'USER': 'postgres',
+ 'PASSWORD': 'secret',
+ 'NAME': 'ldv-spec-db',
+ 'HOST': 'localhost',
+ 'PORT': '5433',
+ },
+}
\ No newline at end of file
diff --git a/ldvspec/lofardata/forms.py b/ldvspec/lofardata/forms.py
index e7d085e76d1b999fd35ded43257d41182bfc9413..8542d95ab7e91267e7212f160110a686ecda0ba1 100644
--- a/ldvspec/lofardata/forms.py
+++ b/ldvspec/lofardata/forms.py
@@ -13,7 +13,7 @@ class WorkSpecificationForm(ModelForm):
filter_names = [filter_name[0] for filter_name in DataProductFilter.objects.all().values_list('field')]
filters = {}
for filter_name in filter_names:
- if filter_name in self.data:
+ if filter_name in self.data and self.data[filter_name] != "":
filters[filter_name] = self.data[filter_name]
return filters
@@ -24,10 +24,21 @@ class WorkSpecificationForm(ModelForm):
class Meta:
model = WorkSpecification
- fields = ['filters', 'selected_workflow', 'processing_site', 'predecessor_specification', 'batch_size']
+ fields = ['filters', 'selected_workflow', 'selected_workflow_tag', 'processing_site',
+ 'predecessor_specification', 'batch_size',
+ 'is_auto_submit']
labels = {
'selected_workflow': 'Selected workflow',
+ 'selected_workflow_tag': 'Workflow tag',
'processing_site': 'Processing Site (ATDB)',
- 'predecessor_specification': 'Predecessor work specification',
- 'batch_size': 'Files per task'
+ 'predecessor_specification': 'Predecessor',
+ 'batch_size': 'Files per task',
+ 'is_auto_submit': 'Auto submit'
}
+ help_texts = {'selected_workflow': "The pipeline to run on the processing site.",
+ 'selected_workflow_tag': "The pipeline's tag as specified in ATDB.",
+ 'processing_site': "The ATDB processing site to run a specific workflow in.",
+ 'predecessor_specification': "The related predecessor of the current work specification.",
+ 'batch_size': "The number of files every task generated by this work specification should have. Example: 10 files in total can be split into 5 tasks of 2 files.",
+ 'is_auto_submit': "By checking this box, the work specification will be directly sent to the processing site and thus does not need inspection."
+ }
diff --git a/ldvspec/lofardata/migrations/0010_dataproductfilter_help_text.py b/ldvspec/lofardata/migrations/0010_dataproductfilter_help_text.py
new file mode 100644
index 0000000000000000000000000000000000000000..39e7ea559bdd1c2f109a7b67cecbac297eab2faa
--- /dev/null
+++ b/ldvspec/lofardata/migrations/0010_dataproductfilter_help_text.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.2 on 2022-11-11 15:40
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('lofardata', '0009_auto_20221110_1307'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='dataproductfilter',
+ name='help_text',
+ field=models.CharField(blank=True, default='', max_length=150),
+ ),
+ ]
diff --git a/ldvspec/lofardata/migrations/0011_workspecification_selected_workflow_tag.py b/ldvspec/lofardata/migrations/0011_workspecification_selected_workflow_tag.py
new file mode 100644
index 0000000000000000000000000000000000000000..2e2737c9f0249dc7b734cccffb0a33e3742f203d
--- /dev/null
+++ b/ldvspec/lofardata/migrations/0011_workspecification_selected_workflow_tag.py
@@ -0,0 +1,18 @@
+# Generated by Django 3.2 on 2022-12-07 11:26
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('lofardata', '0010_dataproductfilter_help_text'),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name='workspecification',
+ name='selected_workflow_tag',
+ field=models.CharField(default='Unknown', max_length=500, null=True),
+ ),
+ ]
diff --git a/ldvspec/lofardata/migrations/0012_merge_20221213_1102.py b/ldvspec/lofardata/migrations/0012_merge_20221213_1102.py
new file mode 100644
index 0000000000000000000000000000000000000000..e6ea54cc063833117a115df74634d6b4bfd82f35
--- /dev/null
+++ b/ldvspec/lofardata/migrations/0012_merge_20221213_1102.py
@@ -0,0 +1,14 @@
+# Generated by Django 3.2 on 2022-12-13 11:02
+
+from django.db import migrations
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ('lofardata', '0010_alter_dataproduct_surl'),
+ ('lofardata', '0011_workspecification_selected_workflow_tag'),
+ ]
+
+ operations = [
+ ]
diff --git a/ldvspec/lofardata/models.py b/ldvspec/lofardata/models.py
index b2515eeec301ce26d7048284bf3786b777734f62..7f0ffbe595a8094cc3fa87835df1f4d2c5433294 100644
--- a/ldvspec/lofardata/models.py
+++ b/ldvspec/lofardata/models.py
@@ -96,6 +96,7 @@ class DataFilterType(models.TextChoices):
class DataProductFilter(models.Model):
field = models.CharField(max_length=100)
name = models.CharField(max_length=20)
+ help_text = models.CharField(max_length=150, blank=True, default="")
lookup_type = models.CharField(max_length=100)
filter_type = models.CharField(max_length=20, choices=DataFilterType.choices, default=DataFilterType.FREEFORM)
@@ -145,6 +146,7 @@ class WorkSpecification(models.Model):
# ATDB Workflow URL
selected_workflow = models.CharField(max_length=500, null=True)
+ selected_workflow_tag = models.CharField(max_length=500, null=True, default="Unknown")
# Task ID's that were created in ATDB
related_tasks = ArrayField(models.IntegerField(), null=True)
@@ -166,6 +168,11 @@ class WorkSpecification(models.Model):
max_length=16, choices=PURGE_POLICY.choices, default=PURGE_POLICY.NO
)
+ def created_by_display_value(self):
+ if self.created_by is None:
+ return "Unknown"
+ return self.created_by.get_full_name() if self.created_by.get_full_name() else self.created_by
+
def __str__(self):
return str(self.id) + ' - ' + str(self.filters) + " (" + str(self.created_on) + ")"
diff --git a/ldvspec/lofardata/static/histogram_amcharts.js b/ldvspec/lofardata/static/histogram_amcharts.js
new file mode 100644
index 0000000000000000000000000000000000000000..61e0a5aced9e29b51786053d6a293c7b653d21a9
--- /dev/null
+++ b/ldvspec/lofardata/static/histogram_amcharts.js
@@ -0,0 +1,103 @@
+am5.ready(function () {
+
+// Create root element
+// https://www.amcharts.com/docs/v5/getting-started/#Root_element
+ var root = am5.Root.new("chartdiv");
+
+
+// Set themes
+// https://www.amcharts.com/docs/v5/concepts/themes/
+ root.setThemes([
+ am5themes_Animated.new(root)
+ ]);
+
+
+// Create chart
+// https://www.amcharts.com/docs/v5/charts/xy-chart/
+ var chart = root.container.children.push(am5xy.XYChart.new(root, {
+ panX: true,
+ panY: true,
+ wheelX: "panX",
+ wheelY: "zoomX",
+ pinchZoomX: true
+ }));
+
+// Add cursor
+// https://www.amcharts.com/docs/v5/charts/xy-chart/cursor/
+ var cursor = chart.set("cursor", am5xy.XYCursor.new(root, {}));
+ cursor.lineY.set("visible", false);
+
+// Create axes
+// https://www.amcharts.com/docs/v5/charts/xy-chart/axes/
+ var xRenderer = am5xy.AxisRendererX.new(root, {minGridDistance: 30});
+ xRenderer.labels.template.setAll({
+ rotation: -90,
+ centerY: am5.p50,
+ centerX: am5.p100,
+ paddingRight: 15
+ });
+
+ var xAxis = chart.xAxes.push(am5xy.CategoryAxis.new(root, {
+ maxDeviation: 0.3,
+ categoryField: xaxis_key, //set in accompanying template
+ renderer: xRenderer,
+ tooltip: am5.Tooltip.new(root, {}),
+ }));
+
+ xAxis.children.push(
+ am5.Label.new(root, {
+ text: xaxis_header,
+ x: am5.p50,
+ })
+ );
+
+ var yAxis = chart.yAxes.push(am5xy.ValueAxis.new(root, {
+ min: 0,
+ max: y_max,
+ strictMinMax: true,
+ renderer: am5xy.AxisRendererY.new(root, {}),
+ }));
+
+ yAxis.children.unshift(
+ am5.Label.new(root, {
+ rotation: -90,
+ text: yaxis_header,
+ y: am5.p50,
+ centerX: am5.p50,
+ })
+ );
+
+
+// Create series
+// https://www.amcharts.com/docs/v5/charts/xy-chart/series/
+ var series = chart.series.push(am5xy.ColumnSeries.new(root, {
+ name: "Series 1",
+ xAxis: xAxis,
+ yAxis: yAxis,
+ valueYField: yaxis_key, //set in accompanying template
+ sequencedInterpolation: true,
+ categoryXField: xaxis_key, //set in accompanying template
+ tooltip: am5.Tooltip.new(root, {
+ labelText: "{valueY}"
+ })
+ }));
+
+ series.columns.template.setAll({cornerRadiusTL: 5, cornerRadiusTR: 5});
+ series.columns.template.adapters.add("fill", function (fill, target) {
+ return chart.get("colors").getIndex(series.columns.indexOf(target));
+ });
+
+ series.columns.template.adapters.add("stroke", function (stroke, target) {
+ return chart.get("colors").getIndex(series.columns.indexOf(target));
+ });
+
+// Set data; defined in accompanying template
+ xAxis.data.setAll(data);
+ series.data.setAll(data);
+
+
+// Make stuff animate on load
+// https://www.amcharts.com/docs/v5/concepts/animations/
+ series.appear(1000);
+ chart.appear(1000, 100);
+});
\ No newline at end of file
diff --git a/ldvspec/lofardata/static/lofardata/styling/dias.css b/ldvspec/lofardata/static/lofardata/styling/dias.css
index d1b6b292f7267382f3b93127abebba6d5a84b957..85987a13fd531e78542bffa48561e895fb031043 100644
--- a/ldvspec/lofardata/static/lofardata/styling/dias.css
+++ b/ldvspec/lofardata/static/lofardata/styling/dias.css
@@ -1035,21 +1035,16 @@ input.section-toggle:checked + label:before {
/* Text styles -------------------------------------------------------*/
html {
- font-family: Raleway;
- font-size: 16px;
+ font-family: Raleway, sans-serif;
+ font-size: 20px;
line-height: 1.5rem;
color: var(--text);
}
-@supports (font-feature-settings: normal) {
- html {
- font-feature-settings: "ss02" on;
- }
-}
@media (max-width: 1440px) {
html {
- font-size: 14px;
+ font-size: 18px;
}
}
@@ -2671,6 +2666,7 @@ input.section-toggle-invert:checked ~ .section-wrapper {
margin: 0.5rem 0 0;
font-size: 14px;
display: block;
+ min-width: fit-content;
}
.input__label--hidden {
@@ -3042,7 +3038,7 @@ input.section-toggle-invert:checked ~ .section-wrapper {
}
.badge {
- --badge-height: 16px;
+ --badge-height: 20px;
background-color: var(--text);
color: white;
border-radius: calc(var(--badge-height) / 2);
@@ -3055,7 +3051,7 @@ input.section-toggle-invert:checked ~ .section-wrapper {
text-align: center;
justify-content: center;
align-items: center;
- font-size: 12px;
+ font-size: 16px;
font-weight: 500;
}
@@ -3691,8 +3687,42 @@ input.section-toggle-invert:checked ~ .section-wrapper {
.custom-atdb-task-modal {
min-height: 15rem;
+ min-width: 50rem;
}
.custom-div-margin {
margin: 1rem;
+}
+
+.custom-tooltip {
+ font-size: 0.75rem;
+ margin-top: calc(var(--spacing) * 0.125);
+}
+
+/*We reset the dias checkbox style because the checkbox label limits inheritance*/
+.custom-checkbox[type="checkbox"]:checked {
+ width: auto;
+ position: relative;
+ display: inline-block;
+ left: 0;
+}
+
+.custom-checkbox[type="checkbox"]:not(:checked) {
+ width: auto;
+ position: relative;
+ display: inline-block;
+ left: 0;
+}
+
+.custom-chart-styling {
+ height: 35rem;
+ width: 50rem;
+}
+
+.custom-input-selection {
+ min-width: 9rem;
+}
+
+.custom-input-filters {
+ min-width: 4rem;
}
\ No newline at end of file
diff --git a/ldvspec/lofardata/static/update_workflow.js b/ldvspec/lofardata/static/update_workflow.js
index 06861df55b0bf6b51082b777817683988049fdd7..7237bdbb5160d503a9105dd4ed66bc15335dc3fa 100644
--- a/ldvspec/lofardata/static/update_workflow.js
+++ b/ldvspec/lofardata/static/update_workflow.js
@@ -1,37 +1,75 @@
-function fetchAvailableWorkflows(ATDBurl) {
- fetch(ATDBurl + 'workflows').then((response) => response.json()).then((data) => {
- $.each(data.results, function (key, value) {
- let workflowDropdownOption = $('<option>', {value: value.workflow_uri}).text(value.workflow_uri);
- $('#id_selected_workflow').append(workflowDropdownOption);
- });
- }).then(() => {
- $('#id_selected_workflow').val($("#id_selected_workflow option:first").val());
- })
-}
-
-
-function updateWorkflows(selectedProcessingSite) {
- $('#id_selected_workflow').empty()
- if (selectedProcessingSite === '') return
- let processingSiteUrl = processingSite.replace('replace', selectedProcessingSite);
- fetch(processingSiteUrl).then(
- (response) => response.json()).then(
- (data) => fetchAvailableWorkflows(data.url)
- )
-}
-
-function selectedProcessingSite(event) {
- let site = event.currentTarget.value
- updateWorkflows(site);
-}
-
-function onReady() {
- let processingSite = $("#id_processing_site"); //#TODO: fix proc site initial on update
- let initialProcessingSite = processingSite[0].value;
- if (initialProcessingSite) {
- updateWorkflows(initialProcessingSite)
- }
- processingSite.on('change', selectedProcessingSite);
-}
-
-$(document).ready(onReady);
\ No newline at end of file
+ (function() {
+ document.addEventListener("DOMContentLoaded", function(event) {
+ ko.applyBindings(new SpecificationViewModel());
+ });
+
+ function SpecificationViewModel() {
+ var self = this;
+
+ self.existing_workflow = existingWorkflow;
+ self.existing_workflow_tag = existingWorkflowTag;
+
+ self.processingSites = ko.observableArray(
+ processingSites
+ );
+
+ self.selectedProcessingSite = ko.observable(existingProcessingSite);
+
+ self.selectedProcessingSite.subscribe((context) => {
+ self.loadWorkflows();
+ });
+
+ self.loadWorkflows = function() {
+
+ const selectedProcessingSite = self.selectedProcessingSite();
+ if (selectedProcessingSite) {
+ const processingSiteUrl = processingSite.replace('replace', selectedProcessingSite);
+ self.isLoading(true);
+
+ fetch(processingSiteUrl).then((response) => response.json()).then((data) => {
+ const workflowUrl = data.url + 'workflows';
+
+ fetch(workflowUrl).then((response) => response.json()).then((data) => {
+
+ const workflows = data.results;
+ self.workflows(workflows);
+ self.selectedTag(self.existing_workflow_tag);
+ self.selectedWorkflow(self.existing_workflow);
+
+ }).finally(() => {
+ self.isLoading(false);
+ });
+ }).catch(() => {
+ self.isLoading(false);
+ });
+ } else {
+ self.workflows([]);
+ self.selectedWorkflow(null);
+ self.selectedTag(null);
+ }
+ }
+
+ self.isLoading = ko.observable(false);
+
+ self.caption = ko.computed(() => {
+ return self.isLoading() ? 'Loading...' : '---------'
+ });
+
+ self.workflows = ko.observableArray();
+
+ self.workflowsByTag = ko.computed(() => {
+ const byTag = self.workflows().filter(workflow => workflow.description === self.selectedTag());
+ return byTag;
+ });
+
+ self.tags = ko.computed(() => {
+ const tags = self.workflows().map(workflow => workflow.description);
+ return [...new Set(tags)];
+ });
+
+ self.selectedTag = ko.observable();
+ self.selectedWorkflow = ko.observable();
+
+ self.loadWorkflows();
+ }
+})();
\ No newline at end of file
diff --git a/ldvspec/lofardata/templates/lofardata/base.html b/ldvspec/lofardata/templates/lofardata/base.html
index abf4ce143f5e7a76e909d4a23455c487d0c7c396..1bfaf1a3a42a9aef2e89d76785f943a6e2f043b1 100644
--- a/ldvspec/lofardata/templates/lofardata/base.html
+++ b/ldvspec/lofardata/templates/lofardata/base.html
@@ -15,7 +15,6 @@
<link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.4.2/css/all.css"
integrity="sha384-/rXc/GQVaYpyDdyxK+ecHPVYJSN9bmVFBvjA/9eOB+pb3F2w2N6fc5qB9Ew5yIns" crossorigin="anonymous">
- <script src="https://ajax.googleapis.com/ajax/libs/jquery/3.6.0/jquery.min.js"></script>
<link rel="stylesheet" type="text/css" href="{% static 'lofardata/styling/dias.css' %}">
diff --git a/ldvspec/lofardata/templates/lofardata/index.html b/ldvspec/lofardata/templates/lofardata/index.html
index 5e0a99b669efc9588417fb6965bc4d661d491377..95d1532297120e94eedb929b0590e9a574b12e0c 100644
--- a/ldvspec/lofardata/templates/lofardata/index.html
+++ b/ldvspec/lofardata/templates/lofardata/index.html
@@ -28,9 +28,17 @@
<a class="tooltip-dias tooltip-dias-bottom"
data-tooltip="The ATDB site where it is processed">m</a>
</div>
+ <div class="table__cell">Created By
+ <a class="tooltip-dias tooltip-dias-bottom"
+ data-tooltip="The user who created the specification">m</a>
+ </div>
<div class="table__cell">Workflow
<a class="tooltip-dias tooltip-dias-bottom" data-tooltip="The pipeline">m</a>
</div>
+ <div class="table__cell">Tag
+ <a class="tooltip-dias tooltip-dias-bottom"
+ data-tooltip="The category to which the workflow belongs ">m</a>
+ </div>
<div class="table__cell table__cell--medium-small-fixed">Successors</div>
<div class="table__cell table__cell--medium-small-fixed">Predecessors</div>
<div class="table__cell">Actions</div>
@@ -58,7 +66,7 @@
{% if specification.get_submission_status_display == "undefined" or specification.get_submission_status_display == "not submitted" %}
{% define "primary" as badgecolor %}
{% endif %}
- <div class="badge badge--{{ badgecolor }}">{{ specification.get_submission_status_display }}</div>
+ <div class="badge badge--{{ badgecolor }} margin-top">{{ specification.get_submission_status_display }}</div>
</div>
<div class="table__cell table__cell--truncate">
<a href="{{ specification.processing_site.url }}"
@@ -67,8 +75,11 @@
<span class="icon icon--external-link-alt margin-left"></span>
</a>
</div>
+ <div class="table__cell table__cell--truncate">{{ specification.created_by_display_value }}</div>
<div class="table__cell table__cell--truncate">{{ specification.selected_workflow }}</div>
+ <div class="table__cell table__cell--truncate">{{ specification.selected_workflow_tag }}</div>
+
<div class="table__cell table__cell--medium-small-fixed">
{% if specification.successor.count %}
@@ -94,9 +105,9 @@
<!-- Actions -->
<div class="table__cell">
<div class="flex-wrapper flex-wrapper--row">
-
<a class="button--icon-button margin-left margin-right"
- href="{% url 'specification-update' pk=specification.pk %}">
+ href="{% url 'specification-update' pk=specification.pk %}"
+ title="Edit this work specification">
<span class="icon icon--pen"></span>
</a>
@@ -107,15 +118,13 @@
</a>
{% else %}
<a class="link--red button--icon-button margin-right"
- href="{% url 'specification-delete' pk=specification.pk %}">
+ href="{% url 'specification-delete' pk=specification.pk %}"
+ title="Delete this work specification">
<span class="icon icon--color-inherit icon--trash-alt"></span>
</a>
{% endif %}
- <form
- method="post"
- action="{% url 'workspecification-submit' pk=specification.pk %}"
- >
+ <form method="post" action="{% url 'workspecification-submit' pk=specification.pk %}">
{% csrf_token %}
{% if specification.get_submission_status_display == "submitted" or specification.get_submission_status_display == "defining" %}
@@ -123,6 +132,11 @@
title="Tasks already present in ATDB">
<span class="icon icon--color-inherit icon--play"></span>
</a>
+ {% elif specification.inputs.surls|length == 0 %}
+ <a class="link--disabled button--icon-button"
+ title="Cannot submit this task to ATDB since it does not contain any files">
+ <span class="icon icon--color-inherit icon--play"></span>
+ </a>
{% else %}
<button type="submit" class="button--icon-button" title="Submit to ATDB">
<span class="icon icon--play"></span>
@@ -143,8 +157,4 @@
<h2>Log in to see/add/update work specifications</h2>
</div>
{% endif %}
- <!-- <script>
- $(document).ready(() => setInterval(() => window.location.reload(), 50000))
- </script> -->
-
{% endblock %}
diff --git a/ldvspec/lofardata/templates/lofardata/workspecification/create_update.html b/ldvspec/lofardata/templates/lofardata/workspecification/create_update.html
index 0796773718cd9188f116a2c68b91e886e2e3fc04..c578f642e856a29a0cfdb8bbf60d77670286290d 100644
--- a/ldvspec/lofardata/templates/lofardata/workspecification/create_update.html
+++ b/ldvspec/lofardata/templates/lofardata/workspecification/create_update.html
@@ -3,6 +3,7 @@
{% load crispy_forms_tags %}
{% load define_action %}
{% load widget_tweaks %}
+{% load json %}
{% block myBlock %}
<div class="overlay">
@@ -20,47 +21,108 @@
{% csrf_token %}
<div class="flex-wrapper flex-wrapper--row flex-wrapper--start">
+ <!-- Standard input fields -->
<div class="custom-div-margin">
<h3 class="text text--primary text--title">Selection</h3>
<div class="flex-wrapper flex-wrapper--row" id="div_id_processing_site">
- <label class="input__label">{{ form.processing_site.label }}</label>
+ <div class="flex-wrapper flex-wrapper--row custom-input-selection">
+ <label class="input__label">{{ form.processing_site.label }}</label>
+ <a class="tooltip-dias tooltip-dias-right custom-tooltip"
+ data-tooltip="{{ form.processing_site.help_text }}">m</a>
+ </div>
<div class="input-select-wrapper icon--inline icon-after icon-after--angle-down">
- {% render_field form.processing_site class="input maxwidth-input input--select margin-left margin-bottom" %}
+ <select class="input input--select margin-left margin-bottom" name="processing_site"
+ data-bind="options:processingSites,
+ optionsCaption: '---------',
+ optionsText: function(item) { return item.name + ' - ' + item.url},
+ optionsValue: 'name',
+ value: selectedProcessingSite"></select>
+ </div>
+ </div>
+
+
+ <div class="flex-wrapper flex-wrapper--row" id="div_id_selected_workflow_tag">
+ <div class="flex-wrapper flex-wrapper--row custom-input-selection">
+ <label class="input__label"
+ for="id_selected_workflow_tag">{{ form.selected_workflow_tag.label_tag }}*</label>
+ <a class="tooltip-dias tooltip-dias-right custom-tooltip"
+ data-tooltip="{{ form.selected_workflow_tag.help_text }}">m</a>
</div>
+ <div class="input-select-wrapper icon--inline icon-after icon-after--angle-down">
+ <select class="input input--select margin-bottom margin-left" id="id_selected_workflow_tag" name="selected_workflow_tag" style="width: 12rem"
+ data-bind="options:tags,
+ optionsCaption: caption,
+ value: selectedTag,
+ disable: isLoading">
+ </select>
+ </div>
</div>
+
+
<div class="flex-wrapper flex-wrapper--row" id="div_id_selected_workflow">
- <label class="input__label"
- for="id_selected_workflow">{{ form.selected_workflow.label }}*</label>
- <div class="input-select-wrapper icon--inline icon-after icon-after--angle-down margin-left margin-bottom">
- <select class="input input--select" id="id_selected_workflow"
- name="selected_workflow">
- <option disabled value="---" selected>Select a processor first</option>
+ <div class="flex-wrapper flex-wrapper--row custom-input-selection">
+ <label class="input__label"
+ for="id_selected_workflow">{{ form.selected_workflow.label }}*</label>
+ <a class="tooltip-dias tooltip-dias-right custom-tooltip"
+ data-tooltip="{{ form.selected_workflow.help_text }}">m</a>
+ </div>
+ <div class="input-select-wrapper icon--inline icon-after icon-after--angle-down">
+ <select class="input input--select margin-bottom margin-left" id="id_selected_workflow" name="selected_workflow" style="width: 12rem"
+ data-bind="options:workflowsByTag,
+ optionsCaption: caption,
+ optionsValue: 'workflow_uri',
+ optionsText: 'workflow_uri',
+ value: selectedWorkflow,
+ disable: isLoading">
</select>
</div>
</div>
+
<div class="flex-wrapper flex-wrapper--row" id="div_id_predecessor">
- <label class="input__label"
- for="id_predecessor">{{ form.predecessor_specification.label }}</label>
+ <div class="flex-wrapper flex-wrapper--row custom-input-selection">
+ <label class="input__label"
+ for="id_predecessor">{{ form.predecessor_specification.label }}</label>
+ <a class="tooltip-dias tooltip-dias-right custom-tooltip"
+ data-tooltip="{{ form.predecessor_specification.help_text }}">m</a>
+ </div>
<div class="input-select-wrapper icon--inline icon-after icon-after--angle-down margin-left margin-bottom">
- {% render_field form.predecessor_specification class="input maxwidth-input input--select" %}
+ {% render_field form.predecessor_specification class="input input--select" %}
</div>
</div>
<div class="flex-wrapper flex-wrapper--row" id="div_id_batch_size">
- <label class="input__label"
- for="id_batch_size">{{ form.batch_size.label_tag }}</label>
- {% render_field form.batch_size class="input maxwidth-input input--select margin-left margin-bottom" type="number" %}
+ <div class="flex-wrapper flex-wrapper--row custom-input-selection">
+ <label class="input__label"
+ for="id_batch_size">{{ form.batch_size.label_tag }}</label>
+ <a class="tooltip-dias tooltip-dias-right custom-tooltip"
+ data-tooltip="{{ form.batch_size.help_text }}">m</a>
+ </div>
+ {% render_field form.batch_size class="input input--select margin-left margin-bottom" type="number" %}
+ </div>
+
+ <div class="flex-wrapper flex-wrapper--row" id="div_id_auto_submit">
+ {% render_field form.is_auto_submit class="input custom-checkbox margin-top" type="checkbox" %}
+ <label class="input__label">{{ form.is_auto_submit.label }}</label>
+ <a class="tooltip-dias tooltip-dias-right custom-tooltip"
+ data-tooltip="{{ form.is_auto_submit.help_text }}">m</a>
</div>
</div>
+ <!-- Filter fields -->
<div class="custom-div-margin">
<h3 class="text text--primary text--title">Filters*</h3>
{% for filter in filters %}
{% if filter.filter_type == 'Free' %}
<div class="flex-wrapper">
- <label class="input__label"
- for="id_{{ filter.field }}">{{ filter.name }}</label>
+ <div class="flex-wrapper flex-wrapper--row custom-input-filters">
+ <label class="input__label"
+ for="id_{{ filter.field }}">{{ filter.name }}</label>
+ {% if filter.help_text %}
+ <a class="tooltip-dias tooltip-dias-right custom-tooltip"
+ data-tooltip="{{ filter.help_text }}">m</a>
+ {% endif %}
+ </div>
<input class="input input--text margin-left margin-bottom"
id="id_{{ filter.field }}" name="{{ filter.field }}"
placeholder="Enter {{ filter.name }}" type="text"
@@ -70,13 +132,21 @@
{% elif filter.filter_type == 'Dropdown' %}
<div class="flex-wrapper flex-wrapper--row"
id="div_id_filter_{{ filter.name }}">
- <label class="input__label"
- for="id_{{ filter.field }}">{{ filter.name }}</label>
+ <div class="flex-wrapper flex-wrapper--row custom-input-filters">
+ <label class="input__label"
+ for="id_{{ filter.field }}">{{ filter.name }}</label>
+ {% if filter.help_text %}
+ <a class="tooltip-dias tooltip-dias-right custom-tooltip"
+ data-tooltip="{{ filter.help_text }}">m</a>
+ {% endif %}
+ </div>
<div class="input-select-wrapper icon--inline icon-after icon-after--angle-down margin-left margin-bottom">
<select class="input input--select"
id="id_{{ filter.field }}"
name="{{ filter.field }}"
- data-filter="{{ filter.lookup_type }}">
+ data-filter="{{ filter.lookup_type }}"
+ style="width: 12rem">
+ {#for some reason, styling is not applied otherwise#}
{% for option in filter.choices %}
{% if filter.default == option.0 %}
<option selected
@@ -100,7 +170,18 @@
type="submit"
name="action"
value="Submit"
- title="Submit the task to inspect the result. Send it later to ATDB.">Submit
+ title="
+ {% if object.pk %}
+ Update the task
+ {% else %}
+ Create the task to inspect the result. Send it later to ATDB.
+ {% endif %}">
+
+ {% if object.pk %}
+ Update
+ {% else %}
+ Create
+ {% endif %}
</button>
<button class="button button--primary margin-right" type="submit"
value="Successor"
@@ -128,8 +209,13 @@
</div>
</div>
</div>
+ <script type='text/javascript' src='https://cdnjs.cloudflare.com/ajax/libs/knockout/3.5.0/knockout-min.js'></script>
<script type="text/javascript">
- var processingSite = "{% url 'processingsite-detail' 'replace' %}"
+ const processingSite = "{% url 'processingsite-detail' 'replace' %}";
+ const existingWorkflow = '{{ object.selected_workflow|default_if_none:"null" }}';
+ const existingWorkflowTag = '{{ object.selected_workflow_tag|default_if_none:"null" }}';
+ const processingSites = {{ processing_sites | json }};
+ const existingProcessingSite = '{{ object.processing_site.name }}';
</script>
<script src="{% static 'update_workflow.js' %}"></script>
{% endblock %}
diff --git a/ldvspec/lofardata/templates/lofardata/workspecification/dataproducts.html b/ldvspec/lofardata/templates/lofardata/workspecification/dataproducts.html
new file mode 100644
index 0000000000000000000000000000000000000000..e10def25bc83ef2039b859cb0d1dcac7b42322db
--- /dev/null
+++ b/ldvspec/lofardata/templates/lofardata/workspecification/dataproducts.html
@@ -0,0 +1,50 @@
+{% extends 'lofardata/index.html' %}
+{% load static %}
+{% load crispy_forms_tags %}
+{% load define_action %}
+{% load widget_tweaks %}
+
+{% block myBlock %}
+ <div class="overlay">
+ <div class="modal-dias-wrapper">
+ <div class="modal-dias modal-dias--fit-content custom-atdb-task-modal">
+ <a class="icon icon--times button--close" href="{% url 'specification-detail' pk %}"></a>
+ {% if dataproduct_info %}
+ <header class="flex-wrapper flex-wrapper--centered flex-wrapper--column">
+ <h2 class="title">Dataproduct information of SAS ID {{ sas_id }}
+ <a class="tooltip-dias tooltip-dias-bottom"
+ data-tooltip="The general data product information for this work specification based on the retrieved information from the SAS ID">m</a>
+ </h2>
+ </header>
+ <div class="table margin-bottom">
+ <div class="table__header">
+ <div class="table__row table__row--dark table__row--padding">
+ <div class="table__cell">Property</div>
+ <div class="table__cell">Value</div>
+ </div>
+ </div>
+ <div class="table__content">
+ {% for key, value in dataproduct_info.items %}
+ <div class="table__row table__row--dark table__row--padding">
+ <div class="table__cell table__cell--title">{{ key }}</div>
+ <div class="table__cell">
+ {% for item in value %}
+ {{ item }}
+ {% if not forloop.last %}&{% endif %}
+ {% endfor %}
+ </div>
+ </div>
+ {% endfor %}
+ </div>
+ </div>
+ {% else %}
+ <div class="flex-wrapper flex-wrapper--centered flex-wrapper--column">
+ <p class="text--red">The information is not present.</p>
+ <p>This information is directly linked to given SAS ID which is incorrect. </p>
+ <p class="text note-text">Please notify 'lautenbach@astron.nl' if this occurs.</p>
+ </div>
+ {% endif %}
+ </div>
+ </div>
+ </div>
+{% endblock %}
\ No newline at end of file
diff --git a/ldvspec/lofardata/templates/lofardata/workspecification/dataset_size_info.html b/ldvspec/lofardata/templates/lofardata/workspecification/dataset_size_info.html
new file mode 100644
index 0000000000000000000000000000000000000000..f8ac2d6777befe21dea2a06bf8dd53dbce9f1788
--- /dev/null
+++ b/ldvspec/lofardata/templates/lofardata/workspecification/dataset_size_info.html
@@ -0,0 +1,54 @@
+{% extends 'lofardata/index.html' %}
+{% load static %}
+{% load crispy_forms_tags %}
+{% load define_action %}
+{% load widget_tweaks %}
+
+{% block myBlock %}
+ <div class="overlay">
+ <div class="modal-dias-wrapper">
+ <div class="modal-dias modal-dias--fit-content custom-atdb-task-modal">
+ <a class="icon icon--times button--close" href="{% url 'specification-detail' object.pk %}"></a>
+ <header class="flex-wrapper flex-wrapper--centered flex-wrapper--column">
+ <h2 class="title">Size distributions
+ <a class="tooltip-dias tooltip-dias-bottom"
+ data-tooltip="The size distribution of the input files of this work specification">m</a>
+ </h2>
+ </header>
+ <div class="custom-chart-styling" id="chartdiv"></div>
+ <div class="flex-wrapper flex-wrapper--row flex-wrapper--space-between">
+ <p>Minimum: {{ min_size }}</p>
+ <p>Maximum: {{ max_size }}</p>
+ </div>
+ </div>
+ </div>
+ </div>
+ <!-- Amcharts -->
+
+ <!-- Data retriever -->
+ <script>
+ const y_max = {{ biggest_bucket }};
+ const xaxis_key = "file_size";
+ const xaxis_header = "File size";
+ const yaxis_key = "total_number_files";
+ const yaxis_header = "Number of files";
+ const xaxis_values = {{ bins|safe }};
+ const yaxis_values = {{ counts|safe }};
+
+ <!-- Construct the data the way AMCharts needs it -->
+ let data = [];
+ for (let i = 0; i < {{ n_bins }}; i++) {
+ let entry = {}
+ entry[xaxis_key] = (xaxis_values[i - 1] ? xaxis_values[i - 1] : 0) + " - " + xaxis_values[i]
+ entry[yaxis_key] = yaxis_values[i]
+ data[i] = entry
+ }
+ </script>
+ <!-- Resources -->
+ <script src="https://cdn.amcharts.com/lib/5/index.js"></script>
+ <script src="https://cdn.amcharts.com/lib/5/xy.js"></script>
+ <script src="https://cdn.amcharts.com/lib/5/themes/Animated.js"></script>
+ <!-- Script code -->
+ <script src="{% static 'histogram_amcharts.js' %}"></script>
+
+{% endblock %}
diff --git a/ldvspec/lofardata/templates/lofardata/workspecification/detail.html b/ldvspec/lofardata/templates/lofardata/workspecification/detail.html
index 77630b328e67c3c72c5812b05a0121880dd8b89a..b2d7784dc3b48cddba0d992b310e5296a86a1b0e 100644
--- a/ldvspec/lofardata/templates/lofardata/workspecification/detail.html
+++ b/ldvspec/lofardata/templates/lofardata/workspecification/detail.html
@@ -15,7 +15,7 @@
<span class="margin-top">Work Specification {{ object.pk }}</span>
<a class="button button--secondary button--icon-button margin-left margin-right"
- href="{% url 'specification-update' pk=object.pk %}">
+ href="{% url 'specification-update' pk=object.pk %}" title="Edit this work specification">
<span class="icon icon--pen"></span>
</a>
@@ -26,33 +26,33 @@
</a>
{% else %}
<a class="button button--secondary button--red button--icon-button margin-right"
- href="{% url 'specification-delete' pk=object.pk %}">
+ href="{% url 'specification-delete' pk=object.pk %}"
+ title="Delete this work specification">
<span class="icon icon--trash-alt"></span>
</a>
{% endif %}
- <form
- method="post"
- action="{% url 'workspecification-submit' pk=object.pk %}"
- >
+ <form method="post" action="{% url 'workspecification-submit' pk=object.pk %}">
{% csrf_token %}
{% if object.get_submission_status_display == "submitted" or object.get_submission_status_display == "defining" %}
- <a
- class="button button--icon-button button--primary button--disabled"
- title="Tasks already present in ATDB"
- href="#"
- >
- <span class="icon icon--play"></span>
+ <a class="button button--icon-button button--primary button--disabled"
+ title="Tasks already present in ATDB"
+ href="#">
+ <span class="icon icon--play"></span>
+ </a>
+ {% elif number_of_files == 0 %}
+ <a class="button button--icon-button button--primary button--disabled"
+ title="No files to process in ATDB"
+ href="#">
+ <span class="icon icon--play"></span>
</a>
{% else %}
- <button
- type="submit"
- class="button button--secondary button--icon-button"
- title="Submit to ATDB"
- >
- <span class="icon icon--play"></span>
- </button>
+ <button type="submit"
+ class="button button--secondary button--icon-button"
+ title="Submit to ATDB">
+ <span class="icon icon--play"></span>
+ </button>
{% endif %}
</form>
@@ -137,60 +137,96 @@
<div class="table__cell"> {{ object.filters }}</div>
</div>
- <div class="table__row table__row--dark table__row--padding">
- <div class="table__cell table__cell--title">Files per task</div>
- <div class="table__cell">{{ object.batch_size }}</div>
- </div>
-
<div class="table__row table__row--dark table__row--padding">
<div class="table__cell table__cell--title">Purge policy</div>
<div class="table__cell">{{ object.purge_policy }}</div>
</div>
{% if object.is_ready or object.is_defined %}
- <div class="table__row table__row--dark table__row--padding">
- <div class="table__cell table__cell--title">Total size</div>
- <div class="table__cell">{{ total_input_size }}</div>
- </div>
-
- <div class="table__row table__row--dark table__row--padding">
- <div class="table__cell table__cell--title">Size per task</div>
- <div class="table__cell">{{ size_per_task }}</div>
- </div>
-
- <div class="table__row table__row--dark table__row--padding">
- <div class="table__cell table__cell--title">Number of files</div>
- <div class="table__cell">{{ number_of_files }}</div>
- </div>
-
<div class="table__row table__row--dark table__row--padding">
<div class="table__cell table__cell--title">Async Task Result</div>
<div class="table__cell">{{ object.async_task_result }}</div>
</div>
+
<div class="table__row table__row--dark table__row--padding">
- <div class="table__cell table__cell--title">Inputs</div>
+ <div class="table__cell table__cell--title">Data Product Information</div>
<div class="table__cell">
<a class="button button--icon-button custom-button-no-padding"
- href="{% url 'specification-inputs' object.pk %}">
- <span class="icon icon--bars"></span>
+ href="{% url 'specification-dataproducts' object.pk %}">
+ <span class="icon icon--file-alt"></span>
</a>
</div>
</div>
<div class="table__row table__row--dark table__row--padding">
- <div class="table__cell table__cell--title">ATDB Tasks</div>
- <div class="table__cell">
- {% if object.related_tasks %}
- <a class="button button--icon-button custom-button-no-padding"
- href="{% url 'specification-tasks' object.pk %}">
- <span class="icon icon--list-ul"></span>
- </a>
+ <div class="table__cell table__cell--title">Number of files</div>
+ {% if number_of_files == 0 %}
+ <div class="table__cell">
+ <a class="tooltip-dias tooltip-dias-right tooltip-dias--without-margin tooltip-dias--red "
+ data-tooltip="No files found for this SAS ID">m</a>
+ </div>
+ {% else %}
+ <div class="table__cell">{{ number_of_files }}</div>
+ {% endif %}
+ </div>
+
+
+ {% if number_of_files > 0 %}
+ <div class="table__row table__row--dark table__row--padding">
+ <div class="table__cell table__cell--title">Files per task</div>
+ {% if object.batch_size == 0 %}
+ <div class="table__cell">{{ number_of_files }}</div>
{% else %}
- <div class="text">-</div>
+ <div class="table__cell">{{ object.batch_size }}</div>
{% endif %}
</div>
- </div>
+
+ <div class="table__row table__row--dark table__row--padding">
+ <div class="table__cell table__cell--title">Total size</div>
+ <div class="table__cell">{{ total_input_size }}</div>
+ </div>
+
+ <div class="table__row table__row--dark table__row--padding">
+ <div class="table__cell table__cell--title">Size per task</div>
+ <div class="table__cell">{{ size_per_task }}</div>
+ </div>
+
+ <div class="table__row table__row--dark table__row--padding">
+ <div class="table__cell table__cell--title">Dataset size distribution</div>
+ <div class="table__cell">
+ <a class="button button--icon-button custom-button-no-padding"
+ href="{% url 'dataset-size-info' object.pk %}">
+ <span class="icon icon--chart-bar"></span>
+ </a>
+ </div>
+ </div>
+
+ <div class="table__row table__row--dark table__row--padding">
+ <div class="table__cell table__cell--title">Inputs</div>
+ <div class="table__cell">
+ <a class="button button--icon-button custom-button-no-padding"
+ href="{% url 'specification-inputs' object.pk %}">
+ <span class="icon icon--bars"></span>
+ </a>
+ </div>
+ </div>
+
+ <div class="table__row table__row--dark table__row--padding">
+ <div class="table__cell table__cell--title">ATDB Tasks</div>
+ <div class="table__cell">
+ {% if object.related_tasks %}
+ <a class="button button--icon-button custom-button-no-padding"
+ href="{% url 'specification-tasks' object.pk %}">
+ <span class="icon icon--list-ul"></span>
+ </a>
+ {% else %}
+ <div class="text">-</div>
+ {% endif %}
+ </div>
+ </div>
+ {% endif %}
+
{% endif %}
</div>
</div>
@@ -198,7 +234,4 @@
</div>
</div>
</div>
- <script>
- $(document).ready(() => setTimeout(() => window.location.reload(), 5000))
- </script>
{% endblock %}
diff --git a/ldvspec/lofardata/templates/lofardata/workspecification/test.json b/ldvspec/lofardata/templates/lofardata/workspecification/test.json
new file mode 100644
index 0000000000000000000000000000000000000000..dfd867d5ff69468546838dead974dbd8df8d772f
--- /dev/null
+++ b/ldvspec/lofardata/templates/lofardata/workspecification/test.json
@@ -0,0 +1,108 @@
+{
+ "count": 9,
+ "next": null,
+ "previous": null,
+ "results": [
+ {
+ "id": 1,
+ "description": null,
+ "workflow_uri": "my_great_workflow_version_1_0",
+ "repository": "https://git.astron.nl/ldv/compress_pipeline.git",
+ "commit_id": "master",
+ "path": "workflow/compress.cwl",
+ "oi_size_fraction": null,
+ "meta_scheduling": null,
+ "default_parameters": null
+ },
+ {
+ "id": 2,
+ "description": null,
+ "workflow_uri": "imaging_compress_v0.1test",
+ "repository": "https://git.astron.nl/ldv/imaging_compress_pipeline.git",
+ "commit_id": "v0.1test",
+ "path": "compress_pipeline.cwl",
+ "oi_size_fraction": null,
+ "meta_scheduling": null,
+ "default_parameters": null
+ },
+ {
+ "id": 3,
+ "description": null,
+ "workflow_uri": "imaging_compress_pipeline_v011",
+ "repository": "https://git.astron.nl/ldv/imaging_compress_pipeline.git",
+ "commit_id": "v0.1.1",
+ "path": "compress_pipeline.cwl",
+ "oi_size_fraction": 0.25,
+ "meta_scheduling": null,
+ "default_parameters": null
+ },
+ {
+ "id": 4,
+ "description": null,
+ "workflow_uri": "ldv_test",
+ "repository": "https://git.astron.nl/ldv/imaging_compress_pipeline.git",
+ "commit_id": "v0.1.11",
+ "path": "download_and_compress_pipeline.cwl",
+ "oi_size_fraction": 0.4,
+ "meta_scheduling": null,
+ "default_parameters": null
+ },
+ {
+ "id": 6,
+ "description": null,
+ "workflow_uri": "prefactor3_calibrator",
+ "repository": "https://git.astron.nl/eosc/prefactor3-cwl",
+ "commit_id": "ldv_v01",
+ "path": "workflows/ldv_prefactor_calibrator.cwl",
+ "oi_size_fraction": 1.0,
+ "meta_scheduling": null,
+ "default_parameters": null
+ },
+ {
+ "id": 7,
+ "description": null,
+ "workflow_uri": "solar_bf_compress_v01",
+ "repository": "https://git.astron.nl/ssw-ksp/solar-bf-compressing.git",
+ "commit_id": "v0.3.1",
+ "path": "pipeline/cwl/workflow/beamformed_compression.cwl",
+ "oi_size_fraction": 1.0,
+ "meta_scheduling": null,
+ "default_parameters": null
+ },
+ {
+ "id": 8,
+ "description": null,
+ "workflow_uri": "prefactor3_target",
+ "repository": "https://git.astron.nl/eosc/prefactor3-cwl",
+ "commit_id": "ldv_v03",
+ "path": "workflows/ldv_prefactor_target.cwl",
+ "oi_size_fraction": 1.0,
+ "meta_scheduling": {
+ "#SBATCH --cpus-per-task": 20
+ },
+ "default_parameters": null
+ },
+ {
+ "id": 9,
+ "description": null,
+ "workflow_uri": "dummy",
+ "repository": "dummy",
+ "commit_id": "dummy",
+ "path": "dummy",
+ "oi_size_fraction": null,
+ "meta_scheduling": null,
+ "default_parameters": null
+ },
+ {
+ "id": 10,
+ "description": null,
+ "workflow_uri": "bf_remove_double_tgz",
+ "repository": "https://git.astron.nl/ldv/bf_double_tgz.git",
+ "commit_id": "v0.3",
+ "path": "workflow/download_and_run_bf_remove.cwl",
+ "oi_size_fraction": 1.0,
+ "meta_scheduling": null,
+ "default_parameters": null
+ }
+ ]
+}
\ No newline at end of file
diff --git a/ldvspec/lofardata/templates/lofardata/workspecification/update.html b/ldvspec/lofardata/templates/lofardata/workspecification/update.html
deleted file mode 100644
index 931a139819f9d1f9607fb8a2236d1f88edb71d1b..0000000000000000000000000000000000000000
--- a/ldvspec/lofardata/templates/lofardata/workspecification/update.html
+++ /dev/null
@@ -1,101 +0,0 @@
-{% extends 'lofardata/index.html' %}
-{% load static %}
-{% load crispy_forms_tags %}
-{% load define_action %}
-{% load widget_tweaks %}
-
-{% block myBlock %}
- <div class="overlay">
- <div class="modal-dias-wrapper">
- <div class="modal-dias modal-dias--fit-content">
- <a class="icon icon--times button--close" href="{% url 'index' %}"></a>
- <header class="flex-wrapper flex-wrapper--centered flex-wrapper--column">
- <h2 class="title text text--primary">Edit Work Specification {{ object.pk }}</h2>
- <form method="post">{% csrf_token %}
- <div class="flex-wrapper flex-wrapper--row flex-wrapper--start">
- <div class="">
- <h3 class="text text--primary text--title">Selection</h3>
- <div class="flex-wrapper flex-wrapper--row" id="div_id_processing_site">
- <label class="input__label">{{ form.processing_site.label_tag }}*</label>
- <div class="input-select-wrapper icon--inline icon-after icon-after--angle-down">
- {% render_field form.processing_site class="input maxwidth-input input--select" %}
- </div>
- </div>
- <div class="flex-wrapper flex-wrapper--row" id="div_id_selected_workflow">
- <label class="input__label"
- for="id_selected_workflow">{{ form.selected_workflow.label_tag }}*</label>
- <div class="input-select-wrapper icon--inline icon-after icon-after--angle-down">
- <select class="input input--select" id="id_selected_workflow"
- name="selected_workflow">
- <option value="---">TODO</option>
- </select>
- </div>
- </div>
- <div class="flex-wrapper flex-wrapper--row" id="div_id_predecessor">
- <label class="input__label"
- for="id_predecessor">{{ form.predecessor_task.label_tag }}</label>
- <input class="input input--text" id="field-name2" placeholder="None"
- type="number">
- </div>
- </div>
-
- {% if filters is not None %}
- <div class="">
- <h3 class="text text--primary text--title">Filters</h3>
- {% for filter in filters %}
- {% if filter.filter_type == 'Free' %}
- <div class="flex-wrapper">
- <label class="input__label"
- for="filter_{{ filter.name }}">{{ filter.name }}</label>
- <input class="input input--text margin-left"
- id="filter_{{ filter.name }}"
- placeholder="Enter {{ filter.name }}" type="text"
- data-filter="{{ filter.lookup_type }}"
- value="{{ filter.default }}">
- </div>
- {% elif filter.filter_type == 'Dropdown' %}
- <div class="flex-wrapper flex-wrapper--row"
- id="div_id_filter_{{ filter.name }}">
- <label class="input__label"
- for="filter_{{ filter.name }}">{{ filter.name }}</label>
- <div class="input-select-wrapper icon--inline icon-after icon-after--angle-down">
- <select class="input input--select"
- id="filter_{{ filter.name }}"
- data-filter="{{ filter.lookup_type }}">
- {% for option in filter.choices %}
- {% if filter.default == option.0 %}
- <option selected
- value="{{ option.0 }}">{{ option.0 }}</option>
- {% else %}
- <option value="{{ option.0 }}">{{ option.0 }}</option>
- {% endif %}
- {% endfor %}
- </select>
- </div>
- </div>
- {% else %}
- <div class="text text--red">Not supported field</div>
- {% endif %}
- {% endfor %}
- </div>
- {% endif %}
-
-
- </div>
- <div class="flex-wrapper flex-wrapper--centered margin-bottom margin-top">
- <button class="button button--primary margin-right" type="submit" value="Submit">Submit
- </button>
- <a class="button button--primary margin-left" href="{% url 'index' %}">Cancel</a>
- </div>
-
- </form>
- </header>
- </div>
- </div>
- </div>
- <script type="text/javascript">
- var processingSite = "{% url 'processingsite-detail' 'replace' %}"
- </script>
- <script src="{% static 'update_workflow.js' %}"></script>
-
-{% endblock %}
\ No newline at end of file
diff --git a/ldvspec/lofardata/templatetags/json.py b/ldvspec/lofardata/templatetags/json.py
new file mode 100644
index 0000000000000000000000000000000000000000..7c7bed945b2dee3c8c06ab878f656ebee58b601f
--- /dev/null
+++ b/ldvspec/lofardata/templatetags/json.py
@@ -0,0 +1,10 @@
+from django import template
+from django.utils.safestring import mark_safe
+
+import json
+
+register = template.Library()
+
+@register.filter(name='json')
+def json_dumps(data):
+ return mark_safe(json.dumps(data))
\ No newline at end of file
diff --git a/ldvspec/lofardata/tests/test_util_funcs.py b/ldvspec/lofardata/tests/test_util_funcs.py
index ec2b6e2664f9a1eab850e8cebd64c13c5f39fee0..d1e54032a0143f1040f8f6bdcc8e1214c2fa9136 100644
--- a/ldvspec/lofardata/tests/test_util_funcs.py
+++ b/ldvspec/lofardata/tests/test_util_funcs.py
@@ -1,7 +1,9 @@
import unittest
+import random
from lofardata.tasks import split_entries_to_batches
-from lofardata.views import compute_size_of_inputs
+from lofardata.views import compute_size_of_inputs, compute_inputs_histogram, format_size
+
class SplitEntries(unittest.TestCase):
def test_no_splitting(self):
@@ -30,14 +32,66 @@ class SplitEntries(unittest.TestCase):
class ComputeInputSizes(unittest.TestCase):
def test_input_sizes(self):
test_data = {
- 'only_a_file': {'class': 'File', 'size': 1},
- 'a_list_of_files': [{'class': 'File', 'size': 1}, {'class': 'File', 'size': 1}],
- 'nested_files': [{'item1': {'class': 'File', 'size': 1}}, {'class': 'File', 'size': 1}],
+ 'only_a_file': {'class': 'File', 'size': 2},
+ 'a_list_of_files': [{'class': 'File', 'size': 2}, {'class': 'File', 'size': 5}],
+ 'nested_files': [{'item1': {'class': 'File', 'size': 1}}, {'class': 'File', 'size': 4}],
'not_a_file': 'bla'
}
- result, number_of_files, average_file_size = compute_size_of_inputs(test_data)
+ total_size, number_of_files, average_file_size = compute_size_of_inputs(test_data)
- self.assertEqual(5, result)
+ self.assertEqual(14, total_size)
self.assertEqual(5, number_of_files)
- self.assertEqual(1.0, average_file_size)
+ self.assertEqual(2.8, average_file_size)
+
+
+class ComputeInputsHistogram(unittest.TestCase):
+ def test_basic(self):
+ test_data = {"surls": [{"size": 33832140800, "surl": "test"},
+ {"size": 21832140800, "surl": "test"},
+ {"size": 33835408000, "surl": "test"},
+ {"size": 53832140800, "surl": "test"},
+ {"size": 29832140800, "surl": "test"},
+ {"size": 30832140801, "surl": "test"}]}
+
+ min_size, max_size, n_bins, counts, biggest_bucket, bins = compute_inputs_histogram(test_data)
+
+ self.assertEqual(6, n_bins)
+ self.assertEqual(21832140800, min_size)
+ self.assertEqual(53832140800, max_size)
+ self.assertEqual(2, biggest_bucket)
+ self.assertListEqual([1, 2, 2, 0, 0, 1], counts)
+ self.assertListEqual(['20.3GB', '25.3GB', '30.3GB', '35.2GB', '40.2GB', '45.2GB', '50.1GB'], bins)
+
+ def test_single_range(self):
+ test_data = {"surls": [{"size": 1, "surl": "test"},
+ {"size": 1, "surl": "test"},
+ {"size": 1, "surl": "test"},
+ {"size": 1, "surl": "test"},
+ {"size": 1, "surl": "test"},
+ {"size": 1, "surl": "test"}]}
+
+ min_size, max_size, n_bins, counts, biggest_bucket, bins = compute_inputs_histogram(test_data)
+
+ self.assertEqual(1, n_bins)
+ self.assertEqual(1, min_size)
+ self.assertEqual(1, max_size)
+ self.assertEqual(6, biggest_bucket)
+ self.assertListEqual([6], counts)
+ self.assertListEqual(['0.5B', '1.5B'], bins)
+
+ def test_extreme_wide_range(self):
+ test_data = {"surls": [{"size": 1, "surl": "test"},
+ {"size": 100000, "surl": "test"},
+ {"size": 100000000, "surl": "test"},
+ {"size": 100000000000, "surl": "test"},
+ {"size": 1000000000000000, "surl": "test"},
+ {"size": 1000000000000000000, "surl": "test"}]}
+
+ min_size, max_size, n_bins, counts, biggest_bucket, bins = compute_inputs_histogram(test_data)
+ self.assertEqual(6, n_bins)
+ self.assertEqual(1, min_size)
+ self.assertEqual(1000000000000000000, max_size)
+ self.assertEqual(5, biggest_bucket)
+ self.assertListEqual([5, 0, 0, 0, 0, 1], counts) # TODO: if this is the case, adapt it to logarithmic scale
+ self.assertListEqual(['1.0B', '148.0PB', '296.1PB', '444.1PB', '592.1PB', '740.1PB', '888.2PB'], bins)
diff --git a/ldvspec/lofardata/urls.py b/ldvspec/lofardata/urls.py
index 9e9a0f9ba460620f4892c09abdf6f5143fe00b00..98ffebcc4267dc90a198fb643804e72bdef68a16 100644
--- a/ldvspec/lofardata/urls.py
+++ b/ldvspec/lofardata/urls.py
@@ -39,6 +39,8 @@ urlpatterns = [
path('specification/delete/<int:pk>/', views.WorkSpecificationDeleteView.as_view(), name='specification-delete'),
path('specification/inputs/<int:pk>/', views.WorkSpecificationInputsView.as_view(), name='specification-inputs'),
path('specification/tasks/<int:pk>/', views.WorkSpecificationATDBTasksView.as_view(), name='specification-tasks'),
+ path('specification/dataset-size-info/<int:pk>/', views.WorkSpecificationDatasetSizeInfoView.as_view(), name='dataset-size-info'),
+ path('specification/dataproducts/<int:pk>', views.DataProductViewPerSasID.as_view(), name='specification-dataproducts'),
# Workaround for injecting the urls from the ModelViewSet, which requires a "Router"
diff --git a/ldvspec/lofardata/views.py b/ldvspec/lofardata/views.py
index eb7f5af1745721adb92eb12bd450ef9247e50615..ac610adb35481de71ecd0dd98129d7bb4bd89028 100644
--- a/ldvspec/lofardata/views.py
+++ b/ldvspec/lofardata/views.py
@@ -1,12 +1,14 @@
+import logging
import time
from typing import Tuple
+import numpy
from django.contrib.auth.models import User
from django.core.exceptions import ObjectDoesNotExist
from django.http import HttpResponseRedirect
from django.shortcuts import redirect, render
from django.urls import reverse
-from django.views.generic import CreateView, DeleteView, DetailView, UpdateView
+from django.views.generic import CreateView, DeleteView, DetailView, UpdateView, TemplateView
from django.views.generic.list import ListView
from django_filters import rest_framework as filters
from rest_framework import generics, status, viewsets
@@ -39,12 +41,12 @@ def compute_size_of_inputs(inputs: dict) -> Tuple[int, int, int]:
number_of_files = 0
if isinstance(inputs, dict) and "size" in inputs:
- total_size = inputs["size"]
number_of_files = 1
+ total_size = inputs["size"]
elif (
- isinstance(inputs, dict)
- or isinstance(inputs, list)
- or isinstance(inputs, tuple)
+ isinstance(inputs, dict)
+ or isinstance(inputs, list)
+ or isinstance(inputs, tuple)
):
values = inputs
if isinstance(inputs, dict):
@@ -55,9 +57,31 @@ def compute_size_of_inputs(inputs: dict) -> Tuple[int, int, int]:
number_of_files += item_count
average_file_size = total_size / number_of_files if number_of_files else 0
+
return total_size, number_of_files, average_file_size
+def compute_inputs_histogram(inputs):
+ # create sizes array
+ if isinstance(inputs, dict):
+ inputs = inputs.values()
+ inputs_sizes = []
+ for entry in inputs:
+ for item in entry:
+ inputs_sizes.append(item['size'])
+ inputs_sizes = numpy.array(inputs_sizes)
+
+ # define histogram values
+ min_size = inputs_sizes.min()
+ max_size = inputs_sizes.max()
+
+ n_bins = 1 if min_size == max_size else (inputs_sizes.__len__() if inputs_sizes.__len__() < 100 else 100)
+ counts, buckets = numpy.histogram(inputs_sizes, bins=n_bins, range=(min_size, max_size))
+ formatted_bins = [format_size(bucket) % bucket for bucket in buckets]
+
+ return min_size, max_size, n_bins, counts.tolist(), counts.max(), formatted_bins
+
+
def format_size(num, suffix="B"):
if num == 0:
return "-"
@@ -107,9 +131,9 @@ def preprocess_filters_specification_view(specification):
dataproduct_filters = DataProductFilter.objects.all()
for dataproduct_filter in dataproduct_filters:
if (
- specification is not None
- and specification.filters
- and dataproduct_filter.field in specification.filters
+ specification is not None
+ and specification.filters
+ and dataproduct_filter.field in specification.filters
):
dataproduct_filter.default = specification.filters[dataproduct_filter.field]
else:
@@ -122,6 +146,24 @@ def preprocess_filters_specification_view(specification):
return dataproduct_filters
+def retrieve_general_dataproduct_information(sas_id):
+ # Per SAS ID, the retrieved data products should have these unique values
+ data_products = DataProduct.objects.filter(obs_id=sas_id).values("dataproduct_source",
+ "dataproduct_type",
+ "project",
+ "location",
+ "activity").distinct()
+ combined_data_products_on_key = {}
+ for data_product in data_products:
+ for key, value in data_product.items():
+ if combined_data_products_on_key.get(key) and value not in combined_data_products_on_key.get(key):
+ combined_data_products_on_key[key].append(value)
+ else:
+ combined_data_products_on_key[key] = [value]
+
+ return combined_data_products_on_key
+
+
class Specifications(ListView):
serializer_class = WorkSpecificationSerializer
template_name = "lofardata/index.html"
@@ -155,6 +197,8 @@ class WorkSpecificationCreateUpdateView(UpdateView):
except ObjectDoesNotExist:
specification = None
context["filters"] = preprocess_filters_specification_view(specification)
+ context["processing_sites"] = list(ATDBProcessingSite.objects.values("name", "url"))
+
return context
def create_successor(self, specification):
@@ -226,6 +270,43 @@ class WorkSpecificationATDBTasksView(DetailView):
model = WorkSpecification
+class WorkSpecificationDatasetSizeInfoView(DetailView):
+ template_name = "lofardata/workspecification/dataset_size_info.html"
+ model = WorkSpecification
+
+ def get_context_data(self, **kwargs):
+ context = super().get_context_data(**kwargs)
+ specification = WorkSpecification.objects.get(pk=context["object"].pk)
+
+ min_size, max_size, n_bins, counts, biggest_bucket, bins = compute_inputs_histogram(specification.inputs)
+
+ context["min_size"] = format_size(min_size)
+ context["max_size"] = format_size(max_size)
+ context["biggest_bucket"] = biggest_bucket
+ context["n_bins"] = n_bins
+ context["counts"] = counts
+ context["bins"] = bins
+
+ return context
+
+
+class DataProductViewPerSasID(TemplateView):
+ template_name = "lofardata/workspecification/dataproducts.html"
+
+ def get_context_data(self, **kwargs):
+ context = super().get_context_data(**kwargs)
+
+ try:
+ specification = WorkSpecification.objects.get(pk=kwargs['pk'])
+ except ObjectDoesNotExist:
+ return context
+
+ sas_id = specification.filters['obs_id']
+ context["sas_id"] = sas_id
+ context["dataproduct_info"] = retrieve_general_dataproduct_information(sas_id)
+ return context
+
+
# ---------- REST API views ----------
class DataProductView(generics.ListCreateAPIView):
model = DataProduct
@@ -294,6 +375,6 @@ class WorkSpecificationViewset(viewsets.ModelViewSet):
# TODO: check that there are some matches in the request?
insert_task_into_atdb.delay(pk)
- time.sleep(1) # allow for some time to pass
+ time.sleep(1) # allow for some time to pass
return redirect("specification-detail", pk=pk)
diff --git a/ldvspec/requirements/base.txt b/ldvspec/requirements/base.txt
index 5ce7dcebce6248b62802046b3cff71477627943d..ec4a4f46a3a4c9565e622a48a08e4dee6010ab4e 100644
--- a/ldvspec/requirements/base.txt
+++ b/ldvspec/requirements/base.txt
@@ -16,4 +16,5 @@ django-uws==0.2.dev355575
django-crispy-forms==1.14.0
crispy-bootstrap5==0.6
humanize==4.4.0
-django-widget-tweaks
\ No newline at end of file
+django-widget-tweaks
+numpy==1.23.0
\ No newline at end of file