merge with master

.gitignore

@@ -94,6 +94,9 @@ ipython_config.py
 #   install all needed dependencies.
 #Pipfile.lock
 
+# IDE
+.vscode
+
 # PEP 582; used by e.g. github.com/David-OConnor/pyflow
 __pypackages__/
 
@@ -120,6 +123,9 @@ venv.bak/
 # Rope project settings
 .ropeproject
 
+# Pycharm settings
+.idea/
+
 # mkdocs documentation
 /site
 

.gitmodules

+[submodule "modules/django-uws"]
+	path = modules/django-uws
+	url = git@git.astron.nl:astron-sdc/django-uws.git

README.md

@@ -6,3 +6,11 @@ It provides a range of services that can be accessed through a REST API.
 * backend (API Gateway): https://git.astron.nl/astron-sdc/esap-api-gateway/-/wikis/ESAP-API-gateway-Overview
 
 * frontend (GUI): https://git.astron.nl/astron-sdc/esap-gui/-/wikis/ESAP-GUI
+
+## Async ESAP
+
+For development with Async ESAP, take a look at the [wiki](https://git.astron.nl/astron-sdc/esap-api-gateway/-/wikis/WIP/Asynchronous-ESAP).
+
+## Contributing
+
+For developer access to this repository, please send a message on the [ESAP channel on Rocket Chat](https://chat.escape2020.de/channel/esap).

compose/docker-compose.async-dev.yml

+services:
+  rabbitmq:
+    image: rabbitmq:3.9-alpine
+    ports:
+      - "5672:5672"
+  postgres:
+    image: postgres:14-alpine
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_PASSWORD: "secret"
+      POSTGRES_USER: "postgres"
+      POSTGRES_DB: "uws_jobs"
+    volumes:
+      - db:/var/lib/postgresql/data
+
+volumes:
+  db:

esap/Dockerfile

 FROM python:3.10-slim
-RUN apt-get update && apt-get install --no-install-recommends -y bash nano mc libmagic1
+RUN apt-get update && apt-get install --no-install-recommends -y bash nano mc libmagic1 git
 
 RUN mkdir /src
 WORKDIR /src

esap/batch/admin.py

+from django.contrib import admin
+
+# Register your models here.

esap/batch/api/urls.py

+from django.urls import path
+from django.contrib import admin
+from rest_framework import routers
+
+from . import views
+
+#router = routers.DefaultRouter()
+#router.register('batch', views.BatchViewSet, 'batch')
+#urlpatterns = router.urls
+
+urlpatterns = [
+    path('', views.IndexView.as_view(), name='index-view'),
+    # example: /esap-api/get-services?dataset=ivoa?keyword=ukidss
+    #path('facilities/search', views.SearchFacilities.as_view(), name='facility-search'),
+    #path('workflows/search', views.SearchWorkflows.as_view(), name='workflows-search'),
+    #path('deploy', views.Deploy.deploy, name='deploy')
+    
+]
+
+

esap/batch/api/views/__init__.py

+from .batch_views import *

esap/batch/api/views/batch_views.py

+import logging
+from urllib.parse import quote_plus as quote_url
+from rest_framework import generics, pagination
+from rest_framework.response import Response
+#from batch.api.services import batch_controller
+from django.views.generic import ListView
+from django_filters import rest_framework as filters
+from rest_framework import viewsets, permissions
+from batch.models import *
+from rest_framework import generics
+#from ..serializers import *
+from django.shortcuts import redirect
+from rest_framework.views import APIView
+from rest_framework.response import Response
+
+logger = logging.getLogger(__name__)
+
+
+class BatchViewSet(viewsets.ModelViewSet):
+    #serializer_class = BatchSerializer
+    permission_classes = [permissions.IsAuthenticated]
+
+    def get_queryset(self):
+        return self.request.user.staging.all()
+
+    #def perform_create(self, serializer):
+        #serializer.save(owner=self.request.user)
+
+
+# example: /esap/batch/
+class IndexView(ListView):
+    queryset = Batch.objects.all()
+    #serializer_class = BatchSerializer
+    template_name = 'batch/index.html'
+
+    # by default this returns the list in an object called object_list, so use 'object_list' in the html page.
+    # but if 'context_object_name' is defined, then this returned list is named and can be accessed that way in html.
+    context_object_name = 'my_batch'
+

esap/batch/apps.py

+from django.apps import AppConfig
+
+
+class BatchConfig(AppConfig):
+    name = 'batch'

esap/batch/batch.py

+
+# definition of the Batch Interface
+batch_schema = {
+    "name": "batch",
+    "title": "Batch Analysis",
+    "type": "object",
+    "properties": {
+        "compute": {
+            "type": "string",
+            "title": "Compute Facility",
+            "default": "jhub_ska",
+            "enum": ["jhub_ska", "jhub_uedin", "spark_uedin"],
+            "enumNames": ["JHub SKA", "JHub Edinburgh", "Spark Cluster Edi"],
+            "uniqueItems": True,
+        },
+    },
+}

esap/batch/database_router.py

+class BatchRouter:
+
+    route_app_labels = {'batch'}
+
+    def db_for_read(self, model, **hints):
+        """
+        Attempts to read batch models go to batch database.
+        """
+        if model._meta.app_label in self.route_app_labels:
+            # return 'batch'
+            return 'default'
+
+    def db_for_write(self, model, **hints):
+        """
+        Writes always go to batch.
+        """
+        #return 'batch'
+        return 'default'
+
+    def allow_relation(self, obj1, obj2, **hints):
+        """
+        Allow relations if a model in the batch apps is
+        involved.
+        """
+        if (
+            obj1._meta.app_label in self.route_app_labels or
+            obj2._meta.app_label in self.route_app_labels
+        ):
+           return True
+        return None
+
+    def allow_migrate(self, db, app_label, model_name=None, **hints):
+        """
+        Make sure the batch apps only appear in the
+        'batch' database.
+        """
+        if app_label in self.route_app_labels:
+            #return db == 'batch'
+            return db == 'default'
+
+        return None

esap/batch/migrations/0001_initial.py

+# Generated by Django 3.1.4 on 2022-04-20 10:42
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    initial = True
+
+    dependencies = [
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='Batch',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('uri', models.CharField(max_length=40)),
+                ('status', models.CharField(max_length=40)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Facility',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=30)),
+                ('description', models.CharField(max_length=240)),
+                ('url', models.CharField(max_length=240)),
+                ('facilitytype', models.CharField(max_length=240)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='Workflow',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('name', models.CharField(max_length=30)),
+                ('description', models.CharField(max_length=240)),
+                ('url', models.CharField(max_length=240)),
+                ('ref', models.CharField(default='HEAD', max_length=240)),
+                ('filepath', models.CharField(blank=True, max_length=240)),
+                ('workflowtype', models.CharField(max_length=240)),
+            ],
+        ),
+        migrations.CreateModel(
+            name='ShoppingCart',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('user', models.PositiveIntegerField()),
+                ('dataset', models.PositiveIntegerField()),
+                ('datatype', models.CharField(max_length=240)),
+                ('facility', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='batch.facility')),
+                ('workflow', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='batch.workflow')),
+            ],
+        ),
+    ]

esap/batch/models.py

+from django.db import models
+from django.db.models import Q
+import django_filters
+
+
+class Batch(models.Model):
+    uri = models.CharField(max_length=40, null=False)
+    status = models.CharField(max_length=40, null=False)
+    
+    
+

esap/batch/templates/batch/base.html

+<!DOCTYPE html>
+{% load static %}
+<html lang="en">
+<head>
+
+    <!-- Required meta tags -->
+    <meta charset="utf-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+
+    <title>{% block myBlockTitle %}ESAP API - Batch Analysis {% endblock %}</title>
+
+    <!-- loads the path to static files -->
+    <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css">
+
+   <script src="https://code.jquery.com/jquery-3.3.1.slim.min.js" integrity="sha384-q8i/X+965DzO0rT7abK41JStQIAqVgRVzpbzo5smXKp4YfRvH+8abtTE1Pi6jizo" crossorigin="anonymous"></script>
+   <script src="https://cdnjs.cloudflare.com/ajax/libs/popper.js/1.14.3/umd/popper.min.js" integrity="sha384-ZMP7rVo3mIykV+2+9J3UJ46jBk0WLaUAdn689aCwoqbBJiSnjAK/l8WvCWPIPm49" crossorigin="anonymous"></script>
+   <script src="https://stackpath.bootstrapcdn.com/bootstrap/4.1.3/js/bootstrap.min.js" integrity="sha384-ChfqqxuZUCnJSK3+MXmPNIyE6ZbWh2IMqE241rYiqJxyMiZ6OW/JmZQ5stwEULTy" crossorigin="anonymous"></script>
+
+    <link rel="stylesheet" type="text/css" href="{% static 'query/style.css' %}"/>
+    <link href='https://fonts.googleapis.com/css?family=Raleway' rel='stylesheet' type='text/css'>
+   <link rel="stylesheet" href="https://use.fontawesome.com/releases/v5.4.2/css/all.css" integrity="sha384-/rXc/GQVaYpyDdyxK+ecHPVYJSN9bmVFBvjA/9eOB+pb3F2w2N6fc5qB9Ew5yIns" crossorigin="anonymous">
+
+    <link rel="icon" href="http://uilennest.net/static/esap_icon.jpg">
+
+    {% block extra_js %}{% endblock %}
+
+</head>
+<body>
+    <nav class="navbar navbar-expand-lg navbar-light bg-light">
+        <div class="container-fluid">
+
+            <!-- Header -->
+            <div class="navbar-header">
+                <a class="navbar-brand mb-0 h1">
+                    <h2>
+                    <img src="{% static 'query/esap_logo.png' %}"   alt="">
+                    ESAP API Gateway - Batch Analysis 
+                    </h2>
+                </a>
+            </div>
+
+        </div>
+    </nav>
+
+    <!-- to add blocks of code -->
+    {% block myBlock %}
+    {% endblock %}
+
+</body>
+</html>
+
+

esap/batch/templates/batch/index.html

+{% extends 'batch/base.html' %}
+{% load static %}
+{% block myBlock %}
+
+<div class="container-fluid details-container">
+    <div class="row">
+        <div class="col-sm-12 col-md-12 col-lg-12">
+            <div class="panel panel-success">
+
+            <div class="panel-body">
+                   &nbsp;
+                <h4>API resources</h4>
+                <table class="table table-striped table-bordered table-sm">
+                    <thead>
+                        <tr>
+                            <th>Function</th>
+                            <th>URL</th>
+                        </tr>
+                    </thead>
+                    <tbody>
+
+                    </tbody>
+                </table>
+
+            </div>
+        </div>
+       </div>
+
+    </div>
+
+</div>
+
+
+<p class="footer" small>version 12 aug 2021</p>
+
+{% endblock %}