diff --git a/esap/Dockerfile b/esap/Dockerfile index 6aba923d497ce97ee9dcb85eca0be46995540856..6150012b8eaeee92b6f151c723afde5be9c07502 100644 --- a/esap/Dockerfile +++ b/esap/Dockerfile @@ -1,6 +1,6 @@ FROM python:3.6.7-slim RUN apt-get update && apt-get install --no-install-recommends -y bash nano mc -ENV PYTHONUNBUFFERED 1 +# RUN apk update && apk add gcc bash nano mc linux-headers musl-dev alpine-sdk openssl-dev libffi-dev RUN mkdir /src WORKDIR /src diff --git a/esap/accounts/__init__.py b/esap/accounts/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/esap/accounts/admin.py b/esap/accounts/admin.py new file mode 100644 index 0000000000000000000000000000000000000000..8c38f3f3dad51e4585f3984282c2a4bec5349c1e --- /dev/null +++ b/esap/accounts/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/esap/accounts/api/__init__.py b/esap/accounts/api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/esap/accounts/api/serializers.py b/esap/accounts/api/serializers.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/esap/accounts/api/urls.py b/esap/accounts/api/urls.py new file mode 100644 index 0000000000000000000000000000000000000000..7eab22293e65ab75106a8118fc26d5211dce79fd --- /dev/null +++ b/esap/accounts/api/urls.py @@ -0,0 +1,9 @@ +from django.urls import path +from knox import views as knox_views +from .views import LoginView + +urlpatterns = [ + path(r'login/', LoginView.as_view(), name='knox_login'), + path(r'logout/', knox_views.LogoutView.as_view(), name='knox_logout'), + path(r'logoutall/', knox_views.LogoutAllView.as_view(), name='knox_logoutall'), +] \ No newline at end of file diff --git a/esap/accounts/api/views.py b/esap/accounts/api/views.py new file mode 100644 index 0000000000000000000000000000000000000000..65ab8611f4e1feb1c9a26c8d47cc32e28fdbddf3 --- /dev/null +++ b/esap/accounts/api/views.py @@ -0,0 +1,15 @@ +from django.contrib.auth import login + +from rest_framework import permissions +from rest_framework.authtoken.serializers import AuthTokenSerializer +from knox.views import LoginView as KnoxLoginView + +class LoginView(KnoxLoginView): + permission_classes = (permissions.AllowAny,) + + def post(self, request, format=None): + serializer = AuthTokenSerializer(data=request.data) + serializer.is_valid(raise_exception=True) + user = serializer.validated_data['user'] + login(request, user) + return super(LoginView, self).post(request, format=None) diff --git a/esap/accounts/apps.py b/esap/accounts/apps.py new file mode 100644 index 0000000000000000000000000000000000000000..9b3fc5a44939430bfb326ca9a33f80e99b06b5be --- /dev/null +++ b/esap/accounts/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class AccountsConfig(AppConfig): + name = 'accounts' diff --git a/esap/accounts/migrations/__init__.py b/esap/accounts/migrations/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/esap/accounts/models.py b/esap/accounts/models.py new file mode 100644 index 0000000000000000000000000000000000000000..0f022274409be346020cd39606cb6355d3c9f04c --- /dev/null +++ b/esap/accounts/models.py @@ -0,0 +1 @@ +from django.db import models \ No newline at end of file diff --git a/esap/accounts/tests.py b/esap/accounts/tests.py new file mode 100644 index 0000000000000000000000000000000000000000..7ce503c2dd97ba78597f6ff6e4393132753573f6 --- /dev/null +++ b/esap/accounts/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/esap/accounts/urls.py b/esap/accounts/urls.py new file mode 100644 index 0000000000000000000000000000000000000000..a5254c8f57841133976faf891460067db5360b2d --- /dev/null +++ b/esap/accounts/urls.py @@ -0,0 +1,9 @@ +from django.urls import path +from django.contrib.auth.views import LoginView +from django.contrib.auth.views import LogoutView + +urlpatterns = [ + path('login/', LoginView.as_view(), name='django_login'), + path('logout/', LogoutView.as_view(), name='django_logout'), + +] \ No newline at end of file diff --git a/esap/accounts/views.py b/esap/accounts/views.py new file mode 100644 index 0000000000000000000000000000000000000000..91ea44a218fbd2f408430959283f0419c921093e --- /dev/null +++ b/esap/accounts/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here. diff --git a/esap/api/business/services/query/alta.py b/esap/api/business/services/query/alta.py index 4bc3b630735ee628414c711323cf9836e2bdd436..c6168ebf68b3e2cb05d5518f0bf6f47993a15e25 100644 --- a/esap/api/business/services/query/alta.py +++ b/esap/api/business/services/query/alta.py @@ -7,6 +7,10 @@ from .query_base import query_base import requests, json +import logging + +logger = logging.getLogger(__name__) + AMP_REPLACEMENT = '_and_' # The request header @@ -77,12 +81,14 @@ class observations_connector(query_base): # iterate over the list of results for observation in observations: + logger.info(observation) # the dataset.select_fields field specifies which fields must be extracted from the response record = {} result = '' select_list = dataset.select_fields.split(',') for select in select_list: + logger.info(select) try: result = result + observation[select] + ',' except: @@ -98,27 +104,29 @@ class observations_connector(query_base): record['result'] = result # some fields to return some rendering information for the frontend. - try: - record['title'] = observation[dataset.title_field] - except: - pass - - try: - record['thumbnail'] = observation[dataset.thumbnail_field] - except: - pass try: + record['title'] = observation[dataset.title_field] + record['thumbnail'] = observation[dataset.thumbnail] + + record['runId'] = observation["runId"] + record['target'] = observation["target"] + record['RA'] = observation["RA"] + record['dec'] = observation["dec"] + record['fov'] = observation["fov"] + record['startTime'] = observation["startTime"] + record['endTime'] = observation["endTime"] record['url'] = "https://alta.astron.nl/science/details/"+observation["runId"] except: pass + logger.info(record) results.append(record) except Exception as error: record['query'] = query - record['dataset'] = dataset.uri record['dataset_name'] = dataset_name + record['dataset'] = dataset.uri record['result'] = str(error) results.append(record) @@ -240,6 +248,7 @@ class dataproducts_connector(query_base): for dataproduct in dataproducts: + logger.info(dataproduct) # the dataset.select_fields field specifies which fields must be extracted from the response record = {} result = '' diff --git a/esap/api/business/services/staging/staging_base.py b/esap/api/business/services/staging/staging_base.py index 642ff868fd4475c4839e3743ae842ee015f5959b..bc77983b0451a8e1246d6b1489914d1cf6acb294 100644 --- a/esap/api/business/services/staging/staging_base.py +++ b/esap/api/business/services/staging/staging_base.py @@ -1,6 +1,6 @@ """ File name: staging_base.py - Author: Nico Vermaas - Astron + Author: Nico Vermaas & Zheng Meyer-Zhao - Astron Date created: 2020-03-09 Description: ESAP staging abstract base class. """ @@ -12,5 +12,9 @@ class staging_base: self.url = url # implement this in the derived service classes - def stage(self, dataset, esap_params, translation_mapping): + def construct_stage_request(self, dataset, esap_params, translation_mapping): pass + + # implement this in the derived service classes + def run_stage_request(self, dataset): + pass \ No newline at end of file diff --git a/esap/api/templates/api/index.html b/esap/api/templates/api/index.html index 9fee0ed301e5c2a0d9aef34cfb555536c115ac01..4e55a66994fcce5030923f0f640ad544c0630403 100644 --- a/esap/api/templates/api/index.html +++ b/esap/api/templates/api/index.html @@ -70,7 +70,7 @@ </div> -<p class="footer" small>ASTRON - version 0.6.5 - 16 apr 2020</p> +<p class="footer" small>ASTRON - version 0.7.0 - 15 jun 2020</p> {% endblock %} diff --git a/esap/esap/configuration/adex.py b/esap/esap/configuration/adex.py index 36219f9708b79c68703b15ef8ac18951ec1ac587..94a7deb7ad5252391d7c23bd753c842d995a36b4 100644 --- a/esap/esap/configuration/adex.py +++ b/esap/esap/configuration/adex.py @@ -27,15 +27,15 @@ datasets_disabled = ['nancay.ivoa.obscore'] # definition of the query query_schema = { - "title": "ESAP Query", + "title": "ASTRON Data Collection Query", "type": "object", "properties": { "institute": { "type": "string", "title": "Institute", - "default": "all", - "enum": ["all","Astron","IVOA"], - "enumNames": ["all","astron","IVOA"] + "default": "Astron", + "enum": ["all", "Astron"], + "enumNames": ["all","ASTRON"] }, "title": { @@ -71,29 +71,17 @@ query_schema = { }, "dataproduct_subtype": { "type": "string", - "title": "DataProduct Subtype", + "title": "DataProduct Type", "default": "continuumMF", - "enum": ["all","continuumMF","imageCube","beamCube"], - "enumNames": ["all","continuumMF","imageCube","beamCube"] + "enum": ["all","uncalibratedVisibility","continuumMF","continuumChunk","calibratedImage","polarisationImage","imageCube","beamCube","polarisationCube","pulsarTimingTimeSeries"], + "enumNames": ["all","uncalibratedVisibility","continuumMF","continuumChunk","calibratedImage","polarisationImage","imageCube","beamCube","polarisationCube","pulsarTimingTimeSeries"] }, - "startdate": { + "access_right": { "type": "string", - "format" : "date", - "title": "Start Date", - "default": "2004-02-07" - } - , - "enddate": { - "type": "string", - "format" : "date", - "title": "End Date", - "default": "2004-02-08" + "title": "Access right", + "default": "public", + "enum": ["all", "public"], + "enumNames": ["all", "public"] }, - "instrument": { - "type": "string", - "title": "Instrument", - "default": "SOHO__MDI", - "enum": ["all", "SOHO__EIT", "SOHO__MDI", "PDMO__COGHA", "HINODE__EIS", "STEREO_A__COR", "STEREO_B__COR"], - } } } diff --git a/esap/esap/configuration/esap_default.py b/esap/esap/configuration/esap_default.py index 2aa6976d3f2a11e3a312c47c39f55c71ec8a0ea5..a7a53848f00a2b76b03172db813d2968165d7106 100644 --- a/esap/esap/configuration/esap_default.py +++ b/esap/esap/configuration/esap_default.py @@ -77,15 +77,13 @@ query_schema = { "startdate": { "type": "string", "format" : "date", - "title": "Start Date", - "default": "2004-02-07" + "title": "Start Date" } , "enddate": { "type": "string", "format" : "date", - "title": "End Date", - "default": "2004-02-08" + "title": "End Date" }, "instrument": { "type": "string", diff --git a/esap/esap/configuration/esap_dev.py b/esap/esap/configuration/esap_dev.py index 3ad41fb36888807e1dfa6d70c232b5283336184f..870d57164ec4bcaf55cee2a56a582055f409f6eb 100644 --- a/esap/esap/configuration/esap_dev.py +++ b/esap/esap/configuration/esap_dev.py @@ -5,7 +5,7 @@ # the url location of the frontend application, # this makes it possible to install multiple instances in different directories on the webserver # that all have their own urls like 'http://esap.astron.nl/esap-gui-dev/queries' -frontend_basename="esap-gui-dev" +frontend_basename="esap-gui" logo = "https://alta.astron.nl/alta-static/images/esap/esap_logo.png" diff --git a/esap/esap/settings/base.py b/esap/esap/settings/base.py index 4e3823bba4e6966d19f1938f659283db38c19bdc..3f6ceedfcfb1c90e1f364d4b7158ddbce1b0fd85 100644 --- a/esap/esap/settings/base.py +++ b/esap/esap/settings/base.py @@ -23,9 +23,13 @@ CORS_ORIGIN_ALLOW_ALL = True # Application definition INSTALLED_APPS = [ + 'accounts', + 'rucio', 'api', + 'knox', 'django.contrib.admin', 'django.contrib.auth', + 'mozilla_django_oidc', 'django.contrib.contenttypes', 'django.contrib.sessions', 'django.contrib.messages', @@ -43,6 +47,7 @@ MIDDLEWARE = [ 'django.middleware.common.CommonMiddleware', 'django.middleware.csrf.CsrfViewMiddleware', 'django.contrib.auth.middleware.AuthenticationMiddleware', + 'mozilla_django_oidc.middleware.SessionRefresh', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware' ] @@ -63,6 +68,7 @@ TEMPLATES = [ 'django.template.context_processors.request', ], }, + 'DIRS': [os.path.join(BASE_DIR, 'templates')], }, ] @@ -71,6 +77,10 @@ WSGI_APPLICATION = 'esap.wsgi.application' REST_FRAMEWORK = { # Use Django's standard `django.contrib.auth` permissions, # or allow read-only access for unauthenticated users. + 'DEFAULT_AUTHENTICATION_CLASSES': [ + #'knox.auth.TokenAuthentication', + #'mozilla_django_oidc.contrib.drf.OIDCAuthentication', + ], 'DEFAULT_PERMISSION_CLASSES': [ 'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly' ], @@ -170,6 +180,10 @@ LOGGING = { 'handlers': ['console'], 'level': 'ERROR', }, + 'mozilla_django_oidc': { + 'handlers': ['console'], + 'level': 'DEBUG', + }, } } @@ -182,3 +196,25 @@ STATIC_ROOT = os.path.join(BASE_DIR, 'static') # configuration settings that can be requested through the REST API CONFIGURATION_DIR = os.path.join(BASE_DIR, 'configuration') CONFIGURATION_FILE = 'esap_default' + +# Settings for mozilla_django_oidc +# use 'mozilla_django_oidc' authentication backend +AUTHENTICATION_BACKENDS = ( + 'django.contrib.auth.backends.ModelBackend', + 'mozilla_django_oidc.auth.OIDCAuthenticationBackend', +) +OIDC_DRF_AUTH_BACKEND = 'mozilla_django_oidc.auth.OIDCAuthenticationBackend' + +OIDC_RP_CLIENT_ID = os.environ['OIDC_RP_CLIENT_ID'] +OIDC_RP_CLIENT_SECRET = os.environ['OIDC_RP_CLIENT_SECRET'] +OIDC_RP_SIGN_ALGO = "RS256" +OIDC_OP_JWKS_ENDPOINT = "https://iam-escape.cloud.cnaf.infn.it/jwk" +OIDC_OP_AUTHORIZATION_ENDPOINT = "https://iam-escape.cloud.cnaf.infn.it/authorize" +OIDC_OP_TOKEN_ENDPOINT = "https://iam-escape.cloud.cnaf.infn.it/token" +OIDC_OP_USER_ENDPOINT = "https://iam-escape.cloud.cnaf.infn.it/userinfo" + +OIDC_STORE_ACCESS_TOKEN = True +OIDC_STORE_ID_TOKEN = True + +LOGIN_REDIRECT_URL = "http://127.0.0.1:3000/login" +LOGOUT_REDIRECT_URL = "http://127.0.0.1:3000/logout" \ No newline at end of file diff --git a/esap/esap/urls.py b/esap/esap/urls.py index 2fa98805deaed15567c53568fc1f7084de5dd181..dda82a9a4a2a28ac0e45a4a5d429508053da404e 100644 --- a/esap/esap/urls.py +++ b/esap/esap/urls.py @@ -14,10 +14,12 @@ Including another URLconf 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) """ from django.contrib import admin -from django.urls import include, path +from django.urls import include, path, re_path urlpatterns = [ path('esap-api/', include('api.urls')), path('esap-api/admin/', admin.site.urls, name='admin-view'), - path('esap-api/api-auth/', include('rest_framework.urls')), + path('api/', include('rucio.api.urls')), + path('accounts/', include('accounts.urls')), + re_path('^oidc/', include('mozilla_django_oidc.urls')), ] diff --git a/esap/rucio/__init__.py b/esap/rucio/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/esap/rucio/admin.py b/esap/rucio/admin.py new file mode 100644 index 0000000000000000000000000000000000000000..8c38f3f3dad51e4585f3984282c2a4bec5349c1e --- /dev/null +++ b/esap/rucio/admin.py @@ -0,0 +1,3 @@ +from django.contrib import admin + +# Register your models here. diff --git a/esap/rucio/api/serializers.py b/esap/rucio/api/serializers.py new file mode 100644 index 0000000000000000000000000000000000000000..ad0994bf4173b700e4bb3396aac5eee6466aedc7 --- /dev/null +++ b/esap/rucio/api/serializers.py @@ -0,0 +1,9 @@ +from rest_framework import serializers + +from rucio.models import Staging + + +class StagingSerializer(serializers.ModelSerializer): + class Meta: + model = Staging + fields = '__all__' \ No newline at end of file diff --git a/esap/rucio/api/urls.py b/esap/rucio/api/urls.py new file mode 100644 index 0000000000000000000000000000000000000000..b4677b58ab534e656c086a1f0577b3637ca21935 --- /dev/null +++ b/esap/rucio/api/urls.py @@ -0,0 +1,8 @@ +from rest_framework import routers + +from .views import StagingViewSet + +router = routers.DefaultRouter() +router.register('staging', StagingViewSet, 'staging') + +urlpatterns = router.urls \ No newline at end of file diff --git a/esap/rucio/api/views.py b/esap/rucio/api/views.py new file mode 100644 index 0000000000000000000000000000000000000000..0b5e330191de4784d788c3439e4478fe4ad62a82 --- /dev/null +++ b/esap/rucio/api/views.py @@ -0,0 +1,14 @@ +from rest_framework import viewsets, permissions + +from .serializers import StagingSerializer + + +class StagingViewSet(viewsets.ModelViewSet): + serializer_class = StagingSerializer + permission_classes = [permissions.IsAuthenticated] + + def get_queryset(self): + return self.request.user.staging.all() + + def perform_create(self, serializer): + serializer.save(owner=self.request.user) \ No newline at end of file diff --git a/esap/rucio/apps.py b/esap/rucio/apps.py new file mode 100644 index 0000000000000000000000000000000000000000..9413591807c58fc8e5a2f14d4651507a75aeaf4f --- /dev/null +++ b/esap/rucio/apps.py @@ -0,0 +1,5 @@ +from django.apps import AppConfig + + +class RucioConfig(AppConfig): + name = 'rucio' diff --git a/esap/rucio/migrations/0001_initial.py b/esap/rucio/migrations/0001_initial.py new file mode 100644 index 0000000000000000000000000000000000000000..87da233c17e243e7dc66c241e12ba18fc4e024a9 --- /dev/null +++ b/esap/rucio/migrations/0001_initial.py @@ -0,0 +1,25 @@ +# Generated by Django 3.0.2 on 2020-05-11 20:15 + +from django.conf import settings +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + initial = True + + dependencies = [ + migrations.swappable_dependency(settings.AUTH_USER_MODEL), + ] + + operations = [ + migrations.CreateModel( + name='Staging', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('task', models.CharField(max_length=255)), + ('owner', models.ForeignKey(null=True, on_delete=django.db.models.deletion.CASCADE, related_name='staging', to=settings.AUTH_USER_MODEL)), + ], + ), + ] diff --git a/esap/rucio/migrations/__init__.py b/esap/rucio/migrations/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/esap/rucio/models.py b/esap/rucio/models.py new file mode 100644 index 0000000000000000000000000000000000000000..5b58d5b7d34a9b1c1a696a50d5714ffad9ebea2a --- /dev/null +++ b/esap/rucio/models.py @@ -0,0 +1,10 @@ +from django.db import models +from django.contrib.auth.models import User + +class Staging(models.Model): + task = models.CharField(max_length=255) + owner = models.ForeignKey( + User, related_name="staging", on_delete=models.CASCADE, null=True) + + def __str__(self): + return self.task \ No newline at end of file diff --git a/esap/rucio/tests.py b/esap/rucio/tests.py new file mode 100644 index 0000000000000000000000000000000000000000..7ce503c2dd97ba78597f6ff6e4393132753573f6 --- /dev/null +++ b/esap/rucio/tests.py @@ -0,0 +1,3 @@ +from django.test import TestCase + +# Create your tests here. diff --git a/esap/rucio/views.py b/esap/rucio/views.py new file mode 100644 index 0000000000000000000000000000000000000000..91ea44a218fbd2f408430959283f0419c921093e --- /dev/null +++ b/esap/rucio/views.py @@ -0,0 +1,3 @@ +from django.shortcuts import render + +# Create your views here.