merging Zheng's changes from esap-gateway-auth into easp-gateway-dev

esap/Dockerfile

 FROM python:3.6.7-slim
 RUN apt-get update && apt-get install --no-install-recommends -y bash nano mc
-ENV PYTHONUNBUFFERED 1
+# RUN apk update && apk add gcc bash nano mc linux-headers musl-dev alpine-sdk openssl-dev libffi-dev
 
 RUN mkdir /src
 WORKDIR /src

esap/accounts/admin.py

+from django.contrib import admin
+
+# Register your models here.

esap/accounts/api/urls.py

+from django.urls import path
+from knox import views as knox_views
+from .views import LoginView
+
+urlpatterns = [
+    path(r'login/', LoginView.as_view(), name='knox_login'),
+    path(r'logout/', knox_views.LogoutView.as_view(), name='knox_logout'),
+    path(r'logoutall/', knox_views.LogoutAllView.as_view(), name='knox_logoutall'),
+]
\ No newline at end of file

esap/accounts/api/views.py

+from django.contrib.auth import login
+
+from rest_framework import permissions
+from rest_framework.authtoken.serializers import AuthTokenSerializer
+from knox.views import LoginView as KnoxLoginView
+
+class LoginView(KnoxLoginView):
+    permission_classes = (permissions.AllowAny,)
+
+    def post(self, request, format=None):
+        serializer = AuthTokenSerializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+        user = serializer.validated_data['user']
+        login(request, user)
+        return super(LoginView, self).post(request, format=None)

esap/accounts/apps.py

+from django.apps import AppConfig
+
+
+class AccountsConfig(AppConfig):
+    name = 'accounts'

esap/accounts/models.py

+from django.db import models
\ No newline at end of file

esap/accounts/tests.py

+from django.test import TestCase
+
+# Create your tests here.

esap/accounts/urls.py

+from django.urls import path
+from django.contrib.auth.views import LoginView
+from django.contrib.auth.views import LogoutView
+
+urlpatterns = [
+    path('login/', LoginView.as_view(), name='django_login'),
+    path('logout/', LogoutView.as_view(), name='django_logout'),
+
+]
\ No newline at end of file

esap/accounts/views.py

+from django.shortcuts import render
+
+# Create your views here.

esap/api/business/services/query/alta.py

@@ -7,6 +7,10 @@
 
 from .query_base import query_base
 import requests, json
+import logging
+
+logger = logging.getLogger(__name__)
+
 AMP_REPLACEMENT = '_and_'
 
 # The request header
@@ -77,12 +81,14 @@ class observations_connector(query_base):
 
             # iterate over the list of results
             for observation in observations:
+                logger.info(observation)
                 # the dataset.select_fields field specifies which fields must be extracted from the response
                 record = {}
                 result = ''
 
                 select_list = dataset.select_fields.split(',')
                 for select in select_list:
+                    logger.info(select)
                     try:
                         result = result + observation[select] + ','
                     except:
@@ -98,27 +104,29 @@ class observations_connector(query_base):
                 record['result'] = result
 
                 # some fields to return some rendering information for the frontend.
-                try:
-                    record['title'] = observation[dataset.title_field]
-                except:
-                    pass
-
-                try:
-                    record['thumbnail'] = observation[dataset.thumbnail_field]
-                except:
-                    pass
 
                 try:
+                    record['title'] = observation[dataset.title_field]
+                    record['thumbnail'] = observation[dataset.thumbnail]
+
+                    record['runId'] = observation["runId"]
+                    record['target'] = observation["target"]
+                    record['RA'] = observation["RA"]
+                    record['dec'] = observation["dec"]
+                    record['fov'] = observation["fov"]
+                    record['startTime'] = observation["startTime"]
+                    record['endTime'] = observation["endTime"]
                     record['url'] = "https://alta.astron.nl/science/details/"+observation["runId"]
                 except:
                     pass
 
+                logger.info(record)
                 results.append(record)
 
         except Exception as error:
             record['query'] = query
-            record['dataset'] = dataset.uri
             record['dataset_name'] = dataset_name
+            record['dataset'] = dataset.uri
             record['result'] =  str(error)
             results.append(record)
 
@@ -240,6 +248,7 @@ class dataproducts_connector(query_base):
 
 
             for dataproduct in dataproducts:
+                logger.info(dataproduct)
                 # the dataset.select_fields field specifies which fields must be extracted from the response
                 record = {}
                 result = ''

esap/api/business/services/staging/staging_base.py

 """
     File name: staging_base.py
-    Author: Nico Vermaas - Astron
+    Author: Nico Vermaas & Zheng Meyer-Zhao - Astron
     Date created: 2020-03-09
     Description:  ESAP staging abstract base class.
 """
@@ -12,5 +12,9 @@ class staging_base:
         self.url = url
 
     # implement this in the derived service classes
-    def stage(self, dataset, esap_params, translation_mapping):
+    def construct_stage_request(self, dataset, esap_params, translation_mapping):
+        pass
+
+    # implement this in the derived service classes
+    def run_stage_request(self, dataset):
         pass
\ No newline at end of file

esap/api/templates/api/index.html

@@ -70,7 +70,7 @@
 </div>
 
 
-<p class="footer" small>ASTRON - version 0.6.5 - 16 apr 2020</p>
+<p class="footer" small>ASTRON - version 0.7.0 - 15 jun 2020</p>
 
 {% endblock %}
 

esap/esap/configuration/adex.py

@@ -27,15 +27,15 @@ datasets_disabled = ['nancay.ivoa.obscore']
 
 # definition of the query
 query_schema = {
-  "title": "ESAP Query",
+  "title": "ASTRON Data Collection Query",
   "type": "object",
   "properties": {
     "institute": {
       "type": "string",
       "title": "Institute",
-      "default": "all",
-      "enum": ["all","Astron","IVOA"],
-      "enumNames": ["all","astron","IVOA"]
+      "default": "Astron",
+      "enum": ["all", "Astron"],
+      "enumNames": ["all","ASTRON"]
     },
 
     "title": {
@@ -71,29 +71,17 @@ query_schema = {
     },
     "dataproduct_subtype": {
       "type": "string",
-      "title": "DataProduct Subtype",
+      "title": "DataProduct Type",
       "default": "continuumMF",
-      "enum": ["all","continuumMF","imageCube","beamCube"],
-      "enumNames": ["all","continuumMF","imageCube","beamCube"]
+      "enum": ["all","uncalibratedVisibility","continuumMF","continuumChunk","calibratedImage","polarisationImage","imageCube","beamCube","polarisationCube","pulsarTimingTimeSeries"],
+      "enumNames": ["all","uncalibratedVisibility","continuumMF","continuumChunk","calibratedImage","polarisationImage","imageCube","beamCube","polarisationCube","pulsarTimingTimeSeries"]
     },
-    "startdate": {
-      "type": "string",
-      "format" : "date",
-      "title": "Start Date",
-      "default": "2004-02-07"
-    }
-    ,
-    "enddate": {
+    "access_right": {
       "type": "string",
-      "format" : "date",
-      "title": "End Date",
-      "default": "2004-02-08"
+      "title": "Access right",
+      "default": "public",
+      "enum": ["all", "public"],
+      "enumNames": ["all", "public"]
     },
-    "instrument": {
-      "type": "string",
-      "title": "Instrument",
-      "default": "SOHO__MDI",
-      "enum": ["all", "SOHO__EIT", "SOHO__MDI", "PDMO__COGHA", "HINODE__EIS", "STEREO_A__COR", "STEREO_B__COR"],
-    }
   }
 }

esap/esap/configuration/esap_default.py

@@ -77,15 +77,13 @@ query_schema = {
     "startdate": {
       "type": "string",
       "format" : "date",
-      "title": "Start Date",
-      "default": "2004-02-07"
+      "title": "Start Date"
     }
     ,
     "enddate": {
       "type": "string",
       "format" : "date",
-      "title": "End Date",
-      "default": "2004-02-08"
+      "title": "End Date"
     },
     "instrument": {
       "type": "string",

esap/esap/configuration/esap_dev.py

@@ -5,7 +5,7 @@
 # the url location of the frontend application,
 # this makes it possible to install multiple instances in different directories on the webserver
 # that all have their own urls like 'http://esap.astron.nl/esap-gui-dev/queries'
-frontend_basename="esap-gui-dev"
+frontend_basename="esap-gui"
 
 logo = "https://alta.astron.nl/alta-static/images/esap/esap_logo.png"
 

esap/esap/settings/base.py

@@ -23,9 +23,13 @@ CORS_ORIGIN_ALLOW_ALL = True
 # Application definition
 
 INSTALLED_APPS = [
+    'accounts',
+    'rucio',
     'api',
+    'knox',
     'django.contrib.admin',
     'django.contrib.auth',
+    'mozilla_django_oidc',
     'django.contrib.contenttypes',
     'django.contrib.sessions',
     'django.contrib.messages',
@@ -43,6 +47,7 @@ MIDDLEWARE = [
     'django.middleware.common.CommonMiddleware',
     'django.middleware.csrf.CsrfViewMiddleware',
     'django.contrib.auth.middleware.AuthenticationMiddleware',
+    'mozilla_django_oidc.middleware.SessionRefresh',
     'django.contrib.messages.middleware.MessageMiddleware',
     'django.middleware.clickjacking.XFrameOptionsMiddleware'
 ]
@@ -63,6 +68,7 @@ TEMPLATES = [
                 'django.template.context_processors.request',
             ],
         },
+        'DIRS': [os.path.join(BASE_DIR, 'templates')],
     },
 ]
 
@@ -71,6 +77,10 @@ WSGI_APPLICATION = 'esap.wsgi.application'
 REST_FRAMEWORK = {
     # Use Django's standard `django.contrib.auth` permissions,
     # or allow read-only access for unauthenticated users.
+    'DEFAULT_AUTHENTICATION_CLASSES': [
+        #'knox.auth.TokenAuthentication',
+        #'mozilla_django_oidc.contrib.drf.OIDCAuthentication',
+    ],
     'DEFAULT_PERMISSION_CLASSES': [
         'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly'
     ],
@@ -170,6 +180,10 @@ LOGGING = {
             'handlers': ['console'],
             'level': 'ERROR',
         },
+        'mozilla_django_oidc': {
+            'handlers': ['console'],
+            'level': 'DEBUG',
+        },
     }
 }
 
@@ -182,3 +196,25 @@ STATIC_ROOT = os.path.join(BASE_DIR, 'static')
 # configuration settings that can be requested through the REST API
 CONFIGURATION_DIR = os.path.join(BASE_DIR, 'configuration')
 CONFIGURATION_FILE = 'esap_default'
+
+# Settings for mozilla_django_oidc
+# use 'mozilla_django_oidc' authentication backend
+AUTHENTICATION_BACKENDS = (
+    'django.contrib.auth.backends.ModelBackend',
+    'mozilla_django_oidc.auth.OIDCAuthenticationBackend',
+)
+OIDC_DRF_AUTH_BACKEND = 'mozilla_django_oidc.auth.OIDCAuthenticationBackend'
+
+OIDC_RP_CLIENT_ID = os.environ['OIDC_RP_CLIENT_ID']
+OIDC_RP_CLIENT_SECRET = os.environ['OIDC_RP_CLIENT_SECRET']
+OIDC_RP_SIGN_ALGO = "RS256"
+OIDC_OP_JWKS_ENDPOINT = "https://iam-escape.cloud.cnaf.infn.it/jwk"
+OIDC_OP_AUTHORIZATION_ENDPOINT = "https://iam-escape.cloud.cnaf.infn.it/authorize"
+OIDC_OP_TOKEN_ENDPOINT = "https://iam-escape.cloud.cnaf.infn.it/token"
+OIDC_OP_USER_ENDPOINT = "https://iam-escape.cloud.cnaf.infn.it/userinfo"
+
+OIDC_STORE_ACCESS_TOKEN = True
+OIDC_STORE_ID_TOKEN = True
+
+LOGIN_REDIRECT_URL = "http://127.0.0.1:3000/login"
+LOGOUT_REDIRECT_URL = "http://127.0.0.1:3000/logout"
\ No newline at end of file