integrating rucio

removing filesystem dependencies

README.md

@@ -3,6 +3,6 @@ ESAP API Gateway is a 'backend' web application written in Django.
 It provides a range of services that can be accessed through a REST API.
 
 ## Documentation 
-* backend (API): https://git.astron.nl/astron-sdc/esap-api-gateway/-/wikis/ESAP-API-gateway
+* backend (API Gateway): https://git.astron.nl/astron-sdc/esap-api-gateway/-/wikis/ESAP-API-gateway-Overview
 
 * frontend (GUI): https://git.astron.nl/astron-sdc/esap-gui/-/wikis/ESAP-GUI

esap/esap/configuration/esap_rucio.py

 import requests
 import urllib.parse
 import json
+from django.conf import settings
 
-rucio_url = "https://escape-rucio.cern.ch"
+ID_TOKEN_KEY = "oidc_id_token"
+ACCESS_TOKEN_KEY = "oidc_access_token"
 
-AUTH_PORT = 32301
-STANDARD_PORT = 32300
-
-RUCIO_AUTH_TOKEN = "<REDACTED>"
-
-
-def validate():
-    url = urllib.parse.urljoin(f"{rucio_url}:{AUTH_PORT}", "auth/validate")
+def validate(token):
+    url = urllib.parse.urljoin(f"{settings.RUCIO_HOST}:{settings.RUCIO_AUTH_PORT}", "auth/validate")
     response = requests.get(
-        url, headers={"X-Rucio-Auth-Token": RUCIO_AUTH_TOKEN}, verify=False
+        url, headers={"X-Rucio-Auth-Token": token}, verify=False
     )
     if response.ok:
         return True
@@ -21,13 +17,15 @@ def validate():
         return False
 
 
-def get_scope_names():
-    # try:
-    validated = validate()
+def get_scope_names(session):
+    token = session.get(ACCESS_TOKEN_KEY, None)
+    if token is None:
+        return [f"Not logged in {session}, {session.keys()}."]
+    validated = validate(token)
     if validated:
-        url = urllib.parse.urljoin(f"{rucio_url}:{STANDARD_PORT}", "scopes")
+        url = urllib.parse.urljoin(f"{settings.RUCIO_HOST}:{settings.RUCIO_PORT}", "scopes")
         response = requests.get(
-            url + "/", headers={"X-Rucio-Auth-Token": RUCIO_AUTH_TOKEN}, verify=False
+            url + "/", headers={"X-Rucio-Auth-Token": token}, verify=False
         )
         if response.ok:
             return json.loads(response.content)
@@ -36,41 +34,60 @@ def get_scope_names():
                 "validated but failed query"
             ]  # , val_response.status_code, val_response.reason]
     else:
-        # , val_response.status_code, val_response.reason]
-        return ["not validated"]
+        return [f"not validated, {len(token)}, {type(token)}, '{token}'."]  # , val_response.status_code, val_response.reason]
     # except Exception as e:
     #     return ["Failed", "Authentication", e]
 
 
-title = "Rucio"
-logo = "http://rucio.cern.ch/images/wide_logo2.png"
+class Config:
+    title = "Rucio"
+    logo = "http://rucio.cern.ch/images/wide_logo2.png"
+
+    # the url location of the frontend application,
+    # this makes it possible to install multiple instances in different directories on the webserver
+    # that all have their own urls like 'http://esap.astron.nl/esap-gui-dev/queries'
+    frontend_basename = "esap-rucio"
+
+    # definition of the navigation bar
+    nav1 = {"title": "Archives", "route": "/archives"}
+    nav2 = {"title": "Query", "route": "/query"}
+    navbar = [nav1, nav2]
+
+    ui_schema = {"catalog": {"ui:widget": "hidden"}}
+
+    def __init__(self, session):
+        self.session=session
+        self.__query_schema = self.gen_query_schema()
 
-# definition of the query
-query_schema = {
-    "name": "rucio",
-    "title": "Rucio Query",
-    "type": "object",
-    "properties": {
-        "scope": {
-            "type": "string",
-            "title": "Scope",
-            "enum": get_scope_names(),
-            "enumNames": get_scope_names(),
-        },
-        "resource_category": {
-            "type": "string",
-            "title": "Category",
-            "enum": ["files", "dids", "replicas"],
-            "enumNames": ["Files", "DIDs", "Replicas"],
-            "default": "dids",
-        },
-        "catalog": {
-            "type": "string",
-            "enum": ["esap_rucio_entities"],
-            "enumNames": ["esap_rucio_entities"],
-            "default": "esap_rucio_entities",
-        },
-    },
-}
+    @property
+    def query_schema(self):
+        return self.__query_schema
 
-ui_schema = {"catalog": {"ui:widget": "hidden"}}
+    def gen_query_schema(self):
+        scope_names = get_scope_names(self.session)
+        return {
+            "name": "rucio",
+            "title": "Rucio Query",
+            "type": "object",
+            "properties": {
+                "scope": {
+                    "type": "string",
+                    "title": "Scope",
+                    "enum": scope_names,
+                    "enumNames": scope_names,
+                },
+                "resource_category": {
+                    "type": "string",
+                    "title": "Category",
+                    "enum": ["files", "dids", "replicas"],
+                    "enumNames": ["Files", "DIDs", "Replicas"],
+                    "default": "dids",
+                },
+                "catalog": {
+                    "type": "string",
+                    "enum": ["esap_rucio_entities"],
+                    "enumNames": ["esap_rucio_entities"],
+                    "default": "esap_rucio_entities",
+                },
+            },
+        }

esap/esap/settings/base.py

@@ -216,6 +216,8 @@ AUTHENTICATION_BACKENDS = (
 )
 OIDC_DRF_AUTH_BACKEND = 'mozilla_django_oidc.auth.OIDCAuthenticationBackend'
 
+# OIDC environment variables
+
 OIDC_RP_CLIENT_ID = os.environ['OIDC_RP_CLIENT_ID']
 OIDC_RP_CLIENT_SECRET = os.environ['OIDC_RP_CLIENT_SECRET']
 OIDC_RP_SIGN_ALGO = "RS256"
@@ -231,6 +233,13 @@ LOGIN_REDIRECT_URL = os.environ['LOGIN_REDIRECT_URL']
 LOGOUT_REDIRECT_URL = os.environ['LOGOUT_REDIRECT_URL']
 LOGIN_REDIRECT_URL_FAILURE = os.environ['LOGIN_REDIRECT_URL_FAILURE']
 
+# Rucio environment variables
+
+RUCIO_AUTH_TOKEN = os.environ['RUCIO_AUTH_TOKEN']
+RUCIO_HOST = os.environ['RUCIO_HOST']
+RUCIO_PORT = os.environ['RUCIO_PORT']
+RUCIO_AUTH_PORT = os.environ['RUCIO_AUTH_PORT']
+
 # Static files (CSS, JavaScript, Images)
 # https://docs.djangoproject.com/en/2.0/howto/static-files/
 
@@ -238,6 +247,6 @@ STATIC_URL = '/static/'
 STATIC_ROOT = os.path.join(BASE_DIR, 'static')
 
 # configuration settings that can be requested through the REST API
-VERSION = "ESAP-API version 23 march 2021"
+VERSION = "ESAP-API version 29 april 2021"
 CONFIGURATION_DIR = os.path.join(BASE_DIR, 'configuration')
 CONFIGURATION_FILE = 'esap_default'

esap/query/api/configuration.py

@@ -20,7 +20,7 @@ def get_datasets_disabled():
     return my_config.datasets_disabled
 
 # return expanded configuration
-def get_configuration(name=None):
+def get_configuration(name=None, session=None):
     result = {}
     result['version'] = settings.VERSION
 
@@ -34,6 +34,12 @@ def get_configuration(name=None):
         else:
             my_config = importlib.import_module(settings.CONFIGURATION_FILE)
 
+        # More sophisticated config files can embed the config in a class called Config
+        try:
+            my_config = my_config.Config(session)
+        except AttributeError as e:
+            pass
+
     except Exception as error:
         return {"configuration error in ": name+ ".py : " + str(error)}
 

esap/query/api/services/rucio.py

@@ -3,7 +3,7 @@
     Date created: 2020-10-15
     Description:  Rucio Service Connector for ESAP.
 """
-
+from django.conf import settings
 from rest_framework import serializers
 from .query_base import query_base
 import requests
@@ -15,11 +15,6 @@ logger = logging.getLogger(__name__)
 
 AMP_REPLACEMENT = "_and_"
 
-# The request header
-RUCIO_HOST = "https://escape-rucio.cern.ch:32300"
-RUCIO_PORT = 32300
-RUCIO_AUTH_TOKEN = "<REDACTED>"
-
 URLPATTERNS = dict(
     scope="{host}/scopes/",
     dids="{host}/dids/{scope}/",
@@ -87,12 +82,12 @@ class rucio_connector(query_base):
         """ use Rucio REST API to query the data lake """
         query_info = query["query_info"]
         url = query_info["url_pattern"].format(
-            host=f"{self.url}:{RUCIO_PORT}", **query_info["url_params"]
+            host=f"{self.url}:{settings.RUCIO_PORT}", **query_info["url_params"]
         )
         response = requests.get(
             url,
             query_info["where"],
-            headers={"X-Rucio-Auth-Token": RUCIO_AUTH_TOKEN},
+            headers={"X-Rucio-Auth-Token": settings.RUCIO_AUTH_TOKEN},
             verify=False,
         )
         results = []

esap/query/api/views/common_views.py

@@ -264,9 +264,9 @@ def ConfigurationView(request):
         pass
 
     try:
-        config_from_settings = configuration.get_configuration(name)
+        config_from_settings = configuration.get_configuration(name, request.session)
     except Exception as error:
-        config_from_settings = "ERROR in configuration: "+str(error)
+        config_from_settings = "ERROR in configuration: "+str(error) + " session => " + str(request.session)
 
     return JsonResponse({'configuration': config_from_settings})
 

esap/query/api/views/query_views.py

@@ -361,7 +361,7 @@ class GetTablesFields(generics.ListAPIView):
 class GetSkyCoordinates(generics.ListAPIView):
     """
     Retrieve a list of fields from the access_url
-    examples: /esap-api/query/get-fields?dataset_uri=vo_reg&access_url=https://vo.astron.nl/tap
+    examples: /esap-api/query/get-sky-coordinates/?target_name=M31
     """
 
     model = DataSet

esap/query/apps.py

@@ -3,11 +3,4 @@ from django.apps import AppConfig
 
 class MyAppConfig(AppConfig):
     name = 'query'
-# experiment to start a samp client on startup... (warning, blocking)
-#    def ready(self):
-#        try:
-#            print('importing samp')
-#            from .api.services import samp
-#            #samp()
-#        except:
-#            pass
\ No newline at end of file
+

esap/query/templates/query/index.html

@@ -70,7 +70,7 @@
 </div>
 
 
-<p class="footer" small>ASTRON - version 26 mar 2021 - 08:00</p>
+<p class="footer" small>ASTRON - version 29 april 2021 - 15:00</p>
 
 {% endblock %}