Skip to content
Snippets Groups Projects
Commit 499ad55b authored by Nico Vermaas's avatar Nico Vermaas
Browse files

integrating rucio

removing filesystem dependencies
parent 73156856
Branches
Tags v0.5.10
1 merge request!58integrating rucio
Pipeline #12422 passed
...@@ -3,6 +3,6 @@ ESAP API Gateway is a 'backend' web application written in Django. ...@@ -3,6 +3,6 @@ ESAP API Gateway is a 'backend' web application written in Django.
It provides a range of services that can be accessed through a REST API. It provides a range of services that can be accessed through a REST API.
## Documentation ## Documentation
* backend (API): https://git.astron.nl/astron-sdc/esap-api-gateway/-/wikis/ESAP-API-gateway * backend (API Gateway): https://git.astron.nl/astron-sdc/esap-api-gateway/-/wikis/ESAP-API-gateway-Overview
* frontend (GUI): https://git.astron.nl/astron-sdc/esap-gui/-/wikis/ESAP-GUI * frontend (GUI): https://git.astron.nl/astron-sdc/esap-gui/-/wikis/ESAP-GUI
import requests import requests
import urllib.parse import urllib.parse
import json import json
from django.conf import settings
rucio_url = "https://escape-rucio.cern.ch" ID_TOKEN_KEY = "oidc_id_token"
ACCESS_TOKEN_KEY = "oidc_access_token"
AUTH_PORT = 32301 def validate(token):
STANDARD_PORT = 32300 url = urllib.parse.urljoin(f"{settings.RUCIO_HOST}:{settings.RUCIO_AUTH_PORT}", "auth/validate")
RUCIO_AUTH_TOKEN = "<REDACTED>"
def validate():
url = urllib.parse.urljoin(f"{rucio_url}:{AUTH_PORT}", "auth/validate")
response = requests.get( response = requests.get(
url, headers={"X-Rucio-Auth-Token": RUCIO_AUTH_TOKEN}, verify=False url, headers={"X-Rucio-Auth-Token": token}, verify=False
) )
if response.ok: if response.ok:
return True return True
...@@ -21,13 +17,15 @@ def validate(): ...@@ -21,13 +17,15 @@ def validate():
return False return False
def get_scope_names(): def get_scope_names(session):
# try: token = session.get(ACCESS_TOKEN_KEY, None)
validated = validate() if token is None:
return [f"Not logged in {session}, {session.keys()}."]
validated = validate(token)
if validated: if validated:
url = urllib.parse.urljoin(f"{rucio_url}:{STANDARD_PORT}", "scopes") url = urllib.parse.urljoin(f"{settings.RUCIO_HOST}:{settings.RUCIO_PORT}", "scopes")
response = requests.get( response = requests.get(
url + "/", headers={"X-Rucio-Auth-Token": RUCIO_AUTH_TOKEN}, verify=False url + "/", headers={"X-Rucio-Auth-Token": token}, verify=False
) )
if response.ok: if response.ok:
return json.loads(response.content) return json.loads(response.content)
...@@ -36,17 +34,38 @@ def get_scope_names(): ...@@ -36,17 +34,38 @@ def get_scope_names():
"validated but failed query" "validated but failed query"
] # , val_response.status_code, val_response.reason] ] # , val_response.status_code, val_response.reason]
else: else:
# , val_response.status_code, val_response.reason] return [f"not validated, {len(token)}, {type(token)}, '{token}'."] # , val_response.status_code, val_response.reason]
return ["not validated"]
# except Exception as e: # except Exception as e:
# return ["Failed", "Authentication", e] # return ["Failed", "Authentication", e]
class Config:
title = "Rucio" title = "Rucio"
logo = "http://rucio.cern.ch/images/wide_logo2.png" logo = "http://rucio.cern.ch/images/wide_logo2.png"
# definition of the query # the url location of the frontend application,
query_schema = { # this makes it possible to install multiple instances in different directories on the webserver
# that all have their own urls like 'http://esap.astron.nl/esap-gui-dev/queries'
frontend_basename = "esap-rucio"
# definition of the navigation bar
nav1 = {"title": "Archives", "route": "/archives"}
nav2 = {"title": "Query", "route": "/query"}
navbar = [nav1, nav2]
ui_schema = {"catalog": {"ui:widget": "hidden"}}
def __init__(self, session):
self.session=session
self.__query_schema = self.gen_query_schema()
@property
def query_schema(self):
return self.__query_schema
def gen_query_schema(self):
scope_names = get_scope_names(self.session)
return {
"name": "rucio", "name": "rucio",
"title": "Rucio Query", "title": "Rucio Query",
"type": "object", "type": "object",
...@@ -54,8 +73,8 @@ query_schema = { ...@@ -54,8 +73,8 @@ query_schema = {
"scope": { "scope": {
"type": "string", "type": "string",
"title": "Scope", "title": "Scope",
"enum": get_scope_names(), "enum": scope_names,
"enumNames": get_scope_names(), "enumNames": scope_names,
}, },
"resource_category": { "resource_category": {
"type": "string", "type": "string",
...@@ -72,5 +91,3 @@ query_schema = { ...@@ -72,5 +91,3 @@ query_schema = {
}, },
}, },
} }
ui_schema = {"catalog": {"ui:widget": "hidden"}}
...@@ -216,6 +216,8 @@ AUTHENTICATION_BACKENDS = ( ...@@ -216,6 +216,8 @@ AUTHENTICATION_BACKENDS = (
) )
OIDC_DRF_AUTH_BACKEND = 'mozilla_django_oidc.auth.OIDCAuthenticationBackend' OIDC_DRF_AUTH_BACKEND = 'mozilla_django_oidc.auth.OIDCAuthenticationBackend'
# OIDC environment variables
OIDC_RP_CLIENT_ID = os.environ['OIDC_RP_CLIENT_ID'] OIDC_RP_CLIENT_ID = os.environ['OIDC_RP_CLIENT_ID']
OIDC_RP_CLIENT_SECRET = os.environ['OIDC_RP_CLIENT_SECRET'] OIDC_RP_CLIENT_SECRET = os.environ['OIDC_RP_CLIENT_SECRET']
OIDC_RP_SIGN_ALGO = "RS256" OIDC_RP_SIGN_ALGO = "RS256"
...@@ -231,6 +233,13 @@ LOGIN_REDIRECT_URL = os.environ['LOGIN_REDIRECT_URL'] ...@@ -231,6 +233,13 @@ LOGIN_REDIRECT_URL = os.environ['LOGIN_REDIRECT_URL']
LOGOUT_REDIRECT_URL = os.environ['LOGOUT_REDIRECT_URL'] LOGOUT_REDIRECT_URL = os.environ['LOGOUT_REDIRECT_URL']
LOGIN_REDIRECT_URL_FAILURE = os.environ['LOGIN_REDIRECT_URL_FAILURE'] LOGIN_REDIRECT_URL_FAILURE = os.environ['LOGIN_REDIRECT_URL_FAILURE']
# Rucio environment variables
RUCIO_AUTH_TOKEN = os.environ['RUCIO_AUTH_TOKEN']
RUCIO_HOST = os.environ['RUCIO_HOST']
RUCIO_PORT = os.environ['RUCIO_PORT']
RUCIO_AUTH_PORT = os.environ['RUCIO_AUTH_PORT']
# Static files (CSS, JavaScript, Images) # Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/2.0/howto/static-files/ # https://docs.djangoproject.com/en/2.0/howto/static-files/
...@@ -238,6 +247,6 @@ STATIC_URL = '/static/' ...@@ -238,6 +247,6 @@ STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'static') STATIC_ROOT = os.path.join(BASE_DIR, 'static')
# configuration settings that can be requested through the REST API # configuration settings that can be requested through the REST API
VERSION = "ESAP-API version 23 march 2021" VERSION = "ESAP-API version 29 april 2021"
CONFIGURATION_DIR = os.path.join(BASE_DIR, 'configuration') CONFIGURATION_DIR = os.path.join(BASE_DIR, 'configuration')
CONFIGURATION_FILE = 'esap_default' CONFIGURATION_FILE = 'esap_default'
...@@ -20,7 +20,7 @@ def get_datasets_disabled(): ...@@ -20,7 +20,7 @@ def get_datasets_disabled():
return my_config.datasets_disabled return my_config.datasets_disabled
# return expanded configuration # return expanded configuration
def get_configuration(name=None): def get_configuration(name=None, session=None):
result = {} result = {}
result['version'] = settings.VERSION result['version'] = settings.VERSION
...@@ -34,6 +34,12 @@ def get_configuration(name=None): ...@@ -34,6 +34,12 @@ def get_configuration(name=None):
else: else:
my_config = importlib.import_module(settings.CONFIGURATION_FILE) my_config = importlib.import_module(settings.CONFIGURATION_FILE)
# More sophisticated config files can embed the config in a class called Config
try:
my_config = my_config.Config(session)
except AttributeError as e:
pass
except Exception as error: except Exception as error:
return {"configuration error in ": name+ ".py : " + str(error)} return {"configuration error in ": name+ ".py : " + str(error)}
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
Date created: 2020-10-15 Date created: 2020-10-15
Description: Rucio Service Connector for ESAP. Description: Rucio Service Connector for ESAP.
""" """
from django.conf import settings
from rest_framework import serializers from rest_framework import serializers
from .query_base import query_base from .query_base import query_base
import requests import requests
...@@ -15,11 +15,6 @@ logger = logging.getLogger(__name__) ...@@ -15,11 +15,6 @@ logger = logging.getLogger(__name__)
AMP_REPLACEMENT = "_and_" AMP_REPLACEMENT = "_and_"
# The request header
RUCIO_HOST = "https://escape-rucio.cern.ch:32300"
RUCIO_PORT = 32300
RUCIO_AUTH_TOKEN = "<REDACTED>"
URLPATTERNS = dict( URLPATTERNS = dict(
scope="{host}/scopes/", scope="{host}/scopes/",
dids="{host}/dids/{scope}/", dids="{host}/dids/{scope}/",
...@@ -87,12 +82,12 @@ class rucio_connector(query_base): ...@@ -87,12 +82,12 @@ class rucio_connector(query_base):
""" use Rucio REST API to query the data lake """ """ use Rucio REST API to query the data lake """
query_info = query["query_info"] query_info = query["query_info"]
url = query_info["url_pattern"].format( url = query_info["url_pattern"].format(
host=f"{self.url}:{RUCIO_PORT}", **query_info["url_params"] host=f"{self.url}:{settings.RUCIO_PORT}", **query_info["url_params"]
) )
response = requests.get( response = requests.get(
url, url,
query_info["where"], query_info["where"],
headers={"X-Rucio-Auth-Token": RUCIO_AUTH_TOKEN}, headers={"X-Rucio-Auth-Token": settings.RUCIO_AUTH_TOKEN},
verify=False, verify=False,
) )
results = [] results = []
......
...@@ -264,9 +264,9 @@ def ConfigurationView(request): ...@@ -264,9 +264,9 @@ def ConfigurationView(request):
pass pass
try: try:
config_from_settings = configuration.get_configuration(name) config_from_settings = configuration.get_configuration(name, request.session)
except Exception as error: except Exception as error:
config_from_settings = "ERROR in configuration: "+str(error) config_from_settings = "ERROR in configuration: "+str(error) + " session => " + str(request.session)
return JsonResponse({'configuration': config_from_settings}) return JsonResponse({'configuration': config_from_settings})
......
...@@ -361,7 +361,7 @@ class GetTablesFields(generics.ListAPIView): ...@@ -361,7 +361,7 @@ class GetTablesFields(generics.ListAPIView):
class GetSkyCoordinates(generics.ListAPIView): class GetSkyCoordinates(generics.ListAPIView):
""" """
Retrieve a list of fields from the access_url Retrieve a list of fields from the access_url
examples: /esap-api/query/get-fields?dataset_uri=vo_reg&access_url=https://vo.astron.nl/tap examples: /esap-api/query/get-sky-coordinates/?target_name=M31
""" """
model = DataSet model = DataSet
......
...@@ -3,11 +3,4 @@ from django.apps import AppConfig ...@@ -3,11 +3,4 @@ from django.apps import AppConfig
class MyAppConfig(AppConfig): class MyAppConfig(AppConfig):
name = 'query' name = 'query'
# experiment to start a samp client on startup... (warning, blocking)
# def ready(self):
# try:
# print('importing samp')
# from .api.services import samp
# #samp()
# except:
# pass
\ No newline at end of file
...@@ -70,7 +70,7 @@ ...@@ -70,7 +70,7 @@
</div> </div>
<p class="footer" small>ASTRON - version 26 mar 2021 - 08:00</p> <p class="footer" small>ASTRON - version 29 april 2021 - 15:00</p>
{% endblock %} {% endblock %}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment