Commit 499ad55b authored by Nico Vermaas's avatar Nico Vermaas

integrating rucio

removing filesystem dependencies
parent 73156856
Pipeline #12422 passed with stage
in 16 seconds
......@@ -3,6 +3,6 @@ ESAP API Gateway is a 'backend' web application written in Django.
It provides a range of services that can be accessed through a REST API.
## Documentation
* backend (API): https://git.astron.nl/astron-sdc/esap-api-gateway/-/wikis/ESAP-API-gateway
* backend (API Gateway): https://git.astron.nl/astron-sdc/esap-api-gateway/-/wikis/ESAP-API-gateway-Overview
* frontend (GUI): https://git.astron.nl/astron-sdc/esap-gui/-/wikis/ESAP-GUI
import requests
import urllib.parse
import json
from django.conf import settings
rucio_url = "https://escape-rucio.cern.ch"
ID_TOKEN_KEY = "oidc_id_token"
ACCESS_TOKEN_KEY = "oidc_access_token"
AUTH_PORT = 32301
STANDARD_PORT = 32300
RUCIO_AUTH_TOKEN = "<REDACTED>"
def validate():
url = urllib.parse.urljoin(f"{rucio_url}:{AUTH_PORT}", "auth/validate")
def validate(token):
url = urllib.parse.urljoin(f"{settings.RUCIO_HOST}:{settings.RUCIO_AUTH_PORT}", "auth/validate")
response = requests.get(
url, headers={"X-Rucio-Auth-Token": RUCIO_AUTH_TOKEN}, verify=False
url, headers={"X-Rucio-Auth-Token": token}, verify=False
)
if response.ok:
return True
......@@ -21,13 +17,15 @@ def validate():
return False
def get_scope_names():
# try:
validated = validate()
def get_scope_names(session):
token = session.get(ACCESS_TOKEN_KEY, None)
if token is None:
return [f"Not logged in {session}, {session.keys()}."]
validated = validate(token)
if validated:
url = urllib.parse.urljoin(f"{rucio_url}:{STANDARD_PORT}", "scopes")
url = urllib.parse.urljoin(f"{settings.RUCIO_HOST}:{settings.RUCIO_PORT}", "scopes")
response = requests.get(
url + "/", headers={"X-Rucio-Auth-Token": RUCIO_AUTH_TOKEN}, verify=False
url + "/", headers={"X-Rucio-Auth-Token": token}, verify=False
)
if response.ok:
return json.loads(response.content)
......@@ -36,41 +34,60 @@ def get_scope_names():
"validated but failed query"
] # , val_response.status_code, val_response.reason]
else:
# , val_response.status_code, val_response.reason]
return ["not validated"]
return [f"not validated, {len(token)}, {type(token)}, '{token}'."] # , val_response.status_code, val_response.reason]
# except Exception as e:
# return ["Failed", "Authentication", e]
title = "Rucio"
logo = "http://rucio.cern.ch/images/wide_logo2.png"
class Config:
title = "Rucio"
logo = "http://rucio.cern.ch/images/wide_logo2.png"
# the url location of the frontend application,
# this makes it possible to install multiple instances in different directories on the webserver
# that all have their own urls like 'http://esap.astron.nl/esap-gui-dev/queries'
frontend_basename = "esap-rucio"
# definition of the navigation bar
nav1 = {"title": "Archives", "route": "/archives"}
nav2 = {"title": "Query", "route": "/query"}
navbar = [nav1, nav2]
ui_schema = {"catalog": {"ui:widget": "hidden"}}
def __init__(self, session):
self.session=session
self.__query_schema = self.gen_query_schema()
# definition of the query
query_schema = {
"name": "rucio",
"title": "Rucio Query",
"type": "object",
"properties": {
"scope": {
"type": "string",
"title": "Scope",
"enum": get_scope_names(),
"enumNames": get_scope_names(),
},
"resource_category": {
"type": "string",
"title": "Category",
"enum": ["files", "dids", "replicas"],
"enumNames": ["Files", "DIDs", "Replicas"],
"default": "dids",
},
"catalog": {
"type": "string",
"enum": ["esap_rucio_entities"],
"enumNames": ["esap_rucio_entities"],
"default": "esap_rucio_entities",
},
},
}
@property
def query_schema(self):
return self.__query_schema
ui_schema = {"catalog": {"ui:widget": "hidden"}}
def gen_query_schema(self):
scope_names = get_scope_names(self.session)
return {
"name": "rucio",
"title": "Rucio Query",
"type": "object",
"properties": {
"scope": {
"type": "string",
"title": "Scope",
"enum": scope_names,
"enumNames": scope_names,
},
"resource_category": {
"type": "string",
"title": "Category",
"enum": ["files", "dids", "replicas"],
"enumNames": ["Files", "DIDs", "Replicas"],
"default": "dids",
},
"catalog": {
"type": "string",
"enum": ["esap_rucio_entities"],
"enumNames": ["esap_rucio_entities"],
"default": "esap_rucio_entities",
},
},
}
......@@ -216,6 +216,8 @@ AUTHENTICATION_BACKENDS = (
)
OIDC_DRF_AUTH_BACKEND = 'mozilla_django_oidc.auth.OIDCAuthenticationBackend'
# OIDC environment variables
OIDC_RP_CLIENT_ID = os.environ['OIDC_RP_CLIENT_ID']
OIDC_RP_CLIENT_SECRET = os.environ['OIDC_RP_CLIENT_SECRET']
OIDC_RP_SIGN_ALGO = "RS256"
......@@ -231,6 +233,13 @@ LOGIN_REDIRECT_URL = os.environ['LOGIN_REDIRECT_URL']
LOGOUT_REDIRECT_URL = os.environ['LOGOUT_REDIRECT_URL']
LOGIN_REDIRECT_URL_FAILURE = os.environ['LOGIN_REDIRECT_URL_FAILURE']
# Rucio environment variables
RUCIO_AUTH_TOKEN = os.environ['RUCIO_AUTH_TOKEN']
RUCIO_HOST = os.environ['RUCIO_HOST']
RUCIO_PORT = os.environ['RUCIO_PORT']
RUCIO_AUTH_PORT = os.environ['RUCIO_AUTH_PORT']
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/2.0/howto/static-files/
......@@ -238,6 +247,6 @@ STATIC_URL = '/static/'
STATIC_ROOT = os.path.join(BASE_DIR, 'static')
# configuration settings that can be requested through the REST API
VERSION = "ESAP-API version 23 march 2021"
VERSION = "ESAP-API version 29 april 2021"
CONFIGURATION_DIR = os.path.join(BASE_DIR, 'configuration')
CONFIGURATION_FILE = 'esap_default'
......@@ -20,7 +20,7 @@ def get_datasets_disabled():
return my_config.datasets_disabled
# return expanded configuration
def get_configuration(name=None):
def get_configuration(name=None, session=None):
result = {}
result['version'] = settings.VERSION
......@@ -34,6 +34,12 @@ def get_configuration(name=None):
else:
my_config = importlib.import_module(settings.CONFIGURATION_FILE)
# More sophisticated config files can embed the config in a class called Config
try:
my_config = my_config.Config(session)
except AttributeError as e:
pass
except Exception as error:
return {"configuration error in ": name+ ".py : " + str(error)}
......
......@@ -3,7 +3,7 @@
Date created: 2020-10-15
Description: Rucio Service Connector for ESAP.
"""
from django.conf import settings
from rest_framework import serializers
from .query_base import query_base
import requests
......@@ -15,11 +15,6 @@ logger = logging.getLogger(__name__)
AMP_REPLACEMENT = "_and_"
# The request header
RUCIO_HOST = "https://escape-rucio.cern.ch:32300"
RUCIO_PORT = 32300
RUCIO_AUTH_TOKEN = "<REDACTED>"
URLPATTERNS = dict(
scope="{host}/scopes/",
dids="{host}/dids/{scope}/",
......@@ -87,12 +82,12 @@ class rucio_connector(query_base):
""" use Rucio REST API to query the data lake """
query_info = query["query_info"]
url = query_info["url_pattern"].format(
host=f"{self.url}:{RUCIO_PORT}", **query_info["url_params"]
host=f"{self.url}:{settings.RUCIO_PORT}", **query_info["url_params"]
)
response = requests.get(
url,
query_info["where"],
headers={"X-Rucio-Auth-Token": RUCIO_AUTH_TOKEN},
headers={"X-Rucio-Auth-Token": settings.RUCIO_AUTH_TOKEN},
verify=False,
)
results = []
......
......@@ -264,9 +264,9 @@ def ConfigurationView(request):
pass
try:
config_from_settings = configuration.get_configuration(name)
config_from_settings = configuration.get_configuration(name, request.session)
except Exception as error:
config_from_settings = "ERROR in configuration: "+str(error)
config_from_settings = "ERROR in configuration: "+str(error) + " session => " + str(request.session)
return JsonResponse({'configuration': config_from_settings})
......
......@@ -361,7 +361,7 @@ class GetTablesFields(generics.ListAPIView):
class GetSkyCoordinates(generics.ListAPIView):
"""
Retrieve a list of fields from the access_url
examples: /esap-api/query/get-fields?dataset_uri=vo_reg&access_url=https://vo.astron.nl/tap
examples: /esap-api/query/get-sky-coordinates/?target_name=M31
"""
model = DataSet
......
......@@ -3,11 +3,4 @@ from django.apps import AppConfig
class MyAppConfig(AppConfig):
name = 'query'
# experiment to start a samp client on startup... (warning, blocking)
# def ready(self):
# try:
# print('importing samp')
# from .api.services import samp
# #samp()
# except:
# pass
\ No newline at end of file
......@@ -70,7 +70,7 @@
</div>
<p class="footer" small>ASTRON - version 26 mar 2021 - 08:00</p>
<p class="footer" small>ASTRON - version 29 april 2021 - 15:00</p>
{% endblock %}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment