Skip to content
Snippets Groups Projects
Commit 85306c70 authored by John Swinbank's avatar John Swinbank
Browse files

Restructure and rename

This moves all code into the esap_client package to avoid name clashes with
other code (e.g.
esap-general#86). It uses
inheritance to reduce duplication. It makes a few small changes to make the
class names idiomatic (ie, using CamelCase).

This is a breaking API change!
parent 78c6df07
No related branches found
No related tags found
1 merge request!11Restructure and rename
Showing
with 49 additions and 351 deletions
...@@ -2,46 +2,19 @@ ...@@ -2,46 +2,19 @@
A Python client for the ESCAPE ESAP User Profile REST API. A Python client for the ESCAPE ESAP User Profile REST API.
The `shopping_client` module, which communicates with the ESCAPE ESAP User Profile REST API is very lightweight. Archive-specific functionality is delegated to "connector" modules like the `zooniverse` module. The `ShoppingClient` class, which communicates with the ESCAPE ESAP User Profile REST API, is very lightweight. Archive-specific functionality is delegated to "connector" classes like `Zooniverse` and `Alta`.
### Installation ### Installation
The client and the Zooniverse client cat be installed using pip: The client and a selection of connector classes client cat be installed using pip:
```sh ```sh
$ pip install git+https://git.astron.nl/astron-sdc/esap-userprofile-python-client.git $ pip install git+https://git.astron.nl/astron-sdc/esap-userprofile-python-client.git
``` ```
### Example - Using the Shopping Client with the Zooniverse connector ### Example
```python See `example.py`.
from shopping_client import shopping_client
from zooniverse import zooniverse
import getpass
# Prompt for Zooniverse account password
zooniverse_password = getpass.getpass("Enter Zooniverse password:")
# Instantiate Zooniverse connector
zc = zooniverse(username="hughdickinson", password=zooniverse_password)
# Instantiate ESAP User Profile shopping client, passing zooniverse connector
sc = shopping_client(host="https://sdc-dev.astron.nl:5555/", connectors=[zc])
# Retrieve basket (prompts to enter access token obtained from ESAP GUI)
res=sc.get_basket(convert_to_pandas=True)
# ... inspect available results ...
# Retrieve data from Zooniverse based on basket item
data = zc.retrieve(res["zooniverse"].loc[3],
generate=False,
wait=True,
convert_to_pandas=True)
# ... analyse data ...
```
## Contributing ## Contributing
......
from .zooniverse import zooniverse
from .shopping_client import shopping_client
from alta.alta_connector import alta_connector
import requests
import json
import io
import getpass
import pandas as pd
from typing import Union, Optional
class alta_connector:
name = "alta"
archive = "apertif"
def basket_item_to_pandas(
self, basket_item: Union[dict, pd.Series], validate: bool = True
) -> Optional[pd.Series]:
"""Convert an item from the shopping basket into a `pd.Series` with
optional validation.
Parameters
----------
basket_item : Union[dict, pd.Series]
A single item from a retrieved shopping basket - either a raw `dict`
or a converted `pd.Series`.
validate : bool
If `True`, check that the data in the shopping item conforms with
the expected format before attempting the conversion.
Returns
-------
Optional[pd.Series]
`pd.Series` containing the data encoded in the shopping item or
`NoneType`.
"""
if validate:
item_data = self.validate_basket_item(basket_item, return_loaded=True)
else:
item_data = json.loads(basket_item["item_data"])
if item_data:
return pd.Series(item_data)
return None
def validate_basket_item(
self, basket_item: Union[dict, pd.Series], return_loaded: bool = False
) -> Union[dict, bool, None]:
"""Check that the data in the shopping item conforms with
the expected format
Parameters
----------
basket_item : Union[dict, pd.Series]
A single item from a retrieved shopping basket - either a raw `dict`
or a converted `pd.Series`.
return_loaded : bool
If `True`, and validation succeeds return the extracted shopping item
as `dict`, otherwise return `True` if validation succeeds and `None`
otherwise.
Returns
-------
Union[dict, bool, None]
If `return_loaded` is `True`, return a `dict` containing the data
encoded in the shopping item when validation succeeds.
Otherwise if `return_loaded` is `True` validation succeeds.
If validation fails return `None`.
"""
item_data = json.loads(basket_item["item_data"])
if "archive" in item_data and item_data["archive"] == self.archive:
if return_loaded:
return item_data
else:
return True
return None
\ No newline at end of file
from astron_vo.astron_vo_connector import astron_vo_connector
from .shopping_client import ShoppingClient
from .alta import Alta
from .astron_vo import AstronVo
from .samp import Samp
from .zooniverse import Zooniverse
from .rucio import Rucio
from .baseConnector import BaseConnector
class Alta(BaseConnector):
name = "alta"
archive = "apertif"
from .baseConnector import BaseConnector
class AstronVo(BaseConnector):
name = "astron_vo"
archive = "astron_vo"
import json import json
import pandas as pd import pandas as pd
from typing import Union, Optional from typing import Union, Optional
class astron_vo_connector: class BaseConnector:
name = "astron_vo"
archive = "astron_vo"
def basket_item_to_pandas( def basket_item_to_pandas(
self, basket_item: Union[dict, pd.Series], validate: bool = True self, basket_item: Union[dict, pd.Series], validate: bool = True
) -> Optional[pd.Series]: ) -> Optional[pd.Series]:
......
from .baseConnector import BaseConnector
class Rucio(BaseConnector):
name = "rucio"
archive = "rucio"
from .baseConnector import BaseConnector
class Samp(BaseConnector):
name = "samp"
archive = "samp"
...@@ -10,8 +10,9 @@ from warnings import warn ...@@ -10,8 +10,9 @@ from warnings import warn
from panoptes_client import Panoptes, Project, Workflow from panoptes_client import Panoptes, Project, Workflow
from panoptes_client.panoptes import PanoptesAPIException from panoptes_client.panoptes import PanoptesAPIException
from .baseConnector import BaseConnector
class zooniverse: class Zooniverse(BaseConnector):
name = "zooniverse" name = "zooniverse"
archive = "zooniverse" archive = "zooniverse"
...@@ -155,7 +156,7 @@ class zooniverse: ...@@ -155,7 +156,7 @@ class zooniverse:
if chunked_retrieve if chunked_retrieve
else pd.read_csv( else pd.read_csv(
io.BytesIO(response.content), io.BytesIO(response.content),
converters=zooniverse.category_converters[ converters=Zooniverse.category_converters[
self._get_item_entry(item, "category") self._get_item_entry(item, "category")
], ],
) )
...@@ -192,7 +193,7 @@ class zooniverse: ...@@ -192,7 +193,7 @@ class zooniverse:
chunk_frames.append( chunk_frames.append(
pd.read_csv( pd.read_csv(
io.BytesIO(chunk), io.BytesIO(chunk),
converters=zooniverse.category_converters[ converters=Zooniverse.category_converters[
self._get_item_entry(item, "category") self._get_item_entry(item, "category")
], ],
header=None if len(chunk_frames) else 0, header=None if len(chunk_frames) else 0,
...@@ -212,7 +213,7 @@ class zooniverse: ...@@ -212,7 +213,7 @@ class zooniverse:
) )
def _get_entity(self, item): def _get_entity(self, item):
entity = zooniverse.entity_types[self._get_item_entry(item, "catalog")].find( entity = Zooniverse.entity_types[self._get_item_entry(item, "catalog")].find(
int(self._get_item_entry(item, self._catalogue_to_id_string(item))) int(self._get_item_entry(item, self._catalogue_to_id_string(item)))
) )
return entity return entity
...@@ -225,66 +226,3 @@ class zooniverse: ...@@ -225,66 +226,3 @@ class zooniverse:
def _catalogue_to_id_string(self, item): def _catalogue_to_id_string(self, item):
return self._get_item_entry(item, "catalog") + "_id" return self._get_item_entry(item, "catalog") + "_id"
def basket_item_to_pandas(
self, basket_item: Union[dict, pd.Series], validate: bool = True
) -> Optional[pd.Series]:
"""Convert an item from the shopping basket into a `pd.Series` with
optional validation.
Parameters
----------
basket_item : Union[dict, pd.Series]
A single item from a retrieved shopping basket - either a raw `dict`
or a converted `pd.Series`.
validate : bool
If `True`, check that the data in the shopping item conforms with
the expected format before attempting the conversion.
Returns
-------
Optional[pd.Series]
`pd.Series` containing the data encoded in the shopping item or
`NoneType`.
"""
if validate:
item_data = self.validate_basket_item(basket_item, return_loaded=True)
else:
item_data = json.loads(basket_item["item_data"])
if item_data:
return pd.Series(item_data)
return None
def validate_basket_item(
self, basket_item: Union[dict, pd.Series], return_loaded: bool = False
) -> Union[dict, bool, None]:
"""Check that the data in the shopping item conforms with
the expected format
Parameters
----------
basket_item : Union[dict, pd.Series]
A single item from a retrieved shopping basket - either a raw `dict`
or a converted `pd.Series`.
return_loaded : bool
If `True`, and validation succeeds return the extracted shopping item
as `dict`, otherwise return `True` if validation succeeds and `None`
otherwise.
Returns
-------
Union[dict, bool, None]
If `return_loaded` is `True`, return a `dict` containing the data
encoded in the shopping item when validation succeeds.
Otherwise if `return_loaded` is `True` validation succeeds.
If validation fails return `None`.
"""
item_data = json.loads(basket_item["item_data"])
if "archive" in item_data and item_data["archive"] == "zooniverse":
if return_loaded:
return item_data
else:
return True
return None
...@@ -14,7 +14,7 @@ import requests ...@@ -14,7 +14,7 @@ import requests
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
class shopping_client: class ShoppingClient:
endpoint = "esap-api/accounts/user-profiles/" endpoint = "esap-api/accounts/user-profiles/"
audience = "rucio" # Audience used by ESAP, might be configurable later audience = "rucio" # Audience used by ESAP, might be configurable later
...@@ -80,7 +80,7 @@ class shopping_client: ...@@ -80,7 +80,7 @@ class shopping_client:
""" """
if self.basket is None or reload: if self.basket is None or reload:
url = urllib.parse.urljoin(self.host, shopping_client.endpoint) url = urllib.parse.urljoin(self.host, ShoppingClient.endpoint)
response = requests.get(url, headers=self._request_header()) response = requests.get(url, headers=self._request_header())
if response.ok: if response.ok:
self.basket = json.loads(response.content)["results"][0][ self.basket = json.loads(response.content)["results"][0][
......
from shopping_client import shopping_client from esap_client import ShoppingClient
from alta import alta_connector from esap_client.connectors import Alta as alta_connector
from astron_vo import astron_vo_connector from esap_client.connectors import AstronVo as astron_vo_connector
from samp import samp_connector from esap_client.connectors import Samp as samp_connector
esap_api_host = "https://sdc-dev.astron.nl:5555/" esap_api_host = "https://sdc-dev.astron.nl:443/"
#access_token = "" #access_token = ""
# Instantiate alta connector # Instantiate alta connector
...@@ -12,7 +12,7 @@ vo = astron_vo_connector() ...@@ -12,7 +12,7 @@ vo = astron_vo_connector()
# Instantiate ESAP User Profile shopping client, passing alta connector # Instantiate ESAP User Profile shopping client, passing alta connector
#sc = shopping_client(host=esap_api_host, token=access_token, connectors=[ac,vo]) #sc = shopping_client(host=esap_api_host, token=access_token, connectors=[ac,vo])
sc = shopping_client(host=esap_api_host, connectors=[ac]) sc = ShoppingClient(host=esap_api_host, connectors=[ac])
# 'apertif'and 'astron_vo' items converted to pandas dataframe # 'apertif'and 'astron_vo' items converted to pandas dataframe
basket_pandas=sc.get_basket(filter_archives=True, convert_to_pandas=True) basket_pandas=sc.get_basket(filter_archives=True, convert_to_pandas=True)
...@@ -29,7 +29,7 @@ print("'apertif'and 'astron_vo' items as json") ...@@ -29,7 +29,7 @@ print("'apertif'and 'astron_vo' items as json")
print(basket_json) print(basket_json)
samp_connector = samp_connector() samp_connector = samp_connector()
sc = shopping_client(host=esap_api_host, token=access_token, connectors=[samp_connector]) sc = ShoppingClient(host=esap_api_host, connectors=[samp_connector])
# "'SAMP' items converted to pandas dataframe:" # "'SAMP' items converted to pandas dataframe:"
basket_pandas=sc.get_basket(convert_to_pandas=True, filter_archives=True) basket_pandas=sc.get_basket(convert_to_pandas=True, filter_archives=True)
......
from rucio_cli.rucio_connector import rucio_connector
import requests
import json
import io
import getpass
import pandas as pd
from typing import Union, Optional
class rucio_connector:
name = "rucio"
archive = "rucio"
def basket_item_to_pandas(
self, basket_item: Union[dict, pd.Series], validate: bool = True
) -> Optional[pd.Series]:
"""Convert an item from the shopping basket into a `pd.Series` with
optional validation.
Parameters
----------
basket_item : Union[dict, pd.Series]
A single item from a retrieved shopping basket - either a raw `dict`
or a converted `pd.Series`.
validate : bool
If `True`, check that the data in the shopping item conforms with
the expected format before attempting the conversion.
Returns
-------
Optional[pd.Series]
`pd.Series` containing the data encoded in the shopping item or
`NoneType`.
"""
if validate:
item_data = self.validate_basket_item(basket_item, return_loaded=True)
else:
item_data = json.loads(basket_item["item_data"])
if item_data:
return pd.Series(item_data)
return None
def validate_basket_item(
self, basket_item: Union[dict, pd.Series], return_loaded: bool = False
) -> Union[dict, bool, None]:
"""Check that the data in the shopping item conforms with
the expected format
Parameters
----------
basket_item : Union[dict, pd.Series]
A single item from a retrieved shopping basket - either a raw `dict`
or a converted `pd.Series`.
return_loaded : bool
If `True`, and validation succeeds return the extracted shopping item
as `dict`, otherwise return `True` if validation succeeds and `None`
otherwise.
Returns
-------
Union[dict, bool, None]
If `return_loaded` is `True`, return a `dict` containing the data
encoded in the shopping item when validation succeeds.
Otherwise if `return_loaded` is `True` validation succeeds.
If validation fails return `None`.
"""
item_data = json.loads(basket_item["item_data"])
if "archive" in item_data and item_data["archive"] == self.archive:
if return_loaded:
return item_data
else:
return True
return None
from samp.samp_connector import samp_connector
\ No newline at end of file
import requests
import json
import io
import getpass
import pandas as pd
from typing import Union, Optional
class samp_connector:
name = "samp"
archive = "samp"
def basket_item_to_pandas(
self, basket_item: Union[dict, pd.Series], validate: bool = True
) -> Optional[pd.Series]:
"""Convert an item from the shopping basket into a `pd.Series` with
optional validation.
Parameters
----------
basket_item : Union[dict, pd.Series]
A single item from a retrieved shopping basket - either a raw `dict`
or a converted `pd.Series`.
validate : bool
If `True`, check that the data in the shopping item conforms with
the expected format before attempting the conversion.
Returns
-------
Optional[pd.Series]
`pd.Series` containing the data encoded in the shopping item or
`NoneType`.
"""
if validate:
item_data = self.validate_basket_item(basket_item, return_loaded=True)
else:
item_data = json.loads(basket_item["item_data"])
if item_data:
return pd.Series(item_data)
return None
def validate_basket_item(
self, basket_item: Union[dict, pd.Series], return_loaded: bool = False
) -> Union[dict, bool, None]:
"""Check that the data in the shopping item conforms with
the expected format
Parameters
----------
basket_item : Union[dict, pd.Series]
A single item from a retrieved shopping basket - either a raw `dict`
or a converted `pd.Series`.
return_loaded : bool
If `True`, and validation succeeds return the extracted shopping item
as `dict`, otherwise return `True` if validation succeeds and `None`
otherwise.
Returns
-------
Union[dict, bool, None]
If `return_loaded` is `True`, return a `dict` containing the data
encoded in the shopping item when validation succeeds.
Otherwise if `return_loaded` is `True` validation succeeds.
If validation fails return `None`.
"""
item_data = json.loads(basket_item["item_data"])
if "archive" in item_data and item_data["archive"] == self.archive:
if return_loaded:
return item_data
else:
return True
return None
\ No newline at end of file
...@@ -4,8 +4,8 @@ with open("README.md", "r") as fh: ...@@ -4,8 +4,8 @@ with open("README.md", "r") as fh:
long_description = fh.read() long_description = fh.read()
setuptools.setup( setuptools.setup(
name="esap-userprofile-python-client", name="esap_client",
version="0.0.4", version="0.0.5",
author="Hugh Dickinson", author="Hugh Dickinson",
author_email="hugh.dickinson@open.ac.uk", author_email="hugh.dickinson@open.ac.uk",
description="Python client for ESAP Data Discovery Shopping Basket", description="Python client for ESAP Data Discovery Shopping Basket",
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment