From 85306c706e7c01a0bf1869c29f5ed8c7a90fec9d Mon Sep 17 00:00:00 2001
From: John Swinbank <swinbank@astron.nl>
Date: Fri, 22 Apr 2022 16:57:08 +0200
Subject: [PATCH] Restructure and rename
This moves all code into the esap_client package to avoid name clashes with
other code (e.g.
https://git.astron.nl/astron-sdc/escape-wp5/esap-general/-/issues/86). It uses
inheritance to reduce duplication. It makes a few small changes to make the
class names idiomatic (ie, using CamelCase).
This is a breaking API change!
---
README.md | 35 +--------
__init__.py | 2 -
alta/__init__.py | 1 -
alta/alta_connector.py | 76 -------------------
astron_vo/__init__.py | 1 -
esap_client/__init__.py | 1 +
esap_client/connectors/__init__.py | 5 ++
esap_client/connectors/alta.py | 5 ++
esap_client/connectors/astron_vo.py | 5 ++
.../connectors/baseConnector.py | 9 +--
esap_client/connectors/rucio.py | 5 ++
esap_client/connectors/samp.py | 5 ++
.../connectors}/zooniverse.py | 72 ++----------------
.../shopping_client.py | 4 +-
alta/alta_example.py => example.py | 16 ++--
rucio_cli/__init__.py | 1 -
rucio_cli/rucio_connector.py | 76 -------------------
samp/__init__.py | 1 -
samp/samp_connector.py | 76 -------------------
setup.py | 4 +-
shopping_client/__init__.py | 1 -
zooniverse/__init__.py | 1 -
22 files changed, 49 insertions(+), 353 deletions(-)
delete mode 100644 __init__.py
delete mode 100644 alta/__init__.py
delete mode 100644 alta/alta_connector.py
delete mode 100644 astron_vo/__init__.py
create mode 100644 esap_client/__init__.py
create mode 100644 esap_client/connectors/__init__.py
create mode 100644 esap_client/connectors/alta.py
create mode 100644 esap_client/connectors/astron_vo.py
rename astron_vo/astron_vo_connector.py => esap_client/connectors/baseConnector.py (96%)
create mode 100644 esap_client/connectors/rucio.py
create mode 100644 esap_client/connectors/samp.py
rename {zooniverse => esap_client/connectors}/zooniverse.py (75%)
rename {shopping_client => esap_client}/shopping_client.py (98%)
rename alta/alta_example.py => example.py (74%)
delete mode 100644 rucio_cli/__init__.py
delete mode 100644 rucio_cli/rucio_connector.py
delete mode 100644 samp/__init__.py
delete mode 100644 samp/samp_connector.py
delete mode 100644 shopping_client/__init__.py
delete mode 100644 zooniverse/__init__.py
diff --git a/README.md b/README.md
index 9151987..71caa38 100644
--- a/README.md
+++ b/README.md
@@ -2,46 +2,19 @@
A Python client for the ESCAPE ESAP User Profile REST API.
-The `shopping_client` module, which communicates with the ESCAPE ESAP User Profile REST API is very lightweight. Archive-specific functionality is delegated to "connector" modules like the `zooniverse` module.
+The `ShoppingClient` class, which communicates with the ESCAPE ESAP User Profile REST API, is very lightweight. Archive-specific functionality is delegated to "connector" classes like `Zooniverse` and `Alta`.
### Installation
-The client and the Zooniverse client cat be installed using pip:
+The client and a selection of connector classes client cat be installed using pip:
```sh
$ pip install git+https://git.astron.nl/astron-sdc/esap-userprofile-python-client.git
```
-### Example - Using the Shopping Client with the Zooniverse connector
+### Example
-```python
-from shopping_client import shopping_client
-from zooniverse import zooniverse
-import getpass
-
-# Prompt for Zooniverse account password
-zooniverse_password = getpass.getpass("Enter Zooniverse password:")
-
-# Instantiate Zooniverse connector
-zc = zooniverse(username="hughdickinson", password=zooniverse_password)
-
-# Instantiate ESAP User Profile shopping client, passing zooniverse connector
-sc = shopping_client(host="https://sdc-dev.astron.nl:5555/", connectors=[zc])
-
-# Retrieve basket (prompts to enter access token obtained from ESAP GUI)
-res=sc.get_basket(convert_to_pandas=True)
-
-# ... inspect available results ...
-
-# Retrieve data from Zooniverse based on basket item
-data = zc.retrieve(res["zooniverse"].loc[3],
- generate=False,
- wait=True,
- convert_to_pandas=True)
-
-# ... analyse data ...
-
-```
+See `example.py`.
## Contributing
diff --git a/__init__.py b/__init__.py
deleted file mode 100644
index adaa6db..0000000
--- a/__init__.py
+++ /dev/null
@@ -1,2 +0,0 @@
-from .zooniverse import zooniverse
-from .shopping_client import shopping_client
diff --git a/alta/__init__.py b/alta/__init__.py
deleted file mode 100644
index 93ccfeb..0000000
--- a/alta/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from alta.alta_connector import alta_connector
diff --git a/alta/alta_connector.py b/alta/alta_connector.py
deleted file mode 100644
index c514c26..0000000
--- a/alta/alta_connector.py
+++ /dev/null
@@ -1,76 +0,0 @@
-import requests
-import json
-import io
-import getpass
-import pandas as pd
-
-from typing import Union, Optional
-
-class alta_connector:
-
- name = "alta"
- archive = "apertif"
-
- def basket_item_to_pandas(
- self, basket_item: Union[dict, pd.Series], validate: bool = True
- ) -> Optional[pd.Series]:
- """Convert an item from the shopping basket into a `pd.Series` with
- optional validation.
-
- Parameters
- ----------
- basket_item : Union[dict, pd.Series]
- A single item from a retrieved shopping basket - either a raw `dict`
- or a converted `pd.Series`.
- validate : bool
- If `True`, check that the data in the shopping item conforms with
- the expected format before attempting the conversion.
-
- Returns
- -------
- Optional[pd.Series]
- `pd.Series` containing the data encoded in the shopping item or
- `NoneType`.
-
- """
- if validate:
- item_data = self.validate_basket_item(basket_item, return_loaded=True)
- else:
- item_data = json.loads(basket_item["item_data"])
- if item_data:
- return pd.Series(item_data)
- return None
-
-
- def validate_basket_item(
- self, basket_item: Union[dict, pd.Series], return_loaded: bool = False
- ) -> Union[dict, bool, None]:
- """Check that the data in the shopping item conforms with
- the expected format
-
- Parameters
- ----------
- basket_item : Union[dict, pd.Series]
- A single item from a retrieved shopping basket - either a raw `dict`
- or a converted `pd.Series`.
- return_loaded : bool
- If `True`, and validation succeeds return the extracted shopping item
- as `dict`, otherwise return `True` if validation succeeds and `None`
- otherwise.
-
- Returns
- -------
- Union[dict, bool, None]
- If `return_loaded` is `True`, return a `dict` containing the data
- encoded in the shopping item when validation succeeds.
- Otherwise if `return_loaded` is `True` validation succeeds.
- If validation fails return `None`.
-
- """
- item_data = json.loads(basket_item["item_data"])
- if "archive" in item_data and item_data["archive"] == self.archive:
- if return_loaded:
- return item_data
- else:
- return True
- return None
\ No newline at end of file
diff --git a/astron_vo/__init__.py b/astron_vo/__init__.py
deleted file mode 100644
index 00582d0..0000000
--- a/astron_vo/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from astron_vo.astron_vo_connector import astron_vo_connector
diff --git a/esap_client/__init__.py b/esap_client/__init__.py
new file mode 100644
index 0000000..d8c3a14
--- /dev/null
+++ b/esap_client/__init__.py
@@ -0,0 +1 @@
+from .shopping_client import ShoppingClient
diff --git a/esap_client/connectors/__init__.py b/esap_client/connectors/__init__.py
new file mode 100644
index 0000000..96cc6c9
--- /dev/null
+++ b/esap_client/connectors/__init__.py
@@ -0,0 +1,5 @@
+from .alta import Alta
+from .astron_vo import AstronVo
+from .samp import Samp
+from .zooniverse import Zooniverse
+from .rucio import Rucio
diff --git a/esap_client/connectors/alta.py b/esap_client/connectors/alta.py
new file mode 100644
index 0000000..9016aaf
--- /dev/null
+++ b/esap_client/connectors/alta.py
@@ -0,0 +1,5 @@
+from .baseConnector import BaseConnector
+
+class Alta(BaseConnector):
+ name = "alta"
+ archive = "apertif"
diff --git a/esap_client/connectors/astron_vo.py b/esap_client/connectors/astron_vo.py
new file mode 100644
index 0000000..de1500e
--- /dev/null
+++ b/esap_client/connectors/astron_vo.py
@@ -0,0 +1,5 @@
+from .baseConnector import BaseConnector
+
+class AstronVo(BaseConnector):
+ name = "astron_vo"
+ archive = "astron_vo"
diff --git a/astron_vo/astron_vo_connector.py b/esap_client/connectors/baseConnector.py
similarity index 96%
rename from astron_vo/astron_vo_connector.py
rename to esap_client/connectors/baseConnector.py
index 32f848a..0bb2086 100644
--- a/astron_vo/astron_vo_connector.py
+++ b/esap_client/connectors/baseConnector.py
@@ -1,14 +1,9 @@
import json
-
import pandas as pd
from typing import Union, Optional
-class astron_vo_connector:
-
- name = "astron_vo"
- archive = "astron_vo"
-
+class BaseConnector:
def basket_item_to_pandas(
self, basket_item: Union[dict, pd.Series], validate: bool = True
) -> Optional[pd.Series]:
@@ -71,4 +66,4 @@ class astron_vo_connector:
return item_data
else:
return True
- return None
\ No newline at end of file
+ return None
diff --git a/esap_client/connectors/rucio.py b/esap_client/connectors/rucio.py
new file mode 100644
index 0000000..a364ebb
--- /dev/null
+++ b/esap_client/connectors/rucio.py
@@ -0,0 +1,5 @@
+from .baseConnector import BaseConnector
+
+class Rucio(BaseConnector):
+ name = "rucio"
+ archive = "rucio"
diff --git a/esap_client/connectors/samp.py b/esap_client/connectors/samp.py
new file mode 100644
index 0000000..0ac3801
--- /dev/null
+++ b/esap_client/connectors/samp.py
@@ -0,0 +1,5 @@
+from .baseConnector import BaseConnector
+
+class Samp(BaseConnector):
+ name = "samp"
+ archive = "samp"
diff --git a/zooniverse/zooniverse.py b/esap_client/connectors/zooniverse.py
similarity index 75%
rename from zooniverse/zooniverse.py
rename to esap_client/connectors/zooniverse.py
index 91210e5..4eb668a 100644
--- a/zooniverse/zooniverse.py
+++ b/esap_client/connectors/zooniverse.py
@@ -10,8 +10,9 @@ from warnings import warn
from panoptes_client import Panoptes, Project, Workflow
from panoptes_client.panoptes import PanoptesAPIException
+from .baseConnector import BaseConnector
-class zooniverse:
+class Zooniverse(BaseConnector):
name = "zooniverse"
archive = "zooniverse"
@@ -155,7 +156,7 @@ class zooniverse:
if chunked_retrieve
else pd.read_csv(
io.BytesIO(response.content),
- converters=zooniverse.category_converters[
+ converters=Zooniverse.category_converters[
self._get_item_entry(item, "category")
],
)
@@ -192,7 +193,7 @@ class zooniverse:
chunk_frames.append(
pd.read_csv(
io.BytesIO(chunk),
- converters=zooniverse.category_converters[
+ converters=Zooniverse.category_converters[
self._get_item_entry(item, "category")
],
header=None if len(chunk_frames) else 0,
@@ -212,7 +213,7 @@ class zooniverse:
)
def _get_entity(self, item):
- entity = zooniverse.entity_types[self._get_item_entry(item, "catalog")].find(
+ entity = Zooniverse.entity_types[self._get_item_entry(item, "catalog")].find(
int(self._get_item_entry(item, self._catalogue_to_id_string(item)))
)
return entity
@@ -225,66 +226,3 @@ class zooniverse:
def _catalogue_to_id_string(self, item):
return self._get_item_entry(item, "catalog") + "_id"
-
- def basket_item_to_pandas(
- self, basket_item: Union[dict, pd.Series], validate: bool = True
- ) -> Optional[pd.Series]:
- """Convert an item from the shopping basket into a `pd.Series` with
- optional validation.
-
- Parameters
- ----------
- basket_item : Union[dict, pd.Series]
- A single item from a retrieved shopping basket - either a raw `dict`
- or a converted `pd.Series`.
- validate : bool
- If `True`, check that the data in the shopping item conforms with
- the expected format before attempting the conversion.
-
- Returns
- -------
- Optional[pd.Series]
- `pd.Series` containing the data encoded in the shopping item or
- `NoneType`.
-
- """
- if validate:
- item_data = self.validate_basket_item(basket_item, return_loaded=True)
- else:
- item_data = json.loads(basket_item["item_data"])
- if item_data:
- return pd.Series(item_data)
- return None
-
- def validate_basket_item(
- self, basket_item: Union[dict, pd.Series], return_loaded: bool = False
- ) -> Union[dict, bool, None]:
- """Check that the data in the shopping item conforms with
- the expected format
-
- Parameters
- ----------
- basket_item : Union[dict, pd.Series]
- A single item from a retrieved shopping basket - either a raw `dict`
- or a converted `pd.Series`.
- return_loaded : bool
- If `True`, and validation succeeds return the extracted shopping item
- as `dict`, otherwise return `True` if validation succeeds and `None`
- otherwise.
-
- Returns
- -------
- Union[dict, bool, None]
- If `return_loaded` is `True`, return a `dict` containing the data
- encoded in the shopping item when validation succeeds.
- Otherwise if `return_loaded` is `True` validation succeeds.
- If validation fails return `None`.
-
- """
- item_data = json.loads(basket_item["item_data"])
- if "archive" in item_data and item_data["archive"] == "zooniverse":
- if return_loaded:
- return item_data
- else:
- return True
- return None
diff --git a/shopping_client/shopping_client.py b/esap_client/shopping_client.py
similarity index 98%
rename from shopping_client/shopping_client.py
rename to esap_client/shopping_client.py
index 476284f..9dfe187 100644
--- a/shopping_client/shopping_client.py
+++ b/esap_client/shopping_client.py
@@ -14,7 +14,7 @@ import requests
logger = logging.getLogger(__name__)
-class shopping_client:
+class ShoppingClient:
endpoint = "esap-api/accounts/user-profiles/"
audience = "rucio" # Audience used by ESAP, might be configurable later
@@ -80,7 +80,7 @@ class shopping_client:
"""
if self.basket is None or reload:
- url = urllib.parse.urljoin(self.host, shopping_client.endpoint)
+ url = urllib.parse.urljoin(self.host, ShoppingClient.endpoint)
response = requests.get(url, headers=self._request_header())
if response.ok:
self.basket = json.loads(response.content)["results"][0][
diff --git a/alta/alta_example.py b/example.py
similarity index 74%
rename from alta/alta_example.py
rename to example.py
index 8a35f67..82b4597 100644
--- a/alta/alta_example.py
+++ b/example.py
@@ -1,9 +1,9 @@
-from shopping_client import shopping_client
-from alta import alta_connector
-from astron_vo import astron_vo_connector
-from samp import samp_connector
+from esap_client import ShoppingClient
+from esap_client.connectors import Alta as alta_connector
+from esap_client.connectors import AstronVo as astron_vo_connector
+from esap_client.connectors import Samp as samp_connector
-esap_api_host = "https://sdc-dev.astron.nl:5555/"
+esap_api_host = "https://sdc-dev.astron.nl:443/"
#access_token = ""
# Instantiate alta connector
@@ -12,7 +12,7 @@ vo = astron_vo_connector()
# Instantiate ESAP User Profile shopping client, passing alta connector
#sc = shopping_client(host=esap_api_host, token=access_token, connectors=[ac,vo])
-sc = shopping_client(host=esap_api_host, connectors=[ac])
+sc = ShoppingClient(host=esap_api_host, connectors=[ac])
# 'apertif'and 'astron_vo' items converted to pandas dataframe
basket_pandas=sc.get_basket(filter_archives=True, convert_to_pandas=True)
@@ -29,7 +29,7 @@ print("'apertif'and 'astron_vo' items as json")
print(basket_json)
samp_connector = samp_connector()
-sc = shopping_client(host=esap_api_host, token=access_token, connectors=[samp_connector])
+sc = ShoppingClient(host=esap_api_host, connectors=[samp_connector])
# "'SAMP' items converted to pandas dataframe:"
basket_pandas=sc.get_basket(convert_to_pandas=True, filter_archives=True)
@@ -42,4 +42,4 @@ print(basket_pandas)
basket_json=sc.get_basket(convert_to_pandas=False, filter_archives=True)
print('------------------------------------')
print("'SAMP' items as json:")
-print(basket_json)
\ No newline at end of file
+print(basket_json)
diff --git a/rucio_cli/__init__.py b/rucio_cli/__init__.py
deleted file mode 100644
index 720a2b2..0000000
--- a/rucio_cli/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from rucio_cli.rucio_connector import rucio_connector
diff --git a/rucio_cli/rucio_connector.py b/rucio_cli/rucio_connector.py
deleted file mode 100644
index 3c98e9c..0000000
--- a/rucio_cli/rucio_connector.py
+++ /dev/null
@@ -1,76 +0,0 @@
-import requests
-import json
-import io
-import getpass
-import pandas as pd
-
-from typing import Union, Optional
-
-class rucio_connector:
-
- name = "rucio"
- archive = "rucio"
-
- def basket_item_to_pandas(
- self, basket_item: Union[dict, pd.Series], validate: bool = True
- ) -> Optional[pd.Series]:
- """Convert an item from the shopping basket into a `pd.Series` with
- optional validation.
-
- Parameters
- ----------
- basket_item : Union[dict, pd.Series]
- A single item from a retrieved shopping basket - either a raw `dict`
- or a converted `pd.Series`.
- validate : bool
- If `True`, check that the data in the shopping item conforms with
- the expected format before attempting the conversion.
-
- Returns
- -------
- Optional[pd.Series]
- `pd.Series` containing the data encoded in the shopping item or
- `NoneType`.
-
- """
- if validate:
- item_data = self.validate_basket_item(basket_item, return_loaded=True)
- else:
- item_data = json.loads(basket_item["item_data"])
- if item_data:
- return pd.Series(item_data)
- return None
-
-
- def validate_basket_item(
- self, basket_item: Union[dict, pd.Series], return_loaded: bool = False
- ) -> Union[dict, bool, None]:
- """Check that the data in the shopping item conforms with
- the expected format
-
- Parameters
- ----------
- basket_item : Union[dict, pd.Series]
- A single item from a retrieved shopping basket - either a raw `dict`
- or a converted `pd.Series`.
- return_loaded : bool
- If `True`, and validation succeeds return the extracted shopping item
- as `dict`, otherwise return `True` if validation succeeds and `None`
- otherwise.
-
- Returns
- -------
- Union[dict, bool, None]
- If `return_loaded` is `True`, return a `dict` containing the data
- encoded in the shopping item when validation succeeds.
- Otherwise if `return_loaded` is `True` validation succeeds.
- If validation fails return `None`.
-
- """
- item_data = json.loads(basket_item["item_data"])
- if "archive" in item_data and item_data["archive"] == self.archive:
- if return_loaded:
- return item_data
- else:
- return True
- return None
diff --git a/samp/__init__.py b/samp/__init__.py
deleted file mode 100644
index e7c8867..0000000
--- a/samp/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from samp.samp_connector import samp_connector
\ No newline at end of file
diff --git a/samp/samp_connector.py b/samp/samp_connector.py
deleted file mode 100644
index 5035b6e..0000000
--- a/samp/samp_connector.py
+++ /dev/null
@@ -1,76 +0,0 @@
-import requests
-import json
-import io
-import getpass
-import pandas as pd
-
-from typing import Union, Optional
-
-class samp_connector:
-
- name = "samp"
- archive = "samp"
-
- def basket_item_to_pandas(
- self, basket_item: Union[dict, pd.Series], validate: bool = True
- ) -> Optional[pd.Series]:
- """Convert an item from the shopping basket into a `pd.Series` with
- optional validation.
-
- Parameters
- ----------
- basket_item : Union[dict, pd.Series]
- A single item from a retrieved shopping basket - either a raw `dict`
- or a converted `pd.Series`.
- validate : bool
- If `True`, check that the data in the shopping item conforms with
- the expected format before attempting the conversion.
-
- Returns
- -------
- Optional[pd.Series]
- `pd.Series` containing the data encoded in the shopping item or
- `NoneType`.
-
- """
- if validate:
- item_data = self.validate_basket_item(basket_item, return_loaded=True)
- else:
- item_data = json.loads(basket_item["item_data"])
- if item_data:
- return pd.Series(item_data)
- return None
-
-
- def validate_basket_item(
- self, basket_item: Union[dict, pd.Series], return_loaded: bool = False
- ) -> Union[dict, bool, None]:
- """Check that the data in the shopping item conforms with
- the expected format
-
- Parameters
- ----------
- basket_item : Union[dict, pd.Series]
- A single item from a retrieved shopping basket - either a raw `dict`
- or a converted `pd.Series`.
- return_loaded : bool
- If `True`, and validation succeeds return the extracted shopping item
- as `dict`, otherwise return `True` if validation succeeds and `None`
- otherwise.
-
- Returns
- -------
- Union[dict, bool, None]
- If `return_loaded` is `True`, return a `dict` containing the data
- encoded in the shopping item when validation succeeds.
- Otherwise if `return_loaded` is `True` validation succeeds.
- If validation fails return `None`.
-
- """
- item_data = json.loads(basket_item["item_data"])
- if "archive" in item_data and item_data["archive"] == self.archive:
- if return_loaded:
- return item_data
- else:
- return True
- return None
\ No newline at end of file
diff --git a/setup.py b/setup.py
index 6393a34..0b99233 100644
--- a/setup.py
+++ b/setup.py
@@ -4,8 +4,8 @@ with open("README.md", "r") as fh:
long_description = fh.read()
setuptools.setup(
- name="esap-userprofile-python-client",
- version="0.0.4",
+ name="esap_client",
+ version="0.0.5",
author="Hugh Dickinson",
author_email="hugh.dickinson@open.ac.uk",
description="Python client for ESAP Data Discovery Shopping Basket",
diff --git a/shopping_client/__init__.py b/shopping_client/__init__.py
deleted file mode 100644
index 3e4b035..0000000
--- a/shopping_client/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from .shopping_client import shopping_client
diff --git a/zooniverse/__init__.py b/zooniverse/__init__.py
deleted file mode 100644
index 1d4ed23..0000000
--- a/zooniverse/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-from .zooniverse import zooniverse
--
GitLab