Merge branch 'make_xmlrpc_use_socks_proxy' into 'master'

Make xmlrpc use socks proxy

See merge request !1

lib/query.py

@@ -4,76 +4,110 @@
 import urllib.request, urllib.parse, urllib.error
 import requests
 from os.path import expanduser, exists
-import xml.etree.ElementTree as ET
+import os
 import xmlrpc.client
-import uuid
-import copy
-import logging
-logger = logging.getLogger(__name__)
-
-path = expanduser("~/.siplibrc")
-user = None
-passw = None
-#host = "lta-ingest-test.lofar.eu:19443"
-host = "lofar-ingest.target.rug.nl:9443"
-
-if not exists(path):
-    # write default file
-    with open(path, 'w') as file:
-        file.write("user=\n")
-        file.write("password=\n")
-        file.write("host=\n")
-
-with open(path,'r') as file:
-        logger.info("Parsing user credentials from %s" % path)
-        for line in file:
-            if line.startswith("user"):
-                user = line.split('=')[1].strip()
-            if line.startswith("password"):
-                passw = line.split('=')[1].strip()
-            if line.startswith("host"):
-                host = line.split('=')[1].strip()
+from configparser import ConfigParser
+
+
+def parse_config(path=None):
+    default_path = expanduser("~/.siplibrc")
+    config = ConfigParser()
+    if path:
+        config_path = path
+    else:
+        config_path = default_path
+
+    found = config.read([config_path])
+    if not found:
+        raise SystemExit(f'Cannot find configuration files {default_path}: please create one')
+    return config
+
+
+DEFAULT_CONFIG = parse_config()
+
+user = DEFAULT_CONFIG['DEFAULT']['user']
+passw = DEFAULT_CONFIG['DEFAULT']['passw']
+host = DEFAULT_CONFIG['DEFAULT']['host']
 
 login_data = {
     'j_username': user,
     'j_password': passw
 }
 
+
+class RequestsTransport(xmlrpc.client.SafeTransport):
+    """
+    Transport method which supports proxies
+    """
+    # change our user agent to reflect Requests
+    user_agent = "Python XMLRPC with Requests (python-requests.org)"
+
+    def __init__(self, use_https=True, cert=None, verify=None, *args, **kwargs):
+        self.cert = cert
+        self.verify = verify
+        self.use_https = use_https
+        xmlrpc.client.SafeTransport.__init__(self, *args, **kwargs)
+
+    def request(self, host, handler, request_body, verbose=False):
+        """
+        Make an xmlrpc request.
+        """
+        headers = {'User-Agent': self.user_agent}
+        url = self._build_url(host, handler)
+        proxies = {}
+        if 'http_proxy' in os.environ:
+            proxies['http_proxy'] = os.environ['http_proxy']
+        if 'https_proxy' in os.environ:
+            proxies['https_proxy'] = os.environ['https_proxy']
+        try:
+            resp = requests.post(url, data=request_body, headers=headers,
+                                 stream=True,
+                                 cert=self.cert, verify=self.verify,
+                                 proxies=proxies)
+        except ValueError:
+            raise
+        except Exception:
+            raise  # something went wrong
+        else:
+            try:
+                resp.raise_for_status()
+            except requests.RequestException as e:
+                raise xmlrpc.client.ProtocolError(url, resp.status_code,
+                                                  str(e), resp.headers)
+            else:
+                self.verbose = verbose
+                return self.parse_response(resp.raw)
+
+    def _build_url(self, host, handler):
+        """
+        Build a url for our request based on the host, handler and use_http
+        property
+        """
+        scheme = 'https' if self.use_https else 'http'
+        return '%s://%s/%s' % (scheme, host, handler.lstrip('/'))
+
+
+class SafeRequestsTransport(RequestsTransport):
+    def __init__(self, *args, cert=None, verify=None, **kwargs):
+        super(SafeRequestsTransport, self).__init__(*args, use_https=True, cert=cert, verify=verify, **kwargs)
+
+
+class UnSafeRequestsTransport(RequestsTransport):
+    def __init__(self, *args, **kwargs):
+        super(UnSafeRequestsTransport, self).__init__(*args, use_https=False, cert=None, verify=None, **kwargs)
+
+
 url = 'https://' + user + ':' + passw + '@' + host
-client = xmlrpc.client.ServerProxy(url)
+client = xmlrpc.client.ServerProxy(url, transport=SafeRequestsTransport())
 
-# id_cache = {}
 
 def _call_idservice(source, userlabel=None):
-
     if userlabel is not None:
         response = client.GetUniqueIDForLabel(source, userlabel)
     else:
         response = client.GetUniqueID(source)
     return response
 
-# for testing:
-    # if userlabel in id_cache:
-    #     print "using existing", userlabel
-    #     response = id_cache.get(userlabel)
-    # else:
-    #     print "creating new", userlabel
-    #     response = {"version": "version",
-    #                 "result": "ok",
-    #                 "id": uuid.uuid1().int>>64,
-    #                 "user_label": userlabel,
-    #                 "data_type": "type",
-    #                 "identifier_source": source,
-    #                 "is_new": True,
-    #                 "error": ''}
-    #     if userlabel is not None:
-    #         print "keeping copy", userlabel
-    #         keeper = copy.deepcopy(response)
-    #         keeper["is_new"] = False
-    #         id_cache[userlabel] = keeper
-    #
-    # return response
-
 
 def create_unique_id(source, userlabel=None):
     """
@@ -103,9 +137,9 @@ def get_unique_id(source, userlabel):
     return response.get('id')
 
 
-
 def get_dataproduct_sip(projectname, dataproductid):
     return client.GetSip(projectname, dataproductid).get("sip")
 
+
 def get_dataproduct_ids(projectname, sasid):
     return client.GetDataProductIDS(projectname, sasid).get("ids")