Skip to content
Snippets Groups Projects
Commit 1443f4e5 authored by Jorrit Schaap's avatar Jorrit Schaap
Browse files

SW-720: replaced obsolete/unmaintained 3rd party package mechanize by requests

parent b5529de2
No related branches found
No related tags found
1 merge request!4Lofar release 4 0 minor fixes
Hide whitespace changes
Inline Side-by-side
LTA/LTAIngest/LTAIngestServer/LTAIngestTransferServer/lib/CMakeLists.txt
+ 1
− 1
View file @ 1443f4e5
find_python_module(mechanize REQUIRED) #sudo pip3 install mechanize
find_python_module(requests REQUIRED) #sudo pip3 install requests
python_install(ingesttransferserver.py
ltacp.py
......
LTA/LTAIngest/LTAIngestServer/LTAIngestTransferServer/lib/momclient.py
+ 44
− 65
View file @ 1443f4e5
#!/usr/bin/env python3
import requests
import logging
import time
import http.cookiejar
import urllib.request, urllib.parse, urllib.error
logger = logging.getLogger()
from lofar.lta.ingest.common.job import jobState2String
from lofar.lta.ingest.server.config import MOM_BASE_URL
from lofar.common import isProductionEnvironment
from lofar.lta.ingest.server.sip import *
from lofar.common.util import humanreadablesize
logger = logging.getLogger()
import mechanize
# ignore ssl errors from expired/selfsigned mom ssl certificate
import ssl
ssl._create_default_https_context = ssl._create_unverified_context
import urllib3
urllib3.disable_warnings()
class MoMClient:
......@@ -34,22 +30,10 @@ class MoMClient:
self.__user = user
self.__password = password
self.__logged_in = False
self.__browser = mechanize.Browser()
cookiejar = http.cookiejar.CookieJar()
self.__browser.set_cookiejar(cookiejar)
self.__browser.set_handle_robots(False)
self.__browser.set_handle_equiv(True)
self.__browser.set_handle_redirect(True)
self.__browser.set_handle_referer(True)
self.__browser.addheaders = [('User-agent', 'Firefox')]
# self.__browser.set_debug_http(logger.level == logging.DEBUG)
# self.__browser.set_debug_redirects(logger.level == logging.DEBUG)
# self.__browser.set_debug_responses(logger.level == logging.DEBUG)
self.__session = None
self.__momURLlogin = MOM_BASE_URL + 'useradministration/user/systemlogin.do'
self.__momUR_security_check = MOM_BASE_URL + 'useradministration/user/j_security_check'
self.__momURLgetSIP = MOM_BASE_URL + 'mom3/interface/importXML2.do'
self.__momURLsetStatus = MOM_BASE_URL + 'mom3/interface/service/setStatusDataProduct.do'
self.__momURLlogout = MOM_BASE_URL + 'useradministration/user/logout.do'
......@@ -58,31 +42,31 @@ class MoMClient:
def login(self):
try:
if self.__logged_in:
return
if self.__session is not None:
self.logout()
logger.debug("logging in to MoM on url: %s", self.__momURLlogin)
self.__browser.open(self.__momURLlogin)
forms = list(self.__browser.forms())
self.__browser.form = forms[0]
self.__browser.form['j_username'] = self.__user
self.__browser.form['j_password'] = self.__password
ret_code = self.__browser.submit().code
if 200 != ret_code:
raise Exception("Logging into MoM failed: http return code = " + ret_code)
session = requests.session()
r = session.get(self.__momURLlogin, verify=False)
if 200 != r.status_code:
raise Exception("Logging into MoM on %s failed: http return code = %s" % (self.__momURLlogin, r.status_code))
r = session.post(self.__momUR_security_check, data={'j_username': self.__user, 'j_password': self.__password}, verify=False)
if 200 != r.status_code:
raise Exception("Logging into MoM on %s failed: http return code = %s" % (self.__momUR_security_check, r.status_code))
logger.debug("logged in on MoM on url: %s", self.__momURLlogin)
self.__logged_in = True
self.__session = session
except Exception as e:
raise Exception("Logging into MoM on %s failed: %s" % (self.__momURLlogin, str(e)))
def logout(self):
try:
if self.__logged_in:
if self.__session is not None:
logger.debug("logging out of MoM on url: %s", self.__momURLlogout)
self.__browser.open(self.__momURLlogout)
self.__browser.close()
self.__logged_in = False
self.__session.get(self.__momURLlogout, verify=False)
self.__session.close()
self.__session = None
logger.debug("logged out of MoM on url: %s", self.__momURLlogout)
except Exception as e:
logger.warning("Logging out of MoM failed: " + str(e))
......@@ -103,11 +87,11 @@ class MoMClient:
self.login()
params = {"exportId" : export_id, "status" : status_id}
statusUrl = self.__momURLsetStatus + '?' + urllib.parse.urlencode(params)
logger.info("updating MoM: " + statusUrl)
response = self.__browser.open(statusUrl)
reply = [line.decode('utf-8') for line in response.readlines()]
if reply == ['ok']:
logger.info("updating MoM on url: %s with params: %s", self.__momURLsetStatus, params)
response = self.__session.get(self.__momURLsetStatus, params=params)
reply = response.text.strip()
if reply == 'ok':
logger.info('MoMClient.setStatus updated status of %s to %s', export_id, jobState2String(int(status_id)))
if message:
......@@ -120,12 +104,11 @@ class MoMClient:
message = message[:97] + '...'
params['message'] = message
logger.info("updating MoM (again to set the message) on url: %s with params: %s", self.__momURLsetStatus, params)
response = self.__session.get(self.__momURLsetStatus, params=params)
reply = response.text.strip()
statusUrl = self.__momURLsetStatus + '?' + urllib.parse.urlencode(params)
logger.info("updating MoM: " + statusUrl)
response = self.__browser.open(statusUrl)
reply = [line.decode('utf-8') for line in response.readlines()]
if reply == ['ok']:
if reply == 'ok':
logger.info('MoMClient.setStatus updated status of %s to %s with message: %s',
export_id,
jobState2String(int(status_id)),
......@@ -137,15 +120,15 @@ class MoMClient:
logger.error('MoMClient.setStatus could not update status of %s to %s using url: %s reply: %s',
export_id,
jobState2String(int(status_id)),
statusUrl,
response.url,
reply)
self.logout()
if 'DOCTYPE HTML PUBLIC' in reply:
logger.error('MoM returned login screen instead of SIP for archive_id=%s mom_id=%s using url %s and data %s', archive_id, mom_id, self.__momURLgetSIP, data)
logger.error('MoM returned login screen instead of SIP for export_id: %s on url %s', export_id, response.url)
wait_secs = (mom_retry + 1) * (mom_retry + 1) * 10
logger.info('Retrying to setStatus for archiveId %s in %s seconds', archive_id, wait_secs)
logger.info('Retrying to setStatus for export_id %s in %s seconds', export_id, wait_secs)
time.sleep(wait_secs)
continue # jump back to for mom_retry in range(self.MAX_MOM_RETRIES)
......@@ -193,15 +176,14 @@ class MoMClient:
while ' ' in xmlcontent:
xmlcontent = xmlcontent.replace(' ', ' ')
data = urllib.parse.urlencode({"command" : "get-sip-with-input", "xmlcontent" : xmlcontent})
# Now get that file-like object again, remembering to mention the data.
response = self.__browser.open(self.__momURLgetSIP, data)
result = response.read().decode('utf-8')
params = {"command": "get-sip-with-input", "xmlcontent" : xmlcontent}
response = self.__session.get(self.__momURLgetSIP, params=params)
result = response.text
result = result.replace('<stationType>Europe</stationType>', '<stationType>International</stationType>')
if 'DOCTYPE HTML PUBLIC' in result:
logger.error('MoM returned login screen instead of SIP for %s %s using url %s and data %s',
archive_id, filename, self.__momURLgetSIP, data)
logger.error('MoM returned login screen instead of SIP for %s %s using url %s and params %s, full url=%s',
archive_id, filename, self.__momURLgetSIP, params, response.url)
# logout, even though we think we should be logged in properly
# it's mom who thinks we should login again, even though we have a proper session.
......@@ -213,7 +195,7 @@ class MoMClient:
# let's give it a final try with just GetSip
# we'll miss some data in the SIP, which we can add ourselves, so the LTA catalogue is up-to-date
# but MoM will miss these parameters. tough luck.
logger.warn("MoMClient.uploadDataAndGetSIP with archiveId %s - StorageTicket %s - FileName %s - Uri %s failed %s times. Trying normal GetSip without uploading data to MoM.", archive_id, storage_ticket, filename, uri, mom_retry)
logger.warning("MoMClient.uploadDataAndGetSIP with archiveId %s - StorageTicket %s - FileName %s - Uri %s failed %s times. Trying normal GetSip without uploading data to MoM.", archive_id, storage_ticket, filename, uri, mom_retry)
result = self.getSIP(archive_id, validate = False)
# add ingest info to sip
result = addIngestInfoToSIP(result, storage_ticket, filesize, md5_checksum, adler32_checksum)
......@@ -266,11 +248,9 @@ class MoMClient:
mom_id = archive_id - 1000000 # stupid mom one million archive_id offset
data = urllib.parse.urlencode({"command" : "GETSIP", "id" : mom_id})
# Now get that file-like object again, remembering to mention the data.
logger.info('%s: GetSip call: %s %s', log_prefix, self.__momURLgetSIP, data)
response = self.__browser.open(self.__momURLgetSIP, data)
result = response.read().decode('utf-8')
# logger.info('%s: GetSip call: %s %s', log_prefix, self.__momURLgetSIP, data)
response = self.__session.get(self.__momURLgetSIP, params={"command" : "GETSIP", "id" : mom_id})
result = response.text
if 'DOCTYPE HTML PUBLIC' in result:
logger.error('%s: MoM returned login screen instead of SIP for archive_id=%s mom_id=%s using url %s and data %s',
......@@ -293,4 +273,3 @@ class MoMClient:
self.logout()
raise Exception("getting SIP from MoM failed: " + str(e))
return ''
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment