From ef5b6dc589ff3ab9a3e92c2fe7241fe86009acce Mon Sep 17 00:00:00 2001
From: Jorrit Schaap <schaap@astron.nl>
Date: Wed, 7 Sep 2016 12:52:27 +0000
Subject: [PATCH] Task #9607: cmdline options
---
LTA/ltastorageoverview/lib/scraper.py | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
diff --git a/LTA/ltastorageoverview/lib/scraper.py b/LTA/ltastorageoverview/lib/scraper.py
index a0ca6dbdbe4..d70c07cff3d 100755
--- a/LTA/ltastorageoverview/lib/scraper.py
+++ b/LTA/ltastorageoverview/lib/scraper.py
@@ -130,7 +130,7 @@ class Location:
self.srmurl,
self.directory) ]
- logger.info(' '.join(cmd))
+ logger.debug(' '.join(cmd))
p = subprocess.Popen(cmd, stdin=open('/dev/null'), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
logs = p.communicate()
# logger.debug('Shell command for %s exited with code %s' % (self.path(), p.returncode))
@@ -326,7 +326,9 @@ def main():
# Check the invocation arguments
parser = OptionParser("%prog [options]", description='runs the lta scraper and stores results in the speficied database.')
+ parser.add_option('-j', '--parallel', dest='parallel', type='int', default=1, help='number of parallel srmls jobs to run, default: %default')
parser.add_option('-V', '--verbose', dest='verbose', action='store_true', help='verbose logging')
+ parser.add_option('-Q', '--log-queries', dest='log_queries', action='store_true', help='log all pqsl queries')
parser.add_option_group(dbcredentials.options_group(parser))
parser.set_defaults(dbcredentials="LTASO")
(options, args) = parser.parse_args()
@@ -338,7 +340,7 @@ def main():
logger.info("Using dbcreds: %s" % dbcreds.stringWithHiddenPassword())
- db = store.LTAStorageDb(dbcreds, options.verbose)
+ db = store.LTAStorageDb(dbcreds, options.log_queries)
populateDbWithLTASitesAndRootDirs(db)
# for each site we want one or more ResultGetterThreads
@@ -377,7 +379,7 @@ def main():
#while numLocationsInQueues() > 0 and (totalNumGetters() <= 4 or
#(os.getloadavg()[0] < 3*multiprocessing.cpu_count() and
#totalNumGetters() < 2.5*multiprocessing.cpu_count())):
- while numLocationsInQueues() > 0 and (totalNumGetters() < 4):
+ while numLocationsInQueues() > 0 and (totalNumGetters() < options.parallel):
with lock:
sitesStats = db.visitStats(datetime.datetime.utcnow() - datetime.timedelta(days=1))
--
GitLab