From 7d28fc79f81ec452c1344fdf78bb482583a512fc Mon Sep 17 00:00:00 2001
From: Jorrit Schaap <schaap@astron.nl>
Date: Wed, 7 Sep 2016 07:03:58 +0000
Subject: [PATCH] Task #8721: run srmls on lexar1..4
---
LTA/ltastorageoverview/lib/scraper.py | 26 +++++++++++++-------------
1 file changed, 13 insertions(+), 13 deletions(-)
diff --git a/LTA/ltastorageoverview/lib/scraper.py b/LTA/ltastorageoverview/lib/scraper.py
index d20820f92d5..a0ca6dbdbe4 100755
--- a/LTA/ltastorageoverview/lib/scraper.py
+++ b/LTA/ltastorageoverview/lib/scraper.py
@@ -35,7 +35,7 @@ import threading
import multiprocessing
from ltastorageoverview import store
from lofar.common.util import humanreadablesize
-from random import random
+from random import random, randint
logger = logging.getLogger()
@@ -123,12 +123,14 @@ class Location:
# the core command: do an srmls call and parse the results
# srmls can only yield max 900 items in a result, hence we can recurse for the next 900 by using the offset
- cmd = ["bash", "-c", "\'source %s;srmls -l -count=900 -offset=%d %s%s\'" % ('/globalhome/ingest/service/bin/init.sh', offset, self.srmurl, self.directory)]
- hostname = socket.gethostname()
- if not 'lexar' in hostname:
- cmd = ['ssh', 'ingest@10.178.1.2'] + cmd
-
- # logger.debug(' '.join(cmd))
+ lexar_nr = randint(1,4)
+ cmd = ['ssh', 'ingest@10.178.1.%d' % lexar_nr, "bash", "-c", "\'source %s;srmls -l -count=900 -offset=%d %s%s\'" % (
+ '/globalhome/ingest/service/bin/init.sh' if lexar_nr <= 2 else '/globalhome/ingest/.grid/.ingest_profile',
+ offset,
+ self.srmurl,
+ self.directory) ]
+
+ logger.info(' '.join(cmd))
p = subprocess.Popen(cmd, stdin=open('/dev/null'), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
logs = p.communicate()
# logger.debug('Shell command for %s exited with code %s' % (self.path(), p.returncode))
@@ -296,9 +298,9 @@ class ResultGetterThread(threading.Thread):
def populateDbWithLTASitesAndRootDirs(db):
if not db.sites():
db.insertSite('target', 'srm://srm.target.rug.nl:8444')
- #db.insertSite('nikhef', 'srm://tbn18.nikhef.nl:8446')
- #db.insertSite('sara', 'srm://srm.grid.sara.nl:8443')
- #db.insertSite('juelich', 'srm://lofar-srm.fz-juelich.de:8443')
+ db.insertSite('nikhef', 'srm://tbn18.nikhef.nl:8446')
+ db.insertSite('sara', 'srm://srm.grid.sara.nl:8443')
+ db.insertSite('juelich', 'srm://lofar-srm.fz-juelich.de:8443')
db.insertSite('poznan', 'srm://lta-head.lofar.psnc.pl:8443')
#db.insertRootDirectory('target', '/lofar/ops')
@@ -338,7 +340,6 @@ def main():
db = store.LTAStorageDb(dbcreds, options.verbose)
populateDbWithLTASitesAndRootDirs(db)
- exit()
# for each site we want one or more ResultGetterThreads
# so make a dict with a list per site based on the locations
@@ -376,8 +377,7 @@ def main():
#while numLocationsInQueues() > 0 and (totalNumGetters() <= 4 or
#(os.getloadavg()[0] < 3*multiprocessing.cpu_count() and
#totalNumGetters() < 2.5*multiprocessing.cpu_count())):
- while numLocationsInQueues() > 0 and (totalNumGetters() < 8):
-
+ while numLocationsInQueues() > 0 and (totalNumGetters() < 4):
with lock:
sitesStats = db.visitStats(datetime.datetime.utcnow() - datetime.timedelta(days=1))
--
GitLab