Task #8721: run srmls on lexar1..4

LTA/ltastorageoverview/lib/scraper.py

@@ -35,7 +35,7 @@ import threading
 import multiprocessing
 from ltastorageoverview import store
 from lofar.common.util import humanreadablesize
-from random import random
+from random import random, randint
 
 logger = logging.getLogger()
 
@@ -123,12 +123,14 @@ class Location:
 
         # the core command: do an srmls call and parse the results
         # srmls can only yield max 900 items in a result, hence we can recurse for the next 900 by using the offset
-        cmd = ["bash", "-c", "\'source %s;srmls -l -count=900 -offset=%d %s%s\'" % ('/globalhome/ingest/service/bin/init.sh', offset, self.srmurl, self.directory)]
-        hostname = socket.gethostname()
-        if not 'lexar' in hostname:
-            cmd = ['ssh', 'ingest@10.178.1.2'] + cmd
-
-        # logger.debug(' '.join(cmd))
+        lexar_nr = randint(1,4)
+        cmd = ['ssh', 'ingest@10.178.1.%d' % lexar_nr, "bash", "-c", "\'source %s;srmls -l -count=900 -offset=%d %s%s\'" % (
+                '/globalhome/ingest/service/bin/init.sh' if lexar_nr <= 2 else '/globalhome/ingest/.grid/.ingest_profile',
+                offset,
+                self.srmurl,
+                self.directory) ]
+
+        logger.info(' '.join(cmd))
         p = subprocess.Popen(cmd, stdin=open('/dev/null'), stdout=subprocess.PIPE, stderr=subprocess.PIPE)
         logs = p.communicate()
         # logger.debug('Shell command for %s exited with code %s' % (self.path(), p.returncode))
@@ -296,9 +298,9 @@ class ResultGetterThread(threading.Thread):
 def populateDbWithLTASitesAndRootDirs(db):
     if not db.sites():
         db.insertSite('target', 'srm://srm.target.rug.nl:8444')
-        #db.insertSite('nikhef', 'srm://tbn18.nikhef.nl:8446')
-        #db.insertSite('sara', 'srm://srm.grid.sara.nl:8443')
-        #db.insertSite('juelich', 'srm://lofar-srm.fz-juelich.de:8443')
+        db.insertSite('nikhef', 'srm://tbn18.nikhef.nl:8446')
+        db.insertSite('sara', 'srm://srm.grid.sara.nl:8443')
+        db.insertSite('juelich', 'srm://lofar-srm.fz-juelich.de:8443')
         db.insertSite('poznan', 'srm://lta-head.lofar.psnc.pl:8443')
 
         #db.insertRootDirectory('target', '/lofar/ops')
@@ -338,7 +340,6 @@ def main():
 
     db = store.LTAStorageDb(dbcreds, options.verbose)
     populateDbWithLTASitesAndRootDirs(db)
-    exit()
     
     # for each site we want one or more ResultGetterThreads
     # so make a dict with a list per site based on the locations
@@ -376,8 +377,7 @@ def main():
             #while numLocationsInQueues() > 0 and (totalNumGetters() <= 4 or
                                                   #(os.getloadavg()[0] < 3*multiprocessing.cpu_count() and
                                                   #totalNumGetters() < 2.5*multiprocessing.cpu_count())):
-            while numLocationsInQueues() > 0 and (totalNumGetters() < 8):
-
+            while numLocationsInQueues() > 0 and (totalNumGetters() < 4):
                 with lock:
                     sitesStats = db.visitStats(datetime.datetime.utcnow() - datetime.timedelta(days=1))