diff --git a/LTA/ltastorageoverview/lib/scraper.py b/LTA/ltastorageoverview/lib/scraper.py index b6729796796d4fc52f05688440ede2697b2681b3..2041325f512e2b56ecd2117407dfbf2be091b140 100755 --- a/LTA/ltastorageoverview/lib/scraper.py +++ b/LTA/ltastorageoverview/lib/scraper.py @@ -88,7 +88,7 @@ class Location: a directory at the storage site. for example: /pnfs/grid.sara.nl/data/lofar/storage ''' self.srmurl = srmurl - self.directory = directory + self.directory = directory.rstrip('/') def path(self): '''returns the full path srmurl + directory''' @@ -127,6 +127,7 @@ class Location: # srmls can only yield max 900 items in a result, hence we can recurse for the next 900 by using the offset lexar_nr = 4 #randint(3,4) lexar_host = 'ingest@10.178.1.%d' % (lexar_nr,) + lexar_host = 'ingest@10.144.4.%d' % (74+lexar_nr,) cmd = ['ssh', '-tt', '-n', '-x', '-q', lexar_host, "bash", "-c", "\'source %s;srmls -l -count=900 -offset=%d %s%s\'" % ( '/globalhome/ingest/service/bin/init.sh' if lexar_nr <= 2 else '/globalhome/ingest/.grid/.ingest_profile', offset, @@ -221,7 +222,7 @@ class LocationResult: A list of files in this location ''' self.location = location - self.subDirectories = [s.rstrip('/') for s in subDirectories] if subDirectories else [] + self.subDirectories = subDirectories if subDirectories else [] self.files = files if files else [] def __str__(self): @@ -458,8 +459,8 @@ def main(): # spawn new ResultGetterThreads # do not overload this host system while (numLocationsInQueues() > 0 and - totalNumGetters() < options.parallel and - os.getloadavg()[0] < 4*multiprocessing.cpu_count()): + totalNumGetters() < options.parallel and + os.getloadavg()[0] < 4*multiprocessing.cpu_count()): sitesStats = db.visitStats(datetime.datetime.utcnow() - VISIT_INTERVAL) for site_name, site_stats in sitesStats.items():