diff --git a/.gitattributes b/.gitattributes
index 4792ccf2df1d3e257f509c0d6cb6903eb0667311..c39cee0011d9b139d9870241d3b42112b5bb5f94 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -2089,6 +2089,9 @@ LTA/ltastorageoverview/test/integration_test_store.sh -text
LTA/ltastorageoverview/test/test_lso_webservice.py -text
LTA/ltastorageoverview/test/test_lso_webservice.run -text
LTA/ltastorageoverview/test/test_lso_webservice.sh -text
+LTA/ltastorageoverview/test/test_scraper.py -text
+LTA/ltastorageoverview/test/test_scraper.run -text
+LTA/ltastorageoverview/test/test_scraper.sh -text
LTA/ltastorageoverview/test/test_store.py -text
LTA/ltastorageoverview/test/test_store.run -text
LTA/ltastorageoverview/test/test_store.sh -text
diff --git a/LTA/ltastorageoverview/lib/scraper.py b/LTA/ltastorageoverview/lib/scraper.py
index 53aac285d4d088f6a0d6279798632da705659c9c..318eb781e086c92d9765345a75affd7aab870d2a 100755
--- a/LTA/ltastorageoverview/lib/scraper.py
+++ b/LTA/ltastorageoverview/lib/scraper.py
@@ -40,6 +40,8 @@ from random import random, randint
logger = logging.getLogger()
VISIT_INTERVAL = datetime.timedelta(days=3)
+LEXAR_HOST = 'ingest@lexar004'
+LEXAR_HOST = 'ingest@10.178.1.4'
class FileInfo:
'''Simple struct to hold filename and size'''
@@ -89,8 +91,14 @@ class Location:
directory : int
a directory at the storage site. for example: /pnfs/grid.sara.nl/data/lofar/storage
'''
- self.srmurl = srmurl
- self.directory = directory.rstrip('/')
+ self.srmurl = srmurl.rstrip('/')
+ self.directory = directory.rstrip('/') if len(directory) > 1 else directory
+
+ if not self.srmurl.startswith('srm://'):
+ raise ValueError('malformed srm url: %s' % (self.srmurl,))
+
+ if not self.directory.startswith('/'):
+ raise ValueError('malformed directory path: "%s". should start with a /' % (self.directory,))
def path(self):
'''returns the full path srmurl + directory'''
@@ -127,11 +135,8 @@ class Location:
# the core command: do an srmls call and parse the results
# srmls can only yield max 900 items in a result, hence we can recurse for the next 900 by using the offset
- lexar_nr = 4 #randint(3,4)
- lexar_host = 'ingest@10.178.1.%d' % (lexar_nr,)
- lexar_host = 'ingest@10.144.4.%d' % (74+lexar_nr,)
- cmd = ['ssh', '-tt', '-n', '-x', '-q', lexar_host, "bash", "-c", "\'source %s;srmls -l -count=900 -offset=%d %s%s\'" % (
- '/globalhome/ingest/service/bin/init.sh' if lexar_nr <= 2 else '/globalhome/ingest/.grid/.ingest_profile',
+ cmd = ['ssh', '-tt', '-n', '-x', '-q', LEXAR_HOST, "bash", "-c",
+ "\'source /globalhome/ingest/.grid/.ingest_profile; srmls -l -count=900 -offset=%d %s%s\'" % (
offset,
self.srmurl,
self.directory) ]
@@ -330,10 +335,8 @@ class ResultGetterThread(threading.Thread):
result_file_tuple = result_file_tuple_dict[key]
known_size = int(known_file['size'])
- known_creation_date = known_file['creation_date']
result_size = result_file_tuple[1]
- result_creation_date = result_file_tuple[2]
if known_size != result_size:
logger.info("%s %s: updating %s (id=%d) size from %d to %d",
diff --git a/LTA/ltastorageoverview/test/CMakeLists.txt b/LTA/ltastorageoverview/test/CMakeLists.txt
index ff8afee6eb7ae4aa5758855f0eca0d95c56c9816..b2142f8b4d86c0e5d7aa44c46a54c483b8e9b092 100644
--- a/LTA/ltastorageoverview/test/CMakeLists.txt
+++ b/LTA/ltastorageoverview/test/CMakeLists.txt
@@ -2,6 +2,7 @@
include(LofarCTest)
lofar_add_test(test_store)
+lofar_add_test(test_scraper)
lofar_add_test(test_lso_webservice)
lofar_add_test(integration_test_store)
diff --git a/LTA/ltastorageoverview/test/test_scraper.py b/LTA/ltastorageoverview/test/test_scraper.py
new file mode 100755
index 0000000000000000000000000000000000000000..ce7d1ff18897b6828aee55698c9787ba9d7c7adb
--- /dev/null
+++ b/LTA/ltastorageoverview/test/test_scraper.py
@@ -0,0 +1,55 @@
+#!/usr/bin/python
+
+# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+# $Id$
+
+import logging
+
+from common_test_ltastoragedb import *
+from lofar.lta.ltastorageoverview import scraper
+
+logger = logging.getLogger(__name__)
+
+class TestLocation(unittest.TestCase):
+ def test_isRoot(self):
+ loc = scraper.Location('srm://srm.grid.sara.nl:8443', '/foo/bar')
+ self.assertFalse(loc.isRoot())
+
+ loc = scraper.Location('srm://srm.grid.sara.nl:8443', '/')
+ self.assertTrue(loc.isRoot())
+
+ def test_malformed_location(self):
+ with self.assertRaises(ValueError) as context:
+ scraper.Location('http://astron.nl', '/foo/bar')
+ self.assertTrue('malformed srm url' in str(context.exception))
+
+ with self.assertRaises(ValueError) as context:
+ scraper.Location('srm://srm.grid.sara.nl:8443', 'some_dir_name')
+ self.assertTrue('malformed directory' in str(context.exception))
+
+
+class TestScraper(CommonLTAStorageDbTest):
+ pass
+
+# run tests if main
+if __name__ == '__main__':
+ logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
+ level=logging.INFO)
+
+ unittest.main()
diff --git a/LTA/ltastorageoverview/test/test_scraper.run b/LTA/ltastorageoverview/test/test_scraper.run
new file mode 100755
index 0000000000000000000000000000000000000000..b47e4dcc95958dcd57c07bc6eec3b5085de62ea1
--- /dev/null
+++ b/LTA/ltastorageoverview/test/test_scraper.run
@@ -0,0 +1,4 @@
+#!/bin/bash
+
+source python-coverage.sh
+python_coverage_test "ltas*" test_scraper.py
diff --git a/LTA/ltastorageoverview/test/test_scraper.sh b/LTA/ltastorageoverview/test/test_scraper.sh
new file mode 100755
index 0000000000000000000000000000000000000000..66ce2e9f7a39c004d9ab8b955db0d59078a7bb2d
--- /dev/null
+++ b/LTA/ltastorageoverview/test/test_scraper.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+./runctest.sh test_scraper