Task #8721: SW-280: processed review comment: added some tests for the scraper...

Task #8721: SW-280: processed review comment: added some tests for the scraper Location class. made some minor changes to the scraper module. Not going to do a full refactoring; I agree some methods in scraper.py are too big, but they are self contained and not part of a public API. So it's good enough for now.

.gitattributes

@@ -2089,6 +2089,9 @@ LTA/ltastorageoverview/test/integration_test_store.sh -text
 LTA/ltastorageoverview/test/test_lso_webservice.py -text
 LTA/ltastorageoverview/test/test_lso_webservice.run -text
 LTA/ltastorageoverview/test/test_lso_webservice.sh -text
+LTA/ltastorageoverview/test/test_scraper.py -text
+LTA/ltastorageoverview/test/test_scraper.run -text
+LTA/ltastorageoverview/test/test_scraper.sh -text
 LTA/ltastorageoverview/test/test_store.py -text
 LTA/ltastorageoverview/test/test_store.run -text
 LTA/ltastorageoverview/test/test_store.sh -text

LTA/ltastorageoverview/lib/scraper.py

@@ -40,6 +40,8 @@ from random import random, randint
 logger = logging.getLogger()
 
 VISIT_INTERVAL = datetime.timedelta(days=3)
+LEXAR_HOST = 'ingest@lexar004'
+LEXAR_HOST = 'ingest@10.178.1.4'
 
 class FileInfo:
     '''Simple struct to hold filename and size'''
@@ -89,8 +91,14 @@ class Location:
         directory : int
             a directory at the storage site. for example: /pnfs/grid.sara.nl/data/lofar/storage
         '''
-        self.srmurl = srmurl
-        self.directory = directory.rstrip('/')
+        self.srmurl = srmurl.rstrip('/')
+        self.directory = directory.rstrip('/') if len(directory) > 1 else directory
+
+        if not self.srmurl.startswith('srm://'):
+            raise ValueError('malformed srm url: %s' % (self.srmurl,))
+
+        if not self.directory.startswith('/'):
+            raise ValueError('malformed directory path: "%s". should start with a /' % (self.directory,))
 
     def path(self):
         '''returns the full path srmurl + directory'''
@@ -127,11 +135,8 @@ class Location:
 
         # the core command: do an srmls call and parse the results
         # srmls can only yield max 900 items in a result, hence we can recurse for the next 900 by using the offset
-        lexar_nr = 4 #randint(3,4)
-        lexar_host = 'ingest@10.178.1.%d' % (lexar_nr,)
-        lexar_host = 'ingest@10.144.4.%d' % (74+lexar_nr,)
-        cmd = ['ssh', '-tt', '-n', '-x', '-q', lexar_host, "bash", "-c", "\'source %s;srmls -l -count=900 -offset=%d %s%s\'" % (
-                '/globalhome/ingest/service/bin/init.sh' if lexar_nr <= 2 else '/globalhome/ingest/.grid/.ingest_profile',
+        cmd = ['ssh', '-tt', '-n', '-x', '-q', LEXAR_HOST, "bash", "-c",
+               "\'source /globalhome/ingest/.grid/.ingest_profile; srmls -l -count=900 -offset=%d %s%s\'" % (
                 offset,
                 self.srmurl,
                 self.directory) ]
@@ -330,10 +335,8 @@ class ResultGetterThread(threading.Thread):
                                 result_file_tuple = result_file_tuple_dict[key]
 
                                 known_size = int(known_file['size'])
-                                known_creation_date = known_file['creation_date']
 
                                 result_size = result_file_tuple[1]
-                                result_creation_date = result_file_tuple[2]
 
                                 if known_size != result_size:
                                     logger.info("%s %s: updating %s (id=%d) size from %d to %d",

LTA/ltastorageoverview/test/CMakeLists.txt

@@ -2,6 +2,7 @@
 include(LofarCTest)
 
 lofar_add_test(test_store)
+lofar_add_test(test_scraper)
 lofar_add_test(test_lso_webservice)
 
 lofar_add_test(integration_test_store)

LTA/ltastorageoverview/test/test_scraper.py

+#!/usr/bin/python
+
+# Copyright (C) 2012-2015    ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.    See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+# $Id$
+
+import logging
+
+from common_test_ltastoragedb import *
+from lofar.lta.ltastorageoverview import scraper
+
+logger = logging.getLogger(__name__)
+
+class TestLocation(unittest.TestCase):
+    def test_isRoot(self):
+        loc = scraper.Location('srm://srm.grid.sara.nl:8443', '/foo/bar')
+        self.assertFalse(loc.isRoot())
+
+        loc = scraper.Location('srm://srm.grid.sara.nl:8443', '/')
+        self.assertTrue(loc.isRoot())
+
+    def test_malformed_location(self):
+        with self.assertRaises(ValueError) as context:
+            scraper.Location('http://astron.nl', '/foo/bar')
+            self.assertTrue('malformed srm url' in str(context.exception))
+
+        with self.assertRaises(ValueError) as context:
+            scraper.Location('srm://srm.grid.sara.nl:8443', 'some_dir_name')
+            self.assertTrue('malformed directory' in str(context.exception))
+
+
+class TestScraper(CommonLTAStorageDbTest):
+    pass
+
+# run tests if main
+if __name__ == '__main__':
+    logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
+                        level=logging.INFO)
+
+    unittest.main()

LTA/ltastorageoverview/test/test_scraper.run

+#!/bin/bash
+
+source python-coverage.sh
+python_coverage_test "ltas*" test_scraper.py

LTA/ltastorageoverview/test/test_scraper.sh

+#!/bin/sh
+
+./runctest.sh test_scraper