From 2ab9e4fdcb48a51851abc219703e9dbd94c2dce3 Mon Sep 17 00:00:00 2001
From: Jorrit Schaap <schaap@astron.nl>
Date: Mon, 26 Oct 2015 13:16:52 +0000
Subject: [PATCH] Task #8291: added TODO's. removed obsolete code
---
LTA/ltastorageoverview/lib/scraper.py | 67 ++-----------------
LTA/ltastorageoverview/lib/store.py | 5 ++
.../lib/webservice/templates/index.html | 1 +
.../lib/webservice/webservice.py | 18 ++---
4 files changed, 18 insertions(+), 73 deletions(-)
diff --git a/LTA/ltastorageoverview/lib/scraper.py b/LTA/ltastorageoverview/lib/scraper.py
index 3bea369b86b..31a29d7d70b 100755
--- a/LTA/ltastorageoverview/lib/scraper.py
+++ b/LTA/ltastorageoverview/lib/scraper.py
@@ -19,6 +19,10 @@
# $Id$
+# TODO: add comments to methods
+# TODO: code cleanup
+# TODO: scraper should be able to process each directory more than once. Requires changes in store.py
+
import subprocess
import logging
import time
@@ -29,20 +33,9 @@ import os.path
import threading
import multiprocessing
from ltastorageoverview import store
+from ltastorageoverview.utils import humanreadablesize
from random import random
-def humanreadablesize(num, suffix='B'):
- """ converts the given size (number) to a human readable string in powers of 1024"""
- try:
- for unit in ['', 'K', 'M', 'G', 'T', 'P', 'E', 'Z']:
- if abs(num) < 1024.0:
- return "%3.1f%s%s" % (num, unit, suffix)
- num /= 1024.0
- return "%.1f%s%s" % (num, 'Y', suffix)
- except TypeError:
- return str(num)
-
-
#logging.basicConfig(filename='scraper.' + time.strftime("%Y-%m-%d") + '.log', level=logging.DEBUG, format="%(asctime)-15s %(levelname)s %(message)s")
logging.basicConfig(level=logging.DEBUG, format="%(asctime)-15s %(levelname)s %(message)s")
logger = logging.getLogger()
@@ -262,55 +255,6 @@ class ResultGetterThread(threading.Thread):
except Exception as e:
logger.error(str(e))
-
-class LocationResultTreeNode:
- '''Helper class to order results in a tree structure,
- so we can get totals (files and sizes) of the whole (or part of) the tree.'''
- def __init__(self, locationResult, parent=None, children=None):
- '''
- Parameters
- ----------
- locationResult : LocationResult
- the locationResult for this tree node
-
- parent : LocationResultTreeNode
- the parent LocationResultTreeNode of this tree node
-
- children : [LocationResultTreeNode]
- the children LocationResultTreeNode's of this tree node
- '''
- self.locationResult = locationResult
- self.parent = parent
- self.children = children if children else []
-
- def __str__(self):
- return "LocationResultTreeNode: loc=%s total # files=%d in total # subDirs=%d with a total size of %s" % (self.locationResult.location.path(), self.totalNrOfFilesInDirAndSubDirs(), self.totalNrOfSubDirs(), humanreadablesize(self.totalFileSizeOfDirAndSubDirs()))
-
- def totalNrOfFilesInDirAndSubDirs(self):
- '''returns the total number of files in this dir and the all subdirs of the tree'''
- return self.locationResult.nrOfFiles() + sum([child.totalNrOfFilesInDirAndSubDirs() for child in self.children])
-
- def totalNrOfSubDirs(self):
- '''returns the total number of subdirs in this dir and the all subdirs of the tree'''
- return self.locationResult.nrOfSubDirs() + sum([child.totalNrOfSubDirs() for child in self.children])
-
- def totalFileSizeOfDirAndSubDirs(self):
- '''returns the total filesize in this dir and the all subdirs of the tree'''
- return self.locationResult.totalFileSizeOfDir() + sum([child.totalFileSizeOfDirAndSubDirs() for child in self.children])
-
- def treeString(self, level=0, maxLevel=sys.maxint):
- '''returns a string tree representation of this node and its subtree
- uses recursion to indent each sublevel.
- can be limited to maxLevel levels deep'''
- return "> {path} # files={numFiles} fileSizeOfDir={fileSize} # filesInTree={numFilesInTree} fileSizeOfTree={fileSizeOfTree}{tree}".format(
- path=self.locationResult.location.path(),
- numFiles=self.locationResult.nrOfFiles(),
- fileSize=humanreadablesize(self.locationResult.totalFileSizeOfDir()),
- numFilesInTree=self.totalNrOfFilesInDirAndSubDirs(),
- fileSizeOfTree=humanreadablesize(self.totalFileSizeOfDirAndSubDirs()),
- tree='\n' + '\n'.join([(level+1)*' ' + child.treeString(level+1, maxLevel) for child in self.children]) if self.children and level < maxLevel else "")
-
-
def main(argv):
'''the main function scanning all locations and gathering the results'''
@@ -335,7 +279,6 @@ def main(argv):
for dir_id in [x[0] for x in db.rootDirectories()]:
db.updateDirectoryLastVisitTime(dir_id, datetime.datetime.utcnow() - datetime.timedelta(days=1000))
-
# for each site we want one or more ResultGetterThreads
# so make a dict with a list per site based on the locations
getters = dict([(site[1],[]) for site in db.sites()])
diff --git a/LTA/ltastorageoverview/lib/store.py b/LTA/ltastorageoverview/lib/store.py
index 68600eb3fcf..71aaa04ceff 100644
--- a/LTA/ltastorageoverview/lib/store.py
+++ b/LTA/ltastorageoverview/lib/store.py
@@ -19,6 +19,11 @@
# $Id$
+# TODO: add comment to methods
+# TODO: reuse connection in methods (take care of exceptions closing the connection)
+# TODO: use generators and yield for faster and more memory efficient processing of results.
+# TODO: use other database? MariaDB? instead of sqlite?
+
import os
import os.path
import sqlite3
diff --git a/LTA/ltastorageoverview/lib/webservice/templates/index.html b/LTA/ltastorageoverview/lib/webservice/templates/index.html
index eecfcc7d6f7..681625af0fb 100644
--- a/LTA/ltastorageoverview/lib/webservice/templates/index.html
+++ b/LTA/ltastorageoverview/lib/webservice/templates/index.html
@@ -1,5 +1,6 @@
<!DOCTYPE html>
<!--$Id$-->
+<!--TODO: use ajax calls to get chart data-->
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8"/>
diff --git a/LTA/ltastorageoverview/lib/webservice/webservice.py b/LTA/ltastorageoverview/lib/webservice/webservice.py
index 5ae0e6fdf2b..65a3f467463 100755
--- a/LTA/ltastorageoverview/lib/webservice/webservice.py
+++ b/LTA/ltastorageoverview/lib/webservice/webservice.py
@@ -19,36 +19,33 @@
# $Id$
+# TODO: add comment to methods
+# TODO: code cleanup
+# TODO: where to store the sqlite database?
+
import sys
import os
import os.path
-from datetime import datetime, timedelta
+from datetime import datetime
from flask import Flask
-from flask import Config
from flask import render_template
from flask import json
-import threading
from ltastorageoverview import store
from ltastorageoverview.utils import humanreadablesize
from ltastorageoverview.utils import monthRanges
app = Flask('LTA storage overview')
-print str(app.config)
-print __file__
-print os.path.dirname(__file__)
app.config.root_path = os.path.dirname(__file__)
-print
-print str(app.config)
db = store.LTAStorageDb('../ltastorageoverview.sqlite')
@app.route('/')
@app.route('/index.html')
def index():
+ # TODO: serve html first, and let client request data via ajax
usages = {}
sites = db.sites()
sites2 = [x for x in sites if x[1] != 'nikhef']
- print sites
sites = [sites2[0], sites2[2], sites2[1]]
total = 0.0
@@ -142,8 +139,7 @@ def get_filesInDirectory(dir_id):
def main(argv):
- #db = store.LTAStorageDb(argv[0] if argv else 'ltastoragedb.sqlite')
- app.run(debug=True,host='0.0.0.0')
+ app.run(debug=False,host='0.0.0.0')
if __name__ == '__main__':
main(sys.argv[1:])
--
GitLab