diff --git a/move_to_rucio.py b/move_to_rucio.py old mode 100644 new mode 100755 index ecd6dfa2fc7f2cd34e6e457156742d8163e61d59..27557b160ad3db5d013817db5052d470a9a17d8d --- a/move_to_rucio.py +++ b/move_to_rucio.py @@ -1,8 +1,39 @@ -# INPUT: srmls of source dir +#!/usr/bin/env python +import os +from rucio.rse.protocols import protocol + +def data_parser(filedata, scope='lofar'): + datafiles = list() + filepaths = list() + lsblocklength = 18 + meta = {} + if not len(filedata[-1].strip()): + filedata.pop() + for idx in range(0,len(filedata),18): + fileprops = filedata[idx].strip().split() + nbytes = int(fileprops[0].strip()) + pth = fileprops[1] + name = os.path.basename(pth) + checkline = filedata[idx+6] + checksum = checkline.split(":")[1].strip() + datafiles.append({'name':name, 'bytes': nbytes, 'adler32':checksum, 'meta':meta}) + filepaths.append(pth) + return filepaths, datafiles +# INPUT: srmls of source dir # step 1: read filenames from file +with open("source_data_list.dat") as sdl: + source_data = sdl.readlines() + +filepaths, filelist_data = data_parser(source_data) # step 2: compute directory hashes +scope = 'lofar' +rdt = protocol.RSEDeterministicTranslation() +for frompath in filepaths: + fname = os.path.basename(frompath) + tosubpath = rdt.path(scope,fname) + print(frompath, tosubpath) # step 3: move data to directories