Select Git revision
messagebus.py
-
Jan David Mol authored
Task #8899: Allow broker options to be specified in FromBus/ToBus constructor (final final fix for tests without auto-reconnect, hopefully)
Jan David Mol authoredTask #8899: Allow broker options to be specified in FromBus/ToBus constructor (final final fix for tests without auto-reconnect, hopefully)
Code owners
Assign users and groups as approvers for specific file changes. Learn more.
create_movescript.py 1.70 KiB
#!/usr/bin/env python
import os
from rucio.rse.protocols import protocol
import cPickle as pickle
def data_parser(filedata, scope='lofar'):
datafiles = list()
filepaths = list()
lsblocklength = 18
meta = {}
if not len(filedata[-1].strip()):
filedata.pop()
for idx in range(lsblocklength,len(filedata),lsblocklength): # skip first because that is the dir itself
fileprops = filedata[idx].strip().split()
nbytes = int(fileprops[0].strip())
pth = fileprops[1]
name = os.path.basename(pth)
checkline = filedata[idx+6]
checksum = checkline.split(":")[1].strip()
datafiles.append({'name':name, 'bytes': nbytes, 'adler32':checksum, 'meta':meta})
filepaths.append(pth)
return filepaths, datafiles
# INPUT: srmls of source dir
# step 1: read filenames from file
with open("source_data_list.dat") as sdl:
source_data = sdl.readlines()
filepaths, filelist_data = data_parser(source_data)
# step 2: compute directory hashes
scope = 'lofar'
srm_root_host = "srm://srm.grid.sara.nl"
rucio_rse_rootdir = "srm://srm.grid.sara.nl/pnfs/grid.sara.nl/data/escape/disk/rucio/sara_dcache"
rdt = protocol.RSEDeterministicTranslation()
move_commands = list()
topaths = list()
for frompath in filepaths:
fname = os.path.basename(frompath)
tosubpath = rdt.path(scope,fname)
from_address = "/".join([srm_root_host, frompath])
to_address = "/".join([rucio_rse_rootdir, tosubpath])
topaths.append(tosubpath[len(scope)+1:]) # strip off scope and slash
move_commands.append("srmmv {src} {tgt}\n".format(src=from_address, tgt=to_address))
with open("data_movescript.sh", 'w') as dms:
dms.writelines(move_commands)
with open("datapaths.pck","wb") as path_pickle:
pickle.dump(topaths, path_pickle)