From d33807c29b2f67ca4ce3181fa6ad3e1bb2bf60f3 Mon Sep 17 00:00:00 2001
From: Yan Grange <grange@astron.nl>
Date: Mon, 24 Aug 2020 14:24:43 +0200
Subject: [PATCH] SDC-45: Adding path hashing step
---
move_to_rucio.py | 33 ++++++++++++++++++++++++++++++++-
1 file changed, 32 insertions(+), 1 deletion(-)
mode change 100644 => 100755 move_to_rucio.py
diff --git a/move_to_rucio.py b/move_to_rucio.py
old mode 100644
new mode 100755
index ecd6dfa..27557b1
--- a/move_to_rucio.py
+++ b/move_to_rucio.py
@@ -1,8 +1,39 @@
-# INPUT: srmls of source dir
+#!/usr/bin/env python
+import os
+from rucio.rse.protocols import protocol
+
+def data_parser(filedata, scope='lofar'):
+ datafiles = list()
+ filepaths = list()
+ lsblocklength = 18
+ meta = {}
+ if not len(filedata[-1].strip()):
+ filedata.pop()
+ for idx in range(0,len(filedata),18):
+ fileprops = filedata[idx].strip().split()
+ nbytes = int(fileprops[0].strip())
+ pth = fileprops[1]
+ name = os.path.basename(pth)
+ checkline = filedata[idx+6]
+ checksum = checkline.split(":")[1].strip()
+ datafiles.append({'name':name, 'bytes': nbytes, 'adler32':checksum, 'meta':meta})
+ filepaths.append(pth)
+ return filepaths, datafiles
+# INPUT: srmls of source dir
# step 1: read filenames from file
+with open("source_data_list.dat") as sdl:
+ source_data = sdl.readlines()
+
+filepaths, filelist_data = data_parser(source_data)
# step 2: compute directory hashes
+scope = 'lofar'
+rdt = protocol.RSEDeterministicTranslation()
+for frompath in filepaths:
+ fname = os.path.basename(frompath)
+ tosubpath = rdt.path(scope,fname)
+ print(frompath, tosubpath)
# step 3: move data to directories
--
GitLab