SDC-45: Initial commit

.gitignore

+./rucio-client.simg

hasher.py

+#!/usr/bin/env python
+import hashlib
+import sys
+
+def hash(scope, name):
+    """
+    Given a LFN, turn it into a sub-directory structure using a hash function.
+    This takes the MD5 of the LFN and uses the first four characters as a subdirectory
+    name.
+    :param scope: Scope of the LFN.
+    :param name: File name of the LFN.
+    :param rse: RSE for PFN (ignored)
+    :param rse_attrs: RSE attributes for PFN (ignored)
+    :param protocol_attrs: RSE protocol attributes for PFN (ignored)
+    :returns: Path for use in the PFN generation.
+    """
+    hstr = hashlib.md5(('%s:%s' % (scope, name)).encode('utf-8')).hexdigest()
+    if scope.startswith('user') or scope.startswith('group'):
+        scope = scope.replace('.', '/')
+    return '%s/%s/%s/%s' % (scope, hstr[0:2], hstr[2:4], name)
+
+if __name__ == "__main__":
+    print(hash(sys.argv[1], sys.argv[2]))

hasher_rucio.py

+#!/usr/bin/env python
+from rucio.rse.protocols import protocol
+rdt = protocol.RSEDeterministicTranslation()
+
+if __name__ == "__main__":
+    print(rdt.path(sys.argv[1], sys.argv[2]))

list_source_directory.sh

+#!/usr/local/bin/env bash
+
+voms-proxy-init --valid 144:00 --hours 144 --vomses /home/ygrange/vomses --vomsdir /home/ygrange/vomses/ --voms escape:/escape/lofar
+srmls -l srm://srm.grid.sara.nl/pnfs/grid.sara.nl/data/escape/disk/LOFAR_testdata/$1 > source_data_list.dat

move_to_rucio.py

+# INPUT: srmls of source dir
+
+# step 1: read filenames from file
+
+# step 2: compute directory hashes
+
+# step 3: move data to directories
+
+# step 4: Add replicas for files using rucio.client.replicaclient the ReplicaClient class (add_replicas)
+
+# step 4: Add dataset for each observation, add the files to the dataset
+
+# step 5: Add container and add the cal/tgt to the container together

register_rucio.py

+#!/usr/bin/env python
+from rucio.client import replicaclient
+rcl = replicaclient.ReplicaClient()
+
+if __name__ == "__main__":
+    RSE = "SARA-DCACHE"
+    filelist = [{'scope':'lofar', 'name':'file1'}, }
+    print(rcl.add_replicas(RSE, filelist))

singulex

+#!/usr/bin/env bash
+export RUCIO_CFG_ACCOUNT=<myaccount>
+singularity exec -B ${HOME}/.rucio/:/opt/rucio/etc -B ${HOME}/.globus/client.crt:/opt/rucio/etc/client.crt -B ${HOME}/.globus/client.key:/opt/rucio/etc/client.key rucio-client.simg $*