Commit 3e4d1098 authored by Yan Grange's avatar Yan Grange 👋

First part of assignment 2 (create and upload)

parent 8a1529f3
#! /home/grange/rucio_dev/singupy.sh
"""
Add replication rule based on attributes: rucio add-rule YOUREXPERIMENT_YOURINSTITUTION_YOURNAME:FILE_TO_BE_UPLOADED 1 QOS=FAST
Please contact Paul or Aleem for any problem/concern.
Check rule status: rucio list-rules YOUREXPERIMENT_YOURINSTITUTION_YOURNAME:FILE_TO_BE_UPLOADED
Discover you files: rucio list-dids YOUREXPERIMENT_YOURINSTITUTION_YOURNAME:* --filter type="ALL"
--filter type is necessary since by default rucio lists the datasets.
Create datasets, upload them, and add different files to them at a later stage.
Locate you file: rucio list-file-replicas YOUREXPERIMENT_YOURINSTITUTION_YOURNAME:FILE_TO_BE_UPLOADED
Create a simple workflow that access the data and perform actions on it (e.g. get checksum and cross-check it).
Delete file.
Delete file from a dataset.
Delete dataset."""
from rucio.client import uploadclient
import os
from string import hexdigits as hxdraw
import random
import logging
import argparse
import subprocess as SP
def create_files(gendir):
os.mkdir(gendir) # fails if already exists, but that's what I want anyhow.
hexdigits = hxdraw[:-6]
obsID = 12345
filelist = list()
for subband in range(244):
somehash = ''.join([random.choice(hexdigits) for a in range(6)])
filename = f"L{obsID}_SB{subband:03}_uv.MS_{somehash}.tar"
with open(os.path.join(gendir, filename), "w") as loffile:
loffile.write(str(random.getrandbits(100000)))
def configure_logger():
logger = logging.getLogger("rucio")
logger.setLevel(logging.INFO)
handler = logging.FileHandler("/home/grange/ruciologs/rucio.log")
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
handler.setFormatter(formatter)
logger.addHandler(handler)
return logger
def upload_files(gendir):
"""
Upload multiple files, automatising the upload procedure, and execute hourly till our next meeting.
Diego and Agustin have already a way to upload or register files. Please, follow up with them in case you face issues.
"""
scope = "LOFAR_ASTRON_GRANGE"
rse = "SARA-DCACHE"
lifetime = 12*3600 # 12 hours in seconds
logger = configure_logger()
uploader = uploadclient.UploadClient(logger=logger)
uploaddata = list()
for fname in os.listdir(gendir):
absfilepath = os.path.abspath(os.path.join(gendir, fname))
uploaddict = {"path" : absfilepath,
"rse" : rse,
"did_scope" : scope,
"lifetime" : lifetime,
"register_after_upload" : True}
uploaddata.append(uploaddict)
voms = "escape:/escape/lofar"
certkeydir = "/home/grange/.rucio"
certpath = os.path.join(certkeydir, "client.crt")
keypath = os.path.join(certkeydir, "client.key")
command_name = ["voms-proxy-init",f"--voms={voms}", f"--cert={certpath}", f"--key={keypath}"]
SP.call(command_name)
uploader.upload(uploaddata)
# For the hourly version, see hourly_push.sh
# $> crontab -l
# 28 * 9-13 * * /home/grange/rucio_dev/hourly_push.sh
# 28 0-11 14 * * /home/grange/rucio_dev/hourly_push.sh
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='Run the second data exercise.')
parser.add_argument("--init", "-i", action="store_true", help="Create dummy data for upload")
parser.add_argument("--data-dir", "-d", help="Directory with the data to upload, or to create with init.",
default="lofar_data")
parser.add_argument("--upload", "-u", help="Upload the files.", action="store_true")
args = parser.parse_args()
if args.init:
create_files(args.data_dir)
elif args.upload:
upload_files(args.data_dir)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment