diff --git a/ingest_assignment_2.py b/ingest_assignment_2.py index b045f2656749ab208378f3dc0921789096039f4e..1d8c5353e36211ca7f1447eb4d2e09eaa9f11825 100755 --- a/ingest_assignment_2.py +++ b/ingest_assignment_2.py @@ -1,22 +1,14 @@ #! /home/grange/rucio_dev/singupy.sh -""" - -Locate you file: rucio list-file-replicas YOUREXPERIMENT_YOURINSTITUTION_YOURNAME:FILE_TO_BE_UPLOADED - -Create a simple workflow that access the data and perform actions on it (e.g. get checksum and cross-check it). - -Delete file. - -Delete file from a dataset. -Delete dataset.""" from rucio import client -from rucio.client import uploadclient +from rucio.client import uploadclient, downloadclient import os from string import hexdigits as hxdraw import random import logging import argparse import subprocess as SP +from rucio.common.utils import adler32 + def voms_proxy_init(): voms = "escape:/escape/lofar" @@ -25,7 +17,7 @@ def voms_proxy_init(): keypath = os.path.join(certkeydir, "client.key") command_name = ["voms-proxy-init",f"--voms={voms}", f"--cert={certpath}", f"--key={keypath}"] - + SP.call(command_name) @@ -40,13 +32,19 @@ def create_files(gendir): with open(os.path.join(gendir, filename), "w") as loffile: loffile.write(str(random.getrandbits(100000))) -def configure_logger(): +def configure_logger(onscreen=False): logger = logging.getLogger("rucio") - logger.setLevel(logging.INFO) + loglvl = logging.INFO + logger.setLevel(loglvl) handler = logging.FileHandler("/home/grange/ruciologs/rucio.log") formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") handler.setFormatter(formatter) logger.addHandler(handler) + if onscreen: + sh = logging.StreamHandler() + sh.setFormatter(formatter) + sh.setLevel(loglvl) + return logger def upload_files(gendir, scope): @@ -149,7 +147,47 @@ def create_dataset(scope, datasetname): uploader.upload(uploaddata) rc.add_files_to_dataset(scope, datasetname, datasetregdata) - +def find_and_download(scope, dataset): + """ + Locate you file: rucio list-file-replicas YOUREXPERIMENT_YOURINSTITUTION_YOURNAME:FILE_TO_BE_UPLOADED + + Create a simple workflow that access the data and perform actions on it (e.g. get checksum and cross-check it). + """ + filename = f"datafile_{dataset_name}_3" + rc = client.Client() + for replica in rc.list_replicas([{"scope":scope, "name":filename}]): + print(replica) + + logger = configure_logger(onscreen=True) + downloader = downloadclient.DownloadClient(logger=logger) + downloader.download_dids([{"did":f"{scope}:{filename}"}]) + + localadler32 = adler32(os.path.join(scope, filename)) + remoteadler32 = rc.get_metadata(scope, filename)['adler32'] + if localadler32 == remoteadler32: + equality = "equal" + else: + equality = "not equal" + print(f"Local adler = {localadler32}, remote adler = {remoteadler32}. Both values are {equality}") + +def delete_data(scope, dataset_name): + """ + Delete file. + Delete file from a dataset. + Delete dataset. + """ + rc = client.Client() + filename = f"datafile_{dataset_name}_1" + rc.set_metadata(scope=scope, name=filename, key='lifetime', value=86400) + print(f'{filename} will be deleted in 24 h.') + + filename2 = [{'scope':scope, 'name':f'datafile_{dataset_name}_2'}] + rc.detach_dids(scope=scope, name=dataset_name, dids=filename2) + print(f'{filename2} detacbed from {dataset_name}.') + + rc.set_metadata(scope=scope, name=dataset_name, key='lifetime', value=86400) + print(f'{dataset_name} will be deleted in 24 h.') + if __name__ == "__main__": parser = argparse.ArgumentParser(description='Run the second data exercise.') parser.add_argument("--init", "-i", action="store_true", help="Create dummy data for upload") @@ -159,9 +197,12 @@ if __name__ == "__main__": parser.add_argument("--replicate", "-r", help="Add replication rule.", action="store_true") parser.add_argument("--showfiles", "-s", help="Show all files in scope", action="store_true") parser.add_argument("--create-dataset", "-c", help="Create data sets", action="store_true") + parser.add_argument("--find-and-download", "-f", help="Find and download data sets", action="store_true") + parser.add_argument("--delete", "--exterminate", "-x", help="Delete files and datasets", action="store_true") + args = parser.parse_args() scope = "LOFAR_ASTRON_GRANGE" - dataset_name = "Dataset_for_assignment_6" + dataset_name = "Dataset_for_assignment_8" if args.init: create_files(args.data_dir) @@ -173,4 +214,8 @@ if __name__ == "__main__": showmyfiles(scope) elif args.create_dataset: create_dataset(scope, dataset_name) + elif args.find_and_download: + find_and_download(scope, dataset_name) + elif args.delete: + delete_data(scope, dataset_name)