last assignments added

ingest_assignment_2.py

 #! /home/grange/rucio_dev/singupy.sh
-"""
-
-Locate you file: rucio list-file-replicas YOUREXPERIMENT_YOURINSTITUTION_YOURNAME:FILE_TO_BE_UPLOADED
-
-Create a simple workflow that access the data and perform actions on it (e.g. get checksum and cross-check it).
-
-Delete file.
-
-Delete file from a dataset.
-Delete dataset."""
 from rucio import client
-from rucio.client import uploadclient
+from rucio.client import uploadclient, downloadclient
 import os
 from string import hexdigits as hxdraw
 import random
 import logging
 import argparse
 import subprocess as SP
+from rucio.common.utils import adler32
+
 
 def voms_proxy_init():
     voms = "escape:/escape/lofar" 
@@ -40,13 +32,19 @@ def create_files(gendir):
         with open(os.path.join(gendir, filename), "w") as loffile:
             loffile.write(str(random.getrandbits(100000)))
 
-def configure_logger():
+def configure_logger(onscreen=False):
     logger = logging.getLogger("rucio")
-    logger.setLevel(logging.INFO)
+    loglvl = logging.INFO
+    logger.setLevel(loglvl)
     handler = logging.FileHandler("/home/grange/ruciologs/rucio.log")
     formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
     handler.setFormatter(formatter)
     logger.addHandler(handler)
+    if onscreen:
+        sh = logging.StreamHandler()
+        sh.setFormatter(formatter)
+        sh.setLevel(loglvl)
+
     return logger
 
 def upload_files(gendir, scope):
@@ -149,6 +147,46 @@ def create_dataset(scope, datasetname):
     uploader.upload(uploaddata)
     rc.add_files_to_dataset(scope, datasetname, datasetregdata)   
 
+def find_and_download(scope, dataset):
+    """
+    Locate you file: rucio list-file-replicas YOUREXPERIMENT_YOURINSTITUTION_YOURNAME:FILE_TO_BE_UPLOADED
+
+    Create a simple workflow that access the data and perform actions on it (e.g. get checksum and cross-check it).
+    """
+    filename = f"datafile_{dataset_name}_3"
+    rc = client.Client()
+    for replica in rc.list_replicas([{"scope":scope, "name":filename}]):
+        print(replica)
+
+    logger = configure_logger(onscreen=True)
+    downloader = downloadclient.DownloadClient(logger=logger)
+    downloader.download_dids([{"did":f"{scope}:{filename}"}])
+
+    localadler32 = adler32(os.path.join(scope, filename))
+    remoteadler32 = rc.get_metadata(scope, filename)['adler32']
+    if localadler32 == remoteadler32:
+        equality = "equal"
+    else:
+        equality = "not equal"
+    print(f"Local adler = {localadler32}, remote adler = {remoteadler32}. Both values are {equality}")
+
+def delete_data(scope, dataset_name):
+    """
+    Delete file.
+    Delete file from a dataset.
+    Delete dataset.
+    """
+    rc = client.Client()
+    filename = f"datafile_{dataset_name}_1"
+    rc.set_metadata(scope=scope, name=filename, key='lifetime', value=86400)
+    print(f'{filename} will be deleted in 24 h.')
+
+    filename2 = [{'scope':scope, 'name':f'datafile_{dataset_name}_2'}]
+    rc.detach_dids(scope=scope, name=dataset_name, dids=filename2)
+    print(f'{filename2} detacbed from {dataset_name}.')
+    
+    rc.set_metadata(scope=scope, name=dataset_name, key='lifetime', value=86400)
+    print(f'{dataset_name} will be deleted in 24 h.')
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(description='Run the second data exercise.')
@@ -159,9 +197,12 @@ if __name__ == "__main__":
     parser.add_argument("--replicate", "-r", help="Add replication rule.", action="store_true")
     parser.add_argument("--showfiles", "-s", help="Show all files in scope", action="store_true")
     parser.add_argument("--create-dataset", "-c", help="Create data sets", action="store_true")
+    parser.add_argument("--find-and-download", "-f", help="Find and download data sets", action="store_true")
+    parser.add_argument("--delete", "--exterminate", "-x", help="Delete files and datasets", action="store_true")
+    
     args = parser.parse_args()
     scope = "LOFAR_ASTRON_GRANGE"
-    dataset_name = "Dataset_for_assignment_6"
+    dataset_name = "Dataset_for_assignment_8"
 
     if args.init:
         create_files(args.data_dir)
@@ -173,4 +214,8 @@ if __name__ == "__main__":
         showmyfiles(scope)
     elif args.create_dataset:
         create_dataset(scope, dataset_name)
+    elif args.find_and_download:
+        find_and_download(scope, dataset_name)
+    elif args.delete:
+        delete_data(scope, dataset_name)