Skip to content
Snippets Groups Projects
Select Git revision
  • 77f81a76fe9da7243368fa91db58aa3c2709f18a
  • master default protected
  • L2SS-1914-fix_job_dispatch
  • TMSS-3170
  • TMSS-3167
  • TMSS-3161
  • TMSS-3158-Front-End-Only-Allow-Changing-Again
  • TMSS-3133
  • TMSS-3319-Fix-Templates
  • test-fix-deploy
  • TMSS-3134
  • TMSS-2872
  • defer-state
  • add-custom-monitoring-points
  • TMSS-3101-Front-End-Only
  • TMSS-984-choices
  • SDC-1400-Front-End-Only
  • TMSS-3079-PII
  • TMSS-2936
  • check-for-max-244-subbands
  • TMSS-2927---Front-End-Only-PXII
  • Before-Remove-TMSS
  • LOFAR-Release-4_4_318 protected
  • LOFAR-Release-4_4_317 protected
  • LOFAR-Release-4_4_316 protected
  • LOFAR-Release-4_4_315 protected
  • LOFAR-Release-4_4_314 protected
  • LOFAR-Release-4_4_313 protected
  • LOFAR-Release-4_4_312 protected
  • LOFAR-Release-4_4_311 protected
  • LOFAR-Release-4_4_310 protected
  • LOFAR-Release-4_4_309 protected
  • LOFAR-Release-4_4_308 protected
  • LOFAR-Release-4_4_307 protected
  • LOFAR-Release-4_4_306 protected
  • LOFAR-Release-4_4_304 protected
  • LOFAR-Release-4_4_303 protected
  • LOFAR-Release-4_4_302 protected
  • LOFAR-Release-4_4_301 protected
  • LOFAR-Release-4_4_300 protected
  • LOFAR-Release-4_4_299 protected
41 results

momqueryrpc.py

Blame
  • create_movescript.py 1.70 KiB
    #!/usr/bin/env python
    import os
    from rucio.rse.protocols import protocol
    import cPickle as pickle
    
    def data_parser(filedata, scope='lofar'):
      datafiles = list()
      filepaths = list()
      lsblocklength = 18
      meta = {} 
      if not len(filedata[-1].strip()):
        filedata.pop()
      for idx in range(lsblocklength,len(filedata),lsblocklength): # skip first because that is the dir itself
        fileprops = filedata[idx].strip().split()
        nbytes = int(fileprops[0].strip())
        pth = fileprops[1]
        name = os.path.basename(pth)
        checkline = filedata[idx+6]
        checksum = checkline.split(":")[1].strip()
        datafiles.append({'name':name, 'bytes': nbytes, 'adler32':checksum, 'meta':meta})
        filepaths.append(pth)
      return filepaths, datafiles
    
    # INPUT: srmls of source dir
    # step 1: read filenames from file
    
    with open("source_data_list.dat") as sdl:
      source_data = sdl.readlines()
    
    filepaths, filelist_data = data_parser(source_data)
    # step 2: compute directory hashes
    scope = 'lofar'
    srm_root_host = "srm://srm.grid.sara.nl"
    rucio_rse_rootdir = "srm://srm.grid.sara.nl/pnfs/grid.sara.nl/data/escape/disk/rucio/sara_dcache"
    rdt = protocol.RSEDeterministicTranslation()
    move_commands = list()
    topaths = list()
    for frompath in filepaths:
      fname = os.path.basename(frompath)
      tosubpath = rdt.path(scope,fname)
      from_address = "/".join([srm_root_host, frompath])
      to_address = "/".join([rucio_rse_rootdir, tosubpath])
      topaths.append(tosubpath[len(scope)+1:]) # strip off scope and slash
      move_commands.append("srmmv {src} {tgt}\n".format(src=from_address, tgt=to_address))
    with open("data_movescript.sh", 'w') as dms:
      dms.writelines(move_commands)
    
    with open("datapaths.pck","wb") as path_pickle:
      pickle.dump(topaths, path_pickle)