Skip to content
Snippets Groups Projects
Select Git revision
  • 81ffed3dbb57c34c8d0b33de9a436750420e809c
  • master default protected
  • tmss
  • python3
  • TMSS-199
  • fix-casacore-version
  • patch-3
  • patch-2
  • patch-1
  • SterVeen-patch-1
10 results

make_cluster_dirs.py

Blame
    • Code owners
    Assign users and groups as approvers for specific file changes. Learn more.
    make_cluster_dirs.py 2.02 KiB
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72 
    

    

    

    #!/usr/bin/env python
import os
from pyautoplot.lofaroffline import *
from numpy import ceil

"""
Create the offline storage cluster file system structure in
testdata/net/... and add a test MS to it.
"""

def ensure_dir(path):
    try:
        os.mkdir(path)
    except (OSError,):
        pass
    
def make_empty_file(path):
    return open(path, mode='a').close()


def create_cluster_dirs(root='testdata/net'):
    os.system('rm -rf '+root)
    ensure_dir(root)
                
    for sub in range(8):
        subdir=os.path.join(root, 'sub%d'%(sub+1,))
        ensure_dir(subdir)
        for lse in range(sub*3, (sub+1)*3):
            lsedir=os.path.join(root, 'sub%d/lse%03d'%(sub+1, lse+1))
            ensure_dir(lsedir)
            for d in range(4):
                path=os.path.join(root, 'sub%d/lse%03d/data%d'%(sub+1, lse+1, d+1))
                #print path
                ensure_dir(path)
                pass
            pass
        pass
    pass


def create_dummy_ms(ms_id, partition, num_subbands, root='testdata/net', exclude_lse=['lse019', 'lse020', 'lse021', 'lse011']):
    storage_nodes = sorted(['lse%03d'%(i+1,) for i in range(24) if 'lse%03d'%(i+1,) not in exclude_lse])
    subcluster_names= ['sub'+str(get_subcluster_number(name)) for name in storage_nodes]

    n = int(ceil(num_subbands*1.0/len(storage_nodes)))
    
    msname='L2010_%05d'%(ms_id,)

    for i,lse in enumerate(storage_nodes):
        ms_dir=os.path.join(root, subcluster_names[i], lse, partition, msname)
        ensure_dir(ms_dir)
        for sb in range(i*n, (i+1)*n):
            if sb < num_subbands:
                make_empty_file(os.path.join(ms_dir, 'L%05d_SB%03d-uv.MS'%(ms_id, sb)))
                pass
            pass
        pass    
    pass




if __name__ == '__main__':
    create_cluster_dirs()
    create_dummy_ms(12345, 'data1', 248)
    create_dummy_ms(20001, 'data1', 248)
    create_dummy_ms(20010, 'data1', 244)
    create_dummy_ms(20020, 'data2', 62)
    create_dummy_ms(20030, 'data3', 248)
    create_dummy_ms(20040, 'data4', 122)
    pass