Commit 97190d37 authored by AK's avatar AK

Background jobs, Calibrators DB, ...

parent 6cdf9525
from background_task import background
from img.models import BeamData, Image
from img.models import BeamData, Image, CalibratorsData
import os
import re
import sys
import glob
from astropy.io import fits
import logging
logging.basicConfig(level=logging.DEBUG, filename='background.log')
def get_data_from_fits(ffile):
with fits.open(ffile) as f:
......@@ -29,17 +31,18 @@ def get_data_from_fits(ffile):
def insert_fits_data(beamdata, path, type):
naxis, dims1, dims2, bmaj, bmin, bpa, ra, dec, pixel_size, field, stokes = get_data_from_fits(path)
insert_image_record(beamdata, path, dims1, ra, dec, pixel_size, bmaj, bmin, bpa, stokes, type=type)
img = insert_image_record(beamdata, path, dims1, ra, dec, pixel_size, bmaj, bmin, bpa, stokes, type=type)
return img
@background()
def test(infos):
print('Yooo here are the stuff you gave me', infos)
def insert_beamdata_record(tid, beam):
beamdata, created = BeamData.objects.get_or_create(taskID=tid, beam=beam)
def insert_beamdata_record(tid, beam, field=''):
beamdata, created = BeamData.objects.get_or_create(taskID=tid, beam=beam, field=field)
beamdata.save()
return beamdata
return beamdata, created
def insert_image_record(beamdata, path, imsize, ra, dec, pixel_size, bmaj, bmin, bpa, stokes, type):
......@@ -55,39 +58,82 @@ def insert_image_record(beamdata, path, imsize, ra, dec, pixel_size, bmaj, bmin,
image.stokes = stokes
image.type = type
image.save()
return image
return image, created
@background()
def update_beamdata():
import socket
if socket.gethostname() == 'dop433':
mnt = '/home/kutkin/mnt/hyperion'
def insert_calibrator_record(tid, beam, name, caltype, field=''):
calibrator, created = CalibratorsData.objects.get_or_create(taskID=tid, beam=beam, name=name, caltype=caltype, field=field)
calibrator.save()
return calibrator, created
def beam_task_from_path(beampath):
task, beam = beampath.split('/')[-2:]
return task, beam
def field_from_path(beampath):
msfiles = glob.glob(beampath + '/raw/[MS][0-9+-]*.MS')
if msfiles:
return re.findall('[MS][\d+-]*', msfiles[0])[0]
else:
mnt = ''
return ''
tidpaths = glob.glob(mnt+'/kutkin/ddcal_processed/[1-9]*')
tids = [_.split('/')[-1] for _ in tidpaths]
beams = ['{:02d}'.format(_) for _ in range(40)]
strings = []
for tid in tids[:]:
for beam in beams[:]:
beamdata = insert_beamdata_record(tid, beam)
print(beamdata)
#sys.exit()
def iterate_over_structure(dirs, func, force=False):
for dir in dirs:
for beampath in glob.iglob(dir + '/[0-9]*/[0-3][0-9]'):
# print(beampath)
tid, beam = beam_task_from_path(beampath)
field = field_from_path(beampath)
print(tid, beam, field)
beamdata, created = insert_beamdata_record(tid, beam, field)
if not created and not force:
logging.debug('Record exists %s -- %s', tid, beam)
continue
func(beampath, beamdata)
ffile = mnt + '/kutkin/ddcal_processed/{tid}/{beam}/restored.fits'.format(tid=tid, beam=beam)
aimlist = glob.glob(mnt + '/kutkin/images_apercal/{tid}_{beam}_*.fits'.format(tid=tid, beam=beam))
for path in aimlist:
type = 'a'
apercal_image = insert_fits_data(beamdata, path, type)
def _apercal(path, beamdata):
afiles = glob.glob(path + '/continuum/image_mf_*.fits')
for path in afiles:
imtype = 'a'
apercal_image, created = insert_fits_data(beamdata, path, imtype)
def _ddcal(path, beamdata):
dimage = path + '/restored.fits'
if os.path.exists(dimage):
imtype = 'd'
ddcal_image, created = insert_fits_data(beamdata, path, imtype)
@background()
def update_apercal_data(dirs=['/happili-01/data', '/happili-01/data2', '/happili-01/data3', '/happili-01/data4'], force=False):
try:
iterate_over_structure(dirs, _apercal, force=force)
logging.info('DB updated')
except Exception as e:
logging.exception(e)
@background()
def update_ddcal_data(dirs=['/kutkin/processed_ddcal', '/kutkin/to_process'], force=False):
try:
iterate_over_structure(dirs, _ddcal, force=force)
logging.info('DB updated')
except Exception as e:
logging.exception(e)
if os.path.exists(ffile):
type = 'd'
ddcal_image = insert_fits_data(beamdata, path, type)
# insert_beamdata_record(tid, beam, is_aimg, is_dimg, imsize, ra, dec, pixel_size, bmaj, bmin, bpa)
# import socket
# if socket.gethostname() == 'dop433':
# mnt = '/home/kutkin/mnt/hyperion'
# else:
# mnt = ''
update_beamdata(repeat=3600)
\ No newline at end of file
update_apercal_data(dirs=['/home/kutkin/mnt/happili-01/data/apertif',
'/home/kutkin/mnt/happili-01/data2/apertif'], repeat=14400, force=True)
update_ddcal_data(dirs = ['/home/kutkin/mnt/hyperion/kutkin/processed_ddcal'], repeat=3600)
\ No newline at end of file
......@@ -9,6 +9,13 @@ class BeamData(models.Model):
]
taskID = models.CharField(max_length=10)
beam = models.IntegerField()
field = models.CharField(max_length=20, default='')
class CalibratorsData(BeamData):
name = models.CharField(max_length=20)
caltype = models.CharField(choices=[('f','fluxcal'), ('p', 'polcal')], max_length=10)
used = models.ForeignKey(BeamData, on_delete=models.DO_NOTHING, related_name='calibrators')
class Image(models.Model):
......
from django.conf.urls import include, url
from django.urls import include, path
from rest_framework.routers import DefaultRouter
from .views import BeamDataViewset, ImageViewset
from .views import BeamDataViewset, ImageViewset, update_apercal_status, update_calibrators_info
mainrouter = DefaultRouter()
mainrouter.register('beamdata', BeamDataViewset)
mainrouter.register('images', ImageViewset)
urlpatterns = [url('', include(mainrouter.urls))]
urlpatterns = [path('', include(mainrouter.urls)), path('apercal_status/<str:tid>/<str:field>', update_apercal_status),
path('calibrators_info/<str:tid>/<str:caltype>/<str:calname>', update_calibrators_info)]
from django.shortcuts import render
from rest_framework import viewsets
from rest_framework.response import Response
from .serializers import BeamDataSerializer, ImageSerializer
from .models import BeamData, Image
from rest_framework.decorators import api_view
# Create your views here.
......@@ -18,3 +20,15 @@ class ImageViewset(viewsets.ModelViewSet):
'path', 'type', 'config', 'imsize', 'bmaj', 'bmin', 'bpa', 'ra', 'dec',
'pixel_size', 'stokes', 'beamdata__taskID', 'beamdata__beam'
]
@api_view(['PUT'])
def update_apercal_status(request, tid, field):
beam_statuses = request.data
print(tid, field, beam_statuses)
return Response(status=200)
@api_view(['PUT'])
def update_calibrators_info(request, tid, caltype, calname):
calibrators_info = request.data
print(calibrators_info, tid, caltype, calname)
return Response(status=200)
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment