dataproductinfo.py

from lofardata.models import DataProduct, WorkSpecification, DataProductFilter, DataFilterType
from django.core.cache import cache

def retrieve_combined_information(sas_id):
    # Per SAS ID, the retrieved data products should have these unique values
    data_products = DataProduct.objects.filter(obs_id=sas_id).values('dataproduct_source',
                                                                     'dataproduct_type',
                                                                     'project',
                                                                     'location',
                                                                     'activity',
                                                                     'antenna_set',
                                                                     'instrument_filter',
                                                                     'dysco_compression').distinct()

    if len(data_products) == 0:
        return {}

    combined_data_products_on_key = combine_dataproducts_on_key(data_products)

    dysco_compressions = combined_data_products_on_key['dysco_compression']
    if len(dysco_compressions) == 0:  # the dysco_compression is unknown i.e. None
        combined_data_products_on_key['dysco_compression'] = [0]
    else:
        true_count = len([dysco_compression for dysco_compression in dysco_compressions if dysco_compression])
        dysco_compression_true_percentage = true_count / len(combined_data_products_on_key['dysco_compression'])
        # put in a list for template convenience
        combined_data_products_on_key['dysco_compression'] = [dysco_compression_true_percentage]

    return combined_data_products_on_key


def combine_dataproducts_on_key(data_products):
    combined_data_products = {}
    for data_product in data_products:
        combined_data_products = fill_unique_nested_dict(data_product, combined_data_products)
    return combined_data_products


def fill_unique_nested_dict(data_product, combined_data_products_on_key):
    for key, value in data_product.items():
        if combined_data_products_on_key.get(key) and value not in combined_data_products_on_key.get(key):
            combined_data_products_on_key[key].append(value)
        else:
            combined_data_products_on_key[key] = [value] if value is not None and value != '' else []
    return combined_data_products_on_key


def count_unspecified_files(workspecification_id: int):
    """ Retrieve amount of unspecified dataproducts for a given WorkSpecification

    Note: could be optimized by putting in the tasks.py where the
    DataProduct array is already available"""
    specification = WorkSpecification.objects.get(pk=workspecification_id)
    filters = specification.filters
    dataproducts = DataProduct.objects.filter(**filters).order_by("surl")
    return len([x for x in dataproducts if x.dataproduct_type == "Unknown"])

def get_distinct_dataproduct_field(dataproduct_field):
    cache_key = f'distinct_dataproduct_{dataproduct_field}'
    distinct_field_values = cache.get(cache_key)

    if not distinct_field_values:
        distinct_field_values = list(DataProduct.objects.distinct(dataproduct_field).values_list(dataproduct_field, flat=True))
        expiration_timeout_infinite = None
        cache.set(cache_key, distinct_field_values, expiration_timeout_infinite)

    return distinct_field_values

def prefill_distinct_dataproduct_field_cache():
    dataproduct_dropdown_filters = DataProductFilter.objects.filter(filter_type=DataFilterType.DROPDOWN).values_list('field', flat=True)
    for dataproduct_dropdown_filter in dataproduct_dropdown_filters:
        get_distinct_dataproduct_field(dataproduct_dropdown_filter)