Skip to content
Snippets Groups Projects
Commit 80a018c8 authored by Jörn Künsemöller's avatar Jörn Künsemöller
Browse files

Task #9091 - fixed schema location, moved to new XSD with integer id

parent 1b58ea7b
No related branches found
No related tags found
No related merge requests found
Show whitespace changes
Inline Side-by-side
lib/CMakeLists.txt
+ 1
− 0
View file @ 80a018c8
......@@ -11,6 +11,7 @@ set(_py_files
validator.py
constants.py
visualizer.py
query.py
)
......
lib/LTA-SIP.xsd
+ 10
− 3
View file @ 80a018c8
<?xml version="1.0" encoding="UTF-8"?>
<xs:schema targetNamespace="http://www.astron.nl/SIP-Lofar" version="2.6.1" xmlns="http://www.astron.nl/SIP-Lofar" xmlns:xs="http://www.w3.org/2001/XMLSchema">
<xs:schema targetNamespace="http://www.astron.nl/SIP-Lofar" version="2.7.0" xmlns="http://www.astron.nl/SIP-Lofar" xmlns:xs="http://www.w3.org/2001/XMLSchema">
<xs:annotation>
<xs:documentation>
XML Schema for data model Submission Information Package LOFAR Long Term Archive
......@@ -112,7 +112,7 @@
<xs:complexType name="IdentifierType">
<xs:sequence>
<xs:element name="source" type="xs:string"/>
<xs:element name="identifier" type="xs:integer"/>
<xs:element name="identifier" type="xs:nonNegativeInteger"/>
<xs:element name="name" minOccurs="0" type="xs:string"/>
</xs:sequence>
</xs:complexType>
......@@ -731,7 +731,12 @@
</xs:sequence>
</xs:complexType>
<xs:annotation>
<xs:documentation>We plan to support three types of file formats currently in the LTA.</xs:documentation>
<xs:documentation>We currently have three well documented fileformats in the LTA:
FITS, CASA and HDF5. There are LOFAR ICDs describing the various types (Correlated, BeamFormed, TBB, InstrumentModel, ...)
and how they are stored in one or more of these file formats.
We also have PULP, which currently isn't well documented, and expect PREFACTOR in 2017, which might get documented.
Next to this we have the UNDOCUMENTED option for data that is in a format that will never be documented.
</xs:documentation>
</xs:annotation>
<xs:simpleType name="FileFormatType">
<xs:restriction base="xs:string">
......@@ -739,6 +744,8 @@
<xs:enumeration value="AIPS++/CASA"/>
<xs:enumeration value="HDF5"/>
<xs:enumeration value="PULP"/>
<xs:enumeration value="PREFACTOR"/>
<xs:enumeration value="UNDOCUMENTED"/>
</xs:restriction>
</xs:simpleType>
<xs:annotation>
......
lib/ltasip.py
+ 1209
− 1207
View file @ 80a018c8
Source diff could not be displayed: it is too large. Options to address this: view the blob.
lib/query.py
+ 36
− 31
View file @ 80a018c8
......@@ -6,6 +6,7 @@ import requests
from os.path import expanduser
import xml.etree.ElementTree as ET
import xmlrpclib
import uuid
path = expanduser("~/.siplibrc")
......@@ -27,44 +28,48 @@ login_data = {
url = 'https://'+user+':'+passw+'@lofar-ingest.target.rug.nl:9443/'
client = xmlrpclib.ServerProxy(url)
def get_unique_id(source):
unique_id = uuid.uuid1().int>>64 #todo: replace by query
return unique_id
def getsip_fromlta_byprojectandltadataproductid(projectname,dpid):
return client.GetSip(projectname,dpid).get("sip")
def getltadataproductids_fromlta_byprojectandsasid(projectname, sasid):
return client.GetDataProductIDS(projectname,sasid).get("ids")
def getsip_frommom_byltadataproductid(id):
id = int(id) - 1000000 # ask Nico...
command = 'GETSIP&id='+str(id)
return __mom_importxml(command)
#def getsip_frommom_byltadataproductid(id):
# id = int(id) - 1000000 # ask Nico...
# command = 'GETSIP&id='+str(id)
# return __mom_importxml(command)
def getsip_frommom_bymomdataproductid(id):
command = 'GETSIP&mom2DPId='+str(id)
return __mom_importxml(command)
#def getsip_frommom_bymomdataproductid(id):
# command = 'GETSIP&mom2DPId='+str(id)
# return __mom_importxml(command)
def getmomdataproductids_frommom_bysasid(id):
command = 'getmom2DPIds&sasid='+str(id)
response = __mom_importxml(command)
#print response
root = ET.fromstring(response)
ids = []
for dpid in root.find('outputdataproducts').findall('mom2dpid'):
#print dpid
ids.append(dpid.text)
if not ids:
print "WARNING: No mom2dpids found. maybe your input id is wrong?"
return ids
#def getmomdataproductids_frommom_bysasid(id):
# command = 'getmom2DPIds&sasid='+str(id)
# response = __mom_importxml(command)
# #print response
# root = ET.fromstring(response)
# ids = []
# for dpid in root.find('outputdataproducts').findall('mom2dpid'):
# #print dpid
# ids.append(dpid.text)
# if not ids:
# print "WARNING: No mom2dpids found. maybe your input id is wrong?"
# return ids
def __mom_importxml(command):
print "Querying MoM:",command
s = requests.session()
r1 = s.get('https://lofar.astron.nl/useradministration/user/forward.do?forward=http://lofar.astron.nl/mom3/interface/importXML2.do')
#print r1.cookies
r2 = s.post('https://lofar.astron.nl/useradministration/user/j_security_check', headers={'referer': 'https://lofar.astron.nl/useradministration/user/forward.do?forward=http://lofar.astron.nl/mom3/interface/importXML2.do'} , cookies=r1.cookies, data=login_data)
#print r2.cookies
r3 = s.get('https://lofar.astron.nl/useradministration/user/forward.do?forward=http://lofar.astron.nl/mom3/interface/importXML2.do', cookies=r1.cookies)
#print r3.cookies
r4 = s.get('https://lofar.astron.nl/mom3/interface/importXML2.do?command='+command.strip(), cookies = r3.cookies)
#print r4.cookies
return r4.content
\ No newline at end of file
#def __mom_importxml(command):
# print "Querying MoM:",command
# s = requests.session()
# r1 = s.get('https://lofar.astron.nl/useradministration/user/forward.do?forward=http://lofar.astron.nl/mom3/interface/importXML2.do')
# #print r1.cookies
# r2 = s.post('https://lofar.astron.nl/useradministration/user/j_security_check', headers={'referer': 'https://lofar.astron.nl/useradministration/user/forward.do?forward=http://lofar.astron.nl/mom3/interface/importXML2.do'} , cookies=r1.cookies, data=login_data)
# #print r2.cookies
# r3 = s.get('https://lofar.astron.nl/useradministration/user/forward.do?forward=http://lofar.astron.nl/mom3/interface/importXML2.do', cookies=r1.cookies)
# #print r3.cookies
# r4 = s.get('https://lofar.astron.nl/mom3/interface/importXML2.do?command='+command.strip(), cookies = r3.cookies)
# #print r4.cookies
# return r4.content
\ No newline at end of file
lib/siplib.py
+ 65
− 38
View file @ 80a018c8
......@@ -29,12 +29,14 @@ import ltasip
import pyxb
import constants
import os
import uuid
import xml.dom.minidom
from pyxb.namespace import XMLSchema_instance as xsi
from pyxb.namespace import XMLNamespaces as xmlns
import query
VERSION = "SIPlib 0.3"
VERSION = "SIPlib 0.4"
d = os.path.dirname(os.path.realpath(__file__))
STATION_CONFIG_PATH = d+'/station_coordinates.conf'
ltasip.Namespace.setPrefix('sip')
......@@ -48,6 +50,16 @@ ltasip.Namespace.setPrefix('sip')
# as per schema definition are optional parameters.
def print_user_warning():
print "!!! You are accessing an object, which is based on code that was auto-generated with the pyxb package."
print "!!! We strongly advise you to only use the datatypes of the siplib wrapper to create your SIP file."
print "!!! If you choose to alter pyxb/ltasip objects or their values directly for some reason, your SIP may "
print "!!! become invalid. - Please make sure to validate your SIP before submission! "
print "!!! Note that the pyxb portion of the code is subject to change whitout being backwards compatible."
print "!!! This means that, should you choose to access pyxb objects e.g. to parse an existing SIP file, things"
print "!!! might break for you without further warning."
print "!!! (You may suppress this warning by setting the flag in the pyxb-related getter/setter functions.)"
# ===============================
# Identifier definition (used for LTA entities, i-e- processes and dataproducts):
......@@ -59,6 +71,10 @@ class Identifier():
):
self.__pyxb_identifier=ltasip.IdentifierType(source=source, identifier=id, name=name)
@classmethod
def request_unique_identifier(cls, source, name=None):
unique_id = query.get_unique_id(source)
return cls(source, unique_id, name)
def get_pyxb_identifier(self):
return self.__pyxb_identifier
......@@ -129,8 +145,6 @@ class Station():
antennafield2=__afield2,
)
def get_pyxb_station(self):
return self.__pyxb_station
......@@ -1215,11 +1229,11 @@ class Sip(object):
project_coinvestigators=None,
):
self.sip = ltasip.ltaSip()
self.__sip = ltasip.ltaSip()
self.sip.sipGeneratorVersion = VERSION
self.__sip.sipGeneratorVersion = VERSION
self.sip.project = ltasip.Project(
self.__sip.project = ltasip.Project(
projectCode=project_code,
primaryInvestigator=project_primaryinvestigator,
contactAuthor=project_contactauthor,
......@@ -1228,33 +1242,43 @@ class Sip(object):
coInvestigator=project_coinvestigators,
)
self.sip.dataProduct=dataproduct.get_pyxb_dataproduct()
self.__sip.dataProduct=dataproduct.get_pyxb_dataproduct()
self.get_prettyxml() # for validation
@classmethod
def from_xml(cls, xml):
newsip = Sip.__new__(Sip)
newsip.sip = ltasip.CreateFromDocument(xml)
newsip._set_pyxb_sip(ltasip.CreateFromDocument(xml), suppress_warning=True)
newsip.get_prettyxml() # for validation
return newsip
def get_pyxb_sip(self, suppress_warning=False):
if not suppress_warning:
print_user_warning()
return self.__sip
def _set_pyxb_sip(self, pyxb_sip, suppress_warning=False):
if not suppress_warning:
print_user_warning()
self.__sip = pyxb_sip
#-------------
# Optional additions
#---
def add_related_dataproduct(self, dataproduct):
self.sip.relatedDataProduct.append(dataproduct.get_pyxb_dataproduct())
self.__sip.relatedDataProduct.append(dataproduct.get_pyxb_dataproduct())
return self.get_prettyxml()
def add_observation(self, observation):
self.sip.observation.append(observation.get_pyxb_observation())
self.__sip.observation.append(observation.get_pyxb_observation())
return self.get_prettyxml()
def add_pipelinerun(self, pipeline):
self.sip.pipelineRun.append(pipeline.get_pyxb_pipeline())
self.__sip.pipelineRun.append(pipeline.get_pyxb_pipeline())
return self.get_prettyxml()
......@@ -1269,7 +1293,7 @@ class Sip(object):
description=description,
**process_map.get_dict()
)
self.sip.unspecifiedProcess.append(up)
self.__sip.unspecifiedProcess.append(up)
return self.get_prettyxml()
......@@ -1278,7 +1302,7 @@ class Sip(object):
identifier,
contents):
self.sip.parset.append(ltasip.Parset(
self.__sip.parset.append(ltasip.Parset(
identifier=identifier.get_pyxb_identifier(),
contents=contents
))
......@@ -1287,42 +1311,45 @@ class Sip(object):
def add_related_dataproduct_with_history(self, relateddataproduct_sip):
# add the dataproduct described by the SIP (if not there)
if not any(x.dataProductIdentifier.identifier == relateddataproduct_sip.sip.dataProduct.dataProductIdentifier.identifier for x in self.sip.relatedDataProduct):
self.sip.relatedDataProduct.append(relateddataproduct_sip.sip.dataProduct)
if not any(x.dataProductIdentifier.identifier == relateddataproduct_sip.__sip.dataProduct.dataProductIdentifier.identifier for x in self.__sip.relatedDataProduct):
self.__sip.relatedDataProduct.append(relateddataproduct_sip.__sip.dataProduct)
else:
print "WARNING: There already exists a dataproduct with id", relateddataproduct_sip.sip.dataProduct.dataProductIdentifier.identifier," - Will try to add any new related items anyway."
if relateddataproduct_sip.sip.relatedDataProduct:
print "WARNING: There already exists a dataproduct with id", relateddataproduct_sip.__sip.dataProduct.dataProductIdentifier.identifier," - Will try to add any new related items anyway."
if relateddataproduct_sip.__sip.relatedDataProduct:
# add related dataproducts (if not there already)
for dp in relateddataproduct_sip.sip.relatedDataProduct:
if not any(x.dataProductIdentifier.identifier == dp.dataProductIdentifier.identifier for x in self.sip.relatedDataProduct):
self.sip.relatedDataProduct.append(dp)
if relateddataproduct_sip.sip.observation:
for dp in relateddataproduct_sip.__sip.relatedDataProduct:
if not any(x.dataProductIdentifier.identifier == dp.dataProductIdentifier.identifier for x in self.__sip.relatedDataProduct):
self.__sip.relatedDataProduct.append(dp)
if relateddataproduct_sip.__sip.observation:
# add related dataproducts (if not there already)
for obs in relateddataproduct_sip.sip.observation:
if not any(x.processIdentifier.identifier == obs.processIdentifier.identifier for x in self.sip.observation):
self.sip.observation.append(obs)
if relateddataproduct_sip.sip.pipelineRun:
for obs in relateddataproduct_sip.__sip.observation:
if not any(x.processIdentifier.identifier == obs.processIdentifier.identifier for x in self.__sip.observation):
self.__sip.observation.append(obs)
if relateddataproduct_sip.__sip.pipelineRun:
# add related pipelineruns (if not there already)
for pipe in relateddataproduct_sip.sip.pipelineRun:
if not any(x.processIdentifier.identifier == pipe.processIdentifier.identifier for x in self.sip.pipelineRun):
self.sip.pipelineRun.append(pipe)
if relateddataproduct_sip.sip.unspecifiedProcess:
for pipe in relateddataproduct_sip.__sip.pipelineRun:
if not any(x.processIdentifier.identifier == pipe.processIdentifier.identifier for x in self.__sip.pipelineRun):
self.__sip.pipelineRun.append(pipe)
if relateddataproduct_sip.__sip.unspecifiedProcess:
# add related unspecified processes (if not there already)
for unspec in relateddataproduct_sip.sip.unspecifiedProcess:
if not any(x.processIdentifier.identifier == unspec.processIdentifier.identifier for x in self.sip.unspecifiedProcess):
self.sip.unspecifiedProcess.append(unspec)
if relateddataproduct_sip.sip.parset:
for unspec in relateddataproduct_sip.__sip.unspecifiedProcess:
if not any(x.processIdentifier.identifier == unspec.processIdentifier.identifier for x in self.__sip.unspecifiedProcess):
self.__sip.unspecifiedProcess.append(unspec)
if relateddataproduct_sip.__sip.parset:
# add related parsets (if not there already)
for par in relateddataproduct_sip.sip.parset:
if not any(x.identifier.identifier == par.identifier.identifier for x in self.sip.parset):
self.sip.parset.append(par)
for par in relateddataproduct_sip.__sip.parset:
if not any(x.identifier.identifier == par.identifier.identifier for x in self.__sip.parset):
self.__sip.parset.append(par)
return self.get_prettyxml()
# this will also validate the document so far
def get_prettyxml(self):
try:
dom = self.sip.toDOM()
#dom.documentElement.setAttributeNS(xsi.uri(), 'xsi:schemaLocation', "http://www.astron.nl/SIP-Lofar LTA-SIP-2.6.1.xsd")
dom = self.__sip.toDOM()
#print "1) "+dom.toprettyxml()
dom.documentElement.setAttribute("xmlns:xsi", "http://www.w3.org/2001/XMLSchema-instance")
dom.documentElement.setAttribute('xsi:schemaLocation', "http://www.astron.nl/SIP-Lofar LTA-SIP-2.7.0.xsd")
#print "2) "+dom.toprettyxml()
return dom.toprettyxml()
except pyxb.ValidationError as err:
#print str(err)
......
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment