adding a draft lofar RD with a sketch of how a grammar that pulls from Groningen could look like.

4125432d · Markus Demleitner · 1e04a1cd · 4125432d · 4125432d · 4125432d
Commit 4125432d authored 5 years ago by Markus Demleitner
--- a/lofar/README
+++ b/lofar/README
+This is a first stab at pulling the LOFAR archive into Obscore, mainly
+by Adrian.  There's a JIRA issue on this: TBD
+Status: Doesn't really work.  Markus as written a custom grammar
+that could be taught to read from the Groningen API and then
+do incremental imports.  Adrian might remember the chat with Markus
+about that.
--- a/lofar/q.rd
+++ b/lofar/q.rd
+<?xml version="1.0"?>
+<resource resdir="lofar" schema="lofar">
+	<meta name="title">LOFAR Test VO Service</meta>
+	<meta name="creationDate">2009-09-30T14:53:00Z</meta>
+	<meta name="creator.name">
+          Renting, G.A.;et al.
+	</meta>
+	<meta name="version">Data Release 2018</meta>
+	<meta name="subject">Catalogs</meta>
+	<meta name="subject">Surveys</meta>
+	<meta name="description">
+	Test Data Service for LOFAR
+        </meta>
+	<meta name="_related"
+		title="Project Website">http://www.astron.nl</meta>
+	<meta name="coverage.waveband">Radio</meta>
+	<meta name="_intro" format="rst"><![CDATA[
+		For advanced queries, you can query
+		`this </__system__/dc_tables/show/tableinfo/lofar.main>`_
+		and many other tables
+		using our \RSTservicelink{/tap}{TAP service}.
+	]]></meta>
+	<meta name="source">Boo</meta>
+	<meta name="facility">LOFAR</meta>
+	<meta name="instrument">Correlator</meta>
+	<meta name="copyright">
+		All publications making use of the LOFAR data
+		should refer to the paper given in source.
+	</meta>
+<!-- you probably want to have <execute every="">... to regularly
+poll and harvest the upstram archive -->
+	<STREAM id="commoncolumns">
+		<column name="raj2000"
+			type="double precision"
+			unit="deg"
+			ucd="pos.eq.ra;meta.main"
+			tablehead="RA"
+			description="Right ascension (J2000.0)"
+			/>
+		<column name="dej2000"
+			type="double precision"
+			unit="deg"
+			ucd="pos.eq.dec;meta.main"
+			tablehead="Dec"
+			description="Declination (J2000.0)"
+			/>
+		<column name="obsdate"
+			unit="d"
+			ucd="time.epoch;obs"
+			tablehead="Date"
+			description="Observation date."
+			displayHint="type=humanDate"
+                        type="double precision"
+                        xtype="mjd"
+			/>
+		<column name="field_name"
+			type="text"
+			ucd="meta.id;obs.field"
+			tablehead="Field"
+			description="Name of observed field (RA/DE)"
+			verbLevel="20"
+			/>
+		<column name="observation_id"
+			required="true"
+			type="smallint"
+			ucd="meta.id;obs.field"
+			tablehead="ObservationID"
+			description="Observation Sequence number"
+			verbLevel="25"
+			/>
+		<meta name="note" tag="5">
+			Flag of Stellar Parameter Pipeline.
+			=== ==================================================================
+			 0  Pipeline converged
+			 1  no convergence.
+			 2  MATISSE oscillates between two values and the mean is given.
+			 3  results of MATISSE at the boundaries or outside the grid and
+			    the DEGAS value has been adopted
+			 4  the metal-poor giants with SNR&lt;20 have been re-run by
+			    DEGAS with a scale factor (ie, internal parameter of DEGAS)
+			    of 0.40
+			=== ==================================================================
+		</meta>
+	</STREAM>
+	<table id="main" onDisk="True" adql="True" primary="lofar_obs_id"
+          mixin="//scs#q3cindex">
+        <mixin
+          accessURL="dlurl"
+          size="10"
+          mime="'application/x-votable+xml;content=datalink'"
+          calibLevel="1"
+          collectionName="'LOFAR_VIS'"
+          coverage="s_region"
+          dec="s_dec"
+          emMax="7e-7"
+          emMin="3.7e-7"
+          emResPower="4000/red_disp_mean"
+          expTime="t_exptime"
+          facilityName="'LOFAR'"
+          fov="1.0"
+          instrumentName="'LOFAR correlator'"
+          oUCD="'phot.flux;em.opt'"
+          productType="'cube'"
+          ra="s_ra"
+          sResolution="0.2778"
+          title="obs_title"
+          tMax="t_min"
+          tMin="t_max"
+          targetClass="'Galaxy'"
+          targetName="target_name"
+          >//obscore#publish</mixin>
+                <mixin>//products#table</mixin>
+		<meta name="description">The LOFAR Visibilities Catalog
+                  for observations 10-250 MHz
+		</meta>
+		<stc>Position ICRS "raj2000" "dej2000"</stc>
+		<column name="lofar_obs_id" type="text"
+			ucd="meta.id;meta.main"
+			tablehead="LOFAR UID"
+			description="Unique Identifier for LOFAR observations"
+			verbLevel="1"/>
+		<column name="process_id" type="text"
+			ucd="meta.id"
+			tablehead="ProcessID"
+			description="Main LOFAR process identifier"
+			verbLevel="1"/>
+                <column name="datalink" type="text"
+                        ucd="meta.ref.url"
+                        tablehead="DL"
+                        description="URL of a datalink document for this dataset"
+                        verbLevel="1" displayHint="type=url"/>
+		<FEED source="commoncolumns"/>
+	</table>
+	<coverage>
+		<updater sourceTable="main" mocOrder="4"/>
+		<temporal>52741 56386</temporal>
+		<spatial>0/8,11 1/16,24,42-43 2/68,72,80,82,88,100,104,113,120,144,146,149,151-152,154,156-157,159-160,162,165 3/7,276-278,280-282,292-294,296,324-326,332,335,340,356,360-362,367,404,406,408,410,420-422,424-426,448,450-451,458,464-465,468-469,484-485,488-490,580-582,588,590,592-593,595,601-603,612,614,620,622,632-633,635,644-645,656-657,659,667,670-671 4/0-1,27,256,265,267-268,270,282,285,292-293,295,514,552,576,776,809,814,1116-1118,1136,1180-1181,1188-1190,1192-1194,1196,1207-1209,1216,1308,1310,1332,1334-1335,1338,1344-1345,1347-1349,1351,1357,1365,1368,1370,1374,1377,1379,1382,1386,1428-1430,1432-1434,1436,1452,1454,1456,1460,1463,1506,1512,1620-1622,1636-1638,1648,1653-1654,1659,1692,1708,1710,1728,1796-1798,1825,1829,1838-1841,1844,1850-1851,1854,1864-1865,1868-1870,1880-1881,1887-1888,1890,1893,1908,1944-1946,1964-1965,1968,1976,1992,2332-2334,2356,2358,2364,2376-2377,2379,2401-2403,2455,2463,2485-2487,2492-2494,2536-2537,2539,2584-2586,2588-2589,2591,2610,2618-2619,2621-2623,2632,2635,2656-2657,2660,2664,2666-2667</spatial>
+	</coverage>
+	<data id="import_main" auto="False" updating="True">
+		<!-- before importing, seed data/laststamp with 0; if
+		the file doesn't exist, nothing gets imported -->
+		<sources>data/laststamp</sources>
+		<customGrammar module="res/grongrammar"/>
+		<make table="main" idmaps="*">
+			<rowmaker id="make_main">
+				<map dest="obsDate">parseDate(@Obsdate, "%Y%m%d")</map>
+				<map dest="pmra">scale(parseFloat(@pmRA), DEG_MAS)</map>
+				<map dest="pmde">scale(parseFloat(@pmDE), DEG_MAS)</map>
+				<map dest="e_pmra">scale(parseFloat(@epmRA), DEG_MAS)</map>
+				<map dest="e_pmde">scale(parseFloat(@epmDE), DEG_MAS)</map>
+				<simplemaps>
+                                        field_name:target,
+					observation_id:observation_id,
+					lofar_obs_id:lofar_obs_id,
+                                        process_id:process_id,
+					datalink:datalink
+				</simplemaps>
+			</rowmaker>
+		</make>
+	</data>
+	<data id="data" auto="false">
+		<!-- this is for publishing all LOFAR DATA releases; add makes as appropriate -->
+		<publish/>
+		<make table="main"/>
+	</data>
+	<service id="cone" allowed="form,scs.xml">
+		<meta name="shortName">lofar_scs</meta>
+		<meta name="title">LOFAR Cone Search</meta>
+		<publish render="form" sets="local,ivo_managed"/>
+		<publish render="scs.xml" sets="ivo_managed"/>
+		<scsCore id="conecore" queriedTable="main">
+			<FEED source="//scs#coreDescs"/>
+			<!--condDesc buildFrom="rv"/>
+			<condDesc buildFrom="Jmag"/>
+			<condDesc buildFrom="snr_k"/-->
+		</scsCore>
+		<outputTable verbLevel="20"/>
+		<meta name="testQuery.ra">324.94</meta>
+		<meta name="testQuery.dec">-35.76</meta>
+		<meta name="testQuery.sr">0.5</meta>
+	</service>
+	<!-- regSuite title="RAVE cone search regression">
+			<regTest title="RAVE cone search delivers plausible data">
+				<url RA="9.172416" DEC="-40.74506"
+					SR="0.002" VERB="3">cone/scs.xml</url>
+				<code>
+					row = self.getFirstVOTableRow()
+					self.assertEqual(row["id_denis"], 'J003641.3-404441')
+					self.assertAlmostEqual(row["rv"], 16.364999771118164)
+					self.assertEqual(row["obsdate"], 52887.0)
+					self.assertEqual(row["id_ppmxl"], 1454951088173346604L)
+					self.assertEqual(row["min_dists"], 'nnnnnnnnnnnnnnnnnnnn')
+					self.assertEqual(row["ti"], None)
+				</code>
+			</regTest>
+	</regSuite -->
+</resource>
--- a/lofar/res/grongrammar.py
+++ b/lofar/res/grongrammar.py
+import random
+from gavo.grammars.customgrammar import CustomRowIterator
+# create data/laststamp and fill it with 0 to start
+class RowIterator(CustomRowIterator):
+  def _iterRows(self):
+    with open(self.sourceToken) as f:
+      lastid = int(f.read())
+    # custom code here
+    for offset in range(10):
+	    yield {"id": lastid, "value": random.random()}
+	    lastid += 1
+    with open(self.sourceToken, "w") as f:
+      f.write("%d\n"%(lastid+1))
+if __name__=="__main__":
+  import sys
+  from gavo.grammars.customgrammar import CustomGrammar
+  ri = RowIterator(CustomGrammar(None), "data/debugstate")
+  for row in ri:
+    print(row)