diff --git a/lofar/README b/lofar/README new file mode 100644 index 0000000000000000000000000000000000000000..a8d4d93a1f545770a868f3a4e69bc0b2c51644ed --- /dev/null +++ b/lofar/README @@ -0,0 +1,8 @@ +This is a first stab at pulling the LOFAR archive into Obscore, mainly +by Adrian. There's a JIRA issue on this: TBD + +Status: Doesn't really work. Markus as written a custom grammar +that could be taught to read from the Groningen API and then +do incremental imports. Adrian might remember the chat with Markus +about that. + diff --git a/lofar/q.rd b/lofar/q.rd new file mode 100644 index 0000000000000000000000000000000000000000..f8d850d2deb3f13e7c0893b4d3a264edebc2fe23 --- /dev/null +++ b/lofar/q.rd @@ -0,0 +1,225 @@ +<?xml version="1.0"?> +<resource resdir="lofar" schema="lofar"> + <meta name="title">LOFAR Test VO Service</meta> + <meta name="creationDate">2009-09-30T14:53:00Z</meta> + <meta name="creator.name"> + Renting, G.A.;et al. + </meta> + <meta name="version">Data Release 2018</meta> + + <meta name="subject">Catalogs</meta> + <meta name="subject">Surveys</meta> + + <meta name="description"> + Test Data Service for LOFAR + </meta> + + + <meta name="_related" + title="Project Website">http://www.astron.nl</meta> + + <meta name="coverage.waveband">Radio</meta> + + <meta name="_intro" format="rst"><![CDATA[ + For advanced queries, you can query + `this </__system__/dc_tables/show/tableinfo/lofar.main>`_ + and many other tables + using our \RSTservicelink{/tap}{TAP service}. + ]]></meta> + + <meta name="source">Boo</meta> + + <meta name="facility">LOFAR</meta> + + <meta name="instrument">Correlator</meta> + + <meta name="copyright"> + All publications making use of the LOFAR data + should refer to the paper given in source. + </meta> + +<!-- you probably want to have <execute every="">... to regularly +poll and harvest the upstram archive --> + + <STREAM id="commoncolumns"> + <column name="raj2000" + type="double precision" + unit="deg" + ucd="pos.eq.ra;meta.main" + tablehead="RA" + description="Right ascension (J2000.0)" + /> + <column name="dej2000" + type="double precision" + unit="deg" + ucd="pos.eq.dec;meta.main" + tablehead="Dec" + description="Declination (J2000.0)" + /> + <column name="obsdate" + unit="d" + ucd="time.epoch;obs" + tablehead="Date" + description="Observation date." + displayHint="type=humanDate" + type="double precision" + xtype="mjd" + /> + <column name="field_name" + type="text" + ucd="meta.id;obs.field" + tablehead="Field" + description="Name of observed field (RA/DE)" + verbLevel="20" + /> + <column name="observation_id" + required="true" + type="smallint" + ucd="meta.id;obs.field" + tablehead="ObservationID" + description="Observation Sequence number" + verbLevel="25" + /> + + <meta name="note" tag="5"> + Flag of Stellar Parameter Pipeline. + + === ================================================================== + 0 Pipeline converged + 1 no convergence. + 2 MATISSE oscillates between two values and the mean is given. + 3 results of MATISSE at the boundaries or outside the grid and + the DEGAS value has been adopted + 4 the metal-poor giants with SNR<20 have been re-run by + DEGAS with a scale factor (ie, internal parameter of DEGAS) + of 0.40 + === ================================================================== + </meta> + </STREAM> + + <table id="main" onDisk="True" adql="True" primary="lofar_obs_id" + mixin="//scs#q3cindex"> + <mixin + accessURL="dlurl" + size="10" + mime="'application/x-votable+xml;content=datalink'" + calibLevel="1" + collectionName="'LOFAR_VIS'" + coverage="s_region" + dec="s_dec" + emMax="7e-7" + emMin="3.7e-7" + emResPower="4000/red_disp_mean" + expTime="t_exptime" + facilityName="'LOFAR'" + fov="1.0" + instrumentName="'LOFAR correlator'" + oUCD="'phot.flux;em.opt'" + productType="'cube'" + ra="s_ra" + sResolution="0.2778" + title="obs_title" + tMax="t_min" + tMin="t_max" + targetClass="'Galaxy'" + targetName="target_name" + >//obscore#publish</mixin> + <mixin>//products#table</mixin> + <meta name="description">The LOFAR Visibilities Catalog + for observations 10-250 MHz + </meta> + + <stc>Position ICRS "raj2000" "dej2000"</stc> + + <column name="lofar_obs_id" type="text" + ucd="meta.id;meta.main" + tablehead="LOFAR UID" + description="Unique Identifier for LOFAR observations" + verbLevel="1"/> + <column name="process_id" type="text" + ucd="meta.id" + tablehead="ProcessID" + description="Main LOFAR process identifier" + verbLevel="1"/> + <column name="datalink" type="text" + ucd="meta.ref.url" + tablehead="DL" + description="URL of a datalink document for this dataset" + verbLevel="1" displayHint="type=url"/> + + <FEED source="commoncolumns"/> + + </table> + + <coverage> + <updater sourceTable="main" mocOrder="4"/> + <temporal>52741 56386</temporal> + <spatial>0/8,11 1/16,24,42-43 2/68,72,80,82,88,100,104,113,120,144,146,149,151-152,154,156-157,159-160,162,165 3/7,276-278,280-282,292-294,296,324-326,332,335,340,356,360-362,367,404,406,408,410,420-422,424-426,448,450-451,458,464-465,468-469,484-485,488-490,580-582,588,590,592-593,595,601-603,612,614,620,622,632-633,635,644-645,656-657,659,667,670-671 4/0-1,27,256,265,267-268,270,282,285,292-293,295,514,552,576,776,809,814,1116-1118,1136,1180-1181,1188-1190,1192-1194,1196,1207-1209,1216,1308,1310,1332,1334-1335,1338,1344-1345,1347-1349,1351,1357,1365,1368,1370,1374,1377,1379,1382,1386,1428-1430,1432-1434,1436,1452,1454,1456,1460,1463,1506,1512,1620-1622,1636-1638,1648,1653-1654,1659,1692,1708,1710,1728,1796-1798,1825,1829,1838-1841,1844,1850-1851,1854,1864-1865,1868-1870,1880-1881,1887-1888,1890,1893,1908,1944-1946,1964-1965,1968,1976,1992,2332-2334,2356,2358,2364,2376-2377,2379,2401-2403,2455,2463,2485-2487,2492-2494,2536-2537,2539,2584-2586,2588-2589,2591,2610,2618-2619,2621-2623,2632,2635,2656-2657,2660,2664,2666-2667</spatial> + </coverage> + + <data id="import_main" auto="False" updating="True"> + <!-- before importing, seed data/laststamp with 0; if + the file doesn't exist, nothing gets imported --> + <sources>data/laststamp</sources> + <customGrammar module="res/grongrammar"/> + <make table="main" idmaps="*"> + <rowmaker id="make_main"> + <map dest="obsDate">parseDate(@Obsdate, "%Y%m%d")</map> + <map dest="pmra">scale(parseFloat(@pmRA), DEG_MAS)</map> + <map dest="pmde">scale(parseFloat(@pmDE), DEG_MAS)</map> + <map dest="e_pmra">scale(parseFloat(@epmRA), DEG_MAS)</map> + <map dest="e_pmde">scale(parseFloat(@epmDE), DEG_MAS)</map> + + <simplemaps> + field_name:target, + observation_id:observation_id, + lofar_obs_id:lofar_obs_id, + process_id:process_id, + datalink:datalink + </simplemaps> + </rowmaker> + </make> + </data> + + <data id="data" auto="false"> + <!-- this is for publishing all LOFAR DATA releases; add makes as appropriate --> + <publish/> + <make table="main"/> + </data> + + <service id="cone" allowed="form,scs.xml"> + <meta name="shortName">lofar_scs</meta> + <meta name="title">LOFAR Cone Search</meta> + <publish render="form" sets="local,ivo_managed"/> + <publish render="scs.xml" sets="ivo_managed"/> + + <scsCore id="conecore" queriedTable="main"> + <FEED source="//scs#coreDescs"/> + <!--condDesc buildFrom="rv"/> + <condDesc buildFrom="Jmag"/> + <condDesc buildFrom="snr_k"/--> + </scsCore> + + <outputTable verbLevel="20"/> + + <meta name="testQuery.ra">324.94</meta> + <meta name="testQuery.dec">-35.76</meta> + <meta name="testQuery.sr">0.5</meta> + </service> + + <!-- regSuite title="RAVE cone search regression"> + <regTest title="RAVE cone search delivers plausible data"> + <url RA="9.172416" DEC="-40.74506" + SR="0.002" VERB="3">cone/scs.xml</url> + <code> + row = self.getFirstVOTableRow() + self.assertEqual(row["id_denis"], 'J003641.3-404441') + self.assertAlmostEqual(row["rv"], 16.364999771118164) + self.assertEqual(row["obsdate"], 52887.0) + self.assertEqual(row["id_ppmxl"], 1454951088173346604L) + self.assertEqual(row["min_dists"], 'nnnnnnnnnnnnnnnnnnnn') + self.assertEqual(row["ti"], None) + </code> + </regTest> + </regSuite --> +</resource> diff --git a/lofar/res/grongrammar.py b/lofar/res/grongrammar.py new file mode 100644 index 0000000000000000000000000000000000000000..7080199c9a72049ae0b07cfab449da8e329aad71 --- /dev/null +++ b/lofar/res/grongrammar.py @@ -0,0 +1,27 @@ +import random + +from gavo.grammars.customgrammar import CustomRowIterator + +# create data/laststamp and fill it with 0 to start + +class RowIterator(CustomRowIterator): + def _iterRows(self): + with open(self.sourceToken) as f: + lastid = int(f.read()) + + # custom code here + for offset in range(10): + yield {"id": lastid, "value": random.random()} + lastid += 1 + + with open(self.sourceToken, "w") as f: + f.write("%d\n"%(lastid+1)) + + +if __name__=="__main__": + import sys + + from gavo.grammars.customgrammar import CustomGrammar + ri = RowIterator(CustomGrammar(None), "data/debugstate") + for row in ri: + print(row)