diff --git a/.idea/workspace.xml b/.idea/workspace.xml index fc81016d3123ef68fc1f7d1bf3af8914b61d4fa3..e7f1e0ec25184c77fa59ae3ef5ea738962c106ea 100644 --- a/.idea/workspace.xml +++ b/.idea/workspace.xml @@ -2,10 +2,49 @@ <project version="4"> <component name="ChangeListManager"> <list default="true" id="39bb9056-a8a2-4105-bdfc-77202926e9ed" name="Default" comment="updated some Helio instruments as example"> - <change type="NEW" beforePath="" afterPath="$PROJECT_DIR$/.idea/workspace.xml" /> - <change type="MODIFICATION" beforePath="$PROJECT_DIR$/esap/esap/configuration/esap_default.py" afterPath="$PROJECT_DIR$/esap/esap/configuration/esap_default.py" /> - <change type="MODIFICATION" beforePath="$PROJECT_DIR$/esap/esap/configuration/esap_dev.py" afterPath="$PROJECT_DIR$/esap/esap/configuration/esap_dev.py" /> + <change type="NEW" beforePath="" afterPath="$PROJECT_DIR$/esap/configuration_assets/esap_solar.png" /> + <change type="NEW" beforePath="" afterPath="$PROJECT_DIR$/esap/query/api/__init__.py" /> + <change type="NEW" beforePath="" afterPath="$PROJECT_DIR$/esap/query/api/business/__init__.py" /> + <change type="NEW" beforePath="" afterPath="$PROJECT_DIR$/esap/query/serializers.py" /> + <change type="DELETED" beforePath="$PROJECT_DIR$/esap/api/business/services/query/alta.py" afterPath="" /> + <change type="DELETED" beforePath="$PROJECT_DIR$/esap/api/business/services/query/helio.py" afterPath="" /> + <change type="DELETED" beforePath="$PROJECT_DIR$/esap/api/business/services/query/query_base.py" afterPath="" /> + <change type="DELETED" beforePath="$PROJECT_DIR$/esap/api/business/services/query/vo.py" afterPath="" /> + <change type="DELETED" beforePath="$PROJECT_DIR$/esap/api/business/services/query/vo_reg.py" afterPath="" /> + <change type="DELETED" beforePath="$PROJECT_DIR$/esap/api/business/services/query/vso.py" afterPath="" /> + <change type="DELETED" beforePath="$PROJECT_DIR$/esap/api/business/services/staging/staging_base.py" afterPath="" /> + <change type="DELETED" beforePath="$PROJECT_DIR$/esap/api/migrations/0001_initial.py" afterPath="" /> + <change type="MODIFICATION" beforePath="$PROJECT_DIR$/.idea/workspace.xml" afterPath="$PROJECT_DIR$/.idea/workspace.xml" /> <change type="MODIFICATION" beforePath="$PROJECT_DIR$/esap/esap/esap.sqlite3" afterPath="$PROJECT_DIR$/esap/esap/esap.sqlite3" /> + <change type="MODIFICATION" beforePath="$PROJECT_DIR$/esap/esap/settings/base.py" afterPath="$PROJECT_DIR$/esap/esap/settings/base.py" /> + <change type="MODIFICATION" beforePath="$PROJECT_DIR$/esap/esap/settings/statics.py" afterPath="$PROJECT_DIR$/esap/esap/settings/statics.py" /> + <change type="MODIFICATION" beforePath="$PROJECT_DIR$/esap/esap/urls.py" afterPath="$PROJECT_DIR$/esap/esap/urls.py" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/__init__.py" afterPath="$PROJECT_DIR$/esap/query/__init__.py" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/admin.py" afterPath="$PROJECT_DIR$/esap/query/admin.py" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/apps.py" afterPath="$PROJECT_DIR$/esap/query/apps.py" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/business/common.py" afterPath="$PROJECT_DIR$/esap/query/api/utils.py" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/business/configuration.py" afterPath="$PROJECT_DIR$/esap/query/api/business/configuration.py" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/business/query_controller.py" afterPath="$PROJECT_DIR$/esap/query/api/business/query_controller.py" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/business/services/__init__.py" afterPath="$PROJECT_DIR$/esap/__init__.py" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/business/staging_controller.py" afterPath="$PROJECT_DIR$/esap/query/api/business/staging_controller.py" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/migrations/__init__.py" afterPath="$PROJECT_DIR$/esap/query/__pycache__/models.cpython-37.pyc.2226543272368" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/models.py" afterPath="$PROJECT_DIR$/esap/query/models.py" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/serializers.py" afterPath="$PROJECT_DIR$/esap/query/api/serializers.py" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/static/api/adex_logo.jpg" afterPath="$PROJECT_DIR$/esap/query/static/query/adex_logo.jpg" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/static/api/esap_logo.png" afterPath="$PROJECT_DIR$/esap/query/static/query/esap_logo.png" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/static/api/favicon.ico" afterPath="$PROJECT_DIR$/esap/query/static/favicon.ico" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/static/api/favicon.png" afterPath="$PROJECT_DIR$/esap/query/static/favicon.png" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/static/api/style.css" afterPath="$PROJECT_DIR$/esap/query/static/query/style.css" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/static/favicon.ico" afterPath="$PROJECT_DIR$/esap/query/static/query/favicon.ico" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/static/favicon.png" afterPath="$PROJECT_DIR$/esap/query/static/query/favicon.png" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/templates/api/base.html" afterPath="$PROJECT_DIR$/esap/query/templates/api/base.html" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/templates/api/index.html" afterPath="$PROJECT_DIR$/esap/query/templates/api/index.html" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/tests.py" afterPath="$PROJECT_DIR$/esap/query/tests.py" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/urls.py" afterPath="$PROJECT_DIR$/esap/query/api/urls.py" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/views/__init__.py" afterPath="$PROJECT_DIR$/esap/query/api/views/__init__.py" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/views/common_views.py" afterPath="$PROJECT_DIR$/esap/query/api/views/common_views.py" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/views/query_views.py" afterPath="$PROJECT_DIR$/esap/query/api/views/query_views.py" /> + <change type="MOVED" beforePath="$PROJECT_DIR$/esap/api/views/staging_views.py" afterPath="$PROJECT_DIR$/esap/query/__pycache__/models.cpython-37.pyc.2530047107504" /> </list> <ignored path=".idea/" /> <manually-removed-from-ignored> @@ -27,105 +66,109 @@ </component> <component name="FileEditorManager"> <leaf SIDE_TABS_SIZE_LIMIT_KEY="300"> - <file leaf-file-name="esap_dev.py" pinned="false" current-in-tab="true"> - <entry file="file://$PROJECT_DIR$/esap/esap/configuration/esap_dev.py"> + <file leaf-file-name="urls.py" pinned="false" current-in-tab="false"> + <entry file="file://$PROJECT_DIR$/esap/query/api/urls.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="850"> - <caret line="73" column="1" lean-forward="true" selection-start-line="73" selection-start-column="1" selection-end-line="73" selection-end-column="1" /> - <folding /> + <state relative-caret-position="265"> + <caret line="11" column="18" lean-forward="false" selection-start-line="11" selection-start-column="18" selection-end-line="11" selection-end-column="18" /> + <folding> + <element signature="e#0#28#0" expanded="true" /> + </folding> </state> </provider> </entry> </file> - <file leaf-file-name="esap_default.py" pinned="false" current-in-tab="false"> - <entry file="file://$PROJECT_DIR$/esap/esap/configuration/esap_default.py"> + <file leaf-file-name="urls.py" pinned="false" current-in-tab="false"> + <entry file="file://$PROJECT_DIR$/esap/esap/urls.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="585"> - <caret line="76" column="6" lean-forward="true" selection-start-line="76" selection-start-column="6" selection-end-line="76" selection-end-column="6" /> - <folding /> + <state relative-caret-position="530"> + <caret line="19" column="25" lean-forward="false" selection-start-line="19" selection-start-column="25" selection-end-line="19" selection-end-column="25" /> + <folding> + <element signature="e#633#665#0" expanded="true" /> + </folding> </state> </provider> </entry> </file> - <file leaf-file-name="adex.py" pinned="false" current-in-tab="false"> - <entry file="file://$PROJECT_DIR$/esap/esap/configuration/adex.py"> + <file leaf-file-name="staging_base.py" pinned="false" current-in-tab="false"> + <entry file="file://$PROJECT_DIR$/esap/query/api/business/services/staging/staging_base.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="5247"> - <caret line="99" column="0" lean-forward="false" selection-start-line="99" selection-start-column="0" selection-end-line="99" selection-end-column="0" /> + <state relative-caret-position="-313"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> <folding /> </state> </provider> </entry> </file> - <file leaf-file-name="models.py" pinned="false" current-in-tab="false"> - <entry file="file://$PROJECT_DIR$/esap/api/models.py"> + <file leaf-file-name="query_controller.py" pinned="false" current-in-tab="true"> + <entry file="file://$PROJECT_DIR$/esap/query/api/business/query_controller.py"> <provider selected="true" editor-type-id="text-editor"> <state relative-caret-position="530"> - <caret line="10" column="21" lean-forward="false" selection-start-line="10" selection-start-column="4" selection-end-line="10" selection-end-column="21" /> + <caret line="10" column="5" lean-forward="false" selection-start-line="10" selection-start-column="5" selection-end-line="10" selection-end-column="5" /> <folding /> </state> </provider> </entry> </file> - <file leaf-file-name="alta.py" pinned="false" current-in-tab="false"> - <entry file="file://$PROJECT_DIR$/esap/api/business/services/query/alta.py"> + <file leaf-file-name="common_views.py" pinned="false" current-in-tab="false"> + <entry file="file://$PROJECT_DIR$/esap/query/api/views/common_views.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="5088"> - <caret line="97" column="0" lean-forward="false" selection-start-line="97" selection-start-column="0" selection-end-line="97" selection-end-column="0" /> + <state relative-caret-position="212"> + <caret line="10" column="15" lean-forward="false" selection-start-line="10" selection-start-column="15" selection-end-line="10" selection-end-column="15" /> <folding /> </state> </provider> </entry> </file> - <file leaf-file-name="vo_reg.py" pinned="false" current-in-tab="false"> - <entry file="file://$PROJECT_DIR$/esap/api/business/services/query/vo_reg.py"> + <file leaf-file-name="query_views.py" pinned="false" current-in-tab="false"> + <entry file="file://$PROJECT_DIR$/esap/query/api/views/query_views.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="5406"> - <caret line="104" column="0" lean-forward="false" selection-start-line="104" selection-start-column="0" selection-end-line="104" selection-end-column="0" /> + <state relative-caret-position="742"> + <caret line="14" column="13" lean-forward="true" selection-start-line="14" selection-start-column="13" selection-end-line="14" selection-end-column="13" /> <folding> - <element signature="e#145#179#0" expanded="true" /> + <element signature="e#0#14#0" expanded="true" /> </folding> </state> </provider> </entry> </file> - <file leaf-file-name="query_controller.py" pinned="false" current-in-tab="false"> - <entry file="file://$PROJECT_DIR$/esap/api/business/query_controller.py"> + <file leaf-file-name="utils.py" pinned="false" current-in-tab="false"> + <entry file="file://$PROJECT_DIR$/esap/query/api/utils.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="2067"> - <caret line="44" column="48" lean-forward="false" selection-start-line="44" selection-start-column="16" selection-end-line="44" selection-end-column="48" /> + <state relative-caret-position="-578"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> <folding /> </state> </provider> </entry> </file> - <file leaf-file-name="dev.py" pinned="false" current-in-tab="false"> - <entry file="file://$PROJECT_DIR$/esap/esap/settings/dev.py"> + <file leaf-file-name="configuration.py" pinned="false" current-in-tab="false"> + <entry file="file://$PROJECT_DIR$/esap/query/api/business/configuration.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="1908"> - <caret line="38" column="30" lean-forward="false" selection-start-line="38" selection-start-column="30" selection-end-line="38" selection-end-column="30" /> + <state relative-caret-position="0"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> <folding> - <element signature="e#58#90#0" expanded="true" /> + <element signature="e#0#14#0" expanded="true" /> </folding> </state> </provider> </entry> </file> - <file leaf-file-name="base.py" pinned="false" current-in-tab="false"> - <entry file="file://$PROJECT_DIR$/esap/esap/settings/base.py"> + <file leaf-file-name="base.html" pinned="false" current-in-tab="false"> + <entry file="file://$PROJECT_DIR$/esap/query/templates/api/base.html"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="993"> - <caret line="182" column="0" lean-forward="false" selection-start-line="182" selection-start-column="0" selection-end-line="182" selection-end-column="0" /> + <state relative-caret-position="-2022"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> <folding /> </state> </provider> </entry> </file> - <file leaf-file-name="vo.py" pinned="false" current-in-tab="false"> - <entry file="file://$PROJECT_DIR$/esap/api/business/services/query/vo.py"> + <file leaf-file-name="index.html" pinned="false" current-in-tab="false"> + <entry file="file://$PROJECT_DIR$/esap/query/templates/api/index.html"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="7208"> - <caret line="137" column="26" lean-forward="false" selection-start-line="137" selection-start-column="26" selection-end-line="137" selection-end-column="26" /> + <state relative-caret-position="2842"> + <caret line="72" column="50" lean-forward="false" selection-start-line="72" selection-start-column="50" selection-end-line="72" selection-end-column="50" /> <folding /> </state> </provider> @@ -142,10 +185,6 @@ </component> <component name="FindInProjectRecents"> <findStrings> - <find>print(s)</find> - <find>esap_service</find> - <find>queryset</find> - <find>-and</find> <find>repl</find> <find>AMP_REPLACEMENT</find> <find>construct</find> @@ -172,6 +211,10 @@ <find>'q</find> <find>syn</find> <find>cat</find> + <find>service_con</find> + <find>connnec</find> + <find>service</find> + <find>api.</find> </findStrings> <replaceStrings> <replace>adex</replace> @@ -183,6 +226,7 @@ <replace>where</replace> <replace>resource_name</replace> <replace>select_fields</replace> + <replace>query.</replace> </replaceStrings> <dirStrings> <dir>R:\source\django\astron_data_explorer</dir> @@ -201,36 +245,14 @@ <component name="IdeDocumentHistory"> <option name="CHANGED_PATHS"> <list> - <option value="$PROJECT_DIR$/esap/api/templates/api/base.html" /> - <option value="$PROJECT_DIR$/esap/api/business/services/vo_service.py" /> - <option value="$PROJECT_DIR$/esap/esap/urls.py" /> - <option value="$PROJECT_DIR$/esap/api/business/services/connectors/alta_observations.py" /> - <option value="$PROJECT_DIR$/esap/api/business/services/connectors/tap_service.py" /> - <option value="$PROJECT_DIR$/esap/api/business/services/connectors/tap_service_connector.py" /> - <option value="$PROJECT_DIR$/esap/api/business/services/tap_service_connector.py" /> - <option value="$PROJECT_DIR$/esap/api/business/services/alta_observations_connector.py" /> - <option value="$PROJECT_DIR$/esap/api/views.py" /> - <option value="$PROJECT_DIR$/esap/api/business/services/esap_service.py" /> - <option value="$PROJECT_DIR$/esap/api/business/algorithms.py" /> - <option value="$PROJECT_DIR$/esap/api/business/services/vo.py" /> - <option value="$PROJECT_DIR$/esap/api/business/services/alta.py" /> - <option value="$PROJECT_DIR$/esap/api/business/services/vso.py" /> - <option value="$PROJECT_DIR$/esap/api/business/services/helio.py" /> - <option value="$PROJECT_DIR$/esap/api/views/staging.py" /> - <option value="$PROJECT_DIR$/esap/api/views/__init__.py" /> - <option value="$PROJECT_DIR$/esap/api/views/query.py" /> - <option value="$PROJECT_DIR$/esap/api/views/discovery.py" /> - <option value="$PROJECT_DIR$/esap/api/business/services/staging_base.py" /> <option value="$PROJECT_DIR$/esap/api/business/services/service_base.py" /> <option value="$PROJECT_DIR$/esap/api/business/staging_controller.py" /> <option value="$PROJECT_DIR$/esap/api/business/service_controller.py" /> <option value="$PROJECT_DIR$/esap/api/business/services/staging/staging_base.py" /> <option value="$PROJECT_DIR$/esap/requirements/base.txt" /> - <option value="$PROJECT_DIR$/esap/esap/settings/base.py" /> <option value="$PROJECT_DIR$/esap/esap/settings/docker_dop457.py" /> <option value="$PROJECT_DIR$/esap/api/business/common.py" /> <option value="$PROJECT_DIR$/esap/esap/configuration/dev.py" /> - <option value="$PROJECT_DIR$/esap/api/models.py" /> <option value="$PROJECT_DIR$/esap/api/admin.py" /> <option value="$PROJECT_DIR$/esap/api/serializers.py" /> <option value="$PROJECT_DIR$/esap/api/urls.py" /> @@ -251,7 +273,29 @@ <option value="$PROJECT_DIR$/esap/api/business/services/query/vo_reg.py" /> <option value="$PROJECT_DIR$/esap/esap/configuration/adex.py" /> <option value="$PROJECT_DIR$/esap/esap/configuration/esap_default.py" /> + <option value="$PROJECT_DIR$/esap/Dockerfile" /> + <option value="$PROJECT_DIR$/esap/run.bat" /> <option value="$PROJECT_DIR$/esap/esap/configuration/esap_dev.py" /> + <option value="$PROJECT_DIR$/esap/api/models.py" /> + <option value="$PROJECT_DIR$/esap/esap/settings/base.py" /> + <option value="$PROJECT_DIR$/esap/esap/settings/statics.py" /> + <option value="$PROJECT_DIR$/esap/query/__init__.py" /> + <option value="$PROJECT_DIR$/esap/query/apps.py" /> + <option value="$PROJECT_DIR$/esap/query/migrations/0005_delete_configuration.py" /> + <option value="$PROJECT_DIR$/esap/query/migrations/0003_configuration.py" /> + <option value="$PROJECT_DIR$/esap/query/migrations/0002_catalog_user_url.py" /> + <option value="$PROJECT_DIR$/esap/query/migrations/0001_initial.py" /> + <option value="$PROJECT_DIR$/esap/query/migrations/0004_auto_20200316_1735.py" /> + <option value="$PROJECT_DIR$/esap/reset_migrations.bat" /> + <option value="$PROJECT_DIR$/esap/query/serializers.py" /> + <option value="$PROJECT_DIR$/esap/query/api/serializers.py" /> + <option value="$PROJECT_DIR$/esap/query/views/__init__.py" /> + <option value="$PROJECT_DIR$/esap/query/views/common_views.py" /> + <option value="$PROJECT_DIR$/esap/query/api/views/common_views.py" /> + <option value="$PROJECT_DIR$/esap/query/api/views/query_views.py" /> + <option value="$PROJECT_DIR$/esap/query/api/urls.py" /> + <option value="$PROJECT_DIR$/esap/esap/urls.py" /> + <option value="$PROJECT_DIR$/esap/query/api/business/query_controller.py" /> </list> </option> </component> @@ -277,7 +321,6 @@ </navigator> <panes> <pane id="Scope" /> - <pane id="Scratches" /> <pane id="ProjectPane"> <subPane> <PATH> @@ -318,7 +361,7 @@ <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> <PATH_ELEMENT> - <option name="myItemId" value="requirements" /> + <option name="myItemId" value="rucio" /> <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> </PATH> @@ -336,7 +379,11 @@ <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> <PATH_ELEMENT> - <option name="myItemId" value="esap" /> + <option name="myItemId" value="rucio" /> + <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> + </PATH_ELEMENT> + <PATH_ELEMENT> + <option name="myItemId" value="migrations" /> <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> </PATH> @@ -354,11 +401,11 @@ <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> <PATH_ELEMENT> - <option name="myItemId" value="esap" /> + <option name="myItemId" value="rucio" /> <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> <PATH_ELEMENT> - <option name="myItemId" value="settings" /> + <option name="myItemId" value="api" /> <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> </PATH> @@ -376,11 +423,7 @@ <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> <PATH_ELEMENT> - <option name="myItemId" value="esap" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="configuration" /> + <option name="myItemId" value="requirements" /> <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> </PATH> @@ -398,7 +441,7 @@ <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> <PATH_ELEMENT> - <option name="myItemId" value="api" /> + <option name="myItemId" value="query" /> <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> </PATH> @@ -416,35 +459,17 @@ <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> <PATH_ELEMENT> - <option name="myItemId" value="api" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="views" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - </PATH> - <PATH> - <PATH_ELEMENT> - <option name="myItemId" value="esap-gateway" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.ProjectViewProjectNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="esap-gateway" /> + <option name="myItemId" value="query" /> <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> <PATH_ELEMENT> - <option name="myItemId" value="esap" /> + <option name="myItemId" value="templates" /> <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> <PATH_ELEMENT> <option name="myItemId" value="api" /> <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="business" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> </PATH> <PATH> <PATH_ELEMENT> @@ -460,15 +485,11 @@ <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> <PATH_ELEMENT> - <option name="myItemId" value="api" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="business" /> + <option name="myItemId" value="query" /> <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> <PATH_ELEMENT> - <option name="myItemId" value="services" /> + <option name="myItemId" value="api" /> <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> </PATH> @@ -486,44 +507,41 @@ <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> <PATH_ELEMENT> - <option name="myItemId" value="api" /> - <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> - </PATH_ELEMENT> - <PATH_ELEMENT> - <option name="myItemId" value="business" /> + <option name="myItemId" value="query" /> <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> <PATH_ELEMENT> - <option name="myItemId" value="services" /> + <option name="myItemId" value="api" /> <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> <PATH_ELEMENT> - <option name="myItemId" value="query" /> + <option name="myItemId" value="business" /> <option name="myItemType" value="com.intellij.ide.projectView.impl.nodes.PsiDirectoryNode" /> </PATH_ELEMENT> </PATH> </subPane> </pane> + <pane id="Scratches" /> </panes> </component> <component name="PropertiesComponent"> - <property name="last_opened_file_path" value="$PROJECT_DIR$/esap/manage.py" /> + <property name="last_opened_file_path" value="$PROJECT_DIR$/esap/query/api" /> <property name="settings.editor.selected.configurable" value="project.propVCSSupport.Mappings" /> </component> <component name="RecentsManager"> + <key name="CopyFile.RECENT_KEYS"> + <recent name="R:\source\django\esap-gateway\esap\query\api" /> + <recent name="R:\source\django\esap-gateway\esap" /> + <recent name="R:\source\django\esap-gateway\esap\esap\settings" /> + <recent name="R:\source\django\esap-gateway\esap\api\business\services\query" /> + <recent name="R:\source\django\esap-gateway\esap\esap\configuration" /> + </key> <key name="MoveFile.RECENT_KEYS"> + <recent name="R:\source\django\esap-gateway\esap\query\api" /> + <recent name="R:\source\django\esap-gateway\esap\api\business\services" /> <recent name="R:\source\django\esap-gateway\esap\api\business\services\staging" /> <recent name="R:\source\django\esap-gateway\esap\api\business\services\query" /> - <recent name="R:\source\django\esap-gateway\esap\api\business\services" /> <recent name="R:\source\django\esap-gateway\esap\api\business\services\connectors" /> - <recent name="R:\source\django\esap-gateway\esap\api" /> - </key> - <key name="CopyFile.RECENT_KEYS"> - <recent name="R:\source\django\esap-gateway\esap\api\business\services\query" /> - <recent name="R:\source\django\esap-gateway\esap\esap\configuration" /> - <recent name="R:\source\django\esap-gateway\esap" /> - <recent name="R:\source\django\esap-gateway\esap\esap" /> - <recent name="R:\source\django\esap-gateway\esap\assets" /> </key> </component> <component name="RunManager" selected="Python.debug esap"> @@ -716,20 +734,6 @@ <option name="presentableId" value="Default" /> <updated>1577692394153</updated> </task> - <task id="LOCAL-00036" summary="adding astropy, numpy and pyvo"> - <created>1579875084876</created> - <option name="number" value="00036" /> - <option name="presentableId" value="LOCAL-00036" /> - <option name="project" value="LOCAL" /> - <updated>1579875084876</updated> - </task> - <task id="LOCAL-00037" summary="adding astropy, numpy and pyvo"> - <created>1579875436329</created> - <option name="number" value="00037" /> - <option name="presentableId" value="LOCAL-00037" /> - <option name="project" value="LOCAL" /> - <updated>1579875436329</updated> - </task> <task id="LOCAL-00038" summary="adding astropy, numpy and pyvo"> <created>1579876085301</created> <option name="number" value="00038" /> @@ -1059,7 +1063,21 @@ <option name="project" value="LOCAL" /> <updated>1590747317774</updated> </task> - <option name="localTasksCounter" value="85" /> + <task id="LOCAL-00085" summary="switch Docker base image from python:3.6.7-alpine to python:3.6.7-slim"> + <created>1591855652224</created> + <option name="number" value="00085" /> + <option name="presentableId" value="LOCAL-00085" /> + <option name="project" value="LOCAL" /> + <updated>1591855652224</updated> + </task> + <task id="LOCAL-00086" summary="merging Zheng's changes from esap-gateway-auth into easp-gateway-dev"> + <created>1592212449479</created> + <option name="number" value="00086" /> + <option name="presentableId" value="LOCAL-00086" /> + <option name="project" value="LOCAL" /> + <updated>1592212449480</updated> + </task> + <option name="localTasksCounter" value="87" /> <servers /> </component> <component name="TodoView"> @@ -1076,25 +1094,25 @@ <editor active="false" /> <layout> <window_info id="PlantUML" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32995737" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" /> - <window_info id="Project" active="true" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.29385027" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" /> - <window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" /> + <window_info id="TODO" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32998884" sideWeight="0.5" order="6" side_tool="false" content_ui="tabs" /> <window_info id="LDAP" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" /> <window_info id="Event Log" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="true" content_ui="tabs" /> - <window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.26702508" sideWeight="0.5" order="8" side_tool="false" content_ui="tabs" /> + <window_info id="Version Control" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.26700112" sideWeight="0.5" order="8" side_tool="false" content_ui="tabs" /> <window_info id="Python Console" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33389074" sideWeight="0.5" order="9" side_tool="false" content_ui="tabs" /> + <window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32936078" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" /> + <window_info id="Terminal" active="true" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.39074692" sideWeight="0.5" order="10" side_tool="false" content_ui="tabs" /> + <window_info id="Project" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.41871658" sideWeight="0.5" order="0" side_tool="false" content_ui="combo" /> + <window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32664436" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" /> <window_info id="Structure" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" /> - <window_info id="Terminal" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="true" show_stripe_button="true" weight="0.36287627" sideWeight="0.5" order="10" side_tool="false" content_ui="tabs" /> <window_info id="Favorites" active="false" anchor="left" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="2" side_tool="true" content_ui="tabs" /> + <window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.28483835" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" /> <window_info id="Cvs" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="4" side_tool="false" content_ui="tabs" /> <window_info id="Messages" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="7" side_tool="false" content_ui="tabs" /> <window_info id="Message" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.33" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" /> <window_info id="Commander" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="0" side_tool="false" content_ui="tabs" /> <window_info id="Inspection" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.4" sideWeight="0.5" order="5" side_tool="false" content_ui="tabs" /> - <window_info id="Run" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32936078" sideWeight="0.5" order="2" side_tool="false" content_ui="tabs" /> <window_info id="Hierarchy" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="2" side_tool="false" content_ui="combo" /> - <window_info id="Find" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.32719395" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" /> <window_info id="Ant Build" active="false" anchor="right" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.25" sideWeight="0.5" order="1" side_tool="false" content_ui="tabs" /> - <window_info id="Debug" active="false" anchor="bottom" auto_hide="false" internal_type="DOCKED" type="DOCKED" visible="false" show_stripe_button="true" weight="0.28483835" sideWeight="0.5" order="3" side_tool="false" content_ui="tabs" /> </layout> </component> <component name="VcsContentAnnotationSettings"> @@ -1102,8 +1120,6 @@ </component> <component name="VcsManagerConfiguration"> <option name="OFFER_MOVE_TO_ANOTHER_CHANGELIST_ON_PARTIAL_COMMIT" value="false" /> - <MESSAGE value="adding start screen to esap-api" /> - <MESSAGE value="testing docker changes" /> <MESSAGE value="working on query" /> <MESSAGE value="refactored create_query and run_query functions away into service classes (vo.py and alta.py)" /> <MESSAGE value="make select and return values from query variable" /> @@ -1127,7 +1143,9 @@ <MESSAGE value="adding VO Registry query" /> <MESSAGE value="adding VO registery of registries" /> <MESSAGE value="updated database with later LOFAR texts" /> - <option name="LAST_COMMIT_MESSAGE" value="updated database with later LOFAR texts" /> + <MESSAGE value="switch Docker base image from python:3.6.7-alpine to python:3.6.7-slim" /> + <MESSAGE value="merging Zheng's changes from esap-gateway-auth into easp-gateway-dev" /> + <option name="LAST_COMMIT_MESSAGE" value="merging Zheng's changes from esap-gateway-auth into easp-gateway-dev" /> </component> <component name="XDebuggerManager"> <breakpoint-manager> @@ -1137,7 +1155,7 @@ <option name="timeStamp" value="1" /> </line-breakpoint> <line-breakpoint enabled="true" suspend="THREAD" type="python-line"> - <url>file://$PROJECT_DIR$/esap/api/views/common_views.py</url> + <url>file://$PROJECT_DIR$/esap/query/api/views/common_views.py</url> <line>259</line> <option name="timeStamp" value="249" /> </line-breakpoint> @@ -1147,350 +1165,401 @@ <watches-manager /> </component> <component name="editorHistoryManager"> - <entry file="file://R:/env_esap/Lib/site-packages/django/db/models/fields/related_descriptors.py"> + <entry file="file://$PROJECT_DIR$/esap/Dockerfile"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="10282"> - <caret line="199" column="0" lean-forward="false" selection-start-line="199" selection-start-column="0" selection-end-line="199" selection-end-column="0" /> + <state relative-caret-position="0"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="22" /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/views_obsolete.py" /> - <entry file="file://$PROJECT_DIR$/esap/api/views/__init__.py"> + <entry file="file://$PROJECT_DIR$/esap/run.bat"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="0"> - <caret line="0" column="6" lean-forward="false" selection-start-line="0" selection-start-column="6" selection-end-line="0" selection-end-column="6" /> + <state relative-caret-position="106"> + <caret line="2" column="17" lean-forward="false" selection-start-line="2" selection-start-column="17" selection-end-line="2" selection-end-column="17" /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/business/services/__init__.py"> + <entry file="file://$PROJECT_DIR$/esap/accounts/api/urls.py"> <provider selected="true" editor-type-id="text-editor"> <state relative-caret-position="0"> <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/business/services/staging/staging_base.py"> + <entry file="file://$PROJECT_DIR$/esap/rucio/api/urls.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="68"> - <caret line="4" column="51" lean-forward="false" selection-start-line="4" selection-start-column="51" selection-end-line="4" selection-end-column="51" /> + <state relative-caret-position="0"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/migrations/0001_initial.py"> + <entry file="file://$PROJECT_DIR$/esap/requirements/dev.txt"> <provider selected="true" editor-type-id="text-editor"> <state relative-caret-position="0"> <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/esap/wsgi.py"> + <entry file="file://$PROJECT_DIR$/esap/esap/settings/dev.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="0"> - <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <state relative-caret-position="1908"> + <caret line="38" column="30" lean-forward="false" selection-start-line="38" selection-start-column="30" selection-end-line="38" selection-end-column="30" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/configuration_assets/schema_datasets.json" /> - <entry file="file://$PROJECT_DIR$/esap/api/business/common.py"> + <entry file="file://$PROJECT_DIR$/esap/esap/configuration/esap_dev.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="663"> - <caret line="25" column="0" lean-forward="false" selection-start-line="25" selection-start-column="0" selection-end-line="25" selection-end-column="0" /> + <state relative-caret-position="265"> + <caret line="5" column="2" lean-forward="false" selection-start-line="5" selection-start-column="2" selection-end-line="5" selection-end-column="2" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/business/staging_controller.py"> + <entry file="file://$PROJECT_DIR$/esap/rucio/views.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="504"> - <caret line="11" column="0" lean-forward="true" selection-start-line="11" selection-start-column="0" selection-end-line="11" selection-end-column="0" /> + <state relative-caret-position="0"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <folding /> </state> </provider> </entry> - <entry file="file://R:/env_esap/Lib/site-packages/django/utils/deprecation.py"> + <entry file="file://$PROJECT_DIR$/esap/rucio/api/views.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="4982"> - <caret line="95" column="22" lean-forward="false" selection-start-line="95" selection-start-column="22" selection-end-line="95" selection-end-column="22" /> + <state relative-caret-position="0"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <folding> + <element signature="e#0#48#0" expanded="false" /> + </folding> </state> </provider> </entry> - <entry file="file://R:/env_esap/Lib/site-packages/django/views/decorators/csrf.py"> + <entry file="file://$PROJECT_DIR$/esap/esap/configuration/esap_default.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="2650"> - <caret line="53" column="0" lean-forward="false" selection-start-line="53" selection-start-column="0" selection-end-line="53" selection-end-column="0" /> + <state relative-caret-position="1325"> + <caret line="25" column="0" lean-forward="false" selection-start-line="25" selection-start-column="0" selection-end-line="25" selection-end-column="0" /> + <folding /> </state> </provider> </entry> - <entry file="file://R:/env_esap/Lib/site-packages/django/db/models/query.py"> + <entry file="file://$PROJECT_DIR$/esap/rucio/models.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="13780"> - <caret line="284" column="0" lean-forward="false" selection-start-line="284" selection-start-column="0" selection-end-line="284" selection-end-column="0" /> + <state relative-caret-position="0"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <folding> + <element signature="e#0#28#0" expanded="false" /> + </folding> </state> </provider> </entry> - <entry file="file://R:/env_esap/Lib/site-packages/rest_framework/views.py"> + <entry file="file://$PROJECT_DIR$/esap/query/api/business/services/query/alta.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="686"> - <caret line="501" column="0" lean-forward="false" selection-start-line="501" selection-start-column="0" selection-end-line="501" selection-end-column="0" /> + <state relative-caret-position="7685"> + <caret line="147" column="48" lean-forward="false" selection-start-line="147" selection-start-column="41" selection-end-line="147" selection-end-column="48" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/esap/settings/__init__.py"> + <entry file="file://$PROJECT_DIR$/esap/query/models.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="0"> - <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <state relative-caret-position="3922"> + <caret line="74" column="16" lean-forward="false" selection-start-line="74" selection-start-column="4" selection-end-line="74" selection-end-column="16" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/esap/configuration/__init__.py" /> - <entry file="file://$PROJECT_DIR$/esap/__init__.py" /> - <entry file="file://$PROJECT_DIR$/esap/esap/wsgi_docker_dop457.py"> + <entry file="file://$PROJECT_DIR$/esap/esap/settings/__init__.py"> <provider selected="true" editor-type-id="text-editor"> <state relative-caret-position="0"> <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/requirements/base.txt"> + <entry file="file://$PROJECT_DIR$/esap/esap/settings/base.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="424"> - <caret line="8" column="16" lean-forward="false" selection-start-line="8" selection-start-column="16" selection-end-line="8" selection-end-column="16" /> + <state relative-caret-position="848"> + <caret line="31" column="26" lean-forward="true" selection-start-line="31" selection-start-column="26" selection-end-line="31" selection-end-column="26" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/admin.py"> + <entry file="file://$PROJECT_DIR$/esap/esap/settings/statics.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="53"> - <caret line="1" column="79" lean-forward="false" selection-start-line="1" selection-start-column="79" selection-end-line="1" selection-end-column="79" /> + <state relative-caret-position="263"> + <caret line="16" column="10" lean-forward="false" selection-start-line="16" selection-start-column="10" selection-end-line="16" selection-end-column="10" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/serializers.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="5247"> - <caret line="101" column="0" lean-forward="false" selection-start-line="101" selection-start-column="0" selection-end-line="101" selection-end-column="0" /> - </state> + <entry file="file://$PROJECT_DIR$/esap/esap/static/query/adex_logo.jpg"> + <provider selected="true" editor-type-id="images"> + <state /> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/urls.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="1484"> - <caret line="30" column="49" lean-forward="false" selection-start-line="30" selection-start-column="49" selection-end-line="30" selection-end-column="49" /> - </state> + <entry file="file://$PROJECT_DIR$/esap/esap/static/query/favicon.png"> + <provider selected="true" editor-type-id="images"> + <state /> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/run_gource.bat"> + <entry file="file://$PROJECT_DIR$/esap/esap/static/query/style.css"> <provider selected="true" editor-type-id="text-editor"> <state relative-caret-position="0"> <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/run.bat"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="53"> - <caret line="1" column="0" lean-forward="false" selection-start-line="1" selection-start-column="0" selection-end-line="1" selection-end-column="0" /> - </state> + <entry file="file://$PROJECT_DIR$/esap/esap/static/query/favicon.ico"> + <provider selected="true" editor-type-id="images"> + <state /> </provider> </entry> - <entry file="file://R:/env_esap/Lib/site-packages/django/core/handlers/exception.py"> - <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="821"> - <caret line="42" column="16" lean-forward="true" selection-start-line="42" selection-start-column="16" selection-end-line="42" selection-end-column="16" /> - </state> + <entry file="file://$PROJECT_DIR$/esap/esap/static/query/esap_logo.png"> + <provider selected="true" editor-type-id="images"> + <state /> + </provider> + </entry> + <entry file="file://$PROJECT_DIR$/esap/esap/static/query/esap_icon.jpg"> + <provider selected="true" editor-type-id="images"> + <state /> </provider> </entry> - <entry file="file://R:/env_esap/Lib/site-packages/rest_framework/generics.py"> + <entry file="file://$PROJECT_DIR$/esap/configuration_assets/esap_logo.png"> + <provider selected="true" editor-type-id="images"> + <state /> + </provider> + </entry> + <entry file="file://$PROJECT_DIR$/esap/configuration_assets/adex_logo.png"> + <provider selected="true" editor-type-id="images"> + <state /> + </provider> + </entry> + <entry file="file://$PROJECT_DIR$/esap/configuration_assets/esap_solar.png"> + <provider selected="true" editor-type-id="images"> + <state /> + </provider> + </entry> + <entry file="file://$PROJECT_DIR$/esap/query/__init__.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="381"> - <caret line="33" column="4" lean-forward="false" selection-start-line="33" selection-start-column="4" selection-end-line="33" selection-end-column="4" /> + <state relative-caret-position="0"> + <caret line="0" column="26" lean-forward="false" selection-start-line="0" selection-start-column="26" selection-end-line="0" selection-end-column="26" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/business/configuration.py"> + <entry file="file://$PROJECT_DIR$/esap/query/apps.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="1060"> - <caret line="23" column="0" lean-forward="false" selection-start-line="23" selection-start-column="0" selection-end-line="23" selection-end-column="0" /> + <state relative-caret-position="212"> + <caret line="4" column="17" lean-forward="false" selection-start-line="4" selection-start-column="17" selection-end-line="4" selection-end-column="17" /> + <folding> + <marker date="1592291120007" expanded="true" signature="65:84" ph="..." /> + </folding> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/Dockerfile"> + <entry file="file://$PROJECT_DIR$/esap/query/migrations/0002_catalog_user_url.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="901"> - <caret line="17" column="19" lean-forward="false" selection-start-line="17" selection-start-column="19" selection-end-line="17" selection-end-column="19" /> + <state relative-caret-position="424"> + <caret line="8" column="15" lean-forward="false" selection-start-line="8" selection-start-column="15" selection-end-line="8" selection-end-column="15" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/tests.py"> + <entry file="file://$PROJECT_DIR$/esap/query/migrations/0003_configuration.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="0"> - <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <state relative-caret-position="424"> + <caret line="8" column="15" lean-forward="false" selection-start-line="8" selection-start-column="15" selection-end-line="8" selection-end-column="15" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/esap/configuration/esap_solar.py"> + <entry file="file://$PROJECT_DIR$/esap/query/migrations/0005_delete_configuration.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="3233"> - <caret line="61" column="1" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="61" selection-end-column="1" /> + <state relative-caret-position="424"> + <caret line="8" column="15" lean-forward="false" selection-start-line="8" selection-start-column="15" selection-end-line="8" selection-end-column="15" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/business/services/query/query_base.py"> + <entry file="file://$PROJECT_DIR$/esap/query/migrations/0004_auto_20200316_1735.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="1007"> - <caret line="19" column="47" lean-forward="false" selection-start-line="19" selection-start-column="47" selection-end-line="19" selection-end-column="47" /> + <state relative-caret-position="424"> + <caret line="8" column="15" lean-forward="false" selection-start-line="8" selection-start-column="15" selection-end-line="8" selection-end-column="15" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/business/services/query/vso.py"> + <entry file="file://$PROJECT_DIR$/esap/rucio/migrations/0001_initial.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="10865"> - <caret line="207" column="12" lean-forward="false" selection-start-line="207" selection-start-column="12" selection-end-line="207" selection-end-column="12" /> + <state relative-caret-position="0"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/business/services/query/helio.py"> + <entry file="file://$PROJECT_DIR$/esap/query/migrations/0001_initial.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="6890"> - <caret line="132" column="0" lean-forward="false" selection-start-line="132" selection-start-column="0" selection-end-line="132" selection-end-column="0" /> + <state relative-caret-position="482"> + <caret line="72" column="14" lean-forward="true" selection-start-line="72" selection-start-column="14" selection-end-line="72" selection-end-column="14" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/templates/api/index.html"> + <entry file="file://$PROJECT_DIR$/esap/reset_migrations_query.bat"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="3816"> - <caret line="72" column="51" lean-forward="false" selection-start-line="72" selection-start-column="51" selection-end-line="72" selection-end-column="51" /> + <state relative-caret-position="477"> + <caret line="9" column="0" lean-forward="false" selection-start-line="9" selection-start-column="0" selection-end-line="9" selection-end-column="0" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/requirements/prod.txt"> + <entry file="file://$PROJECT_DIR$/esap/accounts/api/__init__.py"> <provider selected="true" editor-type-id="text-editor"> <state relative-caret-position="0"> <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/requirements/dev.txt"> + <entry file="file://$PROJECT_DIR$/esap/query/api/__init__.py"> <provider selected="true" editor-type-id="text-editor"> <state relative-caret-position="0"> <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/views/staging_views.py"> + <entry file="file://$PROJECT_DIR$/esap/query/serializers.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="0"> - <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <state relative-caret-position="159"> + <caret line="3" column="0" lean-forward="true" selection-start-line="3" selection-start-column="0" selection-end-line="3" selection-end-column="0" /> + <folding> + <element signature="e#0#38#0" expanded="false" /> + </folding> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/views/common_views.py"> + <entry file="file://$PROJECT_DIR$/esap/rucio/api/serializers.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="5883"> - <caret line="130" column="0" lean-forward="false" selection-start-line="130" selection-start-column="0" selection-end-line="130" selection-end-column="0" /> - <folding /> + <state relative-caret-position="0"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <folding> + <element signature="e#0#38#0" expanded="false" /> + </folding> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/views/query_views.py"> + <entry file="file://$PROJECT_DIR$/esap/query/api/serializers.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="4346"> - <caret line="89" column="73" lean-forward="false" selection-start-line="89" selection-start-column="73" selection-end-line="89" selection-end-column="73" /> + <state relative-caret-position="53"> + <caret line="1" column="10" lean-forward="false" selection-start-line="1" selection-start-column="10" selection-end-line="1" selection-end-column="10" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/esap/settings/docker_dop457.py"> + <entry file="file://$PROJECT_DIR$/esap/query/views/staging_views.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="2226"> - <caret line="43" column="0" lean-forward="false" selection-start-line="43" selection-start-column="0" selection-end-line="43" selection-end-column="0" /> + <state relative-caret-position="0"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/esap/settings/prod.py"> + <entry file="file://$PROJECT_DIR$/esap/query/api/views/__init__.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="1272"> - <caret line="25" column="27" lean-forward="false" selection-start-line="25" selection-start-column="27" selection-end-line="25" selection-end-column="34" /> + <state relative-caret-position="106"> + <caret line="2" column="0" lean-forward="false" selection-start-line="2" selection-start-column="0" selection-end-line="2" selection-end-column="0" /> + <folding> + <element signature="e#0#27#0" expanded="false" /> + </folding> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/esap/configuration/adex.py"> + <entry file="file://$PROJECT_DIR$/esap/query/api/views/common_views.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="5247"> - <caret line="99" column="0" lean-forward="false" selection-start-line="99" selection-start-column="0" selection-end-line="99" selection-end-column="0" /> + <state relative-caret-position="212"> + <caret line="10" column="15" lean-forward="false" selection-start-line="10" selection-start-column="15" selection-end-line="10" selection-end-column="15" /> <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/business/services/query/alta.py"> + <entry file="file://$PROJECT_DIR$/esap/query/templates/api/base.html"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="5088"> - <caret line="97" column="0" lean-forward="false" selection-start-line="97" selection-start-column="0" selection-end-line="97" selection-end-column="0" /> + <state relative-caret-position="-2022"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/business/services/query/vo.py"> + <entry file="file://$PROJECT_DIR$/esap/query/templates/api/index.html"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="7208"> - <caret line="137" column="26" lean-forward="false" selection-start-line="137" selection-start-column="26" selection-end-line="137" selection-end-column="26" /> + <state relative-caret-position="2842"> + <caret line="72" column="50" lean-forward="false" selection-start-line="72" selection-start-column="50" selection-end-line="72" selection-end-column="50" /> <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/models.py"> + <entry file="file://$PROJECT_DIR$/esap/query/api/urls.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="530"> - <caret line="10" column="21" lean-forward="false" selection-start-line="10" selection-start-column="4" selection-end-line="10" selection-end-column="21" /> - <folding /> + <state relative-caret-position="265"> + <caret line="11" column="18" lean-forward="false" selection-start-line="11" selection-start-column="18" selection-end-line="11" selection-end-column="18" /> + <folding> + <element signature="e#0#28#0" expanded="true" /> + </folding> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/business/services/query/vo_reg.py"> + <entry file="file://$PROJECT_DIR$/esap/esap/urls.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="5406"> - <caret line="104" column="0" lean-forward="false" selection-start-line="104" selection-start-column="0" selection-end-line="104" selection-end-column="0" /> + <state relative-caret-position="530"> + <caret line="19" column="25" lean-forward="false" selection-start-line="19" selection-start-column="25" selection-end-line="19" selection-end-column="25" /> <folding> - <element signature="e#145#179#0" expanded="true" /> + <element signature="e#633#665#0" expanded="true" /> </folding> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/api/business/query_controller.py"> + <entry file="file://$PROJECT_DIR$/esap/query/api/business/services/staging/staging_base.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="2067"> - <caret line="44" column="48" lean-forward="false" selection-start-line="44" selection-start-column="16" selection-end-line="44" selection-end-column="48" /> + <state relative-caret-position="-313"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/esap/settings/dev.py"> + <entry file="file://$PROJECT_DIR$/esap/query/api/views/query_views.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="1908"> - <caret line="38" column="30" lean-forward="false" selection-start-line="38" selection-start-column="30" selection-end-line="38" selection-end-column="30" /> + <state relative-caret-position="742"> + <caret line="14" column="13" lean-forward="true" selection-start-line="14" selection-start-column="13" selection-end-line="14" selection-end-column="13" /> <folding> - <element signature="e#58#90#0" expanded="true" /> + <element signature="e#0#14#0" expanded="true" /> </folding> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/esap/settings/base.py"> + <entry file="file://$PROJECT_DIR$/esap/query/api/utils.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="993"> - <caret line="182" column="0" lean-forward="false" selection-start-line="182" selection-start-column="0" selection-end-line="182" selection-end-column="0" /> + <state relative-caret-position="-578"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> <folding /> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/esap/configuration/esap_default.py"> + <entry file="file://$PROJECT_DIR$/esap/query/api/business/configuration.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="585"> - <caret line="76" column="6" lean-forward="true" selection-start-line="76" selection-start-column="6" selection-end-line="76" selection-end-column="6" /> - <folding /> + <state relative-caret-position="0"> + <caret line="0" column="0" lean-forward="false" selection-start-line="0" selection-start-column="0" selection-end-line="0" selection-end-column="0" /> + <folding> + <element signature="e#0#14#0" expanded="true" /> + </folding> </state> </provider> </entry> - <entry file="file://$PROJECT_DIR$/esap/esap/configuration/esap_dev.py"> + <entry file="file://$PROJECT_DIR$/esap/query/api/business/query_controller.py"> <provider selected="true" editor-type-id="text-editor"> - <state relative-caret-position="850"> - <caret line="73" column="1" lean-forward="true" selection-start-line="73" selection-start-column="1" selection-end-line="73" selection-end-column="1" /> + <state relative-caret-position="530"> + <caret line="10" column="5" lean-forward="false" selection-start-line="10" selection-start-column="5" selection-end-line="10" selection-end-column="5" /> <folding /> </state> </provider> diff --git a/esap/api/business/services/__init__.py b/esap/__init__.py similarity index 100% rename from esap/api/business/services/__init__.py rename to esap/__init__.py diff --git a/esap/api/__init__.py b/esap/api/__init__.py deleted file mode 100644 index 9c091b3cab1dceb3fe2266ef4e3825e10a0b6c6a..0000000000000000000000000000000000000000 --- a/esap/api/__init__.py +++ /dev/null @@ -1 +0,0 @@ -default_app_config = 'api.apps.MyAppConfig' \ No newline at end of file diff --git a/esap/api/business/services/query/alta.py b/esap/api/business/services/query/alta.py deleted file mode 100644 index c6168ebf68b3e2cb05d5518f0bf6f47993a15e25..0000000000000000000000000000000000000000 --- a/esap/api/business/services/query/alta.py +++ /dev/null @@ -1,297 +0,0 @@ -""" - File name: alta.py - Author: Nico Vermaas - Astron - Date created: 2020-02-07 - Description: ESAP services for ALTA. -""" - -from .query_base import query_base -import requests, json -import logging - -logger = logging.getLogger(__name__) - -AMP_REPLACEMENT = '_and_' - -# The request header -ALTA_HEADER = { - 'content-type': "application/json", -} - - -class observations_connector(query_base): - """ - The connector to access the ALTA observations dataset - """ - - # Initializer - def __init__(self, url): - self.url = url - - # construct a query for this type of service - def construct_query(self, dataset, esap_query_params, translation_parameters, equinox): - query = '' - where = '' - errors = [] - - for esap_param in esap_query_params: - esap_key = esap_param - value = esap_query_params[esap_key][0] - - try: - dataset_key = translation_parameters[esap_key] - - # because '&' has a special meaning in urls (specifying a parameter) replace it with - # something harmless during serialization. - where = where + dataset_key + '=' + value + AMP_REPLACEMENT - - except Exception as error: - # if the parameter could not be translated, then just continue without that key - errors.append("ERROR: translating key " + esap_key + ' ' + str(error)) - - # cut off the last separation character - where = where[:-len(AMP_REPLACEMENT)] - - # construct the query url - query = self.url + '?' + where - - return query, errors - - - def run_query(self, dataset, dataset_name, query): - """ - # use the ALTA REST API to do a query - :param dataset: the dataset object that must be queried - :param query: the constructed ALTA query (that was probably generated with the above construct_query function) - :return: results: an array of dicts with the following structure; - - """ - results = [] - - # because '&' has a special meaning in urls (specifying a parameter) it had been replaced with - # something harmless during serialization. Replace it again with the & - query = query.replace(AMP_REPLACEMENT,'&') - - # execute the http request to ALTA - response = requests.request("GET", query, headers=ALTA_HEADER) - - try: - json_response = json.loads(response.text) - observations = json_response["results"] - - # iterate over the list of results - for observation in observations: - logger.info(observation) - # the dataset.select_fields field specifies which fields must be extracted from the response - record = {} - result = '' - - select_list = dataset.select_fields.split(',') - for select in select_list: - logger.info(select) - try: - result = result + observation[select] + ',' - except: - pass - - # cut off the last ',' - result = result[:-1] - - #result = "https://alta.astron.nl/science/details/"+observation["runId"] - record['query'] = query - record['dataset'] = dataset.uri - record['dataset_name'] = dataset_name - record['result'] = result - - # some fields to return some rendering information for the frontend. - - try: - record['title'] = observation[dataset.title_field] - record['thumbnail'] = observation[dataset.thumbnail] - - record['runId'] = observation["runId"] - record['target'] = observation["target"] - record['RA'] = observation["RA"] - record['dec'] = observation["dec"] - record['fov'] = observation["fov"] - record['startTime'] = observation["startTime"] - record['endTime'] = observation["endTime"] - record['url'] = "https://alta.astron.nl/science/details/"+observation["runId"] - except: - pass - - logger.info(record) - results.append(record) - - except Exception as error: - record['query'] = query - record['dataset_name'] = dataset_name - record['dataset'] = dataset.uri - record['result'] = str(error) - results.append(record) - - return results - - -# -------------------------------------------------------------------------------------------------------------------- - -class dataproducts_connector(query_base): - """ - The connector to access the ALTA dataproducts dataset - """ - - # Initializer - def __init__(self, url): - self.url = url - - - # construct a query for this type of service - def construct_query(self, dataset, esap_query_params, translation_parameters, equinox): - - query = '' - where = '' - errors = [] - - # add some weird business logic for the ALTA dataproducts query, which cannot really do a cone search - - for esap_param in esap_query_params: - esap_key = esap_param - value = esap_query_params[esap_key][0] - - try: - dataset_key = translation_parameters[esap_key] - - # because '&' has a special meaning in urls (specifying a parameter) replace it with - # something harmless during serialization. - where = where + dataset_key + '=' + value + AMP_REPLACEMENT - - except Exception as error: - # if the parameter could not be translated, then just continue without this parameter - errors.append("ERROR: translating key " + esap_key + ' ' + str(error)) - - # cut off the last separation character - where = where[:-len(AMP_REPLACEMENT)] - - # construct the query url - query = self.url + '?' + where - - return query, errors - - - - def run_query(self, dataset, dataset_name, query): - """ - # use the ALTA REST API to do a query - :param dataset: the dataset object that must be queried - :param query: the constructed ALTA query (that was probably generated with the above construct_query function) - :return: results: an array of dicts with the following structure; - - { - "dataset": "apertif-dataproducts", - "result": "/alta-static//media/190409015_AP_B000/image_mf_02.png" - }, - { - "dataset": "apertif-dataproducts", - "result": "/alta-static//media/190409015_AP_B001/image_mf_02.png" - }, - { - "dataset": "apertif-dataproducts", - "result": "/alta-static//media/190409015_AP_B002/image_mf_02.png" - }, - { - "dataset": "apertif-dataproducts", - "result": "/alta-static//media/190409015_AP_B003/image_mf_01.png" - }, - - example: - /esap-api/run-query/?dataset_uri=apertif-dataproducts&query=https://alta.astron.nl/altapi/observations-flat?view_ra=342.16_and_view_dec=33.94_and_view_fov=10_and_dataProductType=image_and_dataProductSubType=continuumMF - - """ - results = [] - - # because '&' has a special meaning in urls (specifying a parameter) it had been replaced with - # something harmless during serialization. Replace it again with the & - query = query.replace(AMP_REPLACEMENT,'&') - - try: - - # execute the first http request to ALTA to do the cone search on observation level. - response = requests.request("GET", query, headers=ALTA_HEADER) - - json_response = json.loads(response.text) - observations = json_response["results"] - - # iterate over the list of results.. and gather the runid's in a comma separated list for the next query on datasetid - list = [] - for observation in observations: - list.append(observation['runId']) - - runids = ','.join(list) - - # construct a second query on dataproducts for the gathered runId's - query_list = query.split('/observations') - host = query_list[0] - - # there may be additional query parameters in the original query, like dataProductSubType=continuumMF - # copy them over to the secondary query as well. - filter = '&'+query_list[1].split('?')[1] - query2 = host + '/dataproducts-flat?datasetID__in=' + str(runids) + filter - - # shortcut to add an extra selection criterium... handle better later - # query2 = query2 + "&dataProductSubType=continuumMF" - - # execute the secondary query to dataproducts - response = requests.request("GET", query2, headers=ALTA_HEADER) - - json_response = json.loads(response.text) - dataproducts = json_response["results"] - - - for dataproduct in dataproducts: - logger.info(dataproduct) - # the dataset.select_fields field specifies which fields must be extracted from the response - record = {} - result = '' - - select_list = dataset.select_fields.split(',') - for select in select_list: - try: - result = result + dataproduct[select] + ',' - except: - pass - - # cut off the last ',' - result = result[:-1] - - # result = "https://alta.astron.nl/science/details/"+observation["runId"] - record['query'] = query - record['dataset'] = dataset.uri - record['dataset_name'] = dataset_name - record['result'] = result - - # some fields to return some rendering information for the frontend. - try: - record['title'] = dataproduct[dataset.title_field] - except: - pass - - try: - record['thumbnail'] = dataproduct[dataset.thumbnail_field] - except: - pass - - try: - record['url'] = dataproduct[dataset.url_field] - except: - pass - - results.append(record) - - except Exception as error: - record['query'] = query - record['dataset'] = dataset.uri - record['dataset_name'] = dataset_name - record['result'] = str(error) - results.append(record) - - return results \ No newline at end of file diff --git a/esap/api/business/services/query/helio.py b/esap/api/business/services/query/helio.py deleted file mode 100644 index f999e4c4d8509501b33b7f8ddb95ed24e17399a2..0000000000000000000000000000000000000000 --- a/esap/api/business/services/query/helio.py +++ /dev/null @@ -1,137 +0,0 @@ -""" - File name: helio.py - Author: Nico Vermaas - Astron - Date created: 2020-03-30 - Description: ESAP services for Helio. - - # example - # http://msslkz.mssl.ucl.ac.uk/helio-dpas/HelioQueryServlet?STARTTIME=2017-10-30T12:00:00&ENDTIME=2017-10-30T15:00:00&INSTRUMENT=SOHO__EIT - - # Data Provider Access Service - # http://helio-vo.eu/services/interfaces/helio-dpas_uix2.php -""" - -from .query_base import query_base -import requests -from astropy.io.votable import parse_single_table - -AMP_REPLACEMENT = '_and_' - -class helio_connector(query_base): - """ - The connector to access HELIO - """ - - # Initializer - def __init__(self, url): - self.url = url - - # construct a query for this type of service - def construct_query(self, dataset, esap_query_params, translation_parameters, equinox): - where = '' - errors = [] - - for esap_param in esap_query_params: - esap_key = esap_param - value = esap_query_params[esap_key][0] - - # temp dirty hack to add time. - # Replace later with proper timestamp fields in the frontend - if esap_key == 'startdate' or esap_key == 'enddate': - value = value + "T00:00:00" - - try: - dataset_key = translation_parameters[esap_key] - - # because '&' has a special meaning in urls (specifying a parameter) replace it with - # something harmless during serialization. - where = where + dataset_key + '=' + value + AMP_REPLACEMENT - - except Exception as error: - # if the parameter could not be translated, then just continue without that key - errors.append("ERROR: translating key " + esap_key + ' ' + str(error)) - - # cut off the last separation character - where = where[:-len(AMP_REPLACEMENT)] - - query = self.url + '?' + where - - return query, errors - - - def run_query(self, dataset, dataset_name, query): - """ - :param dataset: the dataset object that must be queried - :param query: the constructed query (that was probably generated with the above construct_query function) - """ - - results = [] - - # because '&' has a special meaning in urls (specifying a parameter) it had been replaced with - # something harmless during serialization. Replace it again with the & - query = query.replace(AMP_REPLACEMENT,'&') - - try: - # do the http request - response = requests.request("GET", query) - - # the response is in VOTable format - # write the contents to a file first, because astropy cannot parse xml as stream - # also, xml in memory cannot be larger than the size of the memory. - f = open('helio_votable.xml','w') - f.write(response.text) - f.close() - - # the VOTable from HELIO has the following fields. - # instrument_name - # provider_instrument - # url - # provider - # time_start - # time_end - - table = parse_single_table("helio_votable.xml") - - # iterate over the list of results.. - for table_rec in table.array: - record = {} - result = '' - - select_list = dataset.select_fields.split(',') - for select in select_list: - try: - result = result + table_rec[select].decode("utf-8") + ',' - except: - pass - - # cut off the last ',' - result = result[:-1] - - record['dataset'] = dataset.uri - record['dataset_name'] = dataset_name - record['result'] = result - record['query'] = query - - try: - record['url'] = table_rec[dataset.url_field] - except: - pass - - # some fields to return display information for the frontend. - try: - # split off the filename from the path - filename = record['url'].decode("utf-8").rsplit('/',1) - record['title'] = filename[1] + '.fits' - except: - pass - - results.append(record) - - except Exception as error: - record['dataset'] = dataset.uri - record['dataset_name'] = dataset_name - record['query'] = query - record['result'] = str(error) - results.append(record) - - return results \ No newline at end of file diff --git a/esap/api/business/services/query/query_base.py b/esap/api/business/services/query/query_base.py deleted file mode 100644 index bbefc081902aae23ac3e71f2ec8ffa6358e30104..0000000000000000000000000000000000000000 --- a/esap/api/business/services/query/query_base.py +++ /dev/null @@ -1,21 +0,0 @@ -""" - File name: service_base.py - Author: Nico Vermaas - Astron - Date created: 2020-02-07 - Description: ESAP service abstract base class. - This shows what the services should implement. -""" - -class query_base: - - # Initializer - def __init__(self, url): - self.url = url - - # implement this in the derived service classes - def construct_query(self, dataset, esap_query_params, translation_parameters, equinox): - pass - - # implement this in the derived service classes - def run_query(self, dataset, dataset_name, query): - pass diff --git a/esap/api/business/services/query/vo.py b/esap/api/business/services/query/vo.py deleted file mode 100644 index daba9d274568d92d7b8421a09ab13033a6450f4e..0000000000000000000000000000000000000000 --- a/esap/api/business/services/query/vo.py +++ /dev/null @@ -1,192 +0,0 @@ -""" - File name: vo.py - Author: Nico Vermaas - Astron - Date created: 2020-02-07 - Description: ESAP services for VO. -""" - -from .query_base import query_base -import pyvo as vo - - -def create_cone_search(esap_query_params, translation_parameters, equinox): - """ - Return a cone search subquery when ra, dec and fov are found in the query parameters. - - example: - SELECT TOP 10 * from ivoa.obscore WHERE CONTAINS(POINT('ICRS',s_ra,s_dec), CIRCLE('ICRS',202.48,47.23,4.0))=1 - - :param esap_query_params: - :param translation_parameters: - :return: - """ - - radius = None - try: - ra = float(esap_query_params['ra'][0]) - dec = float(esap_query_params['dec'][0]) - radius = float(esap_query_params['fov'][0]) - except: - pass - - if radius != None: - # found a fov parameter, which indicates a cone search - cone_search = "CONTAINS(POINT('"+equinox+"'," + \ - translation_parameters['ra'] + "," + \ - translation_parameters['dec'] + "), " \ - "CIRCLE('"+equinox+"'," + str(ra) + "," + str(dec) + "," + str(radius) + "))=1" - - # remove ra,dec,fov from the parameters so that they are not used in the where clause - del esap_query_params['ra'] - del esap_query_params['dec'] - del esap_query_params['fov'] - return cone_search - - -class tap_service_connector(query_base): - - # Initializer - def __init__(self, url): - self.url = url - - # cut off the '/sync' resource, because the vo.dal.TAPServices will add it again - if self.url.endswith('/sync'): - self.url = self.url[:-4] - - # construct a query for this type of service - def construct_query(self, dataset, query_params, translation_parameters, equinox): - - esap_query_params = dict(query_params) - query = '' - where = '' - errors = [] - - # cone search is a specific type of query that uses ra, dec and a search radius. - # it is also done with a specific ADQL syntax. - # First check if the incoming query describes a cone search. - - cone_search = create_cone_search(esap_query_params,translation_parameters, equinox) - - for esap_param in esap_query_params: - - esap_key = esap_param - value = esap_query_params[esap_key][0] - - try: - dataset_key = translation_parameters[esap_key] - where = where + dataset_key + "='" + value + "' " - - except Exception as error: - # if the parameter could not be translated, then just continue - errors.append("ERROR: translating key " + esap_key + ' ' + str(error)) - - - - # add sync (or async) specifier - query = self.url + '/sync' \ - - # add fixed ADQL parameters - query = query + "?lang=ADQL&REQUEST=doQuery" - - # add query ADQL parameters (limit to 10 results) - query = query + "&QUERY=SELECT TOP 10 * from " + dataset.resource_name - # query = query + "&QUERY=SELECT TOP 10 " + dataset.select_fields +" from " + dataset.resource_name - - # add ADQL where where - query = query +" WHERE " - if len(where)>0: - # cut off the last separation character - where = where[:-1] - query = query + where - - if len(cone_search)>0: - query = query + cone_search - - return query, errors - - - # run a query - def run_query(self, dataset, dataset_name, query): - """ - # use pyvo to do a vo query - :param dataset: the dataset object that must be queried - :param query: the constructed (adql) query (that was probably generated with the above construct_query function) - :return: results: an array of dicts with the following structure; - { - "dataset": "astron.ivoa.obscore", - "result": "https://vo.astron.nl/getproduct/tgssadr/fits/TGSSADR_R40D60_5x5.MOSAIC.FITS" - }, - { - "dataset": "astron.ivoa.obscore", - "result": "https://vo.astron.nl/getproduct/tgssadr/fits/TGSSADR_R40D62_5x5.MOSAIC.FITS" - }, - """ - - results = [] - - # use pyvo the get to the results - service = vo.dal.TAPService(self.url) - try: - resultset = service.search(query) - except Exception as error: - record = {} - record['query'] = query - record['dataset'] = dataset.uri - record['dataset_name'] = dataset_name - record['result'] = str(error) - results.append(record) - return results - - for row in resultset: - # for the definition of standard fields to return see: - # http://www.ivoa.net/documents/ObsCore/20170509/REC-ObsCore-v1.1-20170509.pdf - - record = {} - result = '' - - # if * then iterate on the full row, otherwise just on the selection - if dataset.select_fields=='*': - values = row.values() - - for value in values: - try: - result = result + value.decode('utf-8') + ',' - except: - try: - result = result + str(value) + ',' - except: - pass - else: - select_list = dataset.select_fields.split(',') - - for select in select_list: - result = result + row[select].decode('utf-8') + ',' - - # cut off the last ',' - result = result[:-1] - record['dataset'] = dataset.uri - record['dataset_name'] = dataset_name - record['result'] = result - record['query'] = query - - # add some fields to return some rendering information for the frontend. - # for ivoa.obscore field names see: http://www.ivoa.net/documents/ObsCore/20170509/REC-ObsCore-v1.1-20170509.pdf - try: - record['title'] = row[dataset.title_field] - except: - pass - - try: - record['thumbnail'] = row[dataset.thumbnail_field] - except: - pass - - try: - record['url'] = row[dataset.url_field] - except: - pass - - - results.append(record) - - return results \ No newline at end of file diff --git a/esap/api/business/services/query/vo_reg.py b/esap/api/business/services/query/vo_reg.py deleted file mode 100644 index fece8e5f3699d085b27248d872a35bd9a5dc51ea..0000000000000000000000000000000000000000 --- a/esap/api/business/services/query/vo_reg.py +++ /dev/null @@ -1,277 +0,0 @@ -""" - File name: vo_reg.py - Author: Nico Vermaas - Astron - Date created: 2020-04-09 - Description: ESAP services for VO registry -""" - -from .query_base import query_base -import pyvo as vo -from pyvo.registry import search as regsearch -import urllib.parse - -def create_cone_search(esap_query_params, translation_parameters, equinox): - """ - Return a cone search subquery when ra, dec and fov are found in the query parameters. - - example: - SELECT TOP 10 * from ivoa.obscore WHERE CONTAINS(POINT('ICRS',s_ra,s_dec), CIRCLE('ICRS',202.48,47.23,4.0))=1 - - :param esap_query_params: - :param translation_parameters: - :return: - """ - - radius = None - try: - ra = float(esap_query_params['ra'][0]) - dec = float(esap_query_params['dec'][0]) - radius = float(esap_query_params['fov'][0]) - except: - pass - - if radius != None: - # found a fov parameter, which indicates a cone search - cone_search = "CONTAINS(POINT('"+equinox+"'," + \ - translation_parameters['ra'] + "," + \ - translation_parameters['dec'] + "), " \ - "CIRCLE('"+equinox+"'," + str(ra) + "," + str(dec) + "," + str(radius) + "))=1" - - # remove ra,dec,fov from the parameters so that they are not used in the where clause - del esap_query_params['ra'] - del esap_query_params['dec'] - del esap_query_params['fov'] - return cone_search - - -class vo_registry_connector(query_base): - - # Initializer - def __init__(self, url): - self.url = url - - # cut off the '/sync' resource, because the vo.dal.TAPServices will add it again - if self.url.endswith('/sync'): - self.url = self.url[:-4] - - - # construct a query for this type of service - def construct_query(self, dataset, query_params, translation_parameters, equinox): - - esap_query_params = dict(query_params) - where = '' - errors = [] - - # cone search is a specific type of query that uses ra, dec and a search radius. - # it is also done with a specific ADQL syntax. - # First check if the incoming query describes a cone search. - - cone_search = create_cone_search(esap_query_params,translation_parameters, equinox) - - for esap_param in esap_query_params: - - esap_key = esap_param - value = esap_query_params[esap_key][0] - - try: - dataset_key = translation_parameters[esap_key] - where = where + dataset_key + "='" + value + "' " - - except Exception as error: - # if the parameter could not be translated, then just continue - errors.append("ERROR: translating key " + esap_key + ' ' + str(error)) - - - results = [] - services = [] - - if "keyword" in esap_query_params: - services = self.search([esap_query_params["keyword"]]) - else : - services = self.search(datamodel='ObsCore') - - for resource in services: - # see row attributes - # https://pyvo.readthedocs.io/en/latest/api/pyvo.registry.regtap.RegistryResource.html#pyvo.registry.regtap.RegistryResource - #resource.describe() - - service = resource.service - - result = {} - # ESAP attributes - result['query_id'] = resource.ivoid - result['dataset'] = dataset.uri - result['dataset_name'] = resource.short_name - result['service_url'] = str(resource.access_url) - result['output_format'] = str(resource.source_format) - result['resource_name'] = str(dataset.resource_name) - result['protocol'] = str(dataset.dataset_catalog.protocol) - result['esap_service'] = str(dataset.dataset_catalog.esap_service) - result['service_connector'] = str(dataset.service_connector) - - # VO RegistryResource attributes - result['service_url'] = str(resource.access_url) - result['access_url'] = str(resource.access_url) - result['content_levels'] = str(resource.content_levels) - result['content_types'] = str(resource.content_types) - result['creators'] = str(resource.creators) - result['ivoid'] = str(resource.ivoid) - result['reference_url'] = str(resource.reference_url) - result['region_of_regard'] = str(resource.region_of_regard) - result['res_description'] = str(resource.res_description) - result['res_title'] = str(resource.res_title) - result['res_type'] = str(resource.res_type) - result['short_name'] = resource.short_name - result['source_format'] = str(resource.source_format) - result['standard_id'] = str(resource.standard_id) - result['waveband'] = ' '.join([str(elem) for elem in resource.waveband]) - #result['waveband'] = str(resource.waveband) - - # add sync (or async) specifier - # query = resource.access_url + '/sync' - - - query = resource.access_url + '/sync' if (self.get_service_type(resource).upper() == "TAP") else resource.access_url - # add fixed ADQL parameters - query_params = {} - query_params["LANG"] = "ADQL" - query_params["REQUEST"] = "doQuery" - query_params["QUERY"] = "SELECT TOP 10 * from " + dataset.resource_name - - - # add ADQL where where - if where: - if len(where)>0: - # cut off the last separation character - - query_params["QUERY"] += " WHERE " + where - - if cone_search: - if len(cone_search)>0: - if len(where)>0: - query_params["QUERY"] += " AND " + cone_search - else: - query_params["QUERY"] += " WHERE " + cone_search - - result['query'] = query + "?" + urllib.parse.urlencode(query_params) - - - results.append(result) - - return results, errors - - - # run a query - def run_query(self, dataset, dataset_name, query): - """ - # use pyvo to do a vo query - :param dataset: the dataset object that must be queried - :param query: the constructed (adql) query (that was probably generated with the above construct_query function) - """ - - results = [] - - # use pyvo the get to the results - - service = vo.dal.TAPService(self.url) - try: - resultset = service.search(query) - except Exception as error: - record = {} - record['query'] = query - record['dataset'] = dataset.uri - record['dataset_name'] = dataset_name - record['result'] = str(error) - results.append(record) - return results - - for row in resultset: - # for the definition of standard fields to return see: - # http://www.ivoa.net/documents/ObsCore/20170509/REC-ObsCore-v1.1-20170509.pdf - - record = {} - result = '' - - # if * then iterate on the full row, otherwise just on the selection - if dataset.select_fields=='*': - values = row.values() - - for value in values: - try: - result = result + value.decode('utf-8') + ',' - except: - try: - result = result + str(value) + ',' - except: - pass - else: - select_list = dataset.select_fields.split(',') - - for select in select_list: - result = result + row[select].decode('utf-8') + ',' - - # cut off the last ',' - result = result[:-1] - record['dataset'] = dataset.uri - record['dataset_name'] = dataset_name - record['result'] = result - record['query'] = query - - # add some fields to return some rendering information for the frontend. - # for ivoa.obscore field names see: http://www.ivoa.net/documents/ObsCore/20170509/REC-ObsCore-v1.1-20170509.pdf - try: - record['title'] = row[dataset.title_field] - except: - pass - - try: - record['thumbnail'] = row[dataset.thumbnail_field] - except: - pass - - try: - record['url'] = row[dataset.url_field] - except: - pass - - - results.append(record) - - return results - - - # Search for a keyword - def search(self, keywords=None, servicetype="tap", datamodel=None, waveband=None, **kwargs): - """ - # Use pyvo to do a Registry search by keyword - :param keyword: The keyword to search for - :param servicetype: The service type that we are searching for (e.g. tap) - """ - - if datamodel: - services = regsearch(datamodel=datamodel) - else: - services = regsearch(keywords=keywords, servicetype=servicetype, waveband=waveband) if waveband else regsearch(keywords=keywords, servicetype=servicetype) - - return services - - - - def get_service_type (self, service): - """ - # Get the IVOA Service Type - :param service: The service -> pyvo.registry.regtap.RegistryResource - """ - servicetype = None - - standards = { - "ivo://ivoa.net/std/tap" : "TAP", - "ivo://ivoa.net/std/sia" : "SIA", - "ivo://ivoa.net/std/obscore" : "OBSCORE", - } - - if not service: - return servicetype - - return standards[service["standard_id"].decode('utf-8').lower()] if service["standard_id"] else None - diff --git a/esap/api/business/services/query/vso.py b/esap/api/business/services/query/vso.py deleted file mode 100644 index ceb386362988814f5e5c1a0d6648b31467c6ef98..0000000000000000000000000000000000000000 --- a/esap/api/business/services/query/vso.py +++ /dev/null @@ -1,210 +0,0 @@ -""" - File name: vso.py - Author: Nico Vermaas - Astron - Date created: 2020-03-06 - Description: ESAP services for Virtual Solar Observatory - - Documentation: - - https://virtualsolar.org/cgi-bin/search?help=1 -""" - -from .query_base import query_base -import requests, json -from datetime import datetime - -AMP_REPLACEMENT = '_and_' - -class vso_connector(query_base): - """ - The connector to access the VSO - """ - - # Initializer - def __init__(self, url): - self.url = url - - # construct a query for this type of service - def construct_query(self, dataset, esap_query_params, translation_parameters, equinox): - - where = '' - errors = [] - - # loop through the list of input parameters and translate them. - for esap_param in esap_query_params: - esap_key = esap_param - value = esap_query_params[esap_key][0] - - # temp dirty hack to cut off instrument from provider (SOHO__EIT => EIT) - if esap_key == 'instrument': - s = value.split('__') - try: - value = s[1] - except: - pass - - - try: - dataset_key = translation_parameters[esap_key] - - # some specific VSO business logic to construct the timerange. - # Convert the ESAP values to a VSO timerange - if dataset_key=="start": - start_date = datetime.strptime(value, "%Y-%m-%d").strftime("%Y%m%d") - elif dataset_key=="end": - end_date = datetime.strptime(value, "%Y-%m-%d").strftime("%Y%m%d") - else: - # because '&' has a special meaning in urls (specifying a parameter) replace it with - # something harmless during serialization. - where = where + dataset_key + '=' + value + AMP_REPLACEMENT - - except Exception as error: - # if the parameter could not be translated, then just continue without this parameter - errors.append("ERROR: translating key " + esap_key + ' ' + str(error)) - - # cut off the last separation character - where = where[:-len(AMP_REPLACEMENT)] - - # construct the query url by first adding the constructed timerange, followed by the rest of the parameters - timerange="timerange="+start_date+"-"+end_date - where = timerange + "_and_" + where - - query = self.url + '?' + where - - return query, errors - - - - def expand_path(self, provider, path): - """ - The output seems to contain a link to a local location? Would that just be a matter of - > adding some http url to it? - > "fileid":"/archive/soho/private/data/processed/eit/lz/2004/01/efz20040107.002047" - - Oh... right. That would only get you the VSO 'Search' method, not the - 'GetData' to actually get you a URL. - - (The VSO is old enough that not all archives had their data online, so we - had to have an 'ordering' process, to tell them to retrieve it from tape - or whatever ... and we didn't want to trigger it happening just because - someone gave too broad of a search ... so it's a separate process. It - also lets people request a tarball vs. a discrete list of URLs when the - archive provides a packaging service) - - Ideally, you would call the 'GetData' method with the list of files that - you're interested in, and how you'd like them packaged, but there's - another way. - - Years ago, for one of the ESA SolarNet projects, I wrote a little CGI that - will attempt to redirect you to the file: - - https://sdac.virtualsolar.org/cgi-bin/get - - You call it using PATH_INFO parameters, with the arguments being the - provider & fileid: - - https://sdac.virtualsolar.org/cgi-bin/get/PROVIDER/FILEID - - (fileid does *not* need to have slashes escaped, only characters that need - URI encoding like '%' or '&') - - As a working example: - - https://sdac.virtualsolar.org/cgi-bin/get/SDAC//archive/soho/private/data/processed/eit/lz/2004/01/efz20040107.014933 - - - :return: - - """ - - path = "https://sdac.virtualsolar.org/cgi-bin/get/"+provider+path - return path - - - def run_query(self, dataset, dataset_name, query): - """ - :param dataset: the dataset object that must be queried - :param query: the constructed query (that was probably generated with the above construct_query function) - :return: results: an array of dicts with the following structure; - - example: - /esap-api/run-query/?dataset_uri=vso&query=https://sdac.virtualsolar.org/cgi/vsoui?timerange=20040107-20040108_and_instrument=eit&waverange=304 - - { - "dataset": "vso", - "result": "SDAC,SOHO", - "title": "/archive/soho/private/data/processed/eit/lz/2004/01/efz20040106.235333", - "url": "/archive/soho/private/data/processed/eit/lz/2004/01/efz20040106.235333" - }, - { - "dataset": "vso", - "result": "SDAC,SOHO", - "title": "/archive/soho/private/data/processed/eit/lz/2004/01/efz20040107.001148", - "url": "/archive/soho/private/data/processed/eit/lz/2004/01/efz20040107.001148" - }, - - - """ - results = [] - - # because '&' has a special meaning in urls (specifying a parameter) it had been replaced with - # something harmless during serialization. Replace it again with the & - query = query.replace(AMP_REPLACEMENT,'&') - - try: - - # do the http request - response = requests.request("GET", query) - json_response = json.loads(response.text) - resultset = json_response["resultset"] - - # iterate over the list of results.. - for json_record in resultset: - record = {} - result = '' - - select_list = dataset.select_fields.split(',') - for select in select_list: - try: - result = result + json_record[select] + ',' - except: - pass - - # cut off the last ',' - result = result[:-1] - - record['dataset'] = dataset.uri - record['dataset_name'] = dataset_name - record['result'] = result - record['query'] = query - - # some fields to return display information for the frontend. - try: - record['title'] = json_record[dataset.title_field] - except: - pass - - try: - # hardcoded thumbnail field, because it is in the deeper 'extra' structure of the json. - extra = json_record['extra'] - record['thumbnail'] = extra['thumbnail'] - except: - pass - - try: - url = json_record[dataset.url_field] - provider = json_record['provider'] - record['url'] = self.expand_path(provider,url) - - except: - pass - - results.append(record) - - except Exception as error: - record['dataset'] = dataset.uri - record['dataset_name'] = dataset_name - record['query'] = query - record['result'] = str(error) - results.append(record) - - return results \ No newline at end of file diff --git a/esap/api/business/services/staging/staging_base.py b/esap/api/business/services/staging/staging_base.py deleted file mode 100644 index bc77983b0451a8e1246d6b1489914d1cf6acb294..0000000000000000000000000000000000000000 --- a/esap/api/business/services/staging/staging_base.py +++ /dev/null @@ -1,20 +0,0 @@ -""" - File name: staging_base.py - Author: Nico Vermaas & Zheng Meyer-Zhao - Astron - Date created: 2020-03-09 - Description: ESAP staging abstract base class. -""" - -class staging_base: - - # Initializer - def __init__(self, url): - self.url = url - - # implement this in the derived service classes - def construct_stage_request(self, dataset, esap_params, translation_mapping): - pass - - # implement this in the derived service classes - def run_stage_request(self, dataset): - pass \ No newline at end of file diff --git a/esap/api/migrations/0001_initial.py b/esap/api/migrations/0001_initial.py deleted file mode 100644 index dd25f729af0e298741acef02d0af1f0b57197851..0000000000000000000000000000000000000000 --- a/esap/api/migrations/0001_initial.py +++ /dev/null @@ -1,75 +0,0 @@ -# Generated by Django 3.0.2 on 2020-03-09 07:46 - -from django.db import migrations, models -import django.db.models.deletion - - -class Migration(migrations.Migration): - - initial = True - - dependencies = [ - ] - - operations = [ - migrations.CreateModel( - name='EsapBaseObject', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('uri', models.CharField(max_length=40)), - ('name', models.CharField(max_length=40)), - ('short_description', models.CharField(max_length=40)), - ('long_description', models.TextField(blank=True, null=True)), - ('retrieval_description', models.TextField(blank=True, null=True)), - ('thumbnail', models.URLField(default='https://alta.astron.nl/alta-static/unknown.jpg')), - ('documentation_url', models.URLField(blank=True, null=True)), - ('institute', models.CharField(max_length=40)), - ], - ), - migrations.CreateModel( - name='ParameterMapping', - fields=[ - ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), - ('uri', models.CharField(max_length=15)), - ('parameters', models.TextField(blank=True, null=True)), - ], - ), - migrations.CreateModel( - name='Archive', - fields=[ - ('esapbaseobject_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='api.EsapBaseObject')), - ('instrument', models.CharField(max_length=30)), - ], - bases=('api.esapbaseobject',), - ), - migrations.CreateModel( - name='Catalog', - fields=[ - ('esapbaseobject_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='api.EsapBaseObject')), - ('esap_service', models.CharField(default='vo', max_length=15)), - ('equinox', models.CharField(choices=[('J2000', 'J2000'), ('ICRS', 'ICRS')], default='ICRS', max_length=10)), - ('protocol', models.CharField(choices=[('adql', 'adql'), ('http', 'http')], max_length=15)), - ('url', models.URLField(null=True)), - ('parameters', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='catalogs', to='api.ParameterMapping')), - ], - bases=('api.esapbaseobject',), - ), - migrations.CreateModel( - name='DataSet', - fields=[ - ('esapbaseobject_ptr', models.OneToOneField(auto_created=True, on_delete=django.db.models.deletion.CASCADE, parent_link=True, primary_key=True, serialize=False, to='api.EsapBaseObject')), - ('datatype', models.CharField(max_length=30)), - ('processing_level', models.CharField(max_length=30)), - ('resource_name', models.CharField(blank=True, max_length=30, null=True)), - ('select_fields', models.CharField(blank=True, default='*', max_length=100, null=True)), - ('title_field', models.CharField(blank=True, max_length=30, null=True)), - ('thumbnail_field', models.CharField(blank=True, max_length=30, null=True)), - ('url_field', models.CharField(blank=True, max_length=30, null=True)), - ('output_format', models.CharField(choices=[('list', 'list'), ('tiles', 'tiles')], default='list', max_length=10)), - ('service_connector', models.CharField(blank=True, max_length=80, null=True)), - ('dataset_archive', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='datasets', to='api.Archive')), - ('dataset_catalog', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='datasets', to='api.Catalog')), - ], - bases=('api.esapbaseobject',), - ), - ] diff --git a/esap/configuration_assets/esap_solar.png b/esap/configuration_assets/esap_solar.png new file mode 100644 index 0000000000000000000000000000000000000000..bbe7615710906312485cde54f973f15766fcfb19 Binary files /dev/null and b/esap/configuration_assets/esap_solar.png differ diff --git a/esap/esap/esap.sqlite3 b/esap/esap/esap.sqlite3 index 05a95a7ebd6f244bd6604cd138ffcca16acbcb0f..97a2ceac52c79ac4f264d6a5e7e1608dd1e72853 100644 Binary files a/esap/esap/esap.sqlite3 and b/esap/esap/esap.sqlite3 differ diff --git a/esap/esap/settings/base.py b/esap/esap/settings/base.py index 3f6ceedfcfb1c90e1f364d4b7158ddbce1b0fd85..50192997505dffff86ab15a56997d805f89086ec 100644 --- a/esap/esap/settings/base.py +++ b/esap/esap/settings/base.py @@ -23,9 +23,9 @@ CORS_ORIGIN_ALLOW_ALL = True # Application definition INSTALLED_APPS = [ + 'query', 'accounts', 'rucio', - 'api', 'knox', 'django.contrib.admin', 'django.contrib.auth', diff --git a/esap/esap/settings/statics.py b/esap/esap/settings/statics.py index ddb2febb5f2152caf543047931fa11acfde64055..8504abe207693f5c0e51b04286cb611ce58531e8 100644 --- a/esap/esap/settings/statics.py +++ b/esap/esap/settings/statics.py @@ -14,7 +14,7 @@ SECRET_KEY = 'cie-((m#n$br$6l53yash45*2^mwuux*2u)bad5(0flx@krnj9' DEBUG = False INSTALLED_APPS = [ - 'api', + 'query', 'django.contrib.staticfiles', ] diff --git a/esap/esap/urls.py b/esap/esap/urls.py index dda82a9a4a2a28ac0e45a4a5d429508053da404e..c51e54d6311e7884eb3c04f08db4febbedb7f83d 100644 --- a/esap/esap/urls.py +++ b/esap/esap/urls.py @@ -17,9 +17,9 @@ from django.contrib import admin from django.urls import include, path, re_path urlpatterns = [ - path('esap-api/', include('api.urls')), + path('esap-api/query/', include('query.api.urls')), + path('esap-api/rucio/', include('rucio.api.urls')), + path('esap-api/accounts/', include('accounts.api.urls')), path('esap-api/admin/', admin.site.urls, name='admin-view'), - path('api/', include('rucio.api.urls')), - path('accounts/', include('accounts.urls')), - re_path('^oidc/', include('mozilla_django_oidc.urls')), + re_path('^esap-api/oidc/', include('mozilla_django_oidc.urls')), ] diff --git a/esap/query/__init__.py b/esap/query/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..6598231de1378a6c4388d35853549001ec415dcf --- /dev/null +++ b/esap/query/__init__.py @@ -0,0 +1 @@ +default_app_config = 'query.apps.MyAppConfig' \ No newline at end of file diff --git a/esap/api/migrations/__init__.py b/esap/query/__pycache__/models.cpython-37.pyc.2226543272368 similarity index 100% rename from esap/api/migrations/__init__.py rename to esap/query/__pycache__/models.cpython-37.pyc.2226543272368 diff --git a/esap/api/views/staging_views.py b/esap/query/__pycache__/models.cpython-37.pyc.2530047107504 similarity index 100% rename from esap/api/views/staging_views.py rename to esap/query/__pycache__/models.cpython-37.pyc.2530047107504 diff --git a/esap/api/admin.py b/esap/query/admin.py similarity index 100% rename from esap/api/admin.py rename to esap/query/admin.py diff --git a/esap/query/api/__init__.py b/esap/query/api/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/esap/query/api/business/__init__.py b/esap/query/api/business/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/esap/api/business/configuration.py b/esap/query/api/business/configuration.py similarity index 100% rename from esap/api/business/configuration.py rename to esap/query/api/business/configuration.py diff --git a/esap/api/business/query_controller.py b/esap/query/api/business/query_controller.py similarity index 99% rename from esap/api/business/query_controller.py rename to esap/query/api/business/query_controller.py index 094ba6a7a136e58244b9a493f1c2003c3747f0b3..5b3b2fa2ed8372935b6c692ee4da115127bdf84c 100644 --- a/esap/api/business/query_controller.py +++ b/esap/query/api/business/query_controller.py @@ -5,11 +5,10 @@ Description: Business logic for ESAP-gateway. These functions are called from the views (views.py). """ -import importlib -import logging import json -from .common import timeit +import logging +from query.api.utils import timeit from .services.query import vo, alta, vso, helio, vo_reg logger = logging.getLogger(__name__) diff --git a/esap/api/business/staging_controller.py b/esap/query/api/business/staging_controller.py similarity index 100% rename from esap/api/business/staging_controller.py rename to esap/query/api/business/staging_controller.py diff --git a/esap/query/api/serializers.py b/esap/query/api/serializers.py new file mode 100644 index 0000000000000000000000000000000000000000..9bacd433325fb629c903455d870a6e9b745b58d9 --- /dev/null +++ b/esap/query/api/serializers.py @@ -0,0 +1,101 @@ +from rest_framework import serializers +from query.models import EsapBaseObject, DataSet, Archive, Catalog, ParameterMapping +import logging + +logger = logging.getLogger(__name__) + +class AdexBaseObjectSerializer(serializers.ModelSerializer): + class Meta(): + model = EsapBaseObject + fields = "__all__" + + +# this is a serializer that uses hyperlinks to produce a navigable REST API +class DataSetSerializer(serializers.HyperlinkedModelSerializer): + + class Meta(): + model = DataSet + fields = "__all__" + + +# this is a serializer that uses uri's in the datasets for easier identification for the frontend +class DataSetModelSerializer(serializers.ModelSerializer): + + class Meta(): + model = DataSet + # fields = "__all__" + fields = ('id', 'uri', 'name', 'resource_name', 'short_description','long_description', 'retrieval_description', + 'thumbnail', 'output_format', 'documentation_url', 'service_connector', 'institute', + 'archive_name_derived', 'archive_uri_derived','catalog_name_derived','catalog_uri_derived', + 'catalog_url_derived', 'catalog_user_url_derived') + + +# this is a serializer that uses hyperlinks to produce a navigable REST API +class ArchiveSerializer(serializers.HyperlinkedModelSerializer): + + datasets = serializers.HyperlinkedRelatedField( + many=True, + read_only=True, + # read_only=False + # queryset=DataSet.objects.all(), + view_name='dataset-detail', + lookup_field='pk', + required=False + ) + + class Meta(): + model = Archive + + fields = ('id', 'uri', 'name', 'short_description', 'long_description', 'retrieval_description', 'thumbnail', + 'documentation_url','instrument','institute','datasets') + + +# this is a serializer that uses uri's in the datasets for easier identification for the frontend +class ArchiveModelSerializer(serializers.ModelSerializer): + + datasets = serializers.StringRelatedField( + many=True, + required=False, + ) + + class Meta(): + model = Archive + + fields = ('id', 'uri', 'name', 'short_description', 'long_description', 'retrieval_description', 'thumbnail', + 'documentation_url','instrument','institute','datasets') + + +# this is a serializer that uses hyperlinks to produce a navigable REST API +class CatalogSerializer(serializers.HyperlinkedModelSerializer): + + dataset = serializers.HyperlinkedRelatedField( + many=False, + required=False, + read_only=True, + view_name='dataset-detail', + lookup_field='pk', + # queryset=DataSet.objects.all(), + ) + + parameters = serializers.HyperlinkedRelatedField( + many=False, + required=False, + read_only=False, + view_name='parametermapping-detail', + lookup_field='pk', + queryset=ParameterMapping.objects.all(), + ) + + class Meta(): + model = Catalog + # fields = "__all__" + fields = ('id', 'uri', 'name', 'short_description', 'long_description', 'retrieval_description', 'thumbnail', + 'url', 'user_url', 'dataset', 'parameters', 'esap_service') + +# this is a serializer that uses hyperlinks to produce a navigable REST API +class ParameterMappingSerializer(serializers.HyperlinkedModelSerializer): + + class Meta(): + model = ParameterMapping + fields = "__all__" + diff --git a/esap/api/urls.py b/esap/query/api/urls.py similarity index 98% rename from esap/api/urls.py rename to esap/query/api/urls.py index f87d665b911c87cff51a5c2bc3987a071e1a0c4b..36676e21d0d83b27a48175cf013843fcf5534bcf 100644 --- a/esap/api/urls.py +++ b/esap/query/api/urls.py @@ -1,6 +1,6 @@ from django.urls import path -from . import views +from query.api import views urlpatterns = [ diff --git a/esap/api/business/common.py b/esap/query/api/utils.py similarity index 100% rename from esap/api/business/common.py rename to esap/query/api/utils.py diff --git a/esap/api/views/__init__.py b/esap/query/api/views/__init__.py similarity index 66% rename from esap/api/views/__init__.py rename to esap/query/api/views/__init__.py index eb9bc3d8408e5e597286cf132eb29264d55927c6..02f762f2c0101527916e7db608a7a89b9688f09a 100644 --- a/esap/api/views/__init__.py +++ b/esap/query/api/views/__init__.py @@ -1,3 +1,2 @@ from .common_views import * from .query_views import * -from .staging_views import * \ No newline at end of file diff --git a/esap/api/views/common_views.py b/esap/query/api/views/common_views.py similarity index 98% rename from esap/api/views/common_views.py rename to esap/query/api/views/common_views.py index 1fa0cde365a5e5c08aeccc1f50c48a93fbcfad6a..0b9afeb6cece96b340c7ce8b26551580ae8ca724 100644 --- a/esap/api/views/common_views.py +++ b/esap/query/api/views/common_views.py @@ -8,10 +8,10 @@ from rest_framework import generics from django_filters import rest_framework as filters -from ..business import configuration +from query.api.business import configuration -from ..models import Archive, DataSet, Catalog, ParameterMapping -from ..serializers import \ +from query.models import Archive, DataSet, Catalog, ParameterMapping +from query.api.serializers import \ ArchiveSerializer, \ ArchiveModelSerializer, \ DataSetSerializer, \ diff --git a/esap/api/views/query_views.py b/esap/query/api/views/query_views.py similarity index 95% rename from esap/api/views/query_views.py rename to esap/query/api/views/query_views.py index 292fb08998a25692dfaaa7726614e8ce636a84e6..3d018a93382e55d33ab16c898ee6dac3054c8edb 100644 --- a/esap/api/views/query_views.py +++ b/esap/query/api/views/query_views.py @@ -1,10 +1,10 @@ import logging -from rest_framework import generics, pagination, status +from rest_framework import generics from rest_framework.response import Response -from ..models import DataSet -from ..business import query_controller, configuration +from query.api.business import query_controller +from query.models import DataSet from . import common_views logger = logging.getLogger(__name__) diff --git a/esap/api/apps.py b/esap/query/apps.py similarity index 77% rename from esap/api/apps.py rename to esap/query/apps.py index 625a267c4fa2b9b3510b1b14feec899c2870e39e..d7bb3aec65dda20ea36a27501911653caa7f6bee 100644 --- a/esap/api/apps.py +++ b/esap/query/apps.py @@ -2,5 +2,5 @@ from django.apps import AppConfig class MyAppConfig(AppConfig): - name = 'api' + name = 'query' diff --git a/esap/api/models.py b/esap/query/models.py similarity index 100% rename from esap/api/models.py rename to esap/query/models.py diff --git a/esap/api/serializers.py b/esap/query/serializers.py similarity index 100% rename from esap/api/serializers.py rename to esap/query/serializers.py diff --git a/esap/api/static/api/favicon.ico b/esap/query/static/favicon.ico similarity index 100% rename from esap/api/static/api/favicon.ico rename to esap/query/static/favicon.ico diff --git a/esap/api/static/api/favicon.png b/esap/query/static/favicon.png similarity index 100% rename from esap/api/static/api/favicon.png rename to esap/query/static/favicon.png diff --git a/esap/api/static/api/adex_logo.jpg b/esap/query/static/query/adex_logo.jpg similarity index 100% rename from esap/api/static/api/adex_logo.jpg rename to esap/query/static/query/adex_logo.jpg diff --git a/esap/api/static/api/esap_logo.png b/esap/query/static/query/esap_logo.png similarity index 100% rename from esap/api/static/api/esap_logo.png rename to esap/query/static/query/esap_logo.png diff --git a/esap/api/static/favicon.ico b/esap/query/static/query/favicon.ico similarity index 100% rename from esap/api/static/favicon.ico rename to esap/query/static/query/favicon.ico diff --git a/esap/api/static/favicon.png b/esap/query/static/query/favicon.png similarity index 100% rename from esap/api/static/favicon.png rename to esap/query/static/query/favicon.png diff --git a/esap/api/static/api/style.css b/esap/query/static/query/style.css similarity index 100% rename from esap/api/static/api/style.css rename to esap/query/static/query/style.css diff --git a/esap/api/templates/api/base.html b/esap/query/templates/api/base.html similarity index 100% rename from esap/api/templates/api/base.html rename to esap/query/templates/api/base.html diff --git a/esap/api/templates/api/index.html b/esap/query/templates/api/index.html similarity index 100% rename from esap/api/templates/api/index.html rename to esap/query/templates/api/index.html diff --git a/esap/api/tests.py b/esap/query/tests.py similarity index 100% rename from esap/api/tests.py rename to esap/query/tests.py