Skip to content
Snippets Groups Projects
Commit 0dd5a607 authored by Klaas Kliffen's avatar Klaas Kliffen :satellite:
Browse files

Merge branch 'add_datatype_filtering' into 'master'

Add option to select datatype during query

See merge request !7
parents e5805d0a 5fab9d69
No related branches found
No related tags found
1 merge request!7Add option to select datatype during query
...@@ -13,6 +13,10 @@ from sqlalchemy.engine.base import Connection, Engine ...@@ -13,6 +13,10 @@ from sqlalchemy.engine.base import Connection, Engine
from sqlalchemy.sql import text from sqlalchemy.sql import text
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
_EXPECTED_STR_PER_DATATYPE = {
'MS': '.MS_',
'BF': '_bf.'
}
def parse_args() -> Namespace: def parse_args() -> Namespace:
...@@ -32,6 +36,7 @@ def parse_args() -> Namespace: ...@@ -32,6 +36,7 @@ def parse_args() -> Namespace:
parser.add_argument( parser.add_argument(
"--save-missing", type=str, help="Directory to store csv with missing files" "--save-missing", type=str, help="Directory to store csv with missing files"
) )
parser.add_argument("--datatype", help="Select a specific datatype", choices=_EXPECTED_STR_PER_DATATYPE.keys())
parser.add_argument("-v", action="store_true", help="Verbose logging") parser.add_argument("-v", action="store_true", help="Verbose logging")
parser.add_argument("-q", action="store_true", help="Quiet logging (only errors)") parser.add_argument("-q", action="store_true", help="Quiet logging (only errors)")
return parser.parse_args() return parser.parse_args()
...@@ -156,11 +161,20 @@ def match_file_lists( ...@@ -156,11 +161,20 @@ def match_file_lists(
return reg_data.intersection(cra_data) return reg_data.intersection(cra_data)
def filter_datatype(size_path_list, datatype):
if datatype is None:
return size_path_list
else:
expected_string = _EXPECTED_STR_PER_DATATYPE[datatype]
return [item for item in size_path_list if expected_string in item]
def gen_csv( def gen_csv(
obs_id: str, obs_id: str,
o_file: Optional[str] = None, o_file: Optional[str] = None,
save_missing: Optional[str] = None, save_missing: Optional[str] = None,
config_file: Optional[str] = None, config_file: Optional[str] = None,
datatype: Optional[str] = None,
): ):
"""Generate CSV file for ATDB """Generate CSV file for ATDB
...@@ -180,7 +194,9 @@ def gen_csv( ...@@ -180,7 +194,9 @@ def gen_csv(
intersection = match_file_lists(registered, crawled, save_missing) intersection = match_file_lists(registered, crawled, save_missing)
write_output("\n".join(intersection), o_file) filtered_intersection = filter_datatype(intersection, datatype)
write_output("\n".join(filtered_intersection), o_file)
def main(): def main():
...@@ -198,6 +214,7 @@ def main(): ...@@ -198,6 +214,7 @@ def main():
o_file=args.o, o_file=args.o,
save_missing=args.save_missing, save_missing=args.save_missing,
config_file=args.c, config_file=args.c,
datatype=args.datatype
) )
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment