From 6f73076c4a8e82dea3c5077a4177f06efa983e5f Mon Sep 17 00:00:00 2001 From: signedav Date: Thu, 30 Nov 2023 11:54:26 +0100 Subject: [PATCH] filter ilidata concerning category preferredDataSource gpkg or pg --- modelbaker/iliwrapper/ilicache.py | 10 ++- tests/test_ilicache.py | 66 +++++++++++++++++-- tests/testdata/ilirepo/invalid/ilidata.xml | 12 ---- .../testdata/ilirepo/usabilityhub/ilidata.xml | 54 ++++----------- 4 files changed, 80 insertions(+), 62 deletions(-) diff --git a/modelbaker/iliwrapper/ilicache.py b/modelbaker/iliwrapper/ilicache.py index d692dc6..b87ae08 100644 --- a/modelbaker/iliwrapper/ilicache.py +++ b/modelbaker/iliwrapper/ilicache.py @@ -437,7 +437,7 @@ class IliDataCache(IliCache): CACHE_PATH = os.path.expanduser("~/.ilidatacache") - def __init__(self, configuration, type="metaconfig", models=None): + def __init__(self, configuration, type="metaconfig", models=None, datasources=[]): IliCache.__init__(self, configuration) self.information_file = "ilidata.xml" @@ -454,6 +454,7 @@ def __init__(self, configuration, type="metaconfig", models=None): if self.base_configuration else [] ) + self.datasources = datasources def process_model_directory(self, path): # download remote and local repositories @@ -512,9 +513,16 @@ def _process_informationfile(self, file, netloc, url): datasource = datasource_code_regex.search( category_value ).group(1) + + # filter for models and types and if datasources are passed and found in the category, they need to be considered if ( not any(model in models for model in self.filter_models) or type != self.type + or ( + self.datasources + and datasource + and datasource not in self.datasources + ) ): continue title = list() diff --git a/tests/test_ilicache.py b/tests/test_ilicache.py index d4357da..fc3d2c4 100644 --- a/tests/test_ilicache.py +++ b/tests/test_ilicache.py @@ -285,6 +285,58 @@ def test_ilimodels_xml_parser_24_local_repo_local_files(self): def test_ilidata_xml_parser_metaconfig_kbs(self): # find kbs metaconfig file according to the model(s) with direct ilidata.xml scan + # pass gpkg as datasource find the ones with gpkg-datasource-category and the ones without datasource-category + ilimetaconfigcache = IliDataCache( + configuration=None, models="KbS_LV95_V1_4", datasources=["gpkg"] + ) + ilimetaconfigcache._process_informationfile( + os.path.join( + test_path, "testdata", "ilirepo", "usabilityhub", "ilidata.xml" + ), + "test_repo", + os.path.join(test_path, "testdata", "ilirepo", "usabilityhub"), + ) + assert "test_repo" in ilimetaconfigcache.repositories.keys() + metaconfigs = { + e["id"] + for e in next(elem for elem in ilimetaconfigcache.repositories.values()) + } + expected_metaconfigs = { + "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0-technical", # no preferred datasource + "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0", # no preferred datasource + "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0_gpkg_localfiletest", # gpkg as preferred datasource + "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0-wrong", # no preferred datasource + "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0_gpkg", # no preferred datasource + } + assert metaconfigs == expected_metaconfigs + + # pass pg as datasource find the ones with pg-datasource-category and the ones without datasource-category + ilimetaconfigcache = IliDataCache( + configuration=None, models="KbS_LV95_V1_4", datasources=["pg"] + ) + ilimetaconfigcache._process_informationfile( + os.path.join( + test_path, "testdata", "ilirepo", "usabilityhub", "ilidata.xml" + ), + "test_repo", + os.path.join(test_path, "testdata", "ilirepo", "usabilityhub"), + ) + assert "test_repo" in ilimetaconfigcache.repositories.keys() + metaconfigs = { + e["id"] + for e in next(elem for elem in ilimetaconfigcache.repositories.values()) + } + expected_metaconfigs = { + "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0-technical", # no preferred datasource + "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0", # no preferred datasource + "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0_localfiletest", # pg as preferred datasource + "ch.opengis.ili.config.KbS_LV95_V1_4_ili2db", # pg as preferred datasource + "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0-wrong", # no preferred datasource + "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0_gpkg", # no preferred datasource + } + assert metaconfigs == expected_metaconfigs + + # don't pass any datasource, means find all ilimetaconfigcache = IliDataCache(configuration=None, models="KbS_LV95_V1_4") ilimetaconfigcache._process_informationfile( os.path.join( @@ -299,13 +351,13 @@ def test_ilidata_xml_parser_metaconfig_kbs(self): for e in next(elem for elem in ilimetaconfigcache.repositories.values()) } expected_metaconfigs = { - "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0-technical", - "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0", - "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0_localfiletest", - "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0_gpkg_localfiletest", - "ch.opengis.ili.config.KbS_LV95_V1_4_ili2db", - "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0-wrong", - "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0_gpkg", + "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0-technical", # no preferred datasource + "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0", # no preferred datasource + "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0_localfiletest", # pg as preferred datasource + "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0_gpkg_localfiletest", # gpkg as preferred datasource + "ch.opengis.ili.config.KbS_LV95_V1_4_ili2db", # pg as preferred datasource + "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0-wrong", # no preferred datasource + "ch.opengis.ili.config.KbS_LV95_V1_4_config_V1_0_gpkg", # no preferred datasource } assert metaconfigs == expected_metaconfigs diff --git a/tests/testdata/ilirepo/invalid/ilidata.xml b/tests/testdata/ilirepo/invalid/ilidata.xml index 4699d80..0f90a97 100644 --- a/tests/testdata/ilirepo/invalid/ilidata.xml +++ b/tests/testdata/ilirepo/invalid/ilidata.xml @@ -34,9 +34,6 @@ http://codes.interlis.ch/type/metaconfig - - http://codes.opengis.ch/modelbaker - @@ -68,9 +65,6 @@ http://codes.interlis.ch/type/metaconfig - - http://codes.opengis.ch/modelbaker - @@ -95,9 +89,6 @@ http://codes.interlis.ch/type/metaconfig - - http://codes.opengis.ch/modelbaker - @@ -133,9 +124,6 @@ http://codes.interlis.ch/type/metaconfig - - http://codes.opengis.ch/modelbaker - diff --git a/tests/testdata/ilirepo/usabilityhub/ilidata.xml b/tests/testdata/ilirepo/usabilityhub/ilidata.xml index dbf781b..f3fbf8c 100644 --- a/tests/testdata/ilirepo/usabilityhub/ilidata.xml +++ b/tests/testdata/ilirepo/usabilityhub/ilidata.xml @@ -36,9 +36,6 @@ http://codes.interlis.ch/type/metaconfig - - http://codes.opengis.ch/modelbaker - @@ -73,9 +70,6 @@ http://codes.interlis.ch/type/metaconfig - - http://codes.opengis.ch/modelbaker - @@ -111,7 +105,7 @@ http://codes.interlis.ch/type/metaconfig - http://codes.opengis.ch/modelbaker + http://codes.modelbaker.ch/preferredDataSource/pg @@ -148,7 +142,7 @@ http://codes.interlis.ch/type/metaconfig - http://codes.opengis.ch/modelbaker + http://codes.modelbaker.ch/preferredDataSource/gpkg @@ -184,9 +178,7 @@ http://codes.interlis.ch/type/metaconfig - - http://codes.opengis.ch/modelbaker - + @@ -211,9 +203,7 @@ http://codes.interlis.ch/type/metaconfig - - http://codes.opengis.ch/modelbaker - + @@ -249,7 +239,7 @@ http://codes.interlis.ch/type/metaconfig - http://codes.opengis.ch/modelbaker + http://codes.modelbaker.ch/preferredDataSource/pg @@ -414,9 +404,7 @@ http://codes.interlis.ch/type/projecttopping - - http://codes.opengis.ch/modelbaker - + @@ -452,9 +440,7 @@ http://codes.interlis.ch/type/projecttopping - - http://codes.opengis.ch/modelbaker - + @@ -554,9 +540,7 @@ http://codes.interlis.ch/type/referenceData - - http://codes.opengis.ch/modelbaker - +           @@ -591,9 +575,7 @@ http://codes.interlis.ch/type/referenceData - - http://codes.opengis.ch/modelbaker - +           @@ -641,9 +623,6 @@ http://codes.interlis.ch/type/metaconfig - - http://codes.opengis.ch/modelbaker - @@ -678,9 +657,6 @@ http://codes.interlis.ch/type/metaconfig - - http://codes.opengis.ch/modelbaker - @@ -716,9 +692,7 @@ http://codes.interlis.ch/type/referenceData - - http://codes.opengis.ch/modelbaker - +                   @@ -764,9 +738,7 @@ http://codes.interlis.ch/type/referenceData - - http://codes.opengis.ch/modelbaker - +                   @@ -812,9 +784,7 @@ http://codes.interlis.ch/type/referenceData - - http://codes.opengis.ch/modelbaker - +