Do not create hash conflicts

fair/parsing/globbing.py

@@ -27,6 +27,7 @@
 import fair.registry.requests as fdp_reg_req
 import fair.configuration as fdp_conf
 import fair.exceptions as fdp_exc
+import fair.registry.versioning as fdp_ver
 import fair.utilities as fdp_util
 
 def glob_read_write(
@@ -64,9 +65,13 @@ def glob_read_write(
     # key-value pairs that contain glob statements.
     for entry in config_dict_sub:
         # We still want to keep the wildcard version in case the
-        # user wants to write to this namespace
+        # user wants to write to this namespace.
+        # Wipe version info for this object to start from beginning
         if not remove_wildcard:
-            _parsed.append(entry)
+            _orig_entry = copy.deepcopy(entry)
+            if 'use' in _orig_entry and 'version' in _orig_entry['use']:
+                del _orig_entry['use']['version']
+            _parsed.append(_orig_entry)
 
         _glob_vals = [(k, v) for k, v in entry.items() if '*' in v]
         if len(_glob_vals) > 1:

fair/parsing/variables.py

@@ -158,20 +158,27 @@ def _tag_check(*args, **kwargs):
 
 
 def subst_versions(local_uri: str, config_yaml_dict: typing.Dict) -> typing.Dict:
-    # dynamic versionables only present in write statement
+    # Check if write block exists, if not return unaltered dict
     if 'write' not in config_yaml_dict:
         return config_yaml_dict
 
     _out_dict = copy.deepcopy(config_yaml_dict)
+    _out_dict['write'] = []
     _obj_type = 'data_product'
 
     _write_statements = config_yaml_dict['write']
 
-    for i, item in enumerate(_write_statements):
-        if _obj_type not in item:
+    for real_item in _write_statements:
+        if _obj_type not in real_item:
             raise fdp_exc.UserConfigError(
-                f"Expected '{_obj_type}' key in object '{item}'"
+                f"Expected '{_obj_type}' key in object '{real_item}'"
             )
+
+        item = {_obj_type: real_item[_obj_type]}
+        item['use'] = real_item['use'] if 'use' in real_item else {}
+
+        if 'version' in real_item:
+            item['use']['version'] = real_item['version']
 
         _params = {"name": item[_obj_type]}
         _results = None
@@ -184,31 +191,42 @@ def subst_versions(local_uri: str, config_yaml_dict: typing.Dict) -> typing.Dict
             # Object does not yet exist on the local registry
             pass
 
-        _latest_version = fdp_ver.get_latest_version(_results)
-
-        # Check for whether format is ['use']['version'] or just ['version']
-        if 'use' in item and 'version' in item['use']:
-            _new_version = _bump_version(item['use']['version'], _latest_version)
-        elif 'version' in item:
-            _new_version = _bump_version(item['version'], _latest_version)
-            # Remove version entry and add back as use: version later
-            del item['version']
-        else:
-            _new_version = _latest_version.bump_patch()
-
-        # Check write product/version not already in registry
-        _params['version'] = str(_new_version)
-        _write_product = fdp_reg_req.get(local_uri, _obj_type, params=_params)
-        if _write_product:
-            raise fdp_exc.UserConfigError(
-                f"Data product '{item[_obj_type]} v{str(_new_version)}' already exists in registry"
-            )
+        # Capture wildcards
+        if '*' in _params['name']:
+            _glob_item = copy.deepcopy(item)
+            _latest_version = fdp_ver.get_latest_version()
+
+            if 'version' in _glob_item['use']:
+                _new_version = _bump_version(_glob_item['use']['version'], _latest_version)
+            else:
+                _new_version = fdp_ver.default_bump(_latest_version)
+
+            _glob_item['use']['version'] = str(_new_version)
+            _out_dict['write'].append(_glob_item)
+
+        # Capture normal data and results of wildcard matches
+        for result in _results:
+            _new_item = copy.deepcopy(item)
+            _latest_version = fdp_ver.get_latest_version([result])
+
+            if 'version' in _new_item['use']:
+                _new_version = _bump_version(_new_item['use']['version'], _latest_version)
+            else:
+                _new_version = fdp_ver.default_bump(_latest_version)
 
-        if 'use' not in _write_statements[i]:
-            _write_statements[i]['use'] = {}
-        _write_statements[i]['use']['version'] = str(_new_version)
+            # Check write product/version not already in registry
+            _params['version'] = str(_new_version)
+            _params['name'] = result['name']
+            _write_product = fdp_reg_req.get(local_uri, _obj_type, params=_params)
+            if _write_product:
+                raise fdp_exc.UserConfigError(
+                    f"Data product '{_new_item[_obj_type]} v{str(_new_version)}' already exists in registry"
+                )
 
-    _out_dict['write'] = _write_statements
+            _new_item['use']['version'] = str(_new_version)
+            _new_item[_obj_type] = result['name']
+
+            _out_dict['write'].append(_new_item)
 
     return _out_dict
 
@@ -222,49 +240,51 @@ def get_read_version(
         return config_yaml_dict
 
     _out_dict = copy.deepcopy(config_yaml_dict)
+    _out_dict['read'] = []
     _obj_type = 'data_product'
 
     _read_statements = config_yaml_dict['read']
 
-    for i, item in enumerate(_read_statements):
-        if _obj_type not in item:
+    for real_item in _read_statements:
+        if _obj_type not in real_item:
             raise fdp_exc.UserConfigError(
-                f"Expected '{_obj_type}' key in object '{item}'"
+                f"Expected '{_obj_type}' key in object '{real_item}'"
             )
+
+        item = {_obj_type: real_item[_obj_type]}
+        item['use'] = real_item['use'] if 'use' in real_item else {}
 
-        _params = {"name": item[_obj_type]}
+        if 'version' in real_item:
+            item['use']['version'] = real_item['version']
 
-        # If version is specified, add as parameter for API call
-        if 'use' in item and 'version' in item['use']:
+        _params = {"name": item[_obj_type]}
+        if 'version' in item['use']:
             _params['version'] = item['use']['version']
-        elif 'version' in item:
-            _params['version'] = item['version']
-            # Remove version entry and add back as use: version later
-            del item['version']
-
         _results = None
 
         _results = fdp_reg_req.get(local_uri, _obj_type, params=_params)
-
         if not _results:
-            if 'version' in _params:
-                _msg = f"'{item[_obj_type]} v{_params['version']}' does not exist in local registry"
-            else:
-                _msg = f"'{item[_obj_type]}' does not exist in local registry"
-
-            raise fdp_exc.RegistryAPICallError(
-                _msg,
-                error_code = 400
+            raise fdp_exc.UserConfigError(
+                f"Data product {_params} does not already exist in registry"
             )
 
+        # Capture normal data and results of wildcard matches
+        for result in _results:
+            _new_item = copy.deepcopy(item)
 
-        _product_version = fdp_ver.get_latest_version(_results)
+            # Check read product/version not already in registry
+            _params['name'] = result['name']
+            _read_product = fdp_reg_req.get(local_uri, _obj_type, params=_params)
+            if not _read_product:
+                raise fdp_exc.UserConfigError(
+                    f"Data product '{_new_item[_obj_type]} v{str(_new_version)}' does not already exist in registry"
+                )
+            _latest_version = fdp_ver.get_latest_version(_read_product)
 
-        if 'use' not in _read_statements[i]:
-            _read_statements[i]['use'] = {}
-        _read_statements[i]['use']['version'] = str(_product_version)
+            _new_item['use']['version'] = str(_latest_version)
+            _new_item[_obj_type] = result['name']
 
-    _out_dict['read'] = _read_statements
+            _out_dict['read'].append(_new_item)
 
     return _out_dict
 

fair/register.py

@@ -259,45 +259,51 @@ def subst_registrations(local_uri: str, input_config: typing.Dict):
 
             )
 
-        _data = {
-            "version": reg['version']
-        }
-
         # If an external object fetch the relevant data_product first
         if 'external_object' in reg:
+            # TODO Not enough info to make unique
+            _ext_data = {
+                "version": reg['version'],
+                "title": reg['title']
+            }
             _results = fdp_req.get(
                 local_uri,
                 'external_object',
-                params={"version": reg['version'], "title": reg['title']}
+                params=_ext_data
             )
 
             if len(_results) > 1 or not _results:
                 raise fdp_exc.InternalError(
-                    f"Expected singular external_object for version '{reg['version']}' "
-                    f""
+                    f"Expected one external_object for '{_ext_data}', "
+                    f"got {len(_results)}"
                 )
 
             if 'data_product' not in _results[0]:
                 raise fdp_exc.RegistryError(
                     f"Expected external_object '{reg['external_object']}' "
                     "to have a data_product"
                 )
-            _data['external_object'] = _results[0]['url']
-
-        _results = fdp_req.get(
-            local_uri,
-            'data_product',
-            params=_data
-        )
+            _data_product_url = _results[0]['data_product']
+            _results[0] = fdp_req.url_get(_data_product_url)
+        else: # data product
+            # TODO Not enough info to make unique
+            _data = {
+                "version": reg['version'],
+                "name": reg['data_product']
+            }
 
-        if len(_results) > 1 or not _results:
-            raise fdp_exc.InternalError(
-                f"Expected singular result for version '{reg['version']}' "
-                f""
+            _results = fdp_req.get(
+                local_uri,
+                'data_product',
+                params=_data
             )
 
-        _namespace_url = _results[0]['namespace']
+            if len(_results) > 1 or not _results:
+                raise fdp_exc.InternalError(
+                    f"Expected one result for {_data}, got {len(_results)}"
+                )
 
+        _namespace_url = _results[0]['namespace']
         _namespace_data = fdp_req.url_get(_namespace_url)
 
         _object_data = {

fair/registry/storage.py

@@ -385,22 +385,16 @@ def store_data_file(
         "hash": _hash,
     }
 
-    try:
-        _post_store_loc = fdp_req.post(
-            uri,
-            "storage_location",
-            data=_storage_loc_data
-        )['url']
-    except fdp_exc.RegistryAPICallError as e:
-        if not e.error_code == 409:
-            raise e
-        else:
-            raise fdp_exc.RegistryAPICallError(
-                f"Cannot post storage_location "
-                f"'{_rel_path}' with hash"
-                f" '{_hash}', object already exists",
-                error_code=409
-            )
+    _search_data = {
+        'hash': _hash
+    }
+
+    _post_store_loc = fdp_req.post_else_get(
+        uri,
+        "storage_location",
+        data=_storage_loc_data,
+        params=_search_data
+    )
 
     _user = store_user(repo_dir, uri)
 

fair/registry/versioning.py

@@ -25,6 +25,17 @@
 import fair.exceptions as fdp_exc
 
 
+BUMP_FUNCS = {
+    "MINOR": "bump_minor",
+    "MAJOR": "bump_major",
+    "PATCH": "bump_patch",
+    "BUILD": "bump_build",
+    "PRERELEASE": "bump_prerelease"
+}
+
+DEFAULT_INCREMENT = "PATCH"
+
+
 def parse_incrementer(incrementer: str) -> str:
     """Convert an incrementer string in a config to the relevant bump function
 
@@ -39,29 +50,21 @@ def parse_incrementer(incrementer: str) -> str:
             relevant member method of VersionInfo for 'bumping' the semantic version
 
     """
-    _bump_dict = {
-        "MINOR": "bump_minor",
-        "MAJOR": "bump_major",
-        "PATCH": "bump_patch",
-        "BUILD": "bump_build",
-        "PRERELEASE": "bump_prerelease"
-    }
-
     # Sanity check to confirm all methods are still present in semver module
-    for func in _bump_dict.values():
+    for func in BUMP_FUNCS.values():
         if func not in dir(semver.VersionInfo):
             raise fdp_exc.InternalError(f"Unrecognised 'semver.VersionInfo' method '{func}'")
 
-    for component in _bump_dict:
+    for component in BUMP_FUNCS:
         if re.findall(r'\$\{\{\s*'+component+r'\s*\}\}', incrementer):
-            return _bump_dict[component]
+            return BUMP_FUNCS[component]
 
     raise fdp_exc.UserConfigError(
         f"Unrecognised version incrementer variable '{incrementer}'"
     )
 
 
-def get_latest_version(results_list: typing.Dict) -> semver.VersionInfo:
+def get_latest_version(results_list: typing.List = None) -> semver.VersionInfo:
     if not results_list:
         return semver.VersionInfo.parse("0.0.0")
 
@@ -75,3 +78,18 @@ def get_latest_version(results_list: typing.Dict) -> semver.VersionInfo:
 
     return max(_versions)
 
+
+def default_bump(version: semver.VersionInfo) -> semver.VersionInfo:
+    """Perform default version bump
+
+    For FAIR-CLI the default version increment is patch
+
+    Parameters
+    ----------
+        version: semver.VersionInfo
+
+    Returns
+    -------
+        new version
+    """
+    return getattr(version, BUMP_FUNCS[DEFAULT_INCREMENT])()