Skip to content

Commit

Permalink
Simplified InformationArchitect rules by remove rule_type and renam…
Browse files Browse the repository at this point in the history 
…ing `rule` to `transformation` instead (#507)

* simplify info rules

* updated tests

* bump version add change log

* update date in changelog

* simplify read

* rename input par
  • Loading branch information
@nikokaoja
nikokaoja committed Jun 21, 2024
1 parent a07773a commit 895b2a2
Show file tree
Hide file tree
Showing 9 changed files with 79 additions and 52 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
.PHONY: run-explorer run-tests run-linters build-ui build-python build-docker run-docker compose-up
version="0.81.11"
version="0.81.12"
run-explorer:
@echo "Running explorer API server..."
# open "http://localhost:8000/static/index.html" || true
Expand Down
2 changes: 1 addition & 1 deletion cognite/neat/_version.py
View file
Original file line number Diff line number Diff line change
@@ -1 +1 @@
__version__ = "0.81.11"
__version__ = "0.81.12"
45 changes: 42 additions & 3 deletions cognite/neat/graph/stores/_base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from cognite.neat.graph.extractors import RdfFileExtractor, TripleExtractors
from cognite.neat.graph.models import Triple
from cognite.neat.graph.transformers import Transformers
from cognite.neat.rules.models.entities import ClassEntity
from cognite.neat.rules.models.information import InformationRules
from cognite.neat.utils import remove_namespace
from cognite.neat.utils.auxiliary import local_import
Expand Down Expand Up @@ -42,6 +43,8 @@ def __init__(
graph: Graph,
rules: InformationRules | None = None,
):
self.rules: InformationRules | None = None

_start = datetime.now(timezone.utc)
self.graph = graph
self.provenance = Provenance(
Expand All @@ -54,11 +57,30 @@ def __init__(
)
]
)
self.rules = rules

if self.rules and self.rules.prefixes:
self._upsert_prefixes(self.rules.prefixes)
if rules:
self.rules = rules
self.base_namespace = self.rules.metadata.namespace
self.provenance.append(
Change.record(
activity=f"{type(self)}.rules",
start=_start,
end=datetime.now(timezone.utc),
description=f"Added rules to graph store as {type(self.rules).__name__}",
)
)

if self.rules.prefixes:
self._upsert_prefixes(self.rules.prefixes)
self.provenance.append(
Change.record(
activity=f"{type(self).__name__}._upsert_prefixes",
start=_start,
end=datetime.now(timezone.utc),
description="Upsert prefixes to graph store",
)
)

else:
self.base_namespace = DEFAULT_NAMESPACE

Expand Down Expand Up @@ -144,6 +166,23 @@ def write(self, extractor: TripleExtractors) -> None:
)
)

def read(self, class_: str) -> list[tuple[str, str, str]]:
"""Read instances for given view from the graph store."""
# PLACEHOLDER: Implement reading instances for a given view
# not yet developed

if not self.rules:
warnings.warn("No rules found for the graph store, returning empty list.", stacklevel=2)
return []

class_entity = ClassEntity(prefix=self.rules.metadata.prefix, suffix=class_)

if class_entity not in [definition.class_ for definition in self.rules.classes.data]:
warnings.warn("Desired type not found in graph!", stacklevel=2)
return []

return []

def _parse_file(
self,
filepath: Path,
Expand Down
4 changes: 2 additions & 2 deletions cognite/neat/rules/analysis/_information_rules.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
from pydantic import ValidationError

from cognite.neat.rules.models import SchemaCompleteness
from cognite.neat.rules.models._rdfpath import TransformationRuleType
from cognite.neat.rules.models._rdfpath import RDFPath
from cognite.neat.rules.models.entities import ClassEntity, EntityTypes, ParentClassEntity, ReferenceEntity
from cognite.neat.rules.models.information import InformationClass, InformationProperty, InformationRules
from cognite.neat.utils.utils import get_inheritance_path
Expand Down Expand Up @@ -191,7 +191,7 @@ def class_property_pairs(
)
continue

if (only_rdfpath and property_.rule_type == TransformationRuleType.rdfpath) or not only_rdfpath:
if (only_rdfpath and isinstance(property_.transformation, RDFPath)) or not only_rdfpath:
processed_properties[property_.property_] = property_
class_property_pairs[class_] = processed_properties

Expand Down
29 changes: 7 additions & 22 deletions cognite/neat/rules/models/information/_rules.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,8 @@
SheetList,
)
from cognite.neat.rules.models._rdfpath import (
AllReferences,
Hop,
RawLookup,
SingleProperty,
SPARQLQuery,
RDFPath,
TransformationRuleType,
Traversal,
parse_rule,
)
from cognite.neat.rules.models._types import (
Expand Down Expand Up @@ -156,9 +151,7 @@ class InformationProperty(SheetEntity):
default: Default value of the property
reference: Reference to the source of the information, HTTP URI
match_type: The match type of the resource being described and the source entity.
rule_type: Rule type for the transformation from source to target representation
of knowledge graph. Defaults to None (no transformation)
rule: Actual rule for the transformation from source to target representation of
transformation: Actual rule for the transformation from source to target representation of
knowledge graph. Defaults to None (no transformation)
"""

Expand All @@ -174,10 +167,7 @@ class InformationProperty(SheetEntity):
default: Any | None = Field(alias="Default", default=None)
reference: URLEntity | ReferenceEntity | None = Field(alias="Reference", default=None, union_mode="left_to_right")
match_type: MatchType | None = Field(alias="Match Type", default=None)
rule_type: str | TransformationRuleType | None = Field(alias="Rule Type", default=None)
rule: str | AllReferences | SingleProperty | Hop | RawLookup | SPARQLQuery | Traversal | None = Field(
alias="Rule", default=None
)
transformation: str | RDFPath | None = Field(alias="Transformation", default=None)
comment: str | None = Field(alias="Comment", default=None)

@field_serializer("max_count", when_used="json-unless-none")
Expand All @@ -193,15 +183,10 @@ def parse_max_count(cls, value: int | float | None) -> int | float | None:
return value

@model_validator(mode="after")
def is_valid_rule(self):
# TODO: Can we skip rule_type and simply try to parse the rule and if it fails, raise an error?
if self.rule_type:
self.rule_type = self.rule_type.lower()
if not self.rule:
raise exceptions.RuleTypeProvidedButRuleMissing(
self.property_, self.class_, self.rule_type
).to_pydantic_custom_error()
self.rule = parse_rule(self.rule, self.rule_type)
def generate_valid_transformation(self):
# TODO: Currently only supporting RDFpath
if self.transformation:
self.transformation = parse_rule(self.transformation, TransformationRuleType.rdfpath)
return self

@model_validator(mode="after")
Expand Down
7 changes: 7 additions & 0 deletions docs/CHANGELOG.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,13 @@ Changes are grouped as follows:
- `Security` in case of vulnerabilities.


## [0.81.12] - 20-06-24
### Added
- Placeholder for `NeatGraphStore.read_view` method
### Improved
- Simplified InformationArchitect rules by remove `rule_type` and renaming `rule` to `transformation` instead


## [0.81.11] - 19-06-24
### Added
- `AssetRelationshipConnector` transformer added
Expand Down
30 changes: 14 additions & 16 deletions docs/terminology/rules.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -157,20 +157,19 @@ For more information about how these different `Rules` objects are used, see the

=== "Information Architect Profile"

| Column | Description | Predefined Value | Mandatory |
|-------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------|-----------|
| Class | Class id that the property is defined for, strongly advise `PascalCase` usage | | Yes |
| Property | Property id, strongly advised to `camelCase` usage | | Yes |
| Name | Human readable name of the property | | No |
| Description | Short description of the property | | Yes |
| Value Type | Value type that the property can hold. It takes either subset of XSD type (see note below) or a class defined | XSD Types or Class id | Yes |
| Min Count | Minimum number of values that the property can hold. If no value is provided, the default value is `0`, which means that the property is optional. | | Yes |
| Max Count | Maximum number of values that the property can hold. If no value is provided, the default value is `inf`, which means that the property can hold any number of values (listable). | | Yes |
| Default | Specifies default value for the property. | | No |
| Rule Type | The rule type that is used to populate the data model | `sparql`, `rdfpath` or `rawlookup` | No |
| Rule | The rule that is used to populate the data model. The rule is provided as a string, which is either SPARQL query or RDFPath query or RAW lookup query | | No |
| Reference | Reference to the source of the property provided as `URI` | | No |
| Match Type | The match type between the source entity and the class | `exact` or `partial` | No |
| Column | Description | Predefined Value | Mandatory |
|----------------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|------------------------------------|-----------|
| Class | Class id that the property is defined for, strongly advise `PascalCase` usage | | Yes |
| Property | Property id, strongly advised to `camelCase` usage | | Yes |
| Name | Human readable name of the property | | No |
| Description | Short description of the property | | Yes |
| Value Type | Value type that the property can hold. It takes either subset of XSD type (see note below) or a class defined | XSD Types or Class id | Yes |
| Min Count | Minimum number of values that the property can hold. If no value is provided, the default value is `0`, which means that the property is optional. | | Yes |
| Max Count | Maximum number of values that the property can hold. If no value is provided, the default value is `inf`, which means that the property can hold any number of values (listable). | | Yes |
| Default | Specifies default value for the property. | | No |
| Transformation | The rule that is used to populate the data model. The rule is provided in a RDFPath query syntax which is converted to downstream solution query (e.g. SPARQL) | | No |
| Reference | Reference to the source of the property provided as `URI` | | No |
| Match Type | The match type between the source entity and the class | `exact` or `partial` | No |

!!! info annotate "XSD Value Types"
The following XSD types are supported:
Expand All @@ -179,8 +178,7 @@ For more information about how these different `Rules` objects are used, see the
`timeseries`, `file` , `sequence` and `json`

!!! info annotate "Data model population rule"
The `Rule Type` and `Rule` columns are used to populate the data model using [NEAT graph store](./graph.md).
They are optional, but if used, both must be provided !
The `Transformation` column are used to populate the data model using [NEAT graph store](./graph.md). Currently we only support `RDFPath` query syntax.

!!! tip annotate "Usage"
More details on **Information Architect** Profile **Properties sheet** usage can be found [here](../tutorials/data-modeling-lifecycle/part-1-knowledge-acquisition.md#information-architect-properties)!
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[tool.poetry]
name = "cognite-neat"
version = "0.81.11"
version = "0.81.12"
readme = "README.md"
description = "Knowledge graph transformation"
authors = [
Expand Down
10 changes: 4 additions & 6 deletions tests/tests_unit/rules/test_models/test_information_rules.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,8 +70,7 @@ def case_insensitive_value_types():
"Default": None,
"Source": None,
"MatchType": None,
"Rule Type": None,
"Rule": None,
"Transformation": None,
}
],
},
Expand Down Expand Up @@ -116,15 +115,14 @@ def invalid_domain_rules_cases():
"Default": None,
"Source": None,
"MatchType": None,
"Rule Type": "rdfpath",
"Rule": None,
"Transformation": ":GeneratingUnit(cim:name)",
}
],
},
(
"Rule type 'rdfpath' provided for property 'name' in class 'GeneratingUnit' but rule is not provided!"
":GeneratingUnit(cim:name) is not a valid rdfpath!"
"\nFor more information visit: "
"https://cognite-neat.readthedocs-hosted.com/en/latest/api/exceptions.html#cognite.neat.rules.exceptions.RuleTypeProvidedButRuleMissing"
"https://cognite-neat.readthedocs-hosted.com/en/latest/api/exceptions.html#cognite.neat.rules.exceptions.NotValidRDFPath"
),
id="missing_rule",
)
Expand Down

0 comments on commit 895b2a2

Please sign in to comment.