Skip to content

Commit

Permalink
[NEAT-179] Remove standardize casing (#395)
Browse files Browse the repository at this point in the history 
* tests: Failing test case with reference edge

* refactor: make test clearer

* tests: fix

* fix: Use correct edge.type

* refactor; fix on reverse

* refactor: handle node type filter moved with reference

* refactor: remove standardize casing
  • Loading branch information
@doctrino
doctrino authored Apr 16, 2024
1 parent da325ac commit de1fcc4
Show file tree
Hide file tree
Showing 8 changed files with 43 additions and 95 deletions.
10 changes: 3 additions & 7 deletions cognite/neat/rules/exporters/_rules2dms.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,8 +40,6 @@ class DMSExporter(CDFExporter[DMSSchema]):
If set, only export components in the given spaces. Defaults to None which means all spaces.
existing_handling (Literal["fail", "skip", "update", "force"], optional): How to handle existing components.
Defaults to "update". See below for details.
standardize_casing(bool, optional): Whether to standardize the casing. This means PascalCase for external ID
of views, containers, and data models, and camelCase for properties.
export_pipeline (bool, optional): Whether to export the pipeline. Defaults to False. This means setting
up transformations, RAW databases and tables to populate the data model.
instance_space (str, optional): The space to use for the instance. Defaults to None.
Expand All @@ -59,14 +57,12 @@ def __init__(
export_components: Component | Collection[Component] = "all",
include_space: set[str] | None = None,
existing_handling: Literal["fail", "skip", "update", "force"] = "update",
standardize_casing: bool = True,
export_pipeline: bool = False,
instance_space: str | None = None,
):
self.export_components = {export_components} if isinstance(export_components, str) else set(export_components)
self.include_space = include_space
self.existing_handling = existing_handling
self.standardize_casing = standardize_casing
self.export_pipeline = export_pipeline
self.instance_space = instance_space
self._schema: DMSSchema | None = None
Expand Down Expand Up @@ -119,11 +115,11 @@ def export(self, rules: Rules) -> DMSSchema:
)
is_new_model = dms_rules.reference is None
if is_new_model or is_solution_model:
return dms_rules.as_schema(self.standardize_casing, self.export_pipeline, self.instance_space)
return dms_rules.as_schema(self.export_pipeline, self.instance_space)

# This is an extension of an existing model.
reference_rules = cast(DMSRules, dms_rules.reference).copy(deep=True)
reference_schema = reference_rules.as_schema(self.standardize_casing, self.export_pipeline)
reference_schema = reference_rules.as_schema(self.export_pipeline)

# Todo Move this to an appropriate location
# Merging Reference with User Rules
Expand All @@ -146,7 +142,7 @@ def export(self, rules: Rules) -> DMSSchema:
property_.reference = None
combined_rules.properties.append(property_)

schema = combined_rules.as_schema(self.standardize_casing, self.export_pipeline, self.instance_space)
schema = combined_rules.as_schema(self.export_pipeline, self.instance_space)

if dms_rules.metadata.extension in (ExtensionCategory.addition, ExtensionCategory.reshape):
# We do not freeze views as they might be changed, even for addition,
Expand Down
22 changes: 6 additions & 16 deletions cognite/neat/rules/models/_rules/_types/_base.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
TransformationRuleType,
parse_rule,
)
from cognite.neat.utils.text import to_pascal

if sys.version_info >= (3, 11):
from enum import StrEnum
Expand Down Expand Up @@ -197,15 +196,14 @@ def from_raw(cls, value: Any) -> "ContainerEntity":
def from_id(cls, container_id: ContainerId) -> "ContainerEntity":
return ContainerEntity(prefix=container_id.space, suffix=container_id.external_id)

def as_id(self, default_space: str | None, standardize_casing: bool = True) -> ContainerId:
def as_id(self, default_space: str | None) -> ContainerId:
if self.space is Undefined and default_space is None:
raise ValueError("Space is Undefined! Set default_space!")

external_id = to_pascal(self.external_id) if standardize_casing else self.external_id
if self.space is Undefined:
return ContainerId(space=cast(str, default_space), external_id=external_id)
return ContainerId(space=cast(str, default_space), external_id=self.external_id)
else:
return ContainerId(space=self.space, external_id=external_id)
return ContainerId(space=self.space, external_id=self.external_id)


class ViewEntity(Entity):
Expand Down Expand Up @@ -233,7 +231,6 @@ def as_id(
allow_none: Literal[False] = False,
default_space: str | None = None,
default_version: str | None = None,
standardize_casing: bool = True,
) -> ViewId:
...

Expand All @@ -243,7 +240,6 @@ def as_id(
allow_none: Literal[True],
default_space: str | None = None,
default_version: str | None = None,
standardize_casing: bool = True,
) -> ViewId | None:
...

Expand All @@ -252,7 +248,6 @@ def as_id(
allow_none: bool = False,
default_space: str | None = None,
default_version: str | None = None,
standardize_casing: bool = True,
) -> ViewId | None:
if self.suffix is Unknown and allow_none:
return None
Expand All @@ -265,8 +260,7 @@ def as_id(
raise ValueError("space is required")
if version is None:
raise ValueError("version is required")
external_id = to_pascal(self.external_id) if standardize_casing else self.external_id
return ViewId(space=space, external_id=external_id, version=version)
return ViewId(space=space, external_id=self.external_id, version=version)


class ViewPropEntity(ViewEntity):
Expand Down Expand Up @@ -303,14 +297,10 @@ def from_prop_id(cls, prop_id: PropertyId) -> "ViewPropEntity":
property_=prop_id.property,
)

def as_prop_id(
self, default_space: str | None = None, default_version: str | None = None, standardize_casing: bool = True
) -> PropertyId:
def as_prop_id(self, default_space: str | None = None, default_version: str | None = None) -> PropertyId:
if self.property_ is None:
raise ValueError("property is required to create PropertyId")
return PropertyId(
source=self.as_id(False, default_space, default_version, standardize_casing), property=self.property_
)
return PropertyId(source=self.as_id(False, default_space, default_version), property=self.property_)

@property
def versioned_id(self) -> str:
Expand Down
83 changes: 28 additions & 55 deletions cognite/neat/rules/models/_rules/dms_architect_rules.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@
import cognite.neat.rules.issues.spreadsheet
from cognite.neat.rules import issues
from cognite.neat.rules.models._rules.domain_rules import DomainRules
from cognite.neat.utils.text import to_camel

from ._types import (
CdfValueType,
Expand Down Expand Up @@ -207,11 +206,11 @@ class DMSContainer(SheetEntity):
reference: ReferenceType = Field(alias="Reference", default=None)
constraint: ContainerListType | None = Field(None, alias="Constraint")

def as_container(self, default_space: str, standardize_casing: bool = True) -> dm.ContainerApply:
container_id = self.container.as_id(default_space, standardize_casing)
def as_container(self, default_space: str) -> dm.ContainerApply:
container_id = self.container.as_id(default_space)
constraints: dict[str, dm.Constraint] = {}
for constraint in self.constraint or []:
requires = dm.RequiresConstraint(constraint.as_id(default_space, standardize_casing))
requires = dm.RequiresConstraint(constraint.as_id(default_space))
constraints[f"{constraint.space}_{constraint.external_id}"] = requires

return dm.ContainerApply(
Expand Down Expand Up @@ -246,18 +245,15 @@ class DMSView(SheetEntity):
filter_: Literal["hasData", "nodeType"] | None = Field(None, alias="Filter")
in_model: bool = Field(True, alias="InModel")

def as_view(self, default_space: str, default_version: str, standardize_casing: bool = True) -> dm.ViewApply:
view_id = self.view.as_id(False, default_space, default_version, standardize_casing)
def as_view(self, default_space: str, default_version: str) -> dm.ViewApply:
view_id = self.view.as_id(False, default_space, default_version)
return dm.ViewApply(
space=view_id.space,
external_id=view_id.external_id,
version=view_id.version or default_version,
name=self.name or None,
description=self.description,
implements=[
parent.as_id(False, default_space, default_version, standardize_casing)
for parent in self.implements or []
]
implements=[parent.as_id(False, default_space, default_version) for parent in self.implements or []]
or None,
properties={},
)
Expand Down Expand Up @@ -564,10 +560,8 @@ def dms_rules_serialization(self, info: SerializationInfo) -> dict[str, Any]:
"Containers" if info.by_alias else "containers": containers,
}

def as_schema(
self, standardize_casing: bool = True, include_pipeline: bool = False, instance_space: str | None = None
) -> DMSSchema:
return _DMSExporter(standardize_casing, include_pipeline, instance_space).to_schema(self)
def as_schema(self, include_pipeline: bool = False, instance_space: str | None = None) -> DMSSchema:
return _DMSExporter(include_pipeline, instance_space).to_schema(self)

def as_information_architect_rules(self) -> "InformationRules":
return _DMSRulesConverter(self).as_information_architect_rules()
Expand Down Expand Up @@ -599,17 +593,12 @@ class _DMSExporter:
(This module cannot have a dependency on the exporter module, as it would create a circular dependency.)
Args
standardize_casing (bool): If True, the casing of the identifiers will be standardized. This means external IDs
are PascalCase and property names are camelCase.
include_pipeline (bool): If True, the pipeline will be included with the schema. Pipeline means the
raw tables and transformations necessary to populate the data model.
instance_space (str): The space to use for the instance. Defaults to None,`Rules.metadata.space` will be used
"""

def __init__(
self, standardize_casing: bool = True, include_pipeline: bool = False, instance_space: str | None = None
):
self.standardize_casing = standardize_casing
def __init__(self, include_pipeline: bool = False, instance_space: str | None = None):
self.include_pipeline = include_pipeline
self.instance_space = instance_space

Expand All @@ -628,9 +617,7 @@ def to_schema(self, rules: DMSRules) -> DMSSchema:
)

views_not_in_model = {
view.view.as_id(False, default_space, default_version, self.standardize_casing)
for view in rules.views
if not view.in_model
view.view.as_id(False, default_space, default_version) for view in rules.views if not view.in_model
}
data_model = rules.metadata.as_data_model()
data_model.views = sorted(
Expand Down Expand Up @@ -675,12 +662,9 @@ def _create_views_with_node_types(
default_space: str,
default_version: str,
) -> tuple[dm.ViewApplyList, dm.NodeApplyList]:
views = dm.ViewApplyList(
[dms_view.as_view(default_space, default_version, self.standardize_casing) for dms_view in dms_views]
)
views = dm.ViewApplyList([dms_view.as_view(default_space, default_version) for dms_view in dms_views])
dms_view_by_id = {
dms_view.view.as_id(False, default_space, default_version, self.standardize_casing): dms_view
for dms_view in dms_views
dms_view.view.as_id(False, default_space, default_version): dms_view for dms_view in dms_views
}

for view in views:
Expand All @@ -694,7 +678,7 @@ def _create_views_with_node_types(
# This is not yet supported in the CDF API, a warning has already been issued, here we convert it to
# a multi-edge connection.
if isinstance(prop.value_type, ViewEntity):
source = prop.value_type.as_id(False, default_space, default_version, self.standardize_casing)
source = prop.value_type.as_id(False, default_space, default_version)
else:
raise ValueError(
"Direct relation must have a view as value type. "
Expand All @@ -709,36 +693,30 @@ def _create_views_with_node_types(
direction="outwards",
)
elif prop.container and prop.container_property and prop.view_property:
container_prop_identifier = (
to_camel(prop.container_property) if self.standardize_casing else prop.container_property
)
container_prop_identifier = prop.container_property
extra_args: dict[str, Any] = {}
if prop.relation == "direct" and isinstance(prop.value_type, ViewEntity):
extra_args["source"] = prop.value_type.as_id(
True, default_space, default_version, self.standardize_casing
)
extra_args["source"] = prop.value_type.as_id(True, default_space, default_version)
elif prop.relation == "direct" and not isinstance(prop.value_type, ViewEntity):
raise ValueError(
"Direct relation must have a view as value type. "
"This should have been validated in the rules"
)
view_property = dm.MappedPropertyApply(
container=prop.container.as_id(default_space, self.standardize_casing),
container=prop.container.as_id(default_space),
container_property_identifier=container_prop_identifier,
**extra_args,
)
elif prop.view and prop.view_property and prop.relation == "multiedge":
if isinstance(prop.value_type, ViewEntity):
source = prop.value_type.as_id(False, default_space, default_version, self.standardize_casing)
source = prop.value_type.as_id(False, default_space, default_version)
else:
raise ValueError(
"Multiedge relation must have a view as value type. "
"This should have been validated in the rules"
)
if isinstance(prop.reference, ReferenceEntity):
ref_view_prop = prop.reference.as_prop_id(
default_space, default_version, self.standardize_casing
)
ref_view_prop = prop.reference.as_prop_id(default_space, default_version)
edge_type = dm.DirectRelationReference(
space=ref_view_prop.source.space,
external_id=f"{ref_view_prop.source.external_id}.{ref_view_prop.property}",
Expand All @@ -756,7 +734,7 @@ def _create_views_with_node_types(
)
elif prop.view and prop.view_property and prop.relation == "reversedirect":
if isinstance(prop.value_type, ViewPropEntity):
source = prop.value_type.as_id(False, default_space, default_version, self.standardize_casing)
source = prop.value_type.as_id(False, default_space, default_version)
else:
raise ValueError(
"Reverse direct relation must have a view as value type. "
Expand All @@ -777,9 +755,7 @@ def _create_views_with_node_types(
issues.dms.ReverseOfDirectRelationListWarning(view_id, prop.property_), stacklevel=2
)
if isinstance(reverse_prop.reference, ReferenceEntity):
ref_view_prop = reverse_prop.reference.as_prop_id(
default_space, default_version, self.standardize_casing
)
ref_view_prop = reverse_prop.reference.as_prop_id(default_space, default_version)
edge_type = dm.DirectRelationReference(
space=ref_view_prop.source.space,
external_id=f"{ref_view_prop.source.external_id}.{ref_view_prop.property}",
Expand Down Expand Up @@ -818,7 +794,7 @@ def _create_views_with_node_types(
continue
else:
continue
prop_name = to_camel(prop.view_property) if self.standardize_casing else prop.view_property
prop_name = prop.view_property
view.properties[prop_name] = view_property

node_types = dm.NodeApplyList([])
Expand All @@ -830,7 +806,7 @@ def _create_views_with_node_types(
if dms_view and isinstance(dms_view.reference, ReferenceEntity):
# If the view is a reference, we implement the reference view,
# and need the filter to match the reference
ref_view = dms_view.reference.as_id(False, default_space, default_version, self.standardize_casing)
ref_view = dms_view.reference.as_id(False, default_space, default_version)
node_type = dm.filters.Equals(
["node", "type"], {"space": ref_view.space, "externalId": ref_view.external_id}
)
Expand Down Expand Up @@ -867,10 +843,7 @@ def _create_containers(
default_space: str,
) -> dm.ContainerApplyList:
containers = dm.ContainerApplyList(
[
dms_container.as_container(default_space, self.standardize_casing)
for dms_container in dms_container or []
]
[dms_container.as_container(default_space) for dms_container in dms_container or []]
)
container_to_drop = set()
for container in containers:
Expand All @@ -887,7 +860,7 @@ def _create_containers(
else:
type_cls = dm.DirectRelation

prop_name = to_camel(prop.container_property) if self.standardize_casing else prop.container_property
prop_name = prop.container_property

if type_cls is dm.DirectRelation:
container.properties[prop_name] = dm.ContainerProperty(
Expand Down Expand Up @@ -948,21 +921,21 @@ def _gather_properties(
container_properties_by_id: dict[dm.ContainerId, list[DMSProperty]] = defaultdict(list)
view_properties_by_id: dict[dm.ViewId, list[DMSProperty]] = defaultdict(list)
for prop in rules.properties:
view_id = prop.view.as_id(False, default_space, default_version, self.standardize_casing)
view_id = prop.view.as_id(False, default_space, default_version)
view_properties_by_id[view_id].append(prop)

if prop.container and prop.container_property:
if prop.relation == "direct" and prop.is_list:
warnings.warn(
issues.dms.DirectRelationListWarning(
container_id=prop.container.as_id(default_space, self.standardize_casing),
view_id=prop.view.as_id(False, default_space, default_version, self.standardize_casing),
container_id=prop.container.as_id(default_space),
view_id=prop.view.as_id(False, default_space, default_version),
property=prop.container_property,
),
stacklevel=2,
)
continue
container_id = prop.container.as_id(default_space, self.standardize_casing)
container_id = prop.container.as_id(default_space)
container_properties_by_id[container_id].append(prop)

return container_properties_by_id, view_properties_by_id
Expand Down
Loading

0 comments on commit de1fcc4

Please sign in to comment.