From 1f16a5e196f2b8ab9b09a28888c6d819db82b770 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sun, 31 May 2020 20:57:07 +0200 Subject: [PATCH 01/39] New Note Representation --- .../representation/json_refactor/note_repr.py | 71 ++++++++++++++ .../json_refactor/test_note_repr.py | 98 +++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 brain_brew/representation/json_refactor/note_repr.py create mode 100644 tests/representation/json_refactor/test_note_repr.py diff --git a/brain_brew/representation/json_refactor/note_repr.py b/brain_brew/representation/json_refactor/note_repr.py new file mode 100644 index 0000000..50d7653 --- /dev/null +++ b/brain_brew/representation/json_refactor/note_repr.py @@ -0,0 +1,71 @@ +import json +from dataclasses import dataclass +from typing import List + + +@dataclass +class OverwritableNoteData: + note_model: str + tags: List[str] + + @staticmethod + def encode_overwritable(obj, json_dict): + if obj.tags is not None and obj.tags != []: + json_dict.setdefault("tags", obj.tags) + if obj.note_model is not None: + json_dict.setdefault("note_model", obj.note_model) + return json_dict + + +@dataclass +class Note(OverwritableNoteData): + fields: List[str] + guid: str + + @classmethod + def from_json(cls, data: dict): + return cls( + fields=data.get("fields"), + guid=data.get("guid"), + note_model=data.get("note_model", None), + tags=data.get("tags", None) + ) + + @staticmethod + def encode(obj): + if isinstance(obj, Note): + json_dict = {"fields": obj.fields, "guid": obj.guid} + super().encode_overwritable(obj, json_dict) + return json_dict + raise TypeError("Cannot encode object as Note", obj) + + def dump_json_to_string(self): + return json.dumps(self, default=Note.encode, sort_keys=False, indent=4, ensure_ascii=False) + + +@dataclass +class NoteGrouping(OverwritableNoteData): + notes: List[Note] + + @classmethod + def from_json(cls, data): + return cls( + notes=list(map(Note.from_json, data.get("notes"))), + note_model=data.get("note_model", None), + tags=data.get("tags", None) + ) + + @staticmethod + def encode(obj): + if isinstance(obj, NoteGrouping): + json_dict = {"notes": [Note.encode(note) for note in obj.notes]} + super().encode_overwritable(obj, json_dict) + return json_dict + raise TypeError("Cannot encode object as NoteGrouping", obj) + + def dump_json_to_string(self): + return json.dumps(self, default=NoteGrouping.encode, sort_keys=False, indent=4, ensure_ascii=False) + + +# @dataclass +# class DeckPartNoteModel: \ No newline at end of file diff --git a/tests/representation/json_refactor/test_note_repr.py b/tests/representation/json_refactor/test_note_repr.py new file mode 100644 index 0000000..e3aaee2 --- /dev/null +++ b/tests/representation/json_refactor/test_note_repr.py @@ -0,0 +1,98 @@ +import json +from typing import List + +import pytest + +from brain_brew.representation.json_refactor.note_repr import Note, NoteGrouping + + +def create_note_json(fields, guid, note_model, tags): + json_data = { + "fields": fields, + "guid": guid + } + + if tags is not None and tags != []: + json_data.setdefault("tags", tags) + if note_model is not None: + json_data.setdefault("note_model", note_model) + + return json_data + + +def create_note_grouping_json(notes, note_model, tags): + json_data = {"notes": notes} + + if tags is not None and tags != []: + json_data.setdefault("tags", tags) + if note_model is not None: + json_data.setdefault("note_model", note_model) + + return json_data + + +@pytest.fixture() +def note_test1(): + return Note(note_model="model", tags=['noun', 'other'], fields=['first'], guid="12345") + + +@pytest.mark.parametrize("fields, guid, note_model, tags", [ + ([], "", "", []), + (None, None, None, None), + (["test", "blah", "whatever"], "1234567890", "model_name", ["noun"]) +]) +class TestNote: + def test_constructor(self, fields: List[str], guid: str, note_model: str, tags: List[str]): + note = Note(fields=fields, guid=guid, note_model=note_model, tags=tags) + + assert isinstance(note, Note) + assert note.fields == fields + assert note.guid == guid + assert note.note_model == note_model + assert note.tags == tags + + def test_from_json(self, fields: List[str], guid: str, note_model: str, tags: List[str]): + json_data = create_note_json(fields, guid, note_model, tags) + note = Note.from_json(json_data) + + assert isinstance(note, Note) + assert note.fields == fields + assert note.guid == guid + assert note.note_model == note_model + assert note.tags == tags or (tags == [] and note.tags is None) + + def test_dump_to_json(self, fields: List[str], guid: str, note_model: str, tags: List[str]): + json_data = create_note_json(fields, guid, note_model, tags) + note = Note.from_json(json_data) + + assert note.dump_json_to_string() == json.dumps(json_data, sort_keys=False, indent=4, ensure_ascii=False) + + +@pytest.mark.parametrize("note_model, tags", [ + ("", []), + (None, None), + ("model_name", ["noun"]) +]) +class TestNoteGrouping: + def test_constructor(self, note_model: str, tags: List[str], note_test1): + group = NoteGrouping(notes=[note_test1], note_model=note_model, tags=tags) + + assert isinstance(group, NoteGrouping) + assert group.notes == [note_test1] + assert group.note_model == note_model + assert group.tags == tags + + def test_from_json(self, note_model: str, tags: List[str], note_test1): + json_data = create_note_grouping_json([Note.encode(note_test1)], note_model, tags) + group = NoteGrouping.from_json(json_data) + + assert isinstance(group, NoteGrouping) + assert group.notes == [note_test1] + assert group.note_model == note_model + assert group.tags == tags or (tags == [] and group.tags is None) + + def test_dump_to_json(self, note_model: str, tags: List[str], note_test1): + json_data = create_note_grouping_json([Note.encode(note_test1)], note_model, tags) + group = NoteGrouping.from_json(json_data) + + assert group.dump_json_to_string() == json.dumps(json_data, sort_keys=False, indent=4, ensure_ascii=False) \ No newline at end of file From 8b591dc579dff243f5e49a97db024dcb2b04cd63 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sun, 7 Jun 2020 18:14:11 +0200 Subject: [PATCH 02/39] First steps towards YAML-ification of DeckParts --- Pipfile | 1 + Pipfile.lock | 64 +++++++++--- .../representation/json_refactor/note_repr.py | 71 -------------- brain_brew/representation/yaml/__init__.py | 0 brain_brew/representation/yaml/my_yaml.py | 6 ++ brain_brew/representation/yaml/note_repr.py | 64 ++++++++++++ .../json_refactor/test_note_repr.py | 98 ------------------- tests/representation/yaml/test_note_repr.py | 67 +++++++++++++ tests/test_files.py | 5 + .../deck_parts/yaml/note/note1.yaml | 9 ++ 10 files changed, 200 insertions(+), 185 deletions(-) delete mode 100644 brain_brew/representation/json_refactor/note_repr.py create mode 100644 brain_brew/representation/yaml/__init__.py create mode 100644 brain_brew/representation/yaml/my_yaml.py create mode 100644 brain_brew/representation/yaml/note_repr.py delete mode 100644 tests/representation/json_refactor/test_note_repr.py create mode 100644 tests/representation/yaml/test_note_repr.py create mode 100644 tests/test_files/deck_parts/yaml/note/note1.yaml diff --git a/Pipfile b/Pipfile index 6c0c31c..679f345 100644 --- a/Pipfile +++ b/Pipfile @@ -11,6 +11,7 @@ args = "==0.1.0" clint = "==0.5.1" coverage = "==4.5.4" PyYAML = "==5.1.2" +ruamel-yaml = "*" [requires] python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index 49d491d..8089dd5 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "659f0f64d2f0d1e78d52f9bfbb8f76167b2b26b5158a1506e3a87af123b36b8e" + "sha256": "a1fdf18fb74ab9ec7c5d2e956b782058ac37275e59250e5067bf531a4bc44c5c" }, "pipfile-spec": 6, "requires": { @@ -86,6 +86,38 @@ ], "index": "Brain Brew", "version": "==5.1.2" + }, + "ruamel-yaml": { + "hashes": [ + "sha256:0962fd7999e064c4865f96fb1e23079075f4a2a14849bcdc5cdba53a24f9759b" + ], + "index": "Brain Brew", + "version": "==0.16.10" + }, + "ruamel.yaml.clib": { + "hashes": [ + "sha256:1e77424825caba5553bbade750cec2277ef130647d685c2b38f68bc03453bac6", + "sha256:392b7c371312abf27fb549ec2d5e0092f7ef6e6c9f767bfb13e83cb903aca0fd", + "sha256:4d55386129291b96483edcb93b381470f7cd69f97585829b048a3d758d31210a", + "sha256:550168c02d8de52ee58c3d8a8193d5a8a9491a5e7b2462d27ac5bf63717574c9", + "sha256:57933a6986a3036257ad7bf283529e7c19c2810ff24c86f4a0cfeb49d2099919", + "sha256:615b0396a7fad02d1f9a0dcf9f01202bf9caefee6265198f252c865f4227fcc6", + "sha256:77556a7aa190be9a2bd83b7ee075d3df5f3c5016d395613671487e79b082d784", + "sha256:7aee724e1ff424757b5bd8f6c5bbdb033a570b2b4683b17ace4dbe61a99a657b", + "sha256:8073c8b92b06b572e4057b583c3d01674ceaf32167801fe545a087d7a1e8bf52", + "sha256:9c6d040d0396c28d3eaaa6cb20152cb3b2f15adf35a0304f4f40a3cf9f1d2448", + "sha256:a0ff786d2a7dbe55f9544b3f6ebbcc495d7e730df92a08434604f6f470b899c5", + "sha256:b1b7fcee6aedcdc7e62c3a73f238b3d080c7ba6650cd808bce8d7761ec484070", + "sha256:b66832ea8077d9b3f6e311c4a53d06273db5dc2db6e8a908550f3c14d67e718c", + "sha256:be018933c2f4ee7de55e7bd7d0d801b3dfb09d21dad0cce8a97995fd3e44be30", + "sha256:d0d3ac228c9bbab08134b4004d748cf9f8743504875b3603b3afbb97e3472947", + "sha256:d10e9dd744cf85c219bf747c75194b624cc7a94f0c80ead624b06bfa9f61d3bc", + "sha256:ea4362548ee0cbc266949d8a441238d9ad3600ca9910c3fe4e82ee3a50706973", + "sha256:ed5b3698a2bb241b7f5cbbe277eaa7fe48b07a58784fba4f75224fd066d253ad", + "sha256:f9dcc1ae73f36e8059589b601e8e4776b9976effd76c21ad6a855a74318efd6e" + ], + "markers": "platform_python_implementation == 'CPython' and python_version < '3.9'", + "version": "==0.2.0" } }, "develop": { @@ -98,25 +130,25 @@ }, "importlib-metadata": { "hashes": [ - "sha256:2a688cbaa90e0cc587f1df48bdc97a6eadccdcd9c35fb3f976a09e3b5016d90f", - "sha256:34513a8a0c4962bc66d35b359558fd8a5e10cd472d37aec5f66858addef32c1e" + "sha256:0505dd08068cfec00f53a74a0ad927676d7757da81b7436a6eefe4c7cf75c545", + "sha256:15ec6c0fd909e893e3a08b3a7c76ecb149122fb14b7efe1199ddd4c7c57ea958" ], "markers": "python_version < '3.8'", - "version": "==1.6.0" + "version": "==1.6.1" }, "more-itertools": { "hashes": [ - "sha256:5dd8bcf33e5f9513ffa06d5ad33d78f31e1931ac9a18f33d37e77a180d393a7c", - "sha256:b1ddb932186d8a6ac451e1d95844b382f55e12686d51ca0c68b6f61f2ab7a507" + "sha256:558bb897a2232f5e4f8e2399089e35aecb746e1f9191b6584a151647e89267be", + "sha256:7818f596b1e87be009031c7653d01acc46ed422e6656b394b0f765ce66ed4982" ], - "version": "==8.2.0" + "version": "==8.3.0" }, "packaging": { "hashes": [ - "sha256:3c292b474fda1671ec57d46d739d072bfd495a4f51ad01a055121d81e952b7a3", - "sha256:82f77b9bee21c1bafbf35a84905d604d5d1223801d639cf3ed140bd651c08752" + "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8", + "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181" ], - "version": "==20.3" + "version": "==20.4" }, "pluggy": { "hashes": [ @@ -149,17 +181,17 @@ }, "six": { "hashes": [ - "sha256:236bdbdce46e6e6a3d61a337c0f8b763ca1e8717c03b369e87a7ec7ce1319c0a", - "sha256:8f3cd2e254d8f793e7f3d6d9df77b92252b52637291d0f0da013c76ea2724b6c" + "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", + "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], - "version": "==1.14.0" + "version": "==1.15.0" }, "wcwidth": { "hashes": [ - "sha256:cafe2186b3c009a04067022ce1dcd79cb38d8d65ee4f4791b8888d6599d1bbe1", - "sha256:ee73862862a156bf77ff92b09034fc4825dd3af9cf81bc5b360668d425f3c5f1" + "sha256:980fbf4f3c196c0f329cdcd1e84c554d6a211f18e252e525a0cf4223154a41d6", + "sha256:edbc2b718b4db6cdf393eefe3a420183947d6aa312505ce6754516f458ff8830" ], - "version": "==0.1.9" + "version": "==0.2.3" }, "zipp": { "hashes": [ diff --git a/brain_brew/representation/json_refactor/note_repr.py b/brain_brew/representation/json_refactor/note_repr.py deleted file mode 100644 index 50d7653..0000000 --- a/brain_brew/representation/json_refactor/note_repr.py +++ /dev/null @@ -1,71 +0,0 @@ -import json -from dataclasses import dataclass -from typing import List - - -@dataclass -class OverwritableNoteData: - note_model: str - tags: List[str] - - @staticmethod - def encode_overwritable(obj, json_dict): - if obj.tags is not None and obj.tags != []: - json_dict.setdefault("tags", obj.tags) - if obj.note_model is not None: - json_dict.setdefault("note_model", obj.note_model) - return json_dict - - -@dataclass -class Note(OverwritableNoteData): - fields: List[str] - guid: str - - @classmethod - def from_json(cls, data: dict): - return cls( - fields=data.get("fields"), - guid=data.get("guid"), - note_model=data.get("note_model", None), - tags=data.get("tags", None) - ) - - @staticmethod - def encode(obj): - if isinstance(obj, Note): - json_dict = {"fields": obj.fields, "guid": obj.guid} - super().encode_overwritable(obj, json_dict) - return json_dict - raise TypeError("Cannot encode object as Note", obj) - - def dump_json_to_string(self): - return json.dumps(self, default=Note.encode, sort_keys=False, indent=4, ensure_ascii=False) - - -@dataclass -class NoteGrouping(OverwritableNoteData): - notes: List[Note] - - @classmethod - def from_json(cls, data): - return cls( - notes=list(map(Note.from_json, data.get("notes"))), - note_model=data.get("note_model", None), - tags=data.get("tags", None) - ) - - @staticmethod - def encode(obj): - if isinstance(obj, NoteGrouping): - json_dict = {"notes": [Note.encode(note) for note in obj.notes]} - super().encode_overwritable(obj, json_dict) - return json_dict - raise TypeError("Cannot encode object as NoteGrouping", obj) - - def dump_json_to_string(self): - return json.dumps(self, default=NoteGrouping.encode, sort_keys=False, indent=4, ensure_ascii=False) - - -# @dataclass -# class DeckPartNoteModel: \ No newline at end of file diff --git a/brain_brew/representation/yaml/__init__.py b/brain_brew/representation/yaml/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/brain_brew/representation/yaml/my_yaml.py b/brain_brew/representation/yaml/my_yaml.py new file mode 100644 index 0000000..67a9c63 --- /dev/null +++ b/brain_brew/representation/yaml/my_yaml.py @@ -0,0 +1,6 @@ +from ruamel.yaml import YAML + +yaml = YAML(typ='safe') +yaml.default_flow_style = False +yaml.indent = 4 +yaml.sort_base_mapping_type_on_output = False diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py new file mode 100644 index 0000000..b6cc403 --- /dev/null +++ b/brain_brew/representation/yaml/note_repr.py @@ -0,0 +1,64 @@ +from brain_brew.representation.yaml.my_yaml import yaml +import json +from dataclasses import dataclass +from typing import List + + +@dataclass +class OverwritableNoteData: + note_model: str + tags: List[str] + + def encode_overwritable(self, json_dict): + if self.tags is not None and self.tags != []: + json_dict.setdefault("tags", self.tags) + if self.note_model is not None: + json_dict.setdefault("note_model", self.note_model) + return json_dict + + +@dataclass +class Note(OverwritableNoteData): + fields: List[str] + guid: str + + @classmethod + def from_dict(cls, data: dict): + return cls( + fields=data.get("fields"), + guid=data.get("guid"), + note_model=data.get("note_model", None), + tags=data.get("tags", None) + ) + + def encode(self): + json_dict = {"guid": self.guid, "fields": self.fields} + super().encode_overwritable(json_dict) + return json_dict + + def dump_to_yaml(self, file): + yaml.dump(self.encode(), file) + + +@dataclass +class NoteGrouping(OverwritableNoteData): + notes: List[Note] + + @classmethod + def from_dict(cls, data): + return cls( + notes=list(map(Note.from_dict, data.get("notes"))), + note_model=data.get("note_model", None), + tags=data.get("tags", None) + ) + + def encode(self): + json_dict = {"notes": [note.encode() for note in self.notes]} + super().encode_overwritable(json_dict) + return json_dict + + def dump_to_yaml(self, file): + yaml.dump(self.encode(), file) + +# @dataclass +# class DeckPartNoteModel: \ No newline at end of file diff --git a/tests/representation/json_refactor/test_note_repr.py b/tests/representation/json_refactor/test_note_repr.py deleted file mode 100644 index e3aaee2..0000000 --- a/tests/representation/json_refactor/test_note_repr.py +++ /dev/null @@ -1,98 +0,0 @@ -import json -from typing import List - -import pytest - -from brain_brew.representation.json_refactor.note_repr import Note, NoteGrouping - - -def create_note_json(fields, guid, note_model, tags): - json_data = { - "fields": fields, - "guid": guid - } - - if tags is not None and tags != []: - json_data.setdefault("tags", tags) - if note_model is not None: - json_data.setdefault("note_model", note_model) - - return json_data - - -def create_note_grouping_json(notes, note_model, tags): - json_data = {"notes": notes} - - if tags is not None and tags != []: - json_data.setdefault("tags", tags) - if note_model is not None: - json_data.setdefault("note_model", note_model) - - return json_data - - -@pytest.fixture() -def note_test1(): - return Note(note_model="model", tags=['noun', 'other'], fields=['first'], guid="12345") - - -@pytest.mark.parametrize("fields, guid, note_model, tags", [ - ([], "", "", []), - (None, None, None, None), - (["test", "blah", "whatever"], "1234567890", "model_name", ["noun"]) -]) -class TestNote: - def test_constructor(self, fields: List[str], guid: str, note_model: str, tags: List[str]): - note = Note(fields=fields, guid=guid, note_model=note_model, tags=tags) - - assert isinstance(note, Note) - assert note.fields == fields - assert note.guid == guid - assert note.note_model == note_model - assert note.tags == tags - - def test_from_json(self, fields: List[str], guid: str, note_model: str, tags: List[str]): - json_data = create_note_json(fields, guid, note_model, tags) - note = Note.from_json(json_data) - - assert isinstance(note, Note) - assert note.fields == fields - assert note.guid == guid - assert note.note_model == note_model - assert note.tags == tags or (tags == [] and note.tags is None) - - def test_dump_to_json(self, fields: List[str], guid: str, note_model: str, tags: List[str]): - json_data = create_note_json(fields, guid, note_model, tags) - note = Note.from_json(json_data) - - assert note.dump_json_to_string() == json.dumps(json_data, sort_keys=False, indent=4, ensure_ascii=False) - - -@pytest.mark.parametrize("note_model, tags", [ - ("", []), - (None, None), - ("model_name", ["noun"]) -]) -class TestNoteGrouping: - def test_constructor(self, note_model: str, tags: List[str], note_test1): - group = NoteGrouping(notes=[note_test1], note_model=note_model, tags=tags) - - assert isinstance(group, NoteGrouping) - assert group.notes == [note_test1] - assert group.note_model == note_model - assert group.tags == tags - - def test_from_json(self, note_model: str, tags: List[str], note_test1): - json_data = create_note_grouping_json([Note.encode(note_test1)], note_model, tags) - group = NoteGrouping.from_json(json_data) - - assert isinstance(group, NoteGrouping) - assert group.notes == [note_test1] - assert group.note_model == note_model - assert group.tags == tags or (tags == [] and group.tags is None) - - def test_dump_to_json(self, note_model: str, tags: List[str], note_test1): - json_data = create_note_grouping_json([Note.encode(note_test1)], note_model, tags) - group = NoteGrouping.from_json(json_data) - - assert group.dump_json_to_string() == json.dumps(json_data, sort_keys=False, indent=4, ensure_ascii=False) \ No newline at end of file diff --git a/tests/representation/yaml/test_note_repr.py b/tests/representation/yaml/test_note_repr.py new file mode 100644 index 0000000..4ac9bc5 --- /dev/null +++ b/tests/representation/yaml/test_note_repr.py @@ -0,0 +1,67 @@ +import json +from typing import List +from ruamel.yaml import round_trip_dump +from brain_brew.representation.yaml.my_yaml import yaml + +import pytest + +from brain_brew.representation.yaml.note_repr import Note, NoteGrouping + + +@pytest.fixture() +def note_test1(): + return Note(note_model="model", tags=['noun', 'other'], fields=['first'], guid="12345") + + +@pytest.mark.parametrize("fields, guid, note_model, tags", [ + ([], "", "", []), + (None, None, None, None), + (["test", "blah", "whatever"], "1234567890x", "model_name", ["noun"]) +]) +class TestNote: + def test_constructor(self, fields: List[str], guid: str, note_model: str, tags: List[str]): + note = Note(fields=fields, guid=guid, note_model=note_model, tags=tags) + + assert isinstance(note, Note) + assert note.fields == fields + assert note.guid == guid + assert note.note_model == note_model + assert note.tags == tags + + def test_encode(self, fields: List[str], guid: str, note_model: str, tags: List[str]): + note = Note(fields=fields, guid=guid, note_model=note_model, tags=tags) + + encoded_dict = note.encode() + + assert "fields" in encoded_dict + assert "guid" in encoded_dict + has_tags = "tags" in encoded_dict + assert has_tags if tags != [] and tags is not None else not has_tags + has_note_model = "note_model" in encoded_dict + assert has_note_model if note_model is not None else not has_note_model + + +class TestNoteFromYaml: + def test_dump_to_yaml(self, tmpdir, fields: List[str], guid: str, note_model: str, tags: List[str]): + folder = tmpdir.mkdir("yaml_files") + file = folder.join("test.yaml") + file.write("test") + + yaml_string = """\ + guid: 7ysf7ysd8f8 + fields: + - test + - blah + - another one + tags: + - noun + - english + note_model: LL Noun + """ + + note = Note(fields=fields, guid=guid, note_model=note_model, tags=tags) + note.dump_to_yaml(file) + # rt_dump = round_trip_dump(Note.encode(self)) + + assert yaml.load(file.read()) == yaml.load(yaml_string) + diff --git a/tests/test_files.py b/tests/test_files.py index 649394e..beda8a4 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -49,3 +49,8 @@ class BuildConfig: class MediaFiles: LOC = "tests/test_files/media_files/" + + class YamlNotes: + LOC = "tests/test_files/yaml/note/" + + TEST1 = LOC + "note1.yaml" diff --git a/tests/test_files/deck_parts/yaml/note/note1.yaml b/tests/test_files/deck_parts/yaml/note/note1.yaml new file mode 100644 index 0000000..7442d62 --- /dev/null +++ b/tests/test_files/deck_parts/yaml/note/note1.yaml @@ -0,0 +1,9 @@ +guid: 7ysf7ysd8f8 +fields: + - test + - blah + - another one +tags: + - noun + - english +note_model: LL Noun \ No newline at end of file From 0cf10278a0fe59bd020ef13fabfd0e381ae66f9d Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sat, 13 Jun 2020 12:32:25 +0200 Subject: [PATCH 03/39] Note and NoteGroupings writing to YAML correctly --- brain_brew/representation/yaml/my_yaml.py | 12 +- brain_brew/representation/yaml/note_repr.py | 42 ++-- tests/representation/yaml/test_note_repr.py | 233 +++++++++++++++----- 3 files changed, 207 insertions(+), 80 deletions(-) diff --git a/brain_brew/representation/yaml/my_yaml.py b/brain_brew/representation/yaml/my_yaml.py index 67a9c63..d6ab588 100644 --- a/brain_brew/representation/yaml/my_yaml.py +++ b/brain_brew/representation/yaml/my_yaml.py @@ -1,6 +1,10 @@ from ruamel.yaml import YAML -yaml = YAML(typ='safe') -yaml.default_flow_style = False -yaml.indent = 4 -yaml.sort_base_mapping_type_on_output = False +yaml_load = YAML(typ='safe') + + +yaml_dump = YAML() +yaml_dump.preserve_quotes = False +yaml_dump.indent(mapping=0, sequence=6, offset=4) +yaml_dump.representer.ignore_aliases = lambda *data: True +# yaml.sort_base_mapping_type_on_output = False diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index b6cc403..8d53370 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -1,19 +1,24 @@ -from brain_brew.representation.yaml.my_yaml import yaml +from brain_brew.representation.yaml.my_yaml import yaml_dump, yaml_load import json from dataclasses import dataclass -from typing import List +from typing import List, Optional + +FIELDS = 'fields' +GUID = 'guid' +TAGS = 'tags' +NOTE_MODEL = 'note_model' @dataclass class OverwritableNoteData: - note_model: str - tags: List[str] + note_model: Optional[str] + tags: Optional[List[str]] def encode_overwritable(self, json_dict): - if self.tags is not None and self.tags != []: - json_dict.setdefault("tags", self.tags) if self.note_model is not None: - json_dict.setdefault("note_model", self.note_model) + json_dict.setdefault(NOTE_MODEL, self.note_model) + if self.tags is not None and self.tags != []: + json_dict.setdefault(TAGS, self.tags) return json_dict @@ -25,19 +30,20 @@ class Note(OverwritableNoteData): @classmethod def from_dict(cls, data: dict): return cls( - fields=data.get("fields"), - guid=data.get("guid"), - note_model=data.get("note_model", None), - tags=data.get("tags", None) + fields=data.get(FIELDS), + guid=data.get(GUID), + note_model=data.get(NOTE_MODEL, None), + tags=data.get(TAGS, None) ) def encode(self): - json_dict = {"guid": self.guid, "fields": self.fields} + json_dict = {FIELDS: self.fields, GUID: self.guid} super().encode_overwritable(json_dict) return json_dict def dump_to_yaml(self, file): - yaml.dump(self.encode(), file) + with open(file, 'w') as fp: + yaml_dump.dump(self.encode(), fp) @dataclass @@ -48,17 +54,19 @@ class NoteGrouping(OverwritableNoteData): def from_dict(cls, data): return cls( notes=list(map(Note.from_dict, data.get("notes"))), - note_model=data.get("note_model", None), - tags=data.get("tags", None) + note_model=data.get(NOTE_MODEL, None), + tags=data.get(TAGS, None) ) def encode(self): - json_dict = {"notes": [note.encode() for note in self.notes]} + json_dict = {} super().encode_overwritable(json_dict) + json_dict.setdefault("notes", [note.encode() for note in self.notes]) return json_dict def dump_to_yaml(self, file): - yaml.dump(self.encode(), file) + with open(file, 'w') as fp: + yaml_dump.dump(self.encode(), fp) # @dataclass # class DeckPartNoteModel: \ No newline at end of file diff --git a/tests/representation/yaml/test_note_repr.py b/tests/representation/yaml/test_note_repr.py index 4ac9bc5..1b294e4 100644 --- a/tests/representation/yaml/test_note_repr.py +++ b/tests/representation/yaml/test_note_repr.py @@ -1,67 +1,182 @@ import json +import sys +from textwrap import dedent from typing import List from ruamel.yaml import round_trip_dump -from brain_brew.representation.yaml.my_yaml import yaml +from brain_brew.representation.yaml.my_yaml import yaml_dump, yaml_load import pytest from brain_brew.representation.yaml.note_repr import Note, NoteGrouping - -@pytest.fixture() -def note_test1(): - return Note(note_model="model", tags=['noun', 'other'], fields=['first'], guid="12345") - - -@pytest.mark.parametrize("fields, guid, note_model, tags", [ - ([], "", "", []), - (None, None, None, None), - (["test", "blah", "whatever"], "1234567890x", "model_name", ["noun"]) -]) -class TestNote: - def test_constructor(self, fields: List[str], guid: str, note_model: str, tags: List[str]): - note = Note(fields=fields, guid=guid, note_model=note_model, tags=tags) - - assert isinstance(note, Note) - assert note.fields == fields - assert note.guid == guid - assert note.note_model == note_model - assert note.tags == tags - - def test_encode(self, fields: List[str], guid: str, note_model: str, tags: List[str]): - note = Note(fields=fields, guid=guid, note_model=note_model, tags=tags) - - encoded_dict = note.encode() - - assert "fields" in encoded_dict - assert "guid" in encoded_dict - has_tags = "tags" in encoded_dict - assert has_tags if tags != [] and tags is not None else not has_tags - has_note_model = "note_model" in encoded_dict - assert has_note_model if note_model is not None else not has_note_model - - -class TestNoteFromYaml: - def test_dump_to_yaml(self, tmpdir, fields: List[str], guid: str, note_model: str, tags: List[str]): - folder = tmpdir.mkdir("yaml_files") - file = folder.join("test.yaml") - file.write("test") - - yaml_string = """\ - guid: 7ysf7ysd8f8 - fields: - - test - - blah - - another one - tags: - - noun - - english - note_model: LL Noun - """ - - note = Note(fields=fields, guid=guid, note_model=note_model, tags=tags) - note.dump_to_yaml(file) - # rt_dump = round_trip_dump(Note.encode(self)) - - assert yaml.load(file.read()) == yaml.load(yaml_string) - +note_values_that_should_work = { + "test1": {"fields": ['first'], "guid": "12345", "note_model": "model_name", "tags": ['noun', 'other']}, + "test2": {"fields": ['english', 'german'], "guid": "sdfhfghsvsdv", "note_model": "LL Test", "tags": ['marked']}, + "no_note_model": {"fields": ['first'], "guid": "12345", "tags": ['noun', 'other']}, + "no_tags1": {"fields": ['first'], "guid": "12345", "note_model": "model_name"}, + "no_tags2": {"fields": ['first'], "guid": "12345", "note_model": "model_name", "tags": []} +} + +note_groupings_that_should_work = { + "nothing_grouped": {"notes": [note_values_that_should_work["test1"], note_values_that_should_work["test2"]]}, + "note_model_grouped": { + "notes": [note_values_that_should_work["no_note_model"], note_values_that_should_work["no_note_model"]], + "note_model": "model_name"} +} + + +########### Notes +@pytest.fixture(params=note_values_that_should_work.values()) +def note_fixtures(request): + return Note.from_dict(request.param) + +# Note Groupings +@pytest.fixture(params=note_groupings_that_should_work.values()) +def note_grouping_fixtures(request): + return NoteGrouping.from_dict(request.param) + + +class TestConstructor: + class TestNote: + @pytest.mark.parametrize("fields, guid, note_model, tags", [ + ([], "", "", []), + (None, None, None, None), + (["test", "blah", "whatever"], "1234567890x", "model_name", ["noun"]) + ]) + def test_constructor(self, fields: List[str], guid: str, note_model: str, tags: List[str]): + note = Note(fields=fields, guid=guid, note_model=note_model, tags=tags) + + assert isinstance(note, Note) + assert note.fields == fields + assert note.guid == guid + assert note.note_model == note_model + assert note.tags == tags + + def test_from_dict(self, note_fixtures): + assert isinstance(note_fixtures, Note) + + class TestNoteGrouping: + def test_constructor(self): + note_grouping = NoteGrouping(notes=[Note.from_dict(note_values_that_should_work["test1"])], note_model=None, tags=None) + + assert isinstance(note_grouping, NoteGrouping) + assert isinstance(note_grouping.notes, List) + + def test_from_dict(self, note_grouping_fixtures): + assert isinstance(note_grouping_fixtures, NoteGrouping) + + +class TestDumpToYaml: + class TestNote: + @staticmethod + def _assert_dump_to_yaml(tmpdir, ystring, note_name): + folder = tmpdir.mkdir("yaml_files") + file = folder.join("test.yaml") + file.write("test") + + note = Note.from_dict(note_values_that_should_work[note_name]) + note.dump_to_yaml(str(file)) + + assert file.read() == ystring + + def test_all1(self, tmpdir): + ystring = dedent('''\ + fields: + - first + guid: '12345' + note_model: model_name + tags: + - noun + - other + ''') + + self._assert_dump_to_yaml(tmpdir, ystring, "test1") + + def test_all2(self, tmpdir): + ystring = dedent('''\ + fields: + - english + - german + guid: sdfhfghsvsdv + note_model: LL Test + tags: + - marked + ''') + + self._assert_dump_to_yaml(tmpdir, ystring, "test2") + + def test_no_note_model(self, tmpdir): + ystring = dedent('''\ + fields: + - first + guid: '12345' + tags: + - noun + - other + ''') + + self._assert_dump_to_yaml(tmpdir, ystring, "no_note_model") + + def test_no_tags(self, tmpdir): + for num, note in enumerate(["no_tags1", "no_tags2"]): + ystring = dedent('''\ + fields: + - first + guid: '12345' + note_model: model_name + ''') + + self._assert_dump_to_yaml(tmpdir.mkdir(str(num)), ystring, note) + + class TestNoteGrouping: + @staticmethod + def _assert_dump_to_yaml(tmpdir, ystring, note_grouping_name): + folder = tmpdir.mkdir("yaml_files") + file = folder.join("test.yaml") + file.write("test") + + note = NoteGrouping.from_dict(note_groupings_that_should_work[note_grouping_name]) + note.dump_to_yaml(str(file)) + + assert file.read() == ystring + + def test_nothing_grouped(self, tmpdir): + ystring = dedent('''\ + notes: + - fields: + - first + guid: '12345' + note_model: model_name + tags: + - noun + - other + - fields: + - english + - german + guid: sdfhfghsvsdv + note_model: LL Test + tags: + - marked + ''') + + self._assert_dump_to_yaml(tmpdir, ystring, "nothing_grouped") + + def test_note_model_grouped(self, tmpdir): + ystring = dedent('''\ + note_model: model_name + notes: + - fields: + - first + guid: '12345' + tags: + - noun + - other + - fields: + - first + guid: '12345' + tags: + - noun + - other + ''') + + self._assert_dump_to_yaml(tmpdir, ystring, "note_model_grouped") From 84404af9b50873458537c8de3276e8ef2fbf0110 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sun, 14 Jun 2020 10:22:26 +0200 Subject: [PATCH 04/39] Test and Requirement Updates --- Pipfile | 2 +- brain_brew/representation/yaml/my_yaml.py | 2 +- brain_brew/representation/yaml/note_repr.py | 59 +++++--- requirements.txt | 1 + tests/representation/yaml/test_note_repr.py | 132 +++++++++++------- .../deck_parts/yaml/note/note1.yaml | 39 ++++-- 6 files changed, 155 insertions(+), 80 deletions(-) diff --git a/Pipfile b/Pipfile index 679f345..e4fe427 100644 --- a/Pipfile +++ b/Pipfile @@ -11,7 +11,7 @@ args = "==0.1.0" clint = "==0.5.1" coverage = "==4.5.4" PyYAML = "==5.1.2" -ruamel-yaml = "*" +ruamel-yaml = "==0.16.10" [requires] python_version = "3.7" diff --git a/brain_brew/representation/yaml/my_yaml.py b/brain_brew/representation/yaml/my_yaml.py index d6ab588..528eab7 100644 --- a/brain_brew/representation/yaml/my_yaml.py +++ b/brain_brew/representation/yaml/my_yaml.py @@ -5,6 +5,6 @@ yaml_dump = YAML() yaml_dump.preserve_quotes = False -yaml_dump.indent(mapping=0, sequence=6, offset=4) +yaml_dump.indent(mapping=0, sequence=4, offset=2) yaml_dump.representer.ignore_aliases = lambda *data: True # yaml.sort_base_mapping_type_on_output = False diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index 8d53370..0e291c3 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -1,29 +1,30 @@ from brain_brew.representation.yaml.my_yaml import yaml_dump, yaml_load import json from dataclasses import dataclass -from typing import List, Optional +from typing import List, Optional, Dict FIELDS = 'fields' GUID = 'guid' TAGS = 'tags' NOTE_MODEL = 'note_model' +NOTES = "notes" @dataclass -class OverwritableNoteData: +class GroupableNoteData: note_model: Optional[str] tags: Optional[List[str]] - def encode_overwritable(self, json_dict): + def encode_groupable(self, data_dict): if self.note_model is not None: - json_dict.setdefault(NOTE_MODEL, self.note_model) + data_dict.setdefault(NOTE_MODEL, self.note_model) if self.tags is not None and self.tags != []: - json_dict.setdefault(TAGS, self.tags) - return json_dict + data_dict.setdefault(TAGS, self.tags) + return data_dict @dataclass -class Note(OverwritableNoteData): +class Note(GroupableNoteData): fields: List[str] guid: str @@ -36,10 +37,10 @@ def from_dict(cls, data: dict): tags=data.get(TAGS, None) ) - def encode(self): - json_dict = {FIELDS: self.fields, GUID: self.guid} - super().encode_overwritable(json_dict) - return json_dict + def encode(self) -> dict: + data_dict = {FIELDS: self.fields, GUID: self.guid} + super().encode_groupable(data_dict) + return data_dict def dump_to_yaml(self, file): with open(file, 'w') as fp: @@ -47,26 +48,42 @@ def dump_to_yaml(self, file): @dataclass -class NoteGrouping(OverwritableNoteData): +class NoteGrouping(GroupableNoteData): notes: List[Note] @classmethod - def from_dict(cls, data): + def from_dict(cls, data: dict): return cls( - notes=list(map(Note.from_dict, data.get("notes"))), + notes=list(map(Note.from_dict, data.get(NOTES))), note_model=data.get(NOTE_MODEL, None), tags=data.get(TAGS, None) ) - def encode(self): - json_dict = {} - super().encode_overwritable(json_dict) - json_dict.setdefault("notes", [note.encode() for note in self.notes]) - return json_dict + def encode(self) -> dict: + data_dict = {} + super().encode_groupable(data_dict) + data_dict.setdefault(NOTES, [note.encode() for note in self.notes]) + return data_dict def dump_to_yaml(self, file): with open(file, 'w') as fp: yaml_dump.dump(self.encode(), fp) -# @dataclass -# class DeckPartNoteModel: \ No newline at end of file + +@dataclass +class DeckPartNoteModel: + note_groupings: List[NoteGrouping] + + @classmethod + def from_list(cls, data: List[dict]): + return cls( + note_groupings=list(map(NoteGrouping.from_dict, data)) + ) + + def encode(self) -> List[dict]: + data_list = [note_grouping.encode() for note_grouping in self.note_groupings] + return data_list + + def dump_to_yaml(self, file): + with open(file, 'w') as fp: + yaml_dump.dump(self.encode(), fp) diff --git a/requirements.txt b/requirements.txt index e83bb7d..72532f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,6 +2,7 @@ args==0.1.0 clint==0.5.1 coverage==4.5.4 PyYAML==5.1.2 +ruamel-yaml==0.16.10 # Testing pytest==5.4.1 \ No newline at end of file diff --git a/tests/representation/yaml/test_note_repr.py b/tests/representation/yaml/test_note_repr.py index 1b294e4..8c6a523 100644 --- a/tests/representation/yaml/test_note_repr.py +++ b/tests/representation/yaml/test_note_repr.py @@ -9,29 +9,30 @@ from brain_brew.representation.yaml.note_repr import Note, NoteGrouping -note_values_that_should_work = { +working_notes = { "test1": {"fields": ['first'], "guid": "12345", "note_model": "model_name", "tags": ['noun', 'other']}, "test2": {"fields": ['english', 'german'], "guid": "sdfhfghsvsdv", "note_model": "LL Test", "tags": ['marked']}, "no_note_model": {"fields": ['first'], "guid": "12345", "tags": ['noun', 'other']}, "no_tags1": {"fields": ['first'], "guid": "12345", "note_model": "model_name"}, - "no_tags2": {"fields": ['first'], "guid": "12345", "note_model": "model_name", "tags": []} + "no_tags2": {"fields": ['first'], "guid": "12345", "note_model": "model_name", "tags": []}, + "no_model_or_tags": {"fields": ['first'], "guid": "12345"} } -note_groupings_that_should_work = { - "nothing_grouped": {"notes": [note_values_that_should_work["test1"], note_values_that_should_work["test2"]]}, - "note_model_grouped": { - "notes": [note_values_that_should_work["no_note_model"], note_values_that_should_work["no_note_model"]], - "note_model": "model_name"} +working_note_groupings = { + "nothing_grouped": {"notes": [working_notes["test1"], working_notes["test2"]]}, + "note_model_grouped": {"notes": [working_notes["no_note_model"], working_notes["no_note_model"]], "note_model": "model_name"}, + "tags_grouped": {"notes": [working_notes["no_tags1"], working_notes["no_tags2"]], "tags": ["noun", "other"]}, + "model_and_tags_grouped": {"notes": [working_notes["no_model_or_tags"], working_notes["no_model_or_tags"]], "note_model": "model_name", "tags": ["noun", "other"]} } ########### Notes -@pytest.fixture(params=note_values_that_should_work.values()) +@pytest.fixture(params=working_notes.values()) def note_fixtures(request): return Note.from_dict(request.param) # Note Groupings -@pytest.fixture(params=note_groupings_that_should_work.values()) +@pytest.fixture(params=working_note_groupings.values()) def note_grouping_fixtures(request): return NoteGrouping.from_dict(request.param) @@ -57,7 +58,7 @@ def test_from_dict(self, note_fixtures): class TestNoteGrouping: def test_constructor(self): - note_grouping = NoteGrouping(notes=[Note.from_dict(note_values_that_should_work["test1"])], note_model=None, tags=None) + note_grouping = NoteGrouping(notes=[Note.from_dict(working_notes["test1"])], note_model=None, tags=None) assert isinstance(note_grouping, NoteGrouping) assert isinstance(note_grouping.notes, List) @@ -74,7 +75,7 @@ def _assert_dump_to_yaml(tmpdir, ystring, note_name): file = folder.join("test.yaml") file.write("test") - note = Note.from_dict(note_values_that_should_work[note_name]) + note = Note.from_dict(working_notes[note_name]) note.dump_to_yaml(str(file)) assert file.read() == ystring @@ -82,12 +83,12 @@ def _assert_dump_to_yaml(tmpdir, ystring, note_name): def test_all1(self, tmpdir): ystring = dedent('''\ fields: - - first + - first guid: '12345' note_model: model_name tags: - - noun - - other + - noun + - other ''') self._assert_dump_to_yaml(tmpdir, ystring, "test1") @@ -95,12 +96,12 @@ def test_all1(self, tmpdir): def test_all2(self, tmpdir): ystring = dedent('''\ fields: - - english - - german + - english + - german guid: sdfhfghsvsdv note_model: LL Test tags: - - marked + - marked ''') self._assert_dump_to_yaml(tmpdir, ystring, "test2") @@ -108,11 +109,11 @@ def test_all2(self, tmpdir): def test_no_note_model(self, tmpdir): ystring = dedent('''\ fields: - - first + - first guid: '12345' tags: - - noun - - other + - noun + - other ''') self._assert_dump_to_yaml(tmpdir, ystring, "no_note_model") @@ -121,7 +122,7 @@ def test_no_tags(self, tmpdir): for num, note in enumerate(["no_tags1", "no_tags2"]): ystring = dedent('''\ fields: - - first + - first guid: '12345' note_model: model_name ''') @@ -135,7 +136,7 @@ def _assert_dump_to_yaml(tmpdir, ystring, note_grouping_name): file = folder.join("test.yaml") file.write("test") - note = NoteGrouping.from_dict(note_groupings_that_should_work[note_grouping_name]) + note = NoteGrouping.from_dict(working_note_groupings[note_grouping_name]) note.dump_to_yaml(str(file)) assert file.read() == ystring @@ -143,20 +144,20 @@ def _assert_dump_to_yaml(tmpdir, ystring, note_grouping_name): def test_nothing_grouped(self, tmpdir): ystring = dedent('''\ notes: - - fields: - - first - guid: '12345' - note_model: model_name - tags: - - noun - - other - - fields: - - english - - german - guid: sdfhfghsvsdv - note_model: LL Test - tags: - - marked + - fields: + - first + guid: '12345' + note_model: model_name + tags: + - noun + - other + - fields: + - english + - german + guid: sdfhfghsvsdv + note_model: LL Test + tags: + - marked ''') self._assert_dump_to_yaml(tmpdir, ystring, "nothing_grouped") @@ -165,18 +166,53 @@ def test_note_model_grouped(self, tmpdir): ystring = dedent('''\ note_model: model_name notes: - - fields: - - first - guid: '12345' - tags: - - noun - - other - - fields: - - first - guid: '12345' - tags: - - noun - - other + - fields: + - first + guid: '12345' + tags: + - noun + - other + - fields: + - first + guid: '12345' + tags: + - noun + - other ''') self._assert_dump_to_yaml(tmpdir, ystring, "note_model_grouped") + + def test_note_tags_grouped(self, tmpdir): + ystring = dedent('''\ + tags: + - noun + - other + notes: + - fields: + - first + guid: '12345' + note_model: model_name + - fields: + - first + guid: '12345' + note_model: model_name + ''') + + self._assert_dump_to_yaml(tmpdir, ystring, "tags_grouped") + + def test_note_model_and_tags_grouped(self, tmpdir): + ystring = dedent('''\ + note_model: model_name + tags: + - noun + - other + notes: + - fields: + - first + guid: '12345' + - fields: + - first + guid: '12345' + ''') + + self._assert_dump_to_yaml(tmpdir, ystring, "model_and_tags_grouped") diff --git a/tests/test_files/deck_parts/yaml/note/note1.yaml b/tests/test_files/deck_parts/yaml/note/note1.yaml index 7442d62..9224b3c 100644 --- a/tests/test_files/deck_parts/yaml/note/note1.yaml +++ b/tests/test_files/deck_parts/yaml/note/note1.yaml @@ -1,9 +1,30 @@ -guid: 7ysf7ysd8f8 -fields: - - test - - blah - - another one -tags: - - noun - - english -note_model: LL Noun \ No newline at end of file +- note_model: LL Noun + tags: + - noun + - english + notes: + - guid: 7ysf7ysd8f8 + fields: + - test + - blah + - another one + - guid: sfkdsfhsd + fields: + - first + - second + - third +- note_model: LL Verb + tags: + - verb + - english + notes: + - guid: dhdfhsdf + fields: + - verby + - boo + - another one + - guid: dfgdfhgjs + fields: + - first + - second + - third From 16ce13eb0d6c00a4308e09f0e7fb342c1fdc4ed5 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sun, 14 Jun 2020 11:15:21 +0200 Subject: [PATCH 05/39] DeckPartNotes Yaml and Tests --- brain_brew/representation/yaml/note_repr.py | 13 +- tests/representation/yaml/test_note_repr.py | 201 +++++++++++++------- 2 files changed, 135 insertions(+), 79 deletions(-) diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index 0e291c3..144ca4b 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -8,6 +8,7 @@ TAGS = 'tags' NOTE_MODEL = 'note_model' NOTES = "notes" +NOTE_GROUPINGS = "note_groupings" @dataclass @@ -71,18 +72,18 @@ def dump_to_yaml(self, file): @dataclass -class DeckPartNoteModel: +class DeckPartNotes: note_groupings: List[NoteGrouping] @classmethod - def from_list(cls, data: List[dict]): + def from_dict(cls, data: dict): return cls( - note_groupings=list(map(NoteGrouping.from_dict, data)) + note_groupings=list(map(NoteGrouping.from_dict, data.get(NOTE_GROUPINGS))) ) - def encode(self) -> List[dict]: - data_list = [note_grouping.encode() for note_grouping in self.note_groupings] - return data_list + def encode(self) -> dict: + data_dict = {NOTE_GROUPINGS: [note_grouping.encode() for note_grouping in self.note_groupings]} + return data_dict def dump_to_yaml(self, file): with open(file, 'w') as fp: diff --git a/tests/representation/yaml/test_note_repr.py b/tests/representation/yaml/test_note_repr.py index 8c6a523..43ce7e8 100644 --- a/tests/representation/yaml/test_note_repr.py +++ b/tests/representation/yaml/test_note_repr.py @@ -7,31 +7,31 @@ import pytest -from brain_brew.representation.yaml.note_repr import Note, NoteGrouping +from brain_brew.representation.yaml.note_repr import Note, NoteGrouping, DeckPartNotes, \ + NOTES, NOTE_GROUPINGS, FIELDS, GUID, NOTE_MODEL, TAGS working_notes = { - "test1": {"fields": ['first'], "guid": "12345", "note_model": "model_name", "tags": ['noun', 'other']}, - "test2": {"fields": ['english', 'german'], "guid": "sdfhfghsvsdv", "note_model": "LL Test", "tags": ['marked']}, - "no_note_model": {"fields": ['first'], "guid": "12345", "tags": ['noun', 'other']}, - "no_tags1": {"fields": ['first'], "guid": "12345", "note_model": "model_name"}, - "no_tags2": {"fields": ['first'], "guid": "12345", "note_model": "model_name", "tags": []}, - "no_model_or_tags": {"fields": ['first'], "guid": "12345"} + "test1": {FIELDS: ['first'], GUID: "12345", NOTE_MODEL: "model_name", TAGS: ['noun', 'other']}, + "test2": {FIELDS: ['english', 'german'], GUID: "sdfhfghsvsdv", NOTE_MODEL: "LL Test", TAGS: ['marked']}, + "no_note_model": {FIELDS: ['first'], GUID: "12345", TAGS: ['noun', 'other']}, + "no_tags1": {FIELDS: ['first'], GUID: "12345", NOTE_MODEL: "model_name"}, + "no_tags2": {FIELDS: ['first'], GUID: "12345", NOTE_MODEL: "model_name", TAGS: []}, + "no_model_or_tags": {FIELDS: ['first'], GUID: "12345"} } working_note_groupings = { - "nothing_grouped": {"notes": [working_notes["test1"], working_notes["test2"]]}, - "note_model_grouped": {"notes": [working_notes["no_note_model"], working_notes["no_note_model"]], "note_model": "model_name"}, - "tags_grouped": {"notes": [working_notes["no_tags1"], working_notes["no_tags2"]], "tags": ["noun", "other"]}, - "model_and_tags_grouped": {"notes": [working_notes["no_model_or_tags"], working_notes["no_model_or_tags"]], "note_model": "model_name", "tags": ["noun", "other"]} + "nothing_grouped": {NOTES: [working_notes["test1"], working_notes["test2"]]}, + "note_model_grouped": {NOTES: [working_notes["no_note_model"], working_notes["no_note_model"]], NOTE_MODEL: "model_name"}, + "tags_grouped": {NOTES: [working_notes["no_tags1"], working_notes["no_tags2"]], TAGS: ["noun", "other"]}, + "model_and_tags_grouped": {NOTES: [working_notes["no_model_or_tags"], working_notes["no_model_or_tags"]], NOTE_MODEL: "model_name", TAGS: ["noun", "other"]} } -########### Notes @pytest.fixture(params=working_notes.values()) def note_fixtures(request): return Note.from_dict(request.param) -# Note Groupings + @pytest.fixture(params=working_note_groupings.values()) def note_grouping_fixtures(request): return NoteGrouping.from_dict(request.param) @@ -66,14 +66,28 @@ def test_constructor(self): def test_from_dict(self, note_grouping_fixtures): assert isinstance(note_grouping_fixtures, NoteGrouping) + class TestDeckPartNote: + def test_constructor(self): + dpn = DeckPartNotes(note_groupings=[NoteGrouping.from_dict(working_note_groupings["nothing_grouped"])]) + assert isinstance(dpn, DeckPartNotes) + + def test_from_dict(self): + dpn = DeckPartNotes.from_dict({NOTE_GROUPINGS: [working_note_groupings["nothing_grouped"]]}) + assert isinstance(dpn, DeckPartNotes) + class TestDumpToYaml: + @staticmethod + def _make_temp_file(tmpdir): + folder = tmpdir.mkdir("yaml_files") + file = folder.join("test.yaml") + file.write("test") + return file + class TestNote: @staticmethod def _assert_dump_to_yaml(tmpdir, ystring, note_name): - folder = tmpdir.mkdir("yaml_files") - file = folder.join("test.yaml") - file.write("test") + file = TestDumpToYaml._make_temp_file(tmpdir) note = Note.from_dict(working_notes[note_name]) note.dump_to_yaml(str(file)) @@ -81,12 +95,12 @@ def _assert_dump_to_yaml(tmpdir, ystring, note_name): assert file.read() == ystring def test_all1(self, tmpdir): - ystring = dedent('''\ - fields: + ystring = dedent(f'''\ + {FIELDS}: - first - guid: '12345' - note_model: model_name - tags: + {GUID}: '12345' + {NOTE_MODEL}: model_name + {TAGS}: - noun - other ''') @@ -94,24 +108,24 @@ def test_all1(self, tmpdir): self._assert_dump_to_yaml(tmpdir, ystring, "test1") def test_all2(self, tmpdir): - ystring = dedent('''\ - fields: + ystring = dedent(f'''\ + {FIELDS}: - english - german - guid: sdfhfghsvsdv - note_model: LL Test - tags: + {GUID}: sdfhfghsvsdv + {NOTE_MODEL}: LL Test + {TAGS}: - marked ''') self._assert_dump_to_yaml(tmpdir, ystring, "test2") def test_no_note_model(self, tmpdir): - ystring = dedent('''\ - fields: + ystring = dedent(f'''\ + {FIELDS}: - first - guid: '12345' - tags: + {GUID}: '12345' + {TAGS}: - noun - other ''') @@ -120,11 +134,11 @@ def test_no_note_model(self, tmpdir): def test_no_tags(self, tmpdir): for num, note in enumerate(["no_tags1", "no_tags2"]): - ystring = dedent('''\ - fields: + ystring = dedent(f'''\ + {FIELDS}: - first - guid: '12345' - note_model: model_name + {GUID}: '12345' + {NOTE_MODEL}: model_name ''') self._assert_dump_to_yaml(tmpdir.mkdir(str(num)), ystring, note) @@ -132,9 +146,7 @@ def test_no_tags(self, tmpdir): class TestNoteGrouping: @staticmethod def _assert_dump_to_yaml(tmpdir, ystring, note_grouping_name): - folder = tmpdir.mkdir("yaml_files") - file = folder.join("test.yaml") - file.write("test") + file = TestDumpToYaml._make_temp_file(tmpdir) note = NoteGrouping.from_dict(working_note_groupings[note_grouping_name]) note.dump_to_yaml(str(file)) @@ -142,40 +154,40 @@ def _assert_dump_to_yaml(tmpdir, ystring, note_grouping_name): assert file.read() == ystring def test_nothing_grouped(self, tmpdir): - ystring = dedent('''\ - notes: - - fields: + ystring = dedent(f'''\ + {NOTES}: + - {FIELDS}: - first - guid: '12345' - note_model: model_name - tags: + {GUID}: '12345' + {NOTE_MODEL}: model_name + {TAGS}: - noun - other - - fields: + - {FIELDS}: - english - german - guid: sdfhfghsvsdv - note_model: LL Test - tags: + {GUID}: sdfhfghsvsdv + {NOTE_MODEL}: LL Test + {TAGS}: - marked ''') self._assert_dump_to_yaml(tmpdir, ystring, "nothing_grouped") def test_note_model_grouped(self, tmpdir): - ystring = dedent('''\ - note_model: model_name - notes: - - fields: + ystring = dedent(f'''\ + {NOTE_MODEL}: model_name + {NOTES}: + - {FIELDS}: - first - guid: '12345' - tags: + {GUID}: '12345' + {TAGS}: - noun - other - - fields: + - {FIELDS}: - first - guid: '12345' - tags: + {GUID}: '12345' + {TAGS}: - noun - other ''') @@ -183,36 +195,79 @@ def test_note_model_grouped(self, tmpdir): self._assert_dump_to_yaml(tmpdir, ystring, "note_model_grouped") def test_note_tags_grouped(self, tmpdir): - ystring = dedent('''\ - tags: + ystring = dedent(f'''\ + {TAGS}: - noun - other - notes: - - fields: + {NOTES}: + - {FIELDS}: - first - guid: '12345' - note_model: model_name - - fields: + {GUID}: '12345' + {NOTE_MODEL}: model_name + - {FIELDS}: - first - guid: '12345' - note_model: model_name + {GUID}: '12345' + {NOTE_MODEL}: model_name ''') self._assert_dump_to_yaml(tmpdir, ystring, "tags_grouped") def test_note_model_and_tags_grouped(self, tmpdir): - ystring = dedent('''\ - note_model: model_name - tags: + ystring = dedent(f'''\ + {NOTE_MODEL}: model_name + {TAGS}: - noun - other - notes: - - fields: + {NOTES}: + - {FIELDS}: - first - guid: '12345' - - fields: + {GUID}: '12345' + - {FIELDS}: - first - guid: '12345' + {GUID}: '12345' ''') self._assert_dump_to_yaml(tmpdir, ystring, "model_and_tags_grouped") + + class TestDeckPartNotes: + @staticmethod + def _assert_dump_to_yaml(tmpdir, ystring, groups: list): + file = TestDumpToYaml._make_temp_file(tmpdir) + + note = DeckPartNotes.from_dict({NOTE_GROUPINGS: [working_note_groupings[name] for name in groups]}) + note.dump_to_yaml(str(file)) + + assert file.read() == ystring + + def test_two_groupings(self, tmpdir): + ystring = dedent(f'''\ + {NOTE_GROUPINGS}: + - {NOTE_MODEL}: model_name + {TAGS}: + - noun + - other + {NOTES}: + - {FIELDS}: + - first + {GUID}: '12345' + - {FIELDS}: + - first + {GUID}: '12345' + - {NOTES}: + - {FIELDS}: + - first + {GUID}: '12345' + {NOTE_MODEL}: model_name + {TAGS}: + - noun + - other + - {FIELDS}: + - english + - german + {GUID}: sdfhfghsvsdv + {NOTE_MODEL}: LL Test + {TAGS}: + - marked + ''') + + self._assert_dump_to_yaml(tmpdir, ystring, ["model_and_tags_grouped", "nothing_grouped"]) From bd05b12332aa54e587879e3633ec8e73e1e42b34 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sun, 14 Jun 2020 12:32:31 +0200 Subject: [PATCH 06/39] NoteGrouping GetAllNotes --- brain_brew/build_tasks/source_csv.py | 2 +- brain_brew/representation/yaml/note_repr.py | 25 ++++++++ tests/representation/yaml/test_note_repr.py | 50 ++++++++++++++- .../deck_parts/yaml/note/note1.yaml | 61 ++++++++++--------- 4 files changed, 104 insertions(+), 34 deletions(-) diff --git a/brain_brew/build_tasks/source_csv.py b/brain_brew/build_tasks/source_csv.py index a870d23..519f249 100644 --- a/brain_brew/build_tasks/source_csv.py +++ b/brain_brew/build_tasks/source_csv.py @@ -145,7 +145,7 @@ def notes_to_source(self) -> Dict[str, dict]: row[CsvKeys.GUID.value] = note[DeckPartNoteKeys.GUID.value] row[CsvKeys.TAGS.value] = self.join_tags(note[DeckPartNoteKeys.TAGS.value]) - formatted_row = self.note_model_mappings_dict[nm_name].note_fields_map_to_csv_row(row) + formatted_row = self.note_model_mappings_dict[nm_name].note_fields_map_to_csv_row(row) # TODO: Do not edit data, make copy csv_data.setdefault(row[CsvKeys.GUID.value], formatted_row) diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index 144ca4b..7b73966 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -70,6 +70,29 @@ def dump_to_yaml(self, file): with open(file, 'w') as fp: yaml_dump.dump(self.encode(), fp) + def verify_groupings(self): + if self.note_model is not None: + modelErrors = ValueError(f"NoteGrouping for 'note_model' {self.note_model} has notes with 'note_model'. " + f"Please remove one of these.") if any([note.note_model for note in self.notes]) else None + + def get_all_notes(self) -> List[Note]: + def join_tags(n_tags): + if self.tags is None and n_tags is None: + return [] + elif self.tags is None: + return n_tags + elif n_tags is None: + return self.tags + else: + return [*n_tags, *self.tags] + + return [Note( + note_model=self.note_model if self.note_model is not None else n.note_model, + tags=join_tags(n.tags), + fields=n.fields, + guid=n.guid + ) for n in self.notes] + @dataclass class DeckPartNotes: @@ -88,3 +111,5 @@ def encode(self) -> dict: def dump_to_yaml(self, file): with open(file, 'w') as fp: yaml_dump.dump(self.encode(), fp) + + diff --git a/tests/representation/yaml/test_note_repr.py b/tests/representation/yaml/test_note_repr.py index 43ce7e8..acadf4f 100644 --- a/tests/representation/yaml/test_note_repr.py +++ b/tests/representation/yaml/test_note_repr.py @@ -14,6 +14,7 @@ "test1": {FIELDS: ['first'], GUID: "12345", NOTE_MODEL: "model_name", TAGS: ['noun', 'other']}, "test2": {FIELDS: ['english', 'german'], GUID: "sdfhfghsvsdv", NOTE_MODEL: "LL Test", TAGS: ['marked']}, "no_note_model": {FIELDS: ['first'], GUID: "12345", TAGS: ['noun', 'other']}, + "no_note_model2": {FIELDS: ['second'], GUID: "67890", TAGS: ['noun', 'other']}, "no_tags1": {FIELDS: ['first'], GUID: "12345", NOTE_MODEL: "model_name"}, "no_tags2": {FIELDS: ['first'], GUID: "12345", NOTE_MODEL: "model_name", TAGS: []}, "no_model_or_tags": {FIELDS: ['first'], GUID: "12345"} @@ -21,8 +22,9 @@ working_note_groupings = { "nothing_grouped": {NOTES: [working_notes["test1"], working_notes["test2"]]}, - "note_model_grouped": {NOTES: [working_notes["no_note_model"], working_notes["no_note_model"]], NOTE_MODEL: "model_name"}, + "note_model_grouped": {NOTES: [working_notes["no_note_model"], working_notes["no_note_model2"]], NOTE_MODEL: "model_name"}, "tags_grouped": {NOTES: [working_notes["no_tags1"], working_notes["no_tags2"]], TAGS: ["noun", "other"]}, + "tags_grouped_as_addition": {NOTES: [working_notes["test1"], working_notes["test2"]], TAGS: ["test", "recent"]}, "model_and_tags_grouped": {NOTES: [working_notes["no_model_or_tags"], working_notes["no_model_or_tags"]], NOTE_MODEL: "model_name", TAGS: ["noun", "other"]} } @@ -185,8 +187,8 @@ def test_note_model_grouped(self, tmpdir): - noun - other - {FIELDS}: - - first - {GUID}: '12345' + - second + {GUID}: '67890' {TAGS}: - noun - other @@ -271,3 +273,45 @@ def test_two_groupings(self, tmpdir): ''') self._assert_dump_to_yaml(tmpdir, ystring, ["model_and_tags_grouped", "nothing_grouped"]) + + +class TestFunctionality: + class TestNoteGrouping: + class TestGetAllNotes: + def test_nothing_grouped(self): + group = NoteGrouping.from_dict(working_note_groupings["nothing_grouped"]) + notes = group.get_all_notes() + assert len(notes) == 2 + + def test_model_grouped(self): + group = NoteGrouping.from_dict(working_note_groupings["note_model_grouped"]) + assert group.note_model == "model_name" + assert all([note.note_model is None for note in group.notes]) + + notes = group.get_all_notes() + assert {note.note_model for note in notes} == {"model_name"} + + def test_tags_grouped(self): + group = NoteGrouping.from_dict(working_note_groupings["tags_grouped"]) + assert group.tags == ["noun", "other"] + assert all([note.tags is None or note.tags == [] for note in group.notes]) + + notes = group.get_all_notes() + assert all([note.tags == ["noun", "other"] for note in notes]) + + def test_tags_grouped_as_addition(self): + group = NoteGrouping.from_dict(working_note_groupings["tags_grouped_as_addition"]) + assert group.tags == ["test", "recent"] + assert all([note.tags is not None for note in group.notes]) + + notes = group.get_all_notes() + assert notes[0].tags == ['noun', 'other', "test", "recent"] + assert notes[1].tags == ['marked', "test", "recent"] + + def test_no_tags(self): + group = NoteGrouping.from_dict(working_note_groupings["tags_grouped"]) + group.tags = None + assert all([note.tags is None or note.tags == [] for note in group.notes]) + + notes = group.get_all_notes() + assert all([note.tags == [] for note in notes]) diff --git a/tests/test_files/deck_parts/yaml/note/note1.yaml b/tests/test_files/deck_parts/yaml/note/note1.yaml index 9224b3c..a79da3d 100644 --- a/tests/test_files/deck_parts/yaml/note/note1.yaml +++ b/tests/test_files/deck_parts/yaml/note/note1.yaml @@ -1,30 +1,31 @@ -- note_model: LL Noun - tags: - - noun - - english - notes: - - guid: 7ysf7ysd8f8 - fields: - - test - - blah - - another one - - guid: sfkdsfhsd - fields: - - first - - second - - third -- note_model: LL Verb - tags: - - verb - - english - notes: - - guid: dhdfhsdf - fields: - - verby - - boo - - another one - - guid: dfgdfhgjs - fields: - - first - - second - - third +note_groupings: + - note_model: LL Noun + tags: + - noun + - english + notes: + - guid: 7ysf7ysd8f8 + fields: + - test + - blah + - another one + - guid: sfkdsfhsd + fields: + - first + - second + - third + - note_model: LL Verb + tags: + - verb + - english + notes: + - guid: dhdfhsdf + fields: + - verby + - boo + - another one + - guid: dfgdfhgjs + fields: + - first + - second + - third From b1edcd2f3bc435c83d5fefb33378769235795a59 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sat, 20 Jun 2020 08:40:16 +0200 Subject: [PATCH 07/39] Minor changes --- Pipfile.lock | 8 ++++---- brain_brew/representation/yaml/note_repr.py | 7 +++++-- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Pipfile.lock b/Pipfile.lock index 8089dd5..09b156b 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "a1fdf18fb74ab9ec7c5d2e956b782058ac37275e59250e5067bf531a4bc44c5c" + "sha256": "db6cc2735bbbbd8ba94783e7605faabf6ec983a5703710bcf39f5176a84ca6f1" }, "pipfile-spec": 6, "requires": { @@ -188,10 +188,10 @@ }, "wcwidth": { "hashes": [ - "sha256:980fbf4f3c196c0f329cdcd1e84c554d6a211f18e252e525a0cf4223154a41d6", - "sha256:edbc2b718b4db6cdf393eefe3a420183947d6aa312505ce6754516f458ff8830" + "sha256:79375666b9954d4a1a10739315816324c3e73110af9d0e102d906fdb0aec009f", + "sha256:8c6b5b6ee1360b842645f336d9e5d68c55817c26d3050f46b235ef2bc650e48f" ], - "version": "==0.2.3" + "version": "==0.2.4" }, "zipp": { "hashes": [ diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index 7b73966..fc56b48 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -71,9 +71,12 @@ def dump_to_yaml(self, file): yaml_dump.dump(self.encode(), fp) def verify_groupings(self): + errors = [] if self.note_model is not None: - modelErrors = ValueError(f"NoteGrouping for 'note_model' {self.note_model} has notes with 'note_model'. " - f"Please remove one of these.") if any([note.note_model for note in self.notes]) else None + if any([note.note_model for note in self.notes]): + errors.append(ValueError(f"NoteGrouping for 'note_model' {self.note_model} has notes with 'note_model'." + f" Please remove one of these.")) + return errors def get_all_notes(self) -> List[Note]: def join_tags(n_tags): From 7ba74f6bb9a21fb0c994fb53bddc101911d65a6d Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sat, 20 Jun 2020 10:22:44 +0200 Subject: [PATCH 08/39] Notes Media References and Note Models; Test Tidy-up; --- .../representation/json/deck_part_notes.py | 12 +- brain_brew/representation/yaml/note_repr.py | 27 +- brain_brew/utils.py | 11 + .../json/test_deck_part_notes.py | 23 -- tests/representation/yaml/test_note_repr.py | 76 ++++- tests/test_helpers.py | 262 ------------------ tests/test_utils.py | 25 ++ 7 files changed, 122 insertions(+), 314 deletions(-) create mode 100644 tests/test_utils.py diff --git a/brain_brew/representation/json/deck_part_notes.py b/brain_brew/representation/json/deck_part_notes.py index 2c0975c..cea5868 100644 --- a/brain_brew/representation/json/deck_part_notes.py +++ b/brain_brew/representation/json/deck_part_notes.py @@ -7,6 +7,7 @@ from brain_brew.representation.json.json_file import JsonFile from brain_brew.constants.deckpart_keys import * from brain_brew.representation.generic.media_file import MediaFile +from brain_brew.utils import find_media_in_field class CANoteKeys(Enum): @@ -202,7 +203,7 @@ def find_all_media_references(self): for note in self._data[DeckPartNoteKeys.NOTES.value]: for field in note[DeckPartNoteKeys.FIELDS.value]: - files_found = self.find_media_in_field(field) + files_found = find_media_in_field(field) if files_found: for filename in files_found: file = self.file_manager.media_file_if_exists(filename) @@ -218,12 +219,3 @@ def find_all_media_references(self): logging.info(f"Found {len(self.referenced_media_files)} referenced media files") - @staticmethod - def find_media_in_field(field_value): - if not isinstance(field_value, str): - return [] - - images = re.findall(r'<\s*?img.*?src="(.*?)"[^>]*?>', field_value) - audio = re.findall(r'\[sound:(.*?)\]', field_value) - - return images + audio diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index fc56b48..a0cf849 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -1,7 +1,9 @@ from brain_brew.representation.yaml.my_yaml import yaml_dump, yaml_load import json from dataclasses import dataclass -from typing import List, Optional, Dict +from typing import List, Optional, Dict, Set + +from brain_brew.utils import find_media_in_field FIELDS = 'fields' GUID = 'guid' @@ -9,6 +11,7 @@ NOTE_MODEL = 'note_model' NOTES = "notes" NOTE_GROUPINGS = "note_groupings" +MEDIA_REFERENCES = "media_references" @dataclass @@ -28,6 +31,7 @@ def encode_groupable(self, data_dict): class Note(GroupableNoteData): fields: List[str] guid: str + media_references: Optional[Set[str]] @classmethod def from_dict(cls, data: dict): @@ -35,7 +39,8 @@ def from_dict(cls, data: dict): fields=data.get(FIELDS), guid=data.get(GUID), note_model=data.get(NOTE_MODEL, None), - tags=data.get(TAGS, None) + tags=data.get(TAGS, None), + media_references=data.get(MEDIA_REFERENCES, None) ) def encode(self) -> dict: @@ -47,6 +52,9 @@ def dump_to_yaml(self, file): with open(file, 'w') as fp: yaml_dump.dump(self.encode(), fp) + def get_media_references(self) -> Set[str]: + return {entry for field in self.fields for entry in find_media_in_field(field)} + @dataclass class NoteGrouping(GroupableNoteData): @@ -70,6 +78,10 @@ def dump_to_yaml(self, file): with open(file, 'w') as fp: yaml_dump.dump(self.encode(), fp) + # TODO: Extract Shared Tags and Note Models + # TODO: Sort notes + # TODO: Set data + def verify_groupings(self): errors = [] if self.note_model is not None: @@ -78,7 +90,10 @@ def verify_groupings(self): f" Please remove one of these.")) return errors - def get_all_notes(self) -> List[Note]: + def get_all_known_note_model_names(self) -> set: + return {self.note_model} if self.note_model else {note.note_model for note in self.notes} + + def get_all_notes_copy(self) -> List[Note]: def join_tags(n_tags): if self.tags is None and n_tags is None: return [] @@ -93,7 +108,8 @@ def join_tags(n_tags): note_model=self.note_model if self.note_model is not None else n.note_model, tags=join_tags(n.tags), fields=n.fields, - guid=n.guid + guid=n.guid, + media_references=n.media_references or n.get_media_references() ) for n in self.notes] @@ -115,4 +131,5 @@ def dump_to_yaml(self, file): with open(file, 'w') as fp: yaml_dump.dump(self.encode(), fp) - + def get_all_known_note_model_names(self): + return {nms for group in self.note_groupings for nms in group.get_all_known_note_model_names()} diff --git a/brain_brew/utils.py b/brain_brew/utils.py index 6c28637..8f6bc3d 100644 --- a/brain_brew/utils.py +++ b/brain_brew/utils.py @@ -3,6 +3,7 @@ import string import random import re +from typing import List def blank_str_if_none(s): @@ -23,6 +24,16 @@ def filename_from_full_path(full_path): return re.findall('[^\\/:*?"<>|\r\n]+$', full_path)[0] +def find_media_in_field(field_value: str) -> List[str]: + if not field_value: + return [] + + images = re.findall(r'<\s*?img.*?src="(.*?)"[^>]*?>', field_value) + audio = re.findall(r'\[sound:(.*?)\]', field_value) + + return images + audio + + def find_all_files_in_directory(directory, recursive=False): found_files = [] for path, dirs, files in os.walk(directory): diff --git a/tests/representation/json/test_deck_part_notes.py b/tests/representation/json/test_deck_part_notes.py index a721be5..ed1be21 100644 --- a/tests/representation/json/test_deck_part_notes.py +++ b/tests/representation/json/test_deck_part_notes.py @@ -3,7 +3,6 @@ from brain_brew.representation.json.json_file import JsonFile from brain_brew.representation.json.deck_part_notes import DeckPartNotes, DeckPartNoteKeys from tests.test_files import TestFiles -from tests.test_helpers import note_models_mock from tests.representation.configuration.test_global_config import global_config @@ -62,28 +61,6 @@ def test_insensitive(self): pass -class TestFindMedia: - @pytest.mark.parametrize("field_value, expected_results", [ - (r'', ["image.png"]), - (r'', ["image.png"]), - (r'< img src="image.png">', ["image.png"]), - (r'< img src="image.png">', ["image.png"]), - (r'', ["image.png"]), - (r'', ["image.png"]), - (r'', ["image.png"]), - (r'words in the field end other stuff', ["image.png"]), - (r'', ["ug-map-saint_barthelemy.png"]), - (r'', - ["ug-map-saint_barthelemy.png", "image.png"]), - (r'[sound:test.mp3]', ["test.mp3"]), - (r'[sound:test.mp3][sound:othersound.mp3]', ["test.mp3", "othersound.mp3"]), - (r'[sound:test.mp3] [sound:othersound.mp3]', ["test.mp3", "othersound.mp3"]), - (r'words in the field [sound:test.mp3] other stuff too [sound:othersound.mp3] end', ["test.mp3", "othersound.mp3"]), - ]) - def test_find_media_in_field(self, field_value, expected_results): - assert DeckPartNotes.find_media_in_field(field_value) == expected_results - - # def test_set_data_from_override(): # assert False # diff --git a/tests/representation/yaml/test_note_repr.py b/tests/representation/yaml/test_note_repr.py index acadf4f..d340d04 100644 --- a/tests/representation/yaml/test_note_repr.py +++ b/tests/representation/yaml/test_note_repr.py @@ -1,7 +1,7 @@ import json import sys from textwrap import dedent -from typing import List +from typing import List, Set from ruamel.yaml import round_trip_dump from brain_brew.representation.yaml.my_yaml import yaml_dump, yaml_load @@ -23,11 +23,18 @@ working_note_groupings = { "nothing_grouped": {NOTES: [working_notes["test1"], working_notes["test2"]]}, "note_model_grouped": {NOTES: [working_notes["no_note_model"], working_notes["no_note_model2"]], NOTE_MODEL: "model_name"}, + "note_model_grouped2": {NOTES: [working_notes["no_note_model"], working_notes["no_note_model2"]], NOTE_MODEL: "different_model"}, "tags_grouped": {NOTES: [working_notes["no_tags1"], working_notes["no_tags2"]], TAGS: ["noun", "other"]}, "tags_grouped_as_addition": {NOTES: [working_notes["test1"], working_notes["test2"]], TAGS: ["test", "recent"]}, "model_and_tags_grouped": {NOTES: [working_notes["no_model_or_tags"], working_notes["no_model_or_tags"]], NOTE_MODEL: "model_name", TAGS: ["noun", "other"]} } +working_dpns = { + "one_group": {NOTE_GROUPINGS: [working_note_groupings["nothing_grouped"]]}, + "two_groups_two_models": {NOTE_GROUPINGS: [working_note_groupings["nothing_grouped"], working_note_groupings["note_model_grouped"]]}, + "two_groups_three_models": {NOTE_GROUPINGS: [working_note_groupings["nothing_grouped"], working_note_groupings["note_model_grouped2"]]}, +} + @pytest.fixture(params=working_notes.values()) def note_fixtures(request): @@ -41,19 +48,21 @@ def note_grouping_fixtures(request): class TestConstructor: class TestNote: - @pytest.mark.parametrize("fields, guid, note_model, tags", [ - ([], "", "", []), - (None, None, None, None), - (["test", "blah", "whatever"], "1234567890x", "model_name", ["noun"]) + @pytest.mark.parametrize("fields, guid, note_model, tags, media", [ + ([], "", "", [], {}), + (None, None, None, None, None), + (["test", "blah", "whatever"], "1234567890x", "model_name", ["noun"], {}), + (["test", "blah", ""], "1234567890x", "model_name", ["noun"], {"animal.jpg"}), ]) - def test_constructor(self, fields: List[str], guid: str, note_model: str, tags: List[str]): - note = Note(fields=fields, guid=guid, note_model=note_model, tags=tags) + def test_constructor(self, fields: List[str], guid: str, note_model: str, tags: List[str], media: Set[str]): + note = Note(fields=fields, guid=guid, note_model=note_model, tags=tags, media_references=media) assert isinstance(note, Note) assert note.fields == fields assert note.guid == guid assert note.note_model == note_model assert note.tags == tags + assert note.media_references == media def test_from_dict(self, note_fixtures): assert isinstance(note_fixtures, Note) @@ -276,11 +285,50 @@ def test_two_groupings(self, tmpdir): class TestFunctionality: - class TestNoteGrouping: - class TestGetAllNotes: + class TestGetMediaReferences: + class TestNote: + @pytest.mark.parametrize("fields, expected_count", [ + ([], 0), + (["nothing", "empty", "can't find nothing here"], 0), + (["", "empty", "can't find nothing here"], 1), + (["", "", ""], 1), + (["", "", ""], 3), + (["", "[sound:test.mp3]", "[sound:test.mp3]"], 2), + ]) + def test_all(self, fields, expected_count): + note = Note(fields=fields, note_model=None, guid="", tags=None, media_references=None) + media_found = note.get_media_references() + assert isinstance(media_found, Set) + assert len(media_found) == expected_count + + class TestGetAllNoteModels: + class TestNoteGrouping: + def test_nothing_grouped(self): + group = NoteGrouping.from_dict(working_note_groupings["nothing_grouped"]) + models = group.get_all_known_note_model_names() + assert models == {'LL Test', 'model_name'} + + def test_grouped(self): + group = NoteGrouping.from_dict(working_note_groupings["note_model_grouped"]) + models = group.get_all_known_note_model_names() + assert models == {'model_name'} + + class TestDeckPartNotes: + def test_two_groups_two_models(self): + dpn = DeckPartNotes.from_dict(working_dpns["two_groups_two_models"]) + models = dpn.get_all_known_note_model_names() + assert models == {'LL Test', 'model_name'} + + def test_two_groups_three_models(self): + dpn = DeckPartNotes.from_dict(working_dpns["two_groups_three_models"]) + models = dpn.get_all_known_note_model_names() + assert models == {'LL Test', 'model_name', 'different_model'} + + class TestGetAllNotes: + class TestNoteGrouping: def test_nothing_grouped(self): group = NoteGrouping.from_dict(working_note_groupings["nothing_grouped"]) - notes = group.get_all_notes() + notes = group.get_all_notes_copy() assert len(notes) == 2 def test_model_grouped(self): @@ -288,7 +336,7 @@ def test_model_grouped(self): assert group.note_model == "model_name" assert all([note.note_model is None for note in group.notes]) - notes = group.get_all_notes() + notes = group.get_all_notes_copy() assert {note.note_model for note in notes} == {"model_name"} def test_tags_grouped(self): @@ -296,7 +344,7 @@ def test_tags_grouped(self): assert group.tags == ["noun", "other"] assert all([note.tags is None or note.tags == [] for note in group.notes]) - notes = group.get_all_notes() + notes = group.get_all_notes_copy() assert all([note.tags == ["noun", "other"] for note in notes]) def test_tags_grouped_as_addition(self): @@ -304,7 +352,7 @@ def test_tags_grouped_as_addition(self): assert group.tags == ["test", "recent"] assert all([note.tags is not None for note in group.notes]) - notes = group.get_all_notes() + notes = group.get_all_notes_copy() assert notes[0].tags == ['noun', 'other', "test", "recent"] assert notes[1].tags == ['marked', "test", "recent"] @@ -313,5 +361,5 @@ def test_no_tags(self): group.tags = None assert all([note.tags is None or note.tags == [] for note in group.notes]) - notes = group.get_all_notes() + notes = group.get_all_notes_copy() assert all([note.tags == [] for note in notes]) diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 45caaeb..fa8ea97 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,268 +1,6 @@ -from tempfile import NamedTemporaryFile, TemporaryDirectory - -import pytest - -from brain_brew.constants.deckpart_keys import DeckPartNoteKeys, NoteFlagKeys from brain_brew.representation.json.json_file import JsonFile -from brain_brew.representation.json.deck_part_header import DeckPartHeader -from brain_brew.representation.json.deck_part_notemodel import DeckPartNoteModel, CANoteModelKeys -from brain_brew.representation.json.deck_part_notes import DeckPartNotes def debug_write_to_target_json(data, json: JsonFile): json.set_data(data) json.write_file() - - -def setup_temp_file_in_folder(file_suffix): - temp_dir = TemporaryDirectory() - temp_file = NamedTemporaryFile(suffix=file_suffix, delete=False, dir=temp_dir.name) - - return temp_dir, temp_file - - -# def get_global_config(note_model_loc: str = "", group_by_note_model=True, extract_shared_tags=False) -> GlobalConfig: -# config = GlobalConfig.get_instance(override=GlobalConfig({ -# ConfigKeys.DECK_PARTS.value: { -# "headers": "", -# "note_models": note_model_loc, -# "notes": "", -# -# ConfigKeys.DECK_PARTS_NOTES_STRUCTURE.value: { -# NoteFlagKeys.GROUP_BY_NOTE_MODEL.value: group_by_note_model, -# NoteFlagKeys.EXTRACT_SHARED_TAGS.value: extract_shared_tags -# } -# }, -# ConfigKeys.FLAGS.value: { -# -# } -# })) -# -# return config - - -# @pytest.fixture() -# def global_config(): -# return GlobalConfig.get_instance(override=GlobalConfig({ -# ConfigKeys.DECK_PARTS.value: { -# "headers": "", -# "note_models": "", -# "notes": "", -# -# ConfigKeys.DECK_PARTS_NOTES_STRUCTURE.value: { -# NoteFlagKeys.GROUP_BY_NOTE_MODEL.value: False, -# NoteFlagKeys.EXTRACT_SHARED_TAGS.value: False -# } -# }, -# ConfigKeys.FLAGS.value: { -# -# } -# })) - - -def make_deck_part_header_mock(data): - return DeckPartHeader( - NamedTemporaryFile(suffix=".json", delete=False).name, - read_now=False, - data_override=data - ) - - -def make_deck_part_note_model_mock(data): - return DeckPartNoteModel( - NamedTemporaryFile(suffix=".json", delete=False).name, - read_now=False, - data_override=data - ) - - -def make_deck_part_notes_mock(data): - return DeckPartNotes( - NamedTemporaryFile(suffix=".json", delete=False).name, - data_override=data - ) - - -@pytest.fixture() -def headers_mock(): - return make_deck_part_header_mock({ - "__type__": "Deck", - "crowdanki_uuid": "72ac74b8-0077-11ea-959e-d8cb8ac9abf0", - "deck_config_uuid": "3cc64d85-e410-11e9-960e-d8cb8ac9abf0", - "name": "LL::1. Vocab" - }) - - -@pytest.fixture() -def note_models_mock(): - return [ - make_deck_part_note_model_mock({ - "__type__": "NoteModel", - CANoteModelKeys.ID.value: "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa", - CANoteModelKeys.NAME.value: "LL Word", - CANoteModelKeys.FIELDS.value: [ - {"name": "Word"}, - {"name": "OtherWord"} - ] - }), - make_deck_part_note_model_mock({ - "__type__": "NoteModel", - CANoteModelKeys.ID.value: "bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb", - CANoteModelKeys.NAME.value: "LL Verb", - CANoteModelKeys.FIELDS.value: [ - {"name": "Word"}, - {"name": "OtherWord"} - ] - }), - make_deck_part_note_model_mock({ - "__type__": "NoteModel", - CANoteModelKeys.ID.value: "cccccccc-cccc-cccc-cccc-cccccccccccc", - CANoteModelKeys.NAME.value: "LL Noun", - CANoteModelKeys.FIELDS.value: [ - {"name": "Word"}, - {"name": "OtherWord"} - ] - }), - make_deck_part_note_model_mock({ - "__type__": "NoteModel", - CANoteModelKeys.ID.value: "dddddddd-dddd-dddd-dddd-dddddddddddd", - CANoteModelKeys.NAME.value: "LL Sentence", - CANoteModelKeys.FIELDS.value: [ - {"name": "Word"}, - {"name": "OtherWord"} - ] - }), - make_deck_part_note_model_mock({ - "__type__": "NoteModel", - CANoteModelKeys.ID.value: "eeeeeeee-eeee-eeee-eeee-eeeeeeeeeeee", - CANoteModelKeys.NAME.value: "Cloze", - CANoteModelKeys.FIELDS.value: [ - {"name": "Word"}, - {"name": "OtherWord"} - ] - }) - ] - - -@pytest.fixture() -def notes_mock(): - return make_deck_part_notes_mock({ - DeckPartNoteKeys.FLAGS.value: { - NoteFlagKeys.GROUP_BY_NOTE_MODEL.value: True, - NoteFlagKeys.EXTRACT_SHARED_TAGS.value: False - }, - "Cloze": { - "notes": [ - { - "fields": [ - "testers", - "" - ], - "guid": "wpoGpMPjTD", - "tags": [] - } - ] - }, - "LL Noun": { - "notes": [ - { - "fields": [ - "banana", - "en banan", - "banano", - "", - "", - "[sound:pronunciation_da_banan.mp3]", - "" - ], - "guid": "sZGs^rTTEr", - "tags": [ - "LL::Grammar::Noun" - ] - } - ] - }, - "LL Sentence": { - "notes": [ - { - "fields": [ - "I don't understand", - "", - "y", - "y", - "jeg", - "forstår", - "ikke", - "", - "", - "mi ne", - "komprenas", - "", - "", - "" - ], - "guid": "x+JWF/d?][", - "tags": [ - "CardType::Sentence" - ] - } - ] - }, - "LL Verb": { - "notes": [ - { - "fields": [ - "to learn", - "at lære", - "lerni", - "", - "", - "[sound:pronunciation_da_lære.mp3]", - "", - "lærer", - "lærte", - "har lært" - ], - "guid": "ONZwlFyCX1", - "tags": [ - "LL::Grammar::Verb" - ] - } - ] - }, - "LL Word": { - "notes": [ - { - "fields": [ - "you", - "du", - "vi", - "", - "Singular, Subjective, 2nd Person", - "[sound:pronunciation_da_du.mp3]", - "" - ], - "guid": "uZFR2ToY#3", - "tags": [ - "LL::Grammar::Pronoun" - ] - }, - { - "fields": [ - "want", - "test", - "", - "", - "", - "", - "" - ], - "guid": "zlEirC^ya/", - "tags": [ - "LL::Grammar::Noun", - "LL::Grammar::Verb" - ] - } - ] - } - }) diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..489dc22 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,25 @@ +import pytest + +from brain_brew.utils import find_media_in_field + + +class TestFindMedia: + @pytest.mark.parametrize("field_value, expected_results", [ + (r'', ["image.png"]), + (r'', ["image.png"]), + (r'< img src="image.png">', ["image.png"]), + (r'< img src="image.png">', ["image.png"]), + (r'', ["image.png"]), + (r'', ["image.png"]), + (r'', ["image.png"]), + (r'words in the field end other stuff', ["image.png"]), + (r'', ["ug-map-saint_barthelemy.png"]), + (r'', + ["ug-map-saint_barthelemy.png", "image.png"]), + (r'[sound:test.mp3]', ["test.mp3"]), + (r'[sound:test.mp3][sound:othersound.mp3]', ["test.mp3", "othersound.mp3"]), + (r'[sound:test.mp3] [sound:othersound.mp3]', ["test.mp3", "othersound.mp3"]), + (r'words in the field [sound:test.mp3] other stuff too [sound:othersound.mp3] end', ["test.mp3", "othersound.mp3"]), + ]) + def test_find_media_in_field(self, field_value, expected_results): + assert find_media_in_field(field_value) == expected_results From 5280e6d04b69dc7834839ff8b5529f78774b5291 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sat, 27 Jun 2020 12:20:00 +0200 Subject: [PATCH 09/39] Yaml and Dataclass'd the FileMapping and NoteModelMappings --- brain_brew/build_tasks/source_csv.py | 8 +- .../configuration/csv_file_mapping.py | 106 ++++++++---------- .../configuration/note_model_mapping.py | 59 +++++----- .../deck_part_transformers/__init__.py | 0 .../tr_notes_crowdanki.py | 14 +++ .../tr_notes_csv_collection.py | 12 ++ .../tr_notes_generic.py | 11 ++ brain_brew/representation/yaml/note_repr.py | 1 - brain_brew/utils.py | 2 + 9 files changed, 114 insertions(+), 99 deletions(-) create mode 100644 brain_brew/representation/deck_part_transformers/__init__.py create mode 100644 brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py create mode 100644 brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py create mode 100644 brain_brew/representation/deck_part_transformers/tr_notes_generic.py diff --git a/brain_brew/build_tasks/source_csv.py b/brain_brew/build_tasks/source_csv.py index 519f249..f72dd7d 100644 --- a/brain_brew/build_tasks/source_csv.py +++ b/brain_brew/build_tasks/source_csv.py @@ -47,11 +47,9 @@ def __init__(self, config_data: dict, read_now=True): self.notes = DeckPartNotes.create(self.get_config(BuildConfigKeys.NOTES), read_now=read_now) nm_mapping = [NoteModelMapping(config, read_now=read_now) - for config in - self.get_config(SourceCsvKeys.NOTE_MODEL_MAPPINGS) - ] - self.note_model_mappings_dict = {mapping.note_model.name: mapping - for mapping in nm_mapping} + for config in self.get_config(SourceCsvKeys.NOTE_MODEL_MAPPINGS)] + + self.note_model_mappings_dict = {mapping.note_model.name: mapping for mapping in nm_mapping} self.csv_file_mappings = [CsvFileMapping(config, read_now=read_now) for config in self.get_config(SourceCsvKeys.CSV_MAPPINGS)] diff --git a/brain_brew/representation/configuration/csv_file_mapping.py b/brain_brew/representation/configuration/csv_file_mapping.py index 26b069b..2291a77 100644 --- a/brain_brew/representation/configuration/csv_file_mapping.py +++ b/brain_brew/representation/configuration/csv_file_mapping.py @@ -1,72 +1,65 @@ import logging +from dataclasses import dataclass, field from enum import Enum -from typing import Dict, List +from typing import Dict, List, Optional, Union from brain_brew.constants.deckpart_keys import DeckPartNoteKeys from brain_brew.interfaces.verifiable import Verifiable from brain_brew.interfaces.writes_file import WritesFile from brain_brew.representation.generic.csv_file import CsvFile, CsvKeys -from brain_brew.representation.generic.generic_file import GenericFile from brain_brew.representation.generic.yaml_file import YamlFile, ConfigKey -from brain_brew.representation.json.deck_part_notemodel import DeckPartNoteModel -from brain_brew.utils import single_item_to_list, generate_anki_guid, list_of_str_to_lowercase +from brain_brew.utils import single_item_to_list, generate_anki_guid -class CsvFileMappingKeys(Enum): - CSV_FILE = "csv" - NOTE_MODEL = "note_model" - SORT_BY_COLUMNS = "sort_by_columns" - REVERSE_SORT = "reverse_sort" - DERIVATIVES = "derivatives" +FILE = "csv" +NOTE_MODEL = "note_model" +SORT_BY_COLUMNS = "sort_by_columns" +REVERSE_SORT = "reverse_sort" +DERIVATIVES = "derivatives" -class CsvFileMappingDerivative(YamlFile): - config_entry = {} - expected_keys = { - CsvFileMappingKeys.CSV_FILE.value: ConfigKey(True, str, None), - CsvFileMappingKeys.SORT_BY_COLUMNS.value: ConfigKey(False, (list, str), None), - CsvFileMappingKeys.REVERSE_SORT.value: ConfigKey(False, bool, None), - CsvFileMappingKeys.DERIVATIVES.value: ConfigKey(False, list, None), +@dataclass +class CsvFileMappingDerivative: + csv_file: CsvFile = field(init=False) + compiled_data: Dict[str, dict] = field(init=False) - CsvFileMappingKeys.NOTE_MODEL.value: ConfigKey(False, str, None), # Optional on Derivatives - } - subconfig_filter = None + file: str + note_model: Optional[str] - csv_file: CsvFile - compiled_data: Dict[str, dict] + sort_by_columns: Optional[Union[list, str]] + reverse_sort: Optional[bool] - sort_by_columns: list - reverse_sort: bool + derivatives: Optional[List['CsvFileMappingDerivative']] - note_model_name: str - derivatives: List['CsvFileMappingDerivative'] - - def __init__(self, config_data, read_now=True): - super().__init__() - - self.setup_config_with_subconfig_replacement(config_data) - self.verify_config_entry() + @classmethod + def from_dict(cls, data: dict): + return cls( + file=data.get(FILE, None), + note_model=data.get(NOTE_MODEL, None), + sort_by_columns=data.get(SORT_BY_COLUMNS, None), + reverse_sort=data.get(REVERSE_SORT, None), + derivatives=list(map(cls.from_dict, data.get(DERIVATIVES, None))) + ) - self.csv_file = CsvFile.create(self.get_config(CsvFileMappingKeys.CSV_FILE), read_now=read_now) + def __post_init__(self): + self.csv_file = CsvFile.create(self.file, read_now=True) # TODO: Fix Read Now - self.sort_by_columns = single_item_to_list(self.get_config(CsvFileMappingKeys.SORT_BY_COLUMNS, [])) - self.reverse_sort = self.get_config(CsvFileMappingKeys.REVERSE_SORT, False) + if self.note_model == "": + self.note_model = None - self.note_model_name = self.get_config(CsvFileMappingKeys.NOTE_MODEL, "") - self.note_model_name = None if self.note_model_name == "" else self.note_model_name + self.sort_by_columns = single_item_to_list(self.sort_by_columns) - self.derivatives = [CsvFileMappingDerivative.create_derivative(config, read_now=read_now) - for config in self.get_config(CsvFileMappingKeys.DERIVATIVES, [])] + if self.reverse_sort is None: + self.reverse_sort = False - @classmethod - def create_derivative(cls, config_data, read_now=True): - return cls(config_data, read_now=read_now) + if self.derivatives is None: + self.derivatives = [] def get_available_columns(self): return self.csv_file.column_headers + [col for der in self.derivatives for col in der.get_available_columns()] def get_used_note_model_names(self) -> List[str]: - nm = [self.note_model_name] if self.note_model_name is not None else [] + nm = [self.note_model] if self.note_model is not None else [] return nm + [name for der in self.derivatives for name in der.get_used_note_model_names()] def _build_data_recursive(self) -> List[dict]: @@ -97,8 +90,8 @@ def _build_data_recursive(self) -> List[dict]: row[der_col] = der_row[der_col] found_match = True # Set Note Model to matching Derivative Note Model - if der.note_model_name is not None: - row.setdefault(DeckPartNoteKeys.NOTE_MODEL.value, der.note_model_name) + if der.note_model is not None: + row.setdefault(DeckPartNoteKeys.NOTE_MODEL.value, der.note_model) break if not found_match: der_match_errors.append(ValueError(f"Cannot match derivative row {der_row} to parent")) @@ -116,24 +109,15 @@ def write_to_csv(self, data_to_set): der.write_to_csv(data_to_set) +@dataclass class CsvFileMapping(CsvFileMappingDerivative, Verifiable, WritesFile): - expected_keys = { - CsvFileMappingKeys.CSV_FILE.value: ConfigKey(True, str, None), - CsvFileMappingKeys.SORT_BY_COLUMNS.value: ConfigKey(False, (list, str), None), - CsvFileMappingKeys.REVERSE_SORT.value: ConfigKey(False, bool, None), - CsvFileMappingKeys.DERIVATIVES.value: ConfigKey(False, list, None), - - CsvFileMappingKeys.NOTE_MODEL.value: ConfigKey(True, str, None), # Required on top level - } - - data_set_has_changed: bool + note_model: str # Override Optional on Parent - def __init__(self, config_data, read_now=True): - super().__init__(config_data, read_now=read_now) + data_set_has_changed: bool = field(init=False, default=False) def verify_contents(self): - if self.note_model_name is None: - raise KeyError(f"Top level Csv Mapping requires key {CsvFileMappingKeys.NOTE_MODEL.value}") + if self.note_model is "": + raise KeyError(f"Top level Csv Mapping requires key {NOTE_MODEL}") def compile_data(self): self.compiled_data = {} @@ -142,9 +126,9 @@ def compile_data(self): data_in_progress = self._build_data_recursive() # Set Note Model if not already set - if self.note_model_name is not None: + if self.note_model is not None: for row in data_in_progress: - row.setdefault(DeckPartNoteKeys.NOTE_MODEL.value, self.note_model_name) + row.setdefault(DeckPartNoteKeys.NOTE_MODEL.value, self.note_model) # Fill in Guid if no Guid guids_generated = 0 diff --git a/brain_brew/representation/configuration/note_model_mapping.py b/brain_brew/representation/configuration/note_model_mapping.py index 4d27325..fa16980 100644 --- a/brain_brew/representation/configuration/note_model_mapping.py +++ b/brain_brew/representation/configuration/note_model_mapping.py @@ -1,5 +1,6 @@ +from dataclasses import dataclass from enum import Enum -from typing import List +from typing import List, Union, Dict from brain_brew.constants.deckpart_keys import DeckPartNoteKeys from brain_brew.interfaces.verifiable import Verifiable @@ -8,10 +9,9 @@ from brain_brew.utils import list_of_str_to_lowercase -class NoteModelMappingKeys(Enum): - NOTE_MODEL = "note_model" - COLUMNS = "csv_columns_to_fields" - PERSONAL_FIELDS = "personal_fields" +NOTE_MODEL = "note_model" +COLUMNS = "csv_columns_to_fields" +PERSONAL_FIELDS = "personal_fields" class FieldMapping: @@ -38,39 +38,34 @@ def __init__(self, field_type: FieldMappingType, field_name: str, value: str): self.value = value -class NoteModelMapping(YamlFile, Verifiable): - config_entry = {} - expected_keys = { - NoteModelMappingKeys.NOTE_MODEL.value: ConfigKey(True, str, None), - NoteModelMappingKeys.COLUMNS.value: ConfigKey(True, dict, None), - NoteModelMappingKeys.PERSONAL_FIELDS.value: ConfigKey(False, list, None), - } - subconfig_filter = None +@dataclass +class NoteModelMappingRepresentation: + note_model: str # TODO: Union[str, list] + columns_to_fields: Dict[str, str] + personal_fields: List[str] - note_model: DeckPartNoteModel = None + +@dataclass +class NoteModelMapping(Verifiable): + note_model: DeckPartNoteModel columns: List[FieldMapping] personal_fields: List[FieldMapping] required_fields_definitions = [DeckPartNoteKeys.GUID.value, DeckPartNoteKeys.TAGS.value] - def __init__(self, config_data: dict, read_now=True): - self.setup_config_with_subconfig_replacement(config_data) - self.verify_config_entry() - - columns = self.get_config(NoteModelMappingKeys.COLUMNS) - personal_fields = self.get_config(NoteModelMappingKeys.PERSONAL_FIELDS, []) - - self.columns = [FieldMapping( - field_type=FieldMapping.FieldMappingType.COLUMN, - field_name=field, - value=columns[field]) for field in columns] - - self.personal_fields = [FieldMapping( - field_type=FieldMapping.FieldMappingType.PERSONAL_FIELD, - field_name=field, - value="") for field in personal_fields] - - self.note_model = DeckPartNoteModel.create(self.get_config(NoteModelMappingKeys.NOTE_MODEL), read_now=read_now) + @classmethod + def from_dict(cls, data: NoteModelMappingRepresentation): + return cls( + columns=[FieldMapping( + field_type=FieldMapping.FieldMappingType.COLUMN, + field_name=field, + value=key) for key, field in data.columns_to_fields.items()], + personal_fields=[FieldMapping( + field_type=FieldMapping.FieldMappingType.PERSONAL_FIELD, + field_name=field, + value="") for field in data.personal_fields], + note_model=DeckPartNoteModel.create(data.note_model, read_now=True) # TODO: Fix read_now + ) def verify_contents(self): errors = [] diff --git a/brain_brew/representation/deck_part_transformers/__init__.py b/brain_brew/representation/deck_part_transformers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py b/brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py new file mode 100644 index 0000000..6e5864d --- /dev/null +++ b/brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py @@ -0,0 +1,14 @@ +from dataclasses import dataclass +from typing import List, Optional + +from brain_brew.representation.deck_part_transformers.tr_notes_generic import TrNotesGeneric + + +@dataclass +class TrNotesCrowdAnki(TrNotesGeneric): + file: str + sort_order: Optional[List[str]] + media: Optional[bool] + useless_note_keys: Optional[dict] + + # crowdanki_file: CrowdAnkiExport ???? diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py new file mode 100644 index 0000000..341f09b --- /dev/null +++ b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py @@ -0,0 +1,12 @@ +from dataclasses import dataclass +from typing import List + +from brain_brew.representation.configuration.csv_file_mapping import CsvFileMapping +from brain_brew.representation.configuration.note_model_mapping import NoteModelMappingRepresentation +from brain_brew.representation.deck_part_transformers.tr_notes_generic import TrNotesGeneric + + +@dataclass +class TrNotesCsvCollection(TrNotesGeneric): + file_mappings: List[CsvFileMapping] + note_model_mappings: List[NoteModelMappingRepresentation] diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_generic.py b/brain_brew/representation/deck_part_transformers/tr_notes_generic.py new file mode 100644 index 0000000..65be791 --- /dev/null +++ b/brain_brew/representation/deck_part_transformers/tr_notes_generic.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass +from typing import Optional + +from brain_brew.representation.yaml.note_repr import DeckPartNotes + + +@dataclass +class TrNotesGeneric: + name: str + data: DeckPartNotes + save_to_file: Optional[str] diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index a0cf849..94b91ed 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -1,5 +1,4 @@ from brain_brew.representation.yaml.my_yaml import yaml_dump, yaml_load -import json from dataclasses import dataclass from typing import List, Optional, Dict, Set diff --git a/brain_brew/utils.py b/brain_brew/utils.py index 8f6bc3d..9d5787e 100644 --- a/brain_brew/utils.py +++ b/brain_brew/utils.py @@ -17,6 +17,8 @@ def list_of_str_to_lowercase(list_of_strings): def single_item_to_list(item): if isinstance(item, list): return item + if item is None: + return [] return [item] From 5e4f3fd5873e4193c43553229f762dc7636deb79 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sun, 28 Jun 2020 12:09:31 +0200 Subject: [PATCH 10/39] Working Representation Reading --- .../configuration/csv_file_mapping.py | 53 +++--- .../configuration/note_model_mapping.py | 17 +- .../tr_notes_crowdanki.py | 14 +- .../tr_notes_csv_collection.py | 34 +++- .../tr_notes_generic.py | 10 +- .../representation/generic/generic_file.py | 2 +- brain_brew/representation/yaml/note_repr.py | 7 +- .../configuration/test_csv_file_mapping.py | 14 +- .../configuration/test_note_model_mapping.py | 10 +- .../deck_part_transformers/__init__.py | 0 .../test_tr_notes_csv_collection.py | 164 ++++++++++++++++++ tests/representation/yaml/__init__.py | 0 tests/representation/yaml/test_note_repr.py | 7 +- 13 files changed, 273 insertions(+), 59 deletions(-) create mode 100644 tests/representation/deck_part_transformers/__init__.py create mode 100644 tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py create mode 100644 tests/representation/yaml/__init__.py diff --git a/brain_brew/representation/configuration/csv_file_mapping.py b/brain_brew/representation/configuration/csv_file_mapping.py index 2291a77..7598018 100644 --- a/brain_brew/representation/configuration/csv_file_mapping.py +++ b/brain_brew/representation/configuration/csv_file_mapping.py @@ -20,41 +20,44 @@ @dataclass class CsvFileMappingDerivative: - csv_file: CsvFile = field(init=False) + @dataclass(init=False) + class Representation: + file: str + note_model: Optional[str] + sort_by_columns: Optional[Union[list, str]] + reverse_sort: Optional[bool] + derivatives: Optional[List['CsvFileMappingDerivative.Representation']] + + def __init__(self, file, note_model=None, sort_by_columns=None, reverse_sort=None, derivatives=None): + self.file = file + self.note_model = note_model + self.sort_by_columns = sort_by_columns + self.reverse_sort = reverse_sort + self.derivatives = list(map(CsvFileMappingDerivative.Representation.from_dict, derivatives)) if derivatives is not None else [] + + @classmethod + def from_dict(cls, data: dict): + return cls(**data) + compiled_data: Dict[str, dict] = field(init=False) - file: str - note_model: Optional[str] + csv_file: CsvFile - sort_by_columns: Optional[Union[list, str]] + note_model: Optional[str] + sort_by_columns: Optional[list] reverse_sort: Optional[bool] - derivatives: Optional[List['CsvFileMappingDerivative']] @classmethod - def from_dict(cls, data: dict): + def from_repr(cls, data: Representation): return cls( - file=data.get(FILE, None), - note_model=data.get(NOTE_MODEL, None), - sort_by_columns=data.get(SORT_BY_COLUMNS, None), - reverse_sort=data.get(REVERSE_SORT, None), - derivatives=list(map(cls.from_dict, data.get(DERIVATIVES, None))) + csv_file=CsvFile.create(data.file, True), # TODO: Fix Read Now + note_model=None if not data.note_model.strip() else data.note_model.strip(), + sort_by_columns=single_item_to_list(data.sort_by_columns), + reverse_sort=data.reverse_sort or False, + derivatives=list(map(cls.from_repr, data.derivatives)) if data.derivatives is not None else [] ) - def __post_init__(self): - self.csv_file = CsvFile.create(self.file, read_now=True) # TODO: Fix Read Now - - if self.note_model == "": - self.note_model = None - - self.sort_by_columns = single_item_to_list(self.sort_by_columns) - - if self.reverse_sort is None: - self.reverse_sort = False - - if self.derivatives is None: - self.derivatives = [] - def get_available_columns(self): return self.csv_file.column_headers + [col for der in self.derivatives for col in der.get_available_columns()] diff --git a/brain_brew/representation/configuration/note_model_mapping.py b/brain_brew/representation/configuration/note_model_mapping.py index fa16980..c3dd513 100644 --- a/brain_brew/representation/configuration/note_model_mapping.py +++ b/brain_brew/representation/configuration/note_model_mapping.py @@ -39,14 +39,17 @@ def __init__(self, field_type: FieldMappingType, field_name: str, value: str): @dataclass -class NoteModelMappingRepresentation: - note_model: str # TODO: Union[str, list] - columns_to_fields: Dict[str, str] - personal_fields: List[str] +class NoteModelMapping(Verifiable): + @dataclass + class Representation: + note_model: str # TODO: Union[str, list] + columns_to_fields: Dict[str, str] + personal_fields: List[str] + @classmethod + def from_dict(cls, data: dict): + return cls(**data) -@dataclass -class NoteModelMapping(Verifiable): note_model: DeckPartNoteModel columns: List[FieldMapping] personal_fields: List[FieldMapping] @@ -54,7 +57,7 @@ class NoteModelMapping(Verifiable): required_fields_definitions = [DeckPartNoteKeys.GUID.value, DeckPartNoteKeys.TAGS.value] @classmethod - def from_dict(cls, data: NoteModelMappingRepresentation): + def from_repr(cls, data: Representation): return cls( columns=[FieldMapping( field_type=FieldMapping.FieldMappingType.COLUMN, diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py b/brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py index 6e5864d..eb09d65 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py @@ -1,14 +1,22 @@ from dataclasses import dataclass -from typing import List, Optional +from typing import List, Optional, Union from brain_brew.representation.deck_part_transformers.tr_notes_generic import TrNotesGeneric +from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport @dataclass class TrNotesCrowdAnki(TrNotesGeneric): - file: str + @dataclass + class Representation: + file: str + sort_order: Optional[Union[str, List[str]]] + media: Optional[bool] + useless_note_keys: Optional[dict] + + crowdanki_file: CrowdAnkiExport sort_order: Optional[List[str]] media: Optional[bool] useless_note_keys: Optional[dict] - # crowdanki_file: CrowdAnkiExport ???? +# TODO: Make Unique classes for Notes <-> CrowdAnki diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py index 341f09b..b2396fd 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py @@ -2,11 +2,41 @@ from typing import List from brain_brew.representation.configuration.csv_file_mapping import CsvFileMapping -from brain_brew.representation.configuration.note_model_mapping import NoteModelMappingRepresentation +from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping from brain_brew.representation.deck_part_transformers.tr_notes_generic import TrNotesGeneric @dataclass class TrNotesCsvCollection(TrNotesGeneric): + @dataclass(init=False) + class Representation(TrNotesGeneric.Representation): + file_mappings: List[CsvFileMapping.Representation] + note_model_mappings: List[NoteModelMapping.Representation] + + def __init__(self, name, file_mappings, note_model_mappings, save_to_file=None): + super().__init__(name, save_to_file) + self.file_mappings = list(map(CsvFileMapping.Representation.from_dict, file_mappings)) + self.note_model_mappings = list(map(NoteModelMapping.Representation.from_dict, note_model_mappings)) + + @classmethod + def from_dict(cls, data: dict): + return cls(**data) + file_mappings: List[CsvFileMapping] - note_model_mappings: List[NoteModelMappingRepresentation] + note_model_mappings: List[NoteModelMapping] + + @classmethod + def from_repr(cls, data: Representation): + return cls( + name=data.name, + save_to_file=data.save_to_file, + file_mappings=list(map(CsvFileMapping.from_repr, data.file_mappings)), + note_model_mappings=list(map(NoteModelMapping.from_repr, data.note_model_mappings)) + ) + + @classmethod + def from_dict(cls, data: dict): + return cls.from_repr(TrNotesCsvCollection.Representation.from_dict(data)) + + +# TODO: Make Unique classes for Notes <-> Csv diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_generic.py b/brain_brew/representation/deck_part_transformers/tr_notes_generic.py index 65be791..a1b7899 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_generic.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_generic.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Optional from brain_brew.representation.yaml.note_repr import DeckPartNotes @@ -6,6 +6,12 @@ @dataclass class TrNotesGeneric: + @dataclass + class Representation: + name: str + save_to_file: Optional[str] + name: str - data: DeckPartNotes save_to_file: Optional[str] + + data: DeckPartNotes = field(init=False) diff --git a/brain_brew/representation/generic/generic_file.py b/brain_brew/representation/generic/generic_file.py index 1543f96..3881815 100644 --- a/brain_brew/representation/generic/generic_file.py +++ b/brain_brew/representation/generic/generic_file.py @@ -27,7 +27,7 @@ def __init__(self, file, read_now, data_override): self.set_data(data_override) elif read_now: if not self.file_exists: - raise FileNotFoundError(file) + return # raise FileNotFoundError(file) # TODO: Fix self.data_state = GenericFile.DataState.READ_IN_DATA self.read_file() diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index 94b91ed..e36959a 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -30,7 +30,7 @@ def encode_groupable(self, data_dict): class Note(GroupableNoteData): fields: List[str] guid: str - media_references: Optional[Set[str]] + # media_references: Optional[Set[str]] @classmethod def from_dict(cls, data: dict): @@ -38,8 +38,7 @@ def from_dict(cls, data: dict): fields=data.get(FIELDS), guid=data.get(GUID), note_model=data.get(NOTE_MODEL, None), - tags=data.get(TAGS, None), - media_references=data.get(MEDIA_REFERENCES, None) + tags=data.get(TAGS, None) ) def encode(self) -> dict: @@ -108,7 +107,7 @@ def join_tags(n_tags): tags=join_tags(n.tags), fields=n.fields, guid=n.guid, - media_references=n.media_references or n.get_media_references() + # media_references=n.media_references or n.get_media_references() ) for n in self.notes] diff --git a/tests/representation/configuration/test_csv_file_mapping.py b/tests/representation/configuration/test_csv_file_mapping.py index ee5454c..59cf8fe 100644 --- a/tests/representation/configuration/test_csv_file_mapping.py +++ b/tests/representation/configuration/test_csv_file_mapping.py @@ -4,8 +4,8 @@ import pytest -from brain_brew.representation.configuration.csv_file_mapping import CsvFileMappingKeys, CsvFileMappingDerivative, \ - CsvFileMapping +from brain_brew.representation.configuration.csv_file_mapping import CsvFileMappingDerivative, CsvFileMapping, \ + SORT_BY_COLUMNS, REVERSE_SORT, NOTE_MODEL, DERIVATIVES, FILE from brain_brew.representation.generic.csv_file import CsvFile from tests.test_file_manager import get_new_file_manager from tests.representation.configuration.test_global_config import global_config @@ -15,16 +15,16 @@ def setup_csv_fm_config(csv: str, sort_by_columns: List[str] = None, reverse_sort: bool = None, note_model_name: str = None, derivatives: List[dict] = None): cfm: dict = { - CsvFileMappingKeys.CSV_FILE.value: csv + FILE: csv } if sort_by_columns is not None: - cfm.setdefault(CsvFileMappingKeys.SORT_BY_COLUMNS.value, sort_by_columns) + cfm.setdefault(SORT_BY_COLUMNS, sort_by_columns) if reverse_sort is not None: - cfm.setdefault(CsvFileMappingKeys.REVERSE_SORT.value, reverse_sort) + cfm.setdefault(REVERSE_SORT, reverse_sort) if note_model_name is not None: - cfm.setdefault(CsvFileMappingKeys.NOTE_MODEL.value, note_model_name) + cfm.setdefault(NOTE_MODEL, note_model_name) if derivatives is not None: - cfm.setdefault(CsvFileMappingKeys.DERIVATIVES.value, derivatives) + cfm.setdefault(DERIVATIVES, derivatives) return cfm diff --git a/tests/representation/configuration/test_note_model_mapping.py b/tests/representation/configuration/test_note_model_mapping.py index b266a9f..313b88a 100644 --- a/tests/representation/configuration/test_note_model_mapping.py +++ b/tests/representation/configuration/test_note_model_mapping.py @@ -3,8 +3,8 @@ import pytest -from brain_brew.representation.configuration.note_model_mapping import NoteModelMappingKeys, NoteModelMapping, \ - FieldMapping +from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping, FieldMapping, \ + NOTE_MODEL, COLUMNS, PERSONAL_FIELDS from brain_brew.representation.generic.csv_file import CsvFile from brain_brew.representation.json.deck_part_notemodel import DeckPartNoteModel from tests.representation.configuration.test_global_config import global_config @@ -14,9 +14,9 @@ def setup_nmm_config(note_model: str, field_mappings: Dict[str, str], personal_fields: List[str]): return { - NoteModelMappingKeys.NOTE_MODEL.value: note_model, - NoteModelMappingKeys.COLUMNS.value: field_mappings, - NoteModelMappingKeys.PERSONAL_FIELDS.value: personal_fields + NOTE_MODEL: note_model, + COLUMNS: field_mappings, + PERSONAL_FIELDS: personal_fields } diff --git a/tests/representation/deck_part_transformers/__init__.py b/tests/representation/deck_part_transformers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py b/tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py new file mode 100644 index 0000000..e50277c --- /dev/null +++ b/tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py @@ -0,0 +1,164 @@ +from textwrap import dedent +from unittest.mock import patch + +from brain_brew.file_manager import FileManager +from brain_brew.representation.deck_part_transformers.tr_notes_csv_collection import TrNotesCsvCollection +from brain_brew.representation.yaml.my_yaml import yaml_dump, yaml_load +from tests.test_file_manager import get_new_file_manager +from tests.representation.configuration.test_global_config import global_config + + +nm_mappings = { + "LL Word": dedent(f'''\ + note_model: LL Word + columns_to_fields: + guid: guid + tags: tags + + english: Word + danish: X Word + danish audio: X Pronunciation (Recording and/or IPA) + esperanto: Y Word + esperanto audio: Y Pronunciation (Recording and/or IPA) + personal_fields: + - picture + - extra + - morphman_focusmorph + '''), + "LL Verb": dedent(f'''\ + note_model: LL Verb + csv_columns_to_fields: + guid: guid + tags: tags + + english: Word + danish: X Word + danish audio: X Pronunciation (Recording and/or IPA) + esperanto: Y Word + esperanto audio: Y Pronunciation (Recording and/or IPA) + + present: Form Present + past: Form Past + present perfect: Form Perfect Present + personal_fields: + - picture + - extra + - morphman_focusmorph + '''), + "LL Noun": dedent(f'''\ + note_model: LL Noun + csv_columns_to_fields: + guid: guid + tags: tags + + english: Word + danish: X Word + danish audio: X Pronunciation (Recording and/or IPA) + esperanto: Y Word + esperanto audio: Y Pronunciation (Recording and/or IPA) + + plural: Plural + indefinite plural: Indefinite Plural + definite plural: Definite Plural + personal_fields: + - picture + - extra + - morphman_focusmorph + ''') +} + +file_mappings = { + "Main1": dedent(f'''\ + file: source/vocab/main.csv + note_model: LL Word + sort_by_columns: [english] + reverse_sort: no + '''), + "Der1": dedent(f'''\ + file: source/vocab/derivatives/danish/danish_verbs.csv + note_model: LL Verb + '''), + "Der2": dedent(f'''\ + file: source/vocab/derivatives/danish/danish_nouns.csv + note_model: LL Noun + ''') +} + + +class TestConstructor: + test_tr_notes = dedent(f'''\ + name: csv_first_attempt + # save_to_file: deckparts/notes/csv_first_attempt.yaml + + note_model_mappings: + - note_model: LL Word + columns_to_fields: + guid: guid + tags: tags + + english: Word + danish: X Word + danish audio: X Pronunciation (Recording and/or IPA) + esperanto: Y Word + esperanto audio: Y Pronunciation (Recording and/or IPA) + personal_fields: + - picture + - extra + - morphman_focusmorph + - note_model: LL Verb + columns_to_fields: + guid: guid + tags: tags + + english: Word + danish: X Word + danish audio: X Pronunciation (Recording and/or IPA) + esperanto: Y Word + esperanto audio: Y Pronunciation (Recording and/or IPA) + + present: Form Present + past: Form Past + present perfect: Form Perfect Present + personal_fields: + - picture + - extra + - morphman_focusmorph + - note_model: LL Noun + columns_to_fields: + guid: guid + tags: tags + + english: Word + danish: X Word + danish audio: X Pronunciation (Recording and/or IPA) + esperanto: Y Word + esperanto audio: Y Pronunciation (Recording and/or IPA) + + plural: Plural + indefinite plural: Indefinite Plural + definite plural: Definite Plural + personal_fields: + - picture + - extra + - morphman_focusmorph + + file_mappings: + - file: source/vocab/main.csv + note_model: LL Word + sort_by_columns: [english] + reverse_sort: no + + derivatives: + - file: source/vocab/derivatives/danish/danish_verbs.csv + note_model: LL Verb + - file: source/vocab/derivatives/danish/danish_nouns.csv + note_model: LL Noun + ''') + + def test_runs(self, global_config): + fm = get_new_file_manager() + data = yaml_load.load(self.test_tr_notes) + + tr_notes = TrNotesCsvCollection.from_dict(data) + + assert isinstance(tr_notes, TrNotesCsvCollection) diff --git a/tests/representation/yaml/__init__.py b/tests/representation/yaml/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/representation/yaml/test_note_repr.py b/tests/representation/yaml/test_note_repr.py index d340d04..eb91850 100644 --- a/tests/representation/yaml/test_note_repr.py +++ b/tests/representation/yaml/test_note_repr.py @@ -55,14 +55,14 @@ class TestNote: (["test", "blah", ""], "1234567890x", "model_name", ["noun"], {"animal.jpg"}), ]) def test_constructor(self, fields: List[str], guid: str, note_model: str, tags: List[str], media: Set[str]): - note = Note(fields=fields, guid=guid, note_model=note_model, tags=tags, media_references=media) + note = Note(fields=fields, guid=guid, note_model=note_model, tags=tags) assert isinstance(note, Note) assert note.fields == fields assert note.guid == guid assert note.note_model == note_model assert note.tags == tags - assert note.media_references == media + # assert note.media_references == media def test_from_dict(self, note_fixtures): assert isinstance(note_fixtures, Note) @@ -296,7 +296,7 @@ class TestNote: (["", "[sound:test.mp3]", "[sound:test.mp3]"], 2), ]) def test_all(self, fields, expected_count): - note = Note(fields=fields, note_model=None, guid="", tags=None, media_references=None) + note = Note(fields=fields, note_model=None, guid="", tags=None,) media_found = note.get_media_references() assert isinstance(media_found, Set) assert len(media_found) == expected_count @@ -363,3 +363,4 @@ def test_no_tags(self): notes = group.get_all_notes_copy() assert all([note.tags == [] for note in notes]) + From f411a7ff0ca7bbc12c2df5ecc786cb97be9d8fee Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sat, 4 Jul 2020 12:26:44 +0200 Subject: [PATCH 11/39] GenericToNotes and NotesToGeneric --- .../tr_notes_crowdanki.py | 19 +++++++++++--- .../tr_notes_csv_collection.py | 26 ++++++++++++------- .../tr_notes_generic.py | 15 ++++++++++- .../test_tr_notes_csv_collection.py | 6 ++--- 4 files changed, 50 insertions(+), 16 deletions(-) diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py b/brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py index eb09d65..26a4ecb 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py @@ -1,12 +1,12 @@ from dataclasses import dataclass from typing import List, Optional, Union -from brain_brew.representation.deck_part_transformers.tr_notes_generic import TrNotesGeneric +from brain_brew.representation.deck_part_transformers.tr_notes_generic import TrNotesToGeneric, TrGenericToNotes from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport @dataclass -class TrNotesCrowdAnki(TrNotesGeneric): +class TrCrowdAnkiToNotes(TrGenericToNotes): @dataclass class Representation: file: str @@ -15,8 +15,21 @@ class Representation: useless_note_keys: Optional[dict] crowdanki_file: CrowdAnkiExport + sort_order: Optional[List[str]] + media: Optional[bool] + useless_note_keys: Optional[Union[dict, list]] + + +@dataclass +class TrNotesToCrowdAnki(TrNotesToGeneric): + @dataclass + class Representation: + file: str + sort_order: Optional[Union[str, List[str]]] + media: Optional[bool] + useless_note_keys: Optional[dict] + sort_order: Optional[List[str]] media: Optional[bool] useless_note_keys: Optional[dict] -# TODO: Make Unique classes for Notes <-> CrowdAnki diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py index b2396fd..94c143f 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py @@ -3,28 +3,36 @@ from brain_brew.representation.configuration.csv_file_mapping import CsvFileMapping from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping -from brain_brew.representation.deck_part_transformers.tr_notes_generic import TrNotesGeneric +from brain_brew.representation.deck_part_transformers.tr_notes_generic import TrNotesToGeneric, TrGenericToNotes @dataclass -class TrNotesCsvCollection(TrNotesGeneric): +class TrCsvCollectionShared: @dataclass(init=False) - class Representation(TrNotesGeneric.Representation): + class Representation: file_mappings: List[CsvFileMapping.Representation] note_model_mappings: List[NoteModelMapping.Representation] - def __init__(self, name, file_mappings, note_model_mappings, save_to_file=None): - super().__init__(name, save_to_file) + def __init__(self, file_mappings, note_model_mappings): self.file_mappings = list(map(CsvFileMapping.Representation.from_dict, file_mappings)) self.note_model_mappings = list(map(NoteModelMapping.Representation.from_dict, note_model_mappings)) + file_mappings: List[CsvFileMapping] + note_model_mappings: List[NoteModelMapping] + + +@dataclass +class TrCsvCollectionToNotes(TrCsvCollectionShared, TrGenericToNotes): + @dataclass(init=False) + class Representation(TrCsvCollectionShared.Representation, TrGenericToNotes.Representation): + def __init__(self, name, file_mappings, note_model_mappings, save_to_file=None): + TrCsvCollectionShared.Representation.__init__(self, file_mappings, note_model_mappings) + TrGenericToNotes.Representation.__init__(self, name, save_to_file) + @classmethod def from_dict(cls, data: dict): return cls(**data) - file_mappings: List[CsvFileMapping] - note_model_mappings: List[NoteModelMapping] - @classmethod def from_repr(cls, data: Representation): return cls( @@ -36,7 +44,7 @@ def from_repr(cls, data: Representation): @classmethod def from_dict(cls, data: dict): - return cls.from_repr(TrNotesCsvCollection.Representation.from_dict(data)) + return cls.from_repr(TrCsvCollectionToNotes.Representation.from_dict(data)) # TODO: Make Unique classes for Notes <-> Csv diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_generic.py b/brain_brew/representation/deck_part_transformers/tr_notes_generic.py index a1b7899..16b2e66 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_generic.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_generic.py @@ -5,13 +5,26 @@ @dataclass -class TrNotesGeneric: +class TrGenericToNotes: @dataclass class Representation: name: str save_to_file: Optional[str] + def __init__(self, name, save_to_file): + self.name = name + self.save_to_file = save_to_file + name: str save_to_file: Optional[str] data: DeckPartNotes = field(init=False) + + +@dataclass +class TrNotesToGeneric: + @dataclass + class Representation: + name: str + + notes: DeckPartNotes diff --git a/tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py b/tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py index e50277c..50ac84e 100644 --- a/tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py +++ b/tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py @@ -2,7 +2,7 @@ from unittest.mock import patch from brain_brew.file_manager import FileManager -from brain_brew.representation.deck_part_transformers.tr_notes_csv_collection import TrNotesCsvCollection +from brain_brew.representation.deck_part_transformers.tr_notes_csv_collection import TrCsvCollectionToNotes from brain_brew.representation.yaml.my_yaml import yaml_dump, yaml_load from tests.test_file_manager import get_new_file_manager from tests.representation.configuration.test_global_config import global_config @@ -159,6 +159,6 @@ def test_runs(self, global_config): fm = get_new_file_manager() data = yaml_load.load(self.test_tr_notes) - tr_notes = TrNotesCsvCollection.from_dict(data) + tr_notes = TrCsvCollectionToNotes.from_dict(data) - assert isinstance(tr_notes, TrNotesCsvCollection) + assert isinstance(tr_notes, TrCsvCollectionToNotes) From 594e091f48e583f6ecaebfdae2b4a6a90dd33209 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sun, 5 Jul 2020 12:11:42 +0200 Subject: [PATCH 12/39] Csv to Notes in progress --- brain_brew/build_tasks/source_csv.py | 33 --------- .../configuration/note_model_mapping.py | 61 +++++++++-------- .../tr_notes_crowdanki.py | 12 ++-- .../tr_notes_csv_collection.py | 67 +++++++++++++++++-- .../tr_notes_generic.py | 5 +- 5 files changed, 105 insertions(+), 73 deletions(-) diff --git a/brain_brew/build_tasks/source_csv.py b/brain_brew/build_tasks/source_csv.py index f72dd7d..ba406f5 100644 --- a/brain_brew/build_tasks/source_csv.py +++ b/brain_brew/build_tasks/source_csv.py @@ -98,39 +98,6 @@ def verify_contents(self): if errors: raise Exception(errors) - def notes_to_deck_parts(self): - csv_data_by_guid: Dict[str, dict] = {} - for csv_map in self.csv_file_mappings: - csv_map.compile_data() - csv_data_by_guid = {**csv_data_by_guid, **csv_map.compiled_data} - csv_rows: List[dict] = list(csv_data_by_guid.values()) - - notes_json = [] - top_level_note_structure = { - DeckPartNoteKeys.FIELDS.value: List[str], - DeckPartNoteKeys.GUID.value: "", - DeckPartNoteKeys.TAGS.value: List[str], - DeckPartNoteKeys.NOTE_MODEL.value: "" - } - - # Get Guid, Tags, NoteTypeName, Fields - for row in csv_rows: - note = top_level_note_structure.copy() - - row_nm: NoteModelMapping = self.note_model_mappings_dict[row[DeckPartNoteKeys.NOTE_MODEL.value]] - - filtered_fields = row_nm.csv_row_map_to_note_fields(row) - - note[DeckPartNoteKeys.NOTE_MODEL.value] = row_nm.note_model.name - note[DeckPartNoteKeys.GUID.value] = filtered_fields.pop(DeckPartNoteKeys.GUID.value) - note[DeckPartNoteKeys.TAGS.value] = self.split_tags(filtered_fields.pop(DeckPartNoteKeys.TAGS.value)) - - note[DeckPartNoteKeys.FIELDS.value] = row_nm.field_values_in_note_model_order(filtered_fields) - - notes_json.append(note) - - return notes_json - def notes_to_source(self) -> Dict[str, dict]: notes_data = self.notes.get_data(deep_copy=True)[DeckPartNoteKeys.NOTES.value] self.verify_notes_match_note_model_mappings(notes_data) diff --git a/brain_brew/representation/configuration/note_model_mapping.py b/brain_brew/representation/configuration/note_model_mapping.py index c3dd513..4920d52 100644 --- a/brain_brew/representation/configuration/note_model_mapping.py +++ b/brain_brew/representation/configuration/note_model_mapping.py @@ -6,8 +6,7 @@ from brain_brew.interfaces.verifiable import Verifiable from brain_brew.representation.generic.yaml_file import YamlFile, ConfigKey from brain_brew.representation.json.deck_part_notemodel import DeckPartNoteModel -from brain_brew.utils import list_of_str_to_lowercase - +from brain_brew.utils import list_of_str_to_lowercase, single_item_to_list NOTE_MODEL = "note_model" COLUMNS = "csv_columns_to_fields" @@ -42,7 +41,7 @@ def __init__(self, field_type: FieldMappingType, field_name: str, value: str): class NoteModelMapping(Verifiable): @dataclass class Representation: - note_model: str # TODO: Union[str, list] + note_models: Union[str, list] columns_to_fields: Dict[str, str] personal_fields: List[str] @@ -50,7 +49,7 @@ class Representation: def from_dict(cls, data: dict): return cls(**data) - note_model: DeckPartNoteModel + note_models: Dict[str, DeckPartNoteModel] columns: List[FieldMapping] personal_fields: List[FieldMapping] @@ -58,6 +57,8 @@ def from_dict(cls, data: dict): @classmethod def from_repr(cls, data: Representation): + note_models = [DeckPartNoteModel.create(model, read_now=True) for model in single_item_to_list(data.note_models)] + return cls( columns=[FieldMapping( field_type=FieldMapping.FieldMappingType.COLUMN, @@ -67,33 +68,37 @@ def from_repr(cls, data: Representation): field_type=FieldMapping.FieldMappingType.PERSONAL_FIELD, field_name=field, value="") for field in data.personal_fields], - note_model=DeckPartNoteModel.create(data.note_model, read_now=True) # TODO: Fix read_now + note_models=dict(map(lambda nm: (nm.name, nm), note_models)) # TODO: Use deck part pool ) def verify_contents(self): errors = [] - # Check for Required Fields - missing = [] - for req in self.required_fields_definitions: - if req not in [field.value for field in self.columns]: - missing.append(req) - - if missing: - errors.append(KeyError(f"""Note model "{self.note_model.name}" to Csv config error: \ - Definitions for fields {missing} are required.""")) - - # Check Fields Align with Note Type - missing, extra = self.note_model.check_field_overlap( - [field.value for field in self.columns if field.value not in self.required_fields_definitions] - ) - missing = [m for m in missing if m not in [field.field_name for field in self.personal_fields]] - - if missing or extra: - raise KeyError( - f"""Note model "{self.note_model.name}" to Csv config error. It expected {self.note_model.fields} \ - but was missing: {missing}, and got extra: {extra} """) - + for model in self.note_models: + # Check for Required Fields + missing = [] + for req in self.required_fields_definitions: + if req not in [field.value for field in self.columns]: + missing.append(req) + + if missing: + errors.append(KeyError(f"""Note model(s) "{model.name}" to Csv config error: \ + Definitions for fields {missing} are required.""")) + + # TODO: Note Model Mappings are allowed extra fields on a specific note model now, since multiple + # TODO: can be applied to a single NMM. Check if ANY are missing and/or ALL have Extra instead + # Check Fields Align with Note Type + missing, extra = model.check_field_overlap( + [field.value for field in self.columns if field.value not in self.required_fields_definitions] + ) + missing = [m for m in missing if m not in [field.field_name for field in self.personal_fields]] + + if missing or extra: + errors.append(KeyError( + f"""Note model "{model.name}" to Csv config error. It expected {model.fields} \ + but was missing: {missing}, and got extra: {extra} """)) + + # TODO: Make sure the same note_model is not defined in multiple NMMs if errors: raise Exception(errors) @@ -142,5 +147,5 @@ def get_relevant_data(self, row): return relevant_data - def field_values_in_note_model_order(self, fields_from_csv): - return [fields_from_csv[field] for field in self.note_model.fields_lowercase] + def field_values_in_note_model_order(self, note_model_name, fields_from_csv): + return [fields_from_csv[field] for field in self.note_models[note_model_name].fields_lowercase] diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py b/brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py index 26a4ecb..3c71206 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py @@ -12,12 +12,12 @@ class Representation: file: str sort_order: Optional[Union[str, List[str]]] media: Optional[bool] - useless_note_keys: Optional[dict] + useless_note_keys: Optional[Union[dict, list]] crowdanki_file: CrowdAnkiExport sort_order: Optional[List[str]] - media: Optional[bool] - useless_note_keys: Optional[Union[dict, list]] + media: bool + useless_note_keys: list @dataclass @@ -27,9 +27,9 @@ class Representation: file: str sort_order: Optional[Union[str, List[str]]] media: Optional[bool] - useless_note_keys: Optional[dict] + useless_note_keys: Optional[dict] # TODO: use default value sort_order: Optional[List[str]] - media: Optional[bool] - useless_note_keys: Optional[dict] + media: bool + useless_note_keys: dict diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py index 94c143f..a66310b 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py @@ -1,9 +1,10 @@ from dataclasses import dataclass -from typing import List +from typing import List, Dict from brain_brew.representation.configuration.csv_file_mapping import CsvFileMapping from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping from brain_brew.representation.deck_part_transformers.tr_notes_generic import TrNotesToGeneric, TrGenericToNotes +from brain_brew.representation.yaml.note_repr import DeckPartNotes, Note @dataclass @@ -17,8 +18,17 @@ def __init__(self, file_mappings, note_model_mappings): self.file_mappings = list(map(CsvFileMapping.Representation.from_dict, file_mappings)) self.note_model_mappings = list(map(NoteModelMapping.Representation.from_dict, note_model_mappings)) + def get_file_mappings(self) -> List[CsvFileMapping]: + return list(map(CsvFileMapping.from_repr, self.file_mappings)) + + def get_note_model_mappings(self) -> Dict[str, NoteModelMapping]: + note_model_mappings: Dict[str, NoteModelMapping] = {} + for nmm in self.note_model_mappings: + if + return dict(map(lambda nmm: (nmm.note_model, NoteModelMapping.from_repr(nmm)), )) + file_mappings: List[CsvFileMapping] - note_model_mappings: List[NoteModelMapping] + note_model_mappings: Dict[str, NoteModelMapping] @dataclass @@ -38,13 +48,60 @@ def from_repr(cls, data: Representation): return cls( name=data.name, save_to_file=data.save_to_file, - file_mappings=list(map(CsvFileMapping.from_repr, data.file_mappings)), - note_model_mappings=list(map(NoteModelMapping.from_repr, data.note_model_mappings)) + file_mappings=data.get_file_mappings(), + note_model_mappings=data.get_note_model_mappings() ) @classmethod def from_dict(cls, data: dict): return cls.from_repr(TrCsvCollectionToNotes.Representation.from_dict(data)) + def notes_to_deck_parts(self): + csv_data_by_guid: Dict[str, dict] = {} + for csv_map in self.file_mappings: + csv_map.compile_data() + csv_data_by_guid = {**csv_data_by_guid, **csv_map.compiled_data} + csv_rows: List[dict] = list(csv_data_by_guid.values()) + + notes_json: List[Note] = [] + + # Get Guid, Tags, NoteTypeName, Fields + for row in csv_rows: + row_nm: NoteModelMapping = self.note_model_mappings_dict[row[DeckPartNoteKeys.NOTE_MODEL.value]] + + filtered_fields = row_nm.csv_row_map_to_note_fields(row) + + note_model = row_nm.note_model.name + guid = filtered_fields.pop(DeckPartNoteKeys.GUID.value) + tags = self.split_tags(filtered_fields.pop(DeckPartNoteKeys.TAGS.value)) + + fields = row_nm.field_values_in_note_model_order(note_model, filtered_fields) + + notes_json.append(Note(guid=guid, tags=tags, note_model=note_model, fields=fields)) + + return notes_json + + +@dataclass +class TrNotesToCsvCollection(TrCsvCollectionShared, TrNotesToGeneric): + @dataclass(init=False) + class Representation(TrCsvCollectionShared.Representation, TrNotesToGeneric.Representation): + def __init__(self, name, file_mappings, note_model_mappings): + TrCsvCollectionShared.Representation.__init__(self, file_mappings, note_model_mappings) + TrNotesToGeneric.Representation.__init__(self, name) + + @classmethod + def from_dict(cls, data: dict): + return cls(**data) -# TODO: Make Unique classes for Notes <-> Csv + @classmethod + def from_repr(cls, data: Representation): + return cls( + notes=DeckPartNotes.create(data.name, read_now=True), #TODO: remove old DeckPartNotes. Use pool of DeckParts + file_mappings=data.get_file_mappings(), + note_model_mappings=data.get_note_model_mappings() + ) + + @classmethod + def from_dict(cls, data: dict): + return cls.from_repr(TrCsvCollectionToNotes.Representation.from_dict(data)) diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_generic.py b/brain_brew/representation/deck_part_transformers/tr_notes_generic.py index 16b2e66..3b137ff 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_generic.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_generic.py @@ -11,7 +11,7 @@ class Representation: name: str save_to_file: Optional[str] - def __init__(self, name, save_to_file): + def __init__(self, name, save_to_file=None): self.name = name self.save_to_file = save_to_file @@ -27,4 +27,7 @@ class TrNotesToGeneric: class Representation: name: str + def __init__(self, name): + self.name = name + notes: DeckPartNotes From 3dfd334bc8da95a6cf84b71107e499c89015ea5f Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sat, 25 Jul 2020 09:42:09 +0200 Subject: [PATCH 13/39] Csv Source Changed to Transformers --- brain_brew/build_tasks/build_task_generic.py | 16 --- brain_brew/build_tasks/source_csv.py | 107 ------------------ .../tr_notes_csv_collection.py | 90 ++++++++++++--- .../tr_notes_generic.py | 19 +++- brain_brew/representation/yaml/note_repr.py | 15 ++- 5 files changed, 108 insertions(+), 139 deletions(-) delete mode 100644 brain_brew/build_tasks/build_task_generic.py diff --git a/brain_brew/build_tasks/build_task_generic.py b/brain_brew/build_tasks/build_task_generic.py deleted file mode 100644 index a796bc9..0000000 --- a/brain_brew/build_tasks/build_task_generic.py +++ /dev/null @@ -1,16 +0,0 @@ -import re - -from brain_brew.representation.configuration.global_config import GlobalConfig - - -class BuildTaskGeneric: - @staticmethod - def split_tags(tags_value: str) -> list: - split = [entry.strip() for entry in re.split(';\s*|,\s*|\s+', tags_value)] - while "" in split: - split.remove("") - return split - - @staticmethod - def join_tags(tags_list: list) -> str: - return GlobalConfig.get_instance().flags.join_values_with.join(tags_list) diff --git a/brain_brew/build_tasks/source_csv.py b/brain_brew/build_tasks/source_csv.py index ba406f5..feb2ac6 100644 --- a/brain_brew/build_tasks/source_csv.py +++ b/brain_brew/build_tasks/source_csv.py @@ -28,110 +28,3 @@ def get_build_keys(): BuildTaskEnum("csv_collection_to_deck_parts", SourceCsv, "source_to_deck_parts", "deck_parts_to_source"), ] - config_entry = {} - expected_keys = { - SourceCsvKeys.NOTES.value: ConfigKey(True, str, None), - SourceCsvKeys.NOTE_MODEL_MAPPINGS.value: ConfigKey(True, list, None), - SourceCsvKeys.CSV_MAPPINGS.value: ConfigKey(True, list, None), - } - subconfig_filter = None - - notes: DeckPartNotes - note_model_mappings_dict: Dict[str, NoteModelMapping] - csv_file_mappings: List[CsvFileMapping] - - def __init__(self, config_data: dict, read_now=True): - self.setup_config_with_subconfig_replacement(config_data) - self.verify_config_entry() - - self.notes = DeckPartNotes.create(self.get_config(BuildConfigKeys.NOTES), read_now=read_now) - - nm_mapping = [NoteModelMapping(config, read_now=read_now) - for config in self.get_config(SourceCsvKeys.NOTE_MODEL_MAPPINGS)] - - self.note_model_mappings_dict = {mapping.note_model.name: mapping for mapping in nm_mapping} - - self.csv_file_mappings = [CsvFileMapping(config, read_now=read_now) - for config in self.get_config(SourceCsvKeys.CSV_MAPPINGS)] - - @classmethod - def from_yaml(cls, yaml_file_name, read_now=True): - config_data = YamlFile.read_file(yaml_file_name) - - return SourceCsv(config_data, read_now=read_now) - - def verify_contents(self): - errors = [] - - for nm in self.note_model_mappings_dict.values(): - try: - nm.verify_contents() - except KeyError as e: - errors.append(e) - - for cfm in self.csv_file_mappings: - # Check all necessary key values are present - try: - cfm.verify_contents() - except KeyError as e: - errors.append(e) - - # Check all references notemodels have a mapping - for csv_map in self.csv_file_mappings: - for nm in csv_map.get_used_note_model_names(): - if nm not in self.note_model_mappings_dict.keys(): - errors.append(f"Missing Note Model Map for {nm}") - - # Check each of the Csvs (or their derivatives) contain all the necessary columns for their stated note model - for cfm in self.csv_file_mappings: - note_model_names = cfm.get_used_note_model_names() - available_columns = cfm.get_available_columns() - - referenced_note_models_maps = [value for key, value in self.note_model_mappings_dict.items() if - key in note_model_names] - for nm_map in referenced_note_models_maps: - missing_columns = [col for col in nm_map.note_model.fields_lowercase if - col not in nm_map.csv_headers_map_to_note_fields(available_columns)] - if missing_columns: - errors.append(KeyError(f"Csvs are missing columns from {nm_map.note_model.name}", missing_columns)) - - if errors: - raise Exception(errors) - - def notes_to_source(self) -> Dict[str, dict]: - notes_data = self.notes.get_data(deep_copy=True)[DeckPartNoteKeys.NOTES.value] - self.verify_notes_match_note_model_mappings(notes_data) - - csv_data: Dict[str, dict] = {} - for note in notes_data: - nm_name = note[DeckPartNoteKeys.NOTE_MODEL.value] - row = self.note_model_mappings_dict[nm_name].note_model.zip_field_to_data( - note[DeckPartNoteKeys.FIELDS.value]) - row[CsvKeys.GUID.value] = note[DeckPartNoteKeys.GUID.value] - row[CsvKeys.TAGS.value] = self.join_tags(note[DeckPartNoteKeys.TAGS.value]) - - formatted_row = self.note_model_mappings_dict[nm_name].note_fields_map_to_csv_row(row) # TODO: Do not edit data, make copy - - csv_data.setdefault(row[CsvKeys.GUID.value], formatted_row) - - return csv_data - - def verify_notes_match_note_model_mappings(self, notes): - note_models_used = {note[DeckPartNoteKeys.NOTE_MODEL.value] for note in notes} - errors = [TypeError(f"Unknown note model type '{model}' in notes '{self.notes.file_location}. " - f"Add {SourceCsvKeys.NOTE_MODEL_MAPPINGS.value} for that model.") - for model in note_models_used if model not in self.note_model_mappings_dict.keys()] - - if errors: - raise Exception(errors) - - def source_to_deck_parts(self): - notes_data = self.notes_to_deck_parts() - self.notes.set_data(notes_data) - - def deck_parts_to_source(self): - csv_data = self.notes_to_source() - - for cfm in self.csv_file_mappings: - cfm.compile_data() - cfm.set_relevant_data(csv_data) diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py index a66310b..1af77a0 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py @@ -22,14 +22,49 @@ def get_file_mappings(self) -> List[CsvFileMapping]: return list(map(CsvFileMapping.from_repr, self.file_mappings)) def get_note_model_mappings(self) -> Dict[str, NoteModelMapping]: - note_model_mappings: Dict[str, NoteModelMapping] = {} - for nmm in self.note_model_mappings: - if - return dict(map(lambda nmm: (nmm.note_model, NoteModelMapping.from_repr(nmm)), )) + return dict(map(lambda nmm: (nmm.note_model, NoteModelMapping.from_repr(nmm)), self.note_model_mappings)) file_mappings: List[CsvFileMapping] note_model_mappings: Dict[str, NoteModelMapping] + def verify_contents(self): + errors = [] + + for nm in self.note_model_mappings.values(): + try: + nm.verify_contents() + except KeyError as e: + errors.append(e) + + for fm in self.file_mappings: + # Check all necessary key values are present + try: + fm.verify_contents() + except KeyError as e: + errors.append(e) + + # Check all referenced note models have a mapping + for csv_map in self.file_mappings: + for nm in csv_map.get_used_note_model_names(): + if nm not in self.note_model_mappings.keys(): + errors.append(f"Missing Note Model Map for {nm}") + + # Check each of the Csvs (or their derivatives) contain all the necessary columns for their stated note model + # for cfm in self.file_mappings: + # note_model_names = cfm.get_used_note_model_names() + # available_columns = cfm.get_available_columns() + # + # referenced_note_models_maps = [value for key, value in self.note_model_mappings.items() if + # key in note_model_names] + # for nm_map in referenced_note_models_maps: + # missing_columns = [col for col in nm_map.note_model.fields_lowercase if + # col not in nm_map.csv_headers_map_to_note_fields(available_columns)] + # if missing_columns: + # errors.append(KeyError(f"Csvs are missing columns from {nm_map.note_model.name}", missing_columns)) + + if errors: + raise Exception(errors) + @dataclass class TrCsvCollectionToNotes(TrCsvCollectionShared, TrGenericToNotes): @@ -63,23 +98,24 @@ def notes_to_deck_parts(self): csv_data_by_guid = {**csv_data_by_guid, **csv_map.compiled_data} csv_rows: List[dict] = list(csv_data_by_guid.values()) - notes_json: List[Note] = [] + deck_part_notes: List[Note] = [] # Get Guid, Tags, NoteTypeName, Fields for row in csv_rows: - row_nm: NoteModelMapping = self.note_model_mappings_dict[row[DeckPartNoteKeys.NOTE_MODEL.value]] + note_model_name = row["note_model"] # TODO: Use object + row_nm: NoteModelMapping = self.note_model_mappings[note_model_name] filtered_fields = row_nm.csv_row_map_to_note_fields(row) - note_model = row_nm.note_model.name - guid = filtered_fields.pop(DeckPartNoteKeys.GUID.value) - tags = self.split_tags(filtered_fields.pop(DeckPartNoteKeys.TAGS.value)) + guid = filtered_fields.pop("guid") + tags = self.split_tags(filtered_fields.pop("tags")) - fields = row_nm.field_values_in_note_model_order(note_model, filtered_fields) + fields = row_nm.field_values_in_note_model_order(note_model_name, filtered_fields) - notes_json.append(Note(guid=guid, tags=tags, note_model=note_model, fields=fields)) + deck_part_notes.append(Note(guid=guid, tags=tags, note_model=note_model_name, fields=fields)) - return notes_json + DeckPartNotes.from_list_of_notes(self.name, deck_part_notes) + # TODO: Save to the singleton holder @dataclass @@ -97,7 +133,7 @@ def from_dict(cls, data: dict): @classmethod def from_repr(cls, data: Representation): return cls( - notes=DeckPartNotes.create(data.name, read_now=True), #TODO: remove old DeckPartNotes. Use pool of DeckParts + notes=DeckPartNotes.create(data.name, read_now=True), # TODO: remove old DeckPartNotes. Use pool of DeckParts file_mappings=data.get_file_mappings(), note_model_mappings=data.get_note_model_mappings() ) @@ -105,3 +141,31 @@ def from_repr(cls, data: Representation): @classmethod def from_dict(cls, data: dict): return cls.from_repr(TrCsvCollectionToNotes.Representation.from_dict(data)) + + def notes_to_source(self): + notes_data = self.notes.get_notes() + self.verify_notes_match_note_model_mappings(notes_data) + + csv_data: Dict[str, dict] = {} + for note in notes_data: + nm_name = note.note_model + row = self.note_model_mappings[nm_name].note_models[nm_name].zip_field_to_data(note.fields) + row["guid"] = note.guid + row["tags"] = self.join_tags(note.tags) + + formatted_row = self.note_model_mappings[nm_name].note_fields_map_to_csv_row(row) # TODO: Do not edit data, make copy + + csv_data.setdefault(row["guid"], formatted_row) + + for fm in self.file_mappings: + fm.compile_data() + fm.set_relevant_data(csv_data) + + def verify_notes_match_note_model_mappings(self, notes: List[Note]): + note_models_used = {note.note_model for note in notes} + errors = [TypeError(f"Unknown note model type '{model}' in deck part '{self.notes.name}'. " + f"Add mapping for that model.") + for model in note_models_used if model not in self.note_model_mappings.keys()] + + if errors: + raise Exception(errors) diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_generic.py b/brain_brew/representation/deck_part_transformers/tr_notes_generic.py index 3b137ff..e20af95 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_generic.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_generic.py @@ -1,11 +1,26 @@ from dataclasses import dataclass, field from typing import Optional +import re from brain_brew.representation.yaml.note_repr import DeckPartNotes +from brain_brew.representation.configuration.global_config import GlobalConfig + + +class TrNotes: + @staticmethod + def split_tags(tags_value: str) -> list: + split = [entry.strip() for entry in re.split(';\s*|,\s*|\s+', tags_value)] + while "" in split: + split.remove("") + return split + + @staticmethod + def join_tags(tags_list: list) -> str: + return GlobalConfig.get_instance().flags.join_values_with.join(tags_list) @dataclass -class TrGenericToNotes: +class TrGenericToNotes(TrNotes): @dataclass class Representation: name: str @@ -22,7 +37,7 @@ def __init__(self, name, save_to_file=None): @dataclass -class TrNotesToGeneric: +class TrNotesToGeneric(TrNotes): @dataclass class Representation: name: str diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index e36959a..f07c3f8 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -113,14 +113,24 @@ def join_tags(n_tags): @dataclass class DeckPartNotes: + name: str note_groupings: List[NoteGrouping] + # TODO: File location and saving @classmethod - def from_dict(cls, data: dict): + def from_dict(cls, name: str, data: dict): return cls( + name=name, note_groupings=list(map(NoteGrouping.from_dict, data.get(NOTE_GROUPINGS))) ) + @classmethod + def from_list_of_notes(cls, name: str, notes: List[Note]): + return cls( + name=name, + note_groupings=[NoteGrouping(note_model=None, tags=None, notes=notes)] + ) + def encode(self) -> dict: data_dict = {NOTE_GROUPINGS: [note_grouping.encode() for note_grouping in self.note_groupings]} return data_dict @@ -131,3 +141,6 @@ def dump_to_yaml(self, file): def get_all_known_note_model_names(self): return {nms for group in self.note_groupings for nms in group.get_all_known_note_model_names()} + + def get_notes(self): + return [note for group in self.note_groupings for note in group.get_all_notes_copy()] From 69038848140be5635d3905a07bf04feddabc5e41 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sat, 25 Jul 2020 12:08:56 +0200 Subject: [PATCH 14/39] Builder changes; Ongoing moving of dependencies --- brain_brew/argument_reader.py | 11 +--- brain_brew/file_manager.py | 18 +++++- brain_brew/main.py | 7 ++- .../representation/build_config/__init__.py | 0 .../build_config/build_tasks.py | 9 +++ .../build_config}/builder.py | 62 ++++++++++--------- .../build_config/generate_csv_collection.py | 18 ++++++ .../deck_part_transformers/tr_generic.py | 4 ++ .../tr_notes_csv_collection.py | 8 +-- .../tr_notes_generic.py | 3 +- brain_brew/representation/yaml/my_yaml.py | 18 ++++++ brain_brew/representation/yaml/note_repr.py | 20 ++++-- tests/test_builder.py | 3 +- 13 files changed, 127 insertions(+), 54 deletions(-) create mode 100644 brain_brew/representation/build_config/__init__.py create mode 100644 brain_brew/representation/build_config/build_tasks.py rename brain_brew/{ => representation/build_config}/builder.py (61%) create mode 100644 brain_brew/representation/build_config/generate_csv_collection.py create mode 100644 brain_brew/representation/deck_part_transformers/tr_generic.py diff --git a/brain_brew/argument_reader.py b/brain_brew/argument_reader.py index 20752a5..1dec7a1 100644 --- a/brain_brew/argument_reader.py +++ b/brain_brew/argument_reader.py @@ -26,14 +26,6 @@ def _set_parser_arguments(self): help="Global config file to use" ) - self.add_argument( - "-r", "--reversed", - action="store_true", - dest="run_reversed", - default=False, - help="Run the builder file in reverse" - ) - def get_parsed(self, override_args=None): parsed_args = self.parse_args(args=override_args) @@ -42,9 +34,8 @@ def get_parsed(self, override_args=None): # Optional config_file = parsed_args.config_file - run_reversed = parsed_args.run_reversed - return builder, config_file, run_reversed + return builder, config_file def error_if_blank(self, arg): if arg == "" or arg is None: diff --git a/brain_brew/file_manager.py b/brain_brew/file_manager.py index 5dae82e..fa5d946 100644 --- a/brain_brew/file_manager.py +++ b/brain_brew/file_manager.py @@ -7,6 +7,7 @@ from brain_brew.representation.configuration.global_config import GlobalConfig from brain_brew.representation.generic.generic_file import GenericFile from brain_brew.representation.generic.media_file import MediaFile +from brain_brew.representation.yaml.my_yaml import YamlRepresentation from brain_brew.utils import filename_from_full_path, find_all_files_in_directory @@ -16,6 +17,7 @@ class FileManager: known_files_dict: Dict[str, GenericFile] known_media_files_dict: Dict[str, MediaFile] + deck_part_pool: Dict[str, YamlRepresentation] write_files_at_end: List[WritesFile] @@ -29,6 +31,7 @@ def __init__(self): self.known_files_dict = {} self.write_files_at_end = [] + self.deck_part_pool = {} self.find_all_deck_part_media_files() @@ -48,7 +51,7 @@ def file_if_exists(self, file_location) -> Union[GenericFile, None]: def register_file(self, full_path, file): if full_path in self.known_files_dict: - raise FileExistsError("File already known to FileManager, cannot be registered twice") + raise FileExistsError(f"File already known to FileManager, cannot be registered twice: {full_path}") self.known_files_dict.setdefault(full_path, file) def media_file_if_exists(self, filename) -> Union[MediaFile, None]: @@ -79,6 +82,16 @@ def find_all_deck_part_media_files(self): logging.debug(f"Media files found: {len(self.known_media_files_dict)}") + def new_deck_part(self, dp: YamlRepresentation): + if dp.name in self.deck_part_pool: + raise KeyError(f"Cannot use same name '{dp.name}' for multiple Deck Parts") + self.deck_part_pool.setdefault(dp.name, dp) + + def deck_part_from_pool(self, name: str): + if name not in self.deck_part_pool: + raise KeyError(f"Cannot find Deck Part '{name}'") + return self.deck_part_pool[name] + def write_to_all(self): files_to_create = [] for location, file in self.known_files_dict.items(): @@ -96,5 +109,8 @@ def write_to_all(self): file.write_file() file.data_state = GenericFile.DataState.READ_IN_DATA + for dp in self.deck_part_pool.values(): + dp.write_to_file() + for filename, media_file in self.known_media_files_dict.items(): media_file.copy_source_to_target() diff --git a/brain_brew/main.py b/brain_brew/main.py index 0833142..b719f2a 100644 --- a/brain_brew/main.py +++ b/brain_brew/main.py @@ -1,7 +1,8 @@ import logging +from brain_brew.representation.build_config.build_tasks import known_build_tasks from brain_brew.argument_reader import BBArgumentReader -from brain_brew.builder import Builder +from brain_brew.representation.build_config.builder import Builder from brain_brew.file_manager import FileManager from brain_brew.representation.configuration.global_config import GlobalConfig from brain_brew.representation.generic.yaml_file import YamlFile @@ -16,7 +17,7 @@ def main(): # Read in Arguments argument_reader = BBArgumentReader() - builder_file_name, global_config_file, run_reversed = argument_reader.get_parsed() + builder_file_name, global_config_file = argument_reader.get_parsed() builder_config = YamlFile.read_file(builder_file_name) # Read in Global Config File @@ -24,7 +25,7 @@ def main(): file_manager = FileManager() # Run chosen Builder - builder = Builder(builder_config, global_config, run_reversed=run_reversed) + builder = Builder.from_dict(builder_config, global_config, file_manager) builder.execute() diff --git a/brain_brew/representation/build_config/__init__.py b/brain_brew/representation/build_config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/brain_brew/representation/build_config/build_tasks.py b/brain_brew/representation/build_config/build_tasks.py new file mode 100644 index 0000000..ad2c10e --- /dev/null +++ b/brain_brew/representation/build_config/build_tasks.py @@ -0,0 +1,9 @@ +from typing import Dict + +known_build_tasks: Dict[str, type] = {} + + +def add_build_task(task_name: str, class_to_call: type): + if task_name in known_build_tasks: + raise KeyError(f"Multiple instances of task name '{task_name}'") + known_build_tasks.setdefault(task_name, class_to_call) diff --git a/brain_brew/builder.py b/brain_brew/representation/build_config/builder.py similarity index 61% rename from brain_brew/builder.py rename to brain_brew/representation/build_config/builder.py index 129a57b..105d948 100644 --- a/brain_brew/builder.py +++ b/brain_brew/representation/build_config/builder.py @@ -1,3 +1,4 @@ +from dataclasses import dataclass, field from enum import Enum from brain_brew.constants.build_config_keys import BuildTaskEnum @@ -10,41 +11,42 @@ from brain_brew.representation.generic.yaml_file import YamlFile, ConfigKey -class BuilderKeys(Enum): - TASKS = "tasks" - REVERSE_RUN_DIRECTION = "reverse" +@dataclass +class Builder: + @dataclass + class Representation: + tasks: list + @classmethod + def from_dict(cls, data: dict): + return cls(**data) -class Builder(YamlFile): - config_entry = {} - expected_keys = { - BuilderKeys.TASKS.value: ConfigKey(True, list, None), - BuilderKeys.REVERSE_RUN_DIRECTION.value: ConfigKey(False, bool, None) - } - subconfig_filter = None - + tasks: list global_config: GlobalConfig - - BUILD_TASK_DEFINITIONS: dict - KNOWN_BUILD_TASK_CLASSES = [SourceCrowdAnki, SourceCsv] - - build_tasks = [] file_manager: FileManager - def __init__(self, config_data, global_config, run_reversed=False, read_now=True): - self.file_manager = FileManager.get_instance() - + @classmethod + def from_repr(cls, data: Representation, global_config, file_manager): + tasks = cls.read_tasks(data.tasks) + return cls( + tasks=tasks, + global_config=global_config, + file_manager=file_manager + ) + + @classmethod + def from_dict(cls, data: dict, global_config, file_manager): + return cls.from_repr(Builder.Representation.from_dict(data), global_config, file_manager) + + @staticmethod + def read_tasks(tasks: list) -> list: self.BUILD_TASK_DEFINITIONS = {build_task.key_name: build_task for source in self.KNOWN_BUILD_TASK_CLASSES for build_task in source.get_build_keys() } - self.setup_config_with_subconfig_replacement(config_data) - self.verify_config_entry() - - self.global_config = global_config - self.reverse_run_direction = run_reversed or self.get_config(BuilderKeys.REVERSE_RUN_DIRECTION, False) + build_tasks = [] # Tasks for key in self.config_entry[BuilderKeys.TASKS.value]: @@ -58,22 +60,24 @@ def __init__(self, config_data, global_config, run_reversed=False, read_now=True definition: BuildTaskEnum = self.BUILD_TASK_DEFINITIONS[task_keys[0]] source = definition.source_type(task[task_keys[0]], read_now) if self.reverse_run_direction: - self.build_tasks.append((source, definition.reverse_task_to_execute)) + build_tasks.append((source, definition.reverse_task_to_execute)) else: - self.build_tasks.append((source, definition.task_to_execute)) + build_tasks.append((source, definition.task_to_execute)) else: raise KeyError(f"Unknown key {key}") # TODO: check this first on all and return all errors if self.reverse_run_direction: - self.build_tasks = list(reversed(self.build_tasks)) + build_tasks = list(reversed(build_tasks)) # Verify tasks - for source, task_to_execute in self.build_tasks: + for source, task_to_execute in build_tasks: if isinstance(source, Verifiable): source.verify_contents() + return build_tasks + def execute(self): - for (source, task_to_execute) in self.build_tasks: + for (source, task_to_execute) in self.tasks: getattr(source, task_to_execute)() self.file_manager.write_to_all() diff --git a/brain_brew/representation/build_config/generate_csv_collection.py b/brain_brew/representation/build_config/generate_csv_collection.py new file mode 100644 index 0000000..cef775b --- /dev/null +++ b/brain_brew/representation/build_config/generate_csv_collection.py @@ -0,0 +1,18 @@ +from dataclasses import dataclass + +from brain_brew.representation.build_config.build_tasks import add_build_task +from brain_brew.representation.deck_part_transformers.tr_notes_csv_collection import TrNotesToCsvCollection + + +@dataclass +class GenerateCsvCollection: + @dataclass + class Representation: + file: str + notes: dict + + file: str + notes: TrNotesToCsvCollection + + +add_build_task("Generate Csv Collection", GenerateCsvCollection) diff --git a/brain_brew/representation/deck_part_transformers/tr_generic.py b/brain_brew/representation/deck_part_transformers/tr_generic.py new file mode 100644 index 0000000..d26cc0b --- /dev/null +++ b/brain_brew/representation/deck_part_transformers/tr_generic.py @@ -0,0 +1,4 @@ + +class TrGeneric: + def execute(self): + raise NotImplemented() diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py index 1af77a0..108d46a 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py @@ -91,7 +91,7 @@ def from_repr(cls, data: Representation): def from_dict(cls, data: dict): return cls.from_repr(TrCsvCollectionToNotes.Representation.from_dict(data)) - def notes_to_deck_parts(self): + def execute(self): csv_data_by_guid: Dict[str, dict] = {} for csv_map in self.file_mappings: csv_map.compile_data() @@ -114,7 +114,7 @@ def notes_to_deck_parts(self): deck_part_notes.append(Note(guid=guid, tags=tags, note_model=note_model_name, fields=fields)) - DeckPartNotes.from_list_of_notes(self.name, deck_part_notes) + DeckPartNotes.from_list_of_notes(self.name, self.save_to_file, deck_part_notes) # TODO: Save to the singleton holder @@ -133,7 +133,7 @@ def from_dict(cls, data: dict): @classmethod def from_repr(cls, data: Representation): return cls( - notes=DeckPartNotes.create(data.name, read_now=True), # TODO: remove old DeckPartNotes. Use pool of DeckParts + notes=DeckPartNotes.from_deck_part_pool(data.name), file_mappings=data.get_file_mappings(), note_model_mappings=data.get_note_model_mappings() ) @@ -142,7 +142,7 @@ def from_repr(cls, data: Representation): def from_dict(cls, data: dict): return cls.from_repr(TrCsvCollectionToNotes.Representation.from_dict(data)) - def notes_to_source(self): + def execute(self): notes_data = self.notes.get_notes() self.verify_notes_match_note_model_mappings(notes_data) diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_generic.py b/brain_brew/representation/deck_part_transformers/tr_notes_generic.py index e20af95..34bcf1b 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_generic.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_generic.py @@ -2,11 +2,12 @@ from typing import Optional import re +from brain_brew.representation.deck_part_transformers.tr_generic import TrGeneric from brain_brew.representation.yaml.note_repr import DeckPartNotes from brain_brew.representation.configuration.global_config import GlobalConfig -class TrNotes: +class TrNotes(TrGeneric): @staticmethod def split_tags(tags_value: str) -> list: split = [entry.strip() for entry in re.split(';\s*|,\s*|\s+', tags_value)] diff --git a/brain_brew/representation/yaml/my_yaml.py b/brain_brew/representation/yaml/my_yaml.py index 528eab7..309200a 100644 --- a/brain_brew/representation/yaml/my_yaml.py +++ b/brain_brew/representation/yaml/my_yaml.py @@ -1,5 +1,10 @@ +from dataclasses import dataclass +from typing import Optional + from ruamel.yaml import YAML +from brain_brew.file_manager import FileManager + yaml_load = YAML(typ='safe') @@ -8,3 +13,16 @@ yaml_dump.indent(mapping=0, sequence=4, offset=2) yaml_dump.representer.ignore_aliases = lambda *data: True # yaml.sort_base_mapping_type_on_output = False + + +@dataclass +class YamlRepresentation: + name: str + save_to_file: Optional[str] + + @classmethod + def from_deck_part_pool(cls, name: str) -> 'YamlRepresentation': + return FileManager.get_instance().deck_part_from_pool(name) + + def write_to_file(self): + raise NotImplemented diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index f07c3f8..4bdd3c0 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -1,4 +1,5 @@ -from brain_brew.representation.yaml.my_yaml import yaml_dump, yaml_load +from brain_brew.file_manager import FileManager +from brain_brew.representation.yaml.my_yaml import yaml_dump, yaml_load, YamlRepresentation from dataclasses import dataclass from typing import List, Optional, Dict, Set @@ -112,25 +113,36 @@ def join_tags(n_tags): @dataclass -class DeckPartNotes: +class DeckPartNotes(YamlRepresentation): name: str + save_to_file: Optional[str] note_groupings: List[NoteGrouping] # TODO: File location and saving @classmethod - def from_dict(cls, name: str, data: dict): + def from_deck_part_pool(cls, name: str) -> 'DeckPartNotes': + return FileManager.get_instance().deck_part_from_pool(name) + + @classmethod + def from_dict(cls, name: str, save_to_file: Optional[str], data: dict): return cls( name=name, + save_to_file=save_to_file, note_groupings=list(map(NoteGrouping.from_dict, data.get(NOTE_GROUPINGS))) ) @classmethod - def from_list_of_notes(cls, name: str, notes: List[Note]): + def from_list_of_notes(cls, name: str, save_to_file: Optional[str], notes: List[Note]): return cls( name=name, + save_to_file=save_to_file, note_groupings=[NoteGrouping(note_model=None, tags=None, notes=notes)] ) + def write_to_file(self): + if self.save_to_file is not None: + self.dump_to_yaml(self.save_to_file) + def encode(self) -> dict: data_dict = {NOTE_GROUPINGS: [note_grouping.encode() for note_grouping in self.note_groupings]} return data_dict diff --git a/tests/test_builder.py b/tests/test_builder.py index dd2e1eb..2a68e36 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -2,10 +2,9 @@ from brain_brew.build_tasks.source_crowd_anki import SourceCrowdAnki from brain_brew.build_tasks.source_csv import SourceCsv -from brain_brew.builder import Builder +from brain_brew.representation.build_config.builder import Builder from brain_brew.representation.generic.yaml_file import YamlFile from tests.test_files import TestFiles -from tests.representation.configuration.test_global_config import global_config class TestConstructor: From e74569bf15b481458b35addb20d1077bcc8a1822 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sun, 26 Jul 2020 12:07:16 +0200 Subject: [PATCH 15/39] BuildTask Generation. Each build task may now have multiple names, and ignores case and separating characters --- brain_brew/main.py | 9 +- .../representation/build_config/build_task.py | 40 +++++++++ .../build_config/build_tasks.py | 9 -- .../representation/build_config/builder.py | 83 ------------------- .../build_config/generate_csv_collection.py | 18 ---- .../build_config/generate_deck_parts.py | 12 +++ .../build_config/task_builder.py | 73 ++++++++++++++++ .../build_config/top_level_task_builder.py | 10 +++ .../deck_part_transformers/tr_generic.py | 4 - .../tr_notes_csv_collection.py | 9 +- .../tr_notes_generic.py | 3 +- brain_brew/representation/yaml/my_yaml.py | 2 +- brain_brew/utils.py | 4 + tests/test_builder.py | 19 ++--- tests/test_files/build_files/builder1.yaml | 71 ++++++++++++---- tests/test_utils.py | 13 ++- 16 files changed, 227 insertions(+), 152 deletions(-) create mode 100644 brain_brew/representation/build_config/build_task.py delete mode 100644 brain_brew/representation/build_config/build_tasks.py delete mode 100644 brain_brew/representation/build_config/builder.py delete mode 100644 brain_brew/representation/build_config/generate_csv_collection.py create mode 100644 brain_brew/representation/build_config/generate_deck_parts.py create mode 100644 brain_brew/representation/build_config/task_builder.py create mode 100644 brain_brew/representation/build_config/top_level_task_builder.py delete mode 100644 brain_brew/representation/deck_part_transformers/tr_generic.py diff --git a/brain_brew/main.py b/brain_brew/main.py index b719f2a..ef28fd5 100644 --- a/brain_brew/main.py +++ b/brain_brew/main.py @@ -1,8 +1,7 @@ import logging -from brain_brew.representation.build_config.build_tasks import known_build_tasks from brain_brew.argument_reader import BBArgumentReader -from brain_brew.representation.build_config.builder import Builder +from brain_brew.representation.build_config.top_level_task_builder import TopLevelTaskBuilder from brain_brew.file_manager import FileManager from brain_brew.representation.configuration.global_config import GlobalConfig from brain_brew.representation.generic.yaml_file import YamlFile @@ -24,8 +23,10 @@ def main(): global_config = GlobalConfig.from_yaml(global_config_file) if global_config_file else GlobalConfig.get_default() file_manager = FileManager() - # Run chosen Builder - builder = Builder.from_dict(builder_config, global_config, file_manager) + # Parse Build Config File + builder = TopLevelTaskBuilder.from_dict(builder_config, global_config, file_manager) + + # If all good, execute it builder.execute() diff --git a/brain_brew/representation/build_config/build_task.py b/brain_brew/representation/build_config/build_task.py new file mode 100644 index 0000000..9d52099 --- /dev/null +++ b/brain_brew/representation/build_config/build_task.py @@ -0,0 +1,40 @@ +from typing import Dict, List, Type + +from brain_brew.utils import str_to_lowercase_no_separators + + +class BuildTask(object): + task_names: List[str] + + def execute(self): + raise NotImplemented() + + @classmethod + def from_dict(cls, data: dict): + raise NotImplemented() + + @classmethod + def get_all_build_tasks(cls) -> Dict[str, Type['BuildTask']]: + subclasses: List[Type[BuildTask]] = cls.__subclasses__() + known_build_tasks: Dict[str, Type[BuildTask]] = {} + + for sc in subclasses: + for original_task_name in sc.task_names: + task_name = str_to_lowercase_no_separators(original_task_name) + + if task_name in known_build_tasks: + raise KeyError(f"Multiple instances of task name '{task_name}'") + elif task_name == "" or task_name is None: + raise KeyError(f"Unknown task name {original_task_name}") + + known_build_tasks.setdefault(task_name, sc) + + return known_build_tasks + + +class TopLevelBuildTask(BuildTask): + pass + + +class GenerateDeckPartBuildTask(BuildTask): + pass diff --git a/brain_brew/representation/build_config/build_tasks.py b/brain_brew/representation/build_config/build_tasks.py deleted file mode 100644 index ad2c10e..0000000 --- a/brain_brew/representation/build_config/build_tasks.py +++ /dev/null @@ -1,9 +0,0 @@ -from typing import Dict - -known_build_tasks: Dict[str, type] = {} - - -def add_build_task(task_name: str, class_to_call: type): - if task_name in known_build_tasks: - raise KeyError(f"Multiple instances of task name '{task_name}'") - known_build_tasks.setdefault(task_name, class_to_call) diff --git a/brain_brew/representation/build_config/builder.py b/brain_brew/representation/build_config/builder.py deleted file mode 100644 index 105d948..0000000 --- a/brain_brew/representation/build_config/builder.py +++ /dev/null @@ -1,83 +0,0 @@ -from dataclasses import dataclass, field -from enum import Enum - -from brain_brew.constants.build_config_keys import BuildTaskEnum -from brain_brew.build_tasks.source_crowd_anki import SourceCrowdAnki -from brain_brew.build_tasks.source_csv import SourceCsv -from brain_brew.file_manager import FileManager -from brain_brew.interfaces.verifiable import Verifiable -from brain_brew.utils import single_item_to_list -from brain_brew.representation.configuration.global_config import GlobalConfig -from brain_brew.representation.generic.yaml_file import YamlFile, ConfigKey - - -@dataclass -class Builder: - @dataclass - class Representation: - tasks: list - - @classmethod - def from_dict(cls, data: dict): - return cls(**data) - - tasks: list - global_config: GlobalConfig - file_manager: FileManager - - @classmethod - def from_repr(cls, data: Representation, global_config, file_manager): - tasks = cls.read_tasks(data.tasks) - return cls( - tasks=tasks, - global_config=global_config, - file_manager=file_manager - ) - - @classmethod - def from_dict(cls, data: dict, global_config, file_manager): - return cls.from_repr(Builder.Representation.from_dict(data), global_config, file_manager) - - @staticmethod - def read_tasks(tasks: list) -> list: - self.BUILD_TASK_DEFINITIONS = {build_task.key_name: build_task - for source in self.KNOWN_BUILD_TASK_CLASSES - for build_task in source.get_build_keys() - } - - - build_tasks = [] - - # Tasks - for key in self.config_entry[BuilderKeys.TASKS.value]: - tasks = single_item_to_list(key) - for task in tasks: - task_keys = list(task.keys()) - if len(task_keys) != 1: - raise KeyError(f"Task should only contain 1 entry, but contains {task_keys} instead", task) - - if task_keys[0] in self.BUILD_TASK_DEFINITIONS.keys(): - definition: BuildTaskEnum = self.BUILD_TASK_DEFINITIONS[task_keys[0]] - source = definition.source_type(task[task_keys[0]], read_now) - if self.reverse_run_direction: - build_tasks.append((source, definition.reverse_task_to_execute)) - else: - build_tasks.append((source, definition.task_to_execute)) - else: - raise KeyError(f"Unknown key {key}") # TODO: check this first on all and return all errors - - if self.reverse_run_direction: - build_tasks = list(reversed(build_tasks)) - - # Verify tasks - for source, task_to_execute in build_tasks: - if isinstance(source, Verifiable): - source.verify_contents() - - return build_tasks - - def execute(self): - for (source, task_to_execute) in self.tasks: - getattr(source, task_to_execute)() - - self.file_manager.write_to_all() diff --git a/brain_brew/representation/build_config/generate_csv_collection.py b/brain_brew/representation/build_config/generate_csv_collection.py deleted file mode 100644 index cef775b..0000000 --- a/brain_brew/representation/build_config/generate_csv_collection.py +++ /dev/null @@ -1,18 +0,0 @@ -from dataclasses import dataclass - -from brain_brew.representation.build_config.build_tasks import add_build_task -from brain_brew.representation.deck_part_transformers.tr_notes_csv_collection import TrNotesToCsvCollection - - -@dataclass -class GenerateCsvCollection: - @dataclass - class Representation: - file: str - notes: dict - - file: str - notes: TrNotesToCsvCollection - - -add_build_task("Generate Csv Collection", GenerateCsvCollection) diff --git a/brain_brew/representation/build_config/generate_deck_parts.py b/brain_brew/representation/build_config/generate_deck_parts.py new file mode 100644 index 0000000..cfe4b6e --- /dev/null +++ b/brain_brew/representation/build_config/generate_deck_parts.py @@ -0,0 +1,12 @@ +from typing import Dict, Type + +from brain_brew.representation.build_config.build_task import TopLevelBuildTask, GenerateDeckPartBuildTask, BuildTask +from brain_brew.representation.build_config.task_builder import TaskBuilder + + +class GenerateDeckParts(TaskBuilder, TopLevelBuildTask): + task_names = ["Generate Deck Parts", "Generate Deck Part", "Deck Part", "Deck Parts"] + + @classmethod + def known_task_dict(cls) -> Dict[str, Type[BuildTask]]: + return GenerateDeckPartBuildTask.get_all_build_tasks() diff --git a/brain_brew/representation/build_config/task_builder.py b/brain_brew/representation/build_config/task_builder.py new file mode 100644 index 0000000..4dfb759 --- /dev/null +++ b/brain_brew/representation/build_config/task_builder.py @@ -0,0 +1,73 @@ +from dataclasses import dataclass +from typing import Dict, List, Type + +from brain_brew.file_manager import FileManager +from brain_brew.interfaces.verifiable import Verifiable +from brain_brew.representation.build_config.build_task import BuildTask +from brain_brew.representation.configuration.global_config import GlobalConfig +from brain_brew.utils import str_to_lowercase_no_separators + + +@dataclass +class TaskBuilder: + @dataclass + class Representation: + tasks: list + + @classmethod + def from_dict(cls, data: dict): + return cls(**data) + + tasks: List[BuildTask] + global_config: GlobalConfig + file_manager: FileManager + + @classmethod + def from_repr(cls, data: Representation, global_config, file_manager): + tasks = cls.read_tasks(data.tasks) + return cls( + tasks=tasks, + global_config=global_config, + file_manager=file_manager + ) + + @classmethod + def from_dict(cls, data: dict, global_config, file_manager): + return cls.from_repr(TaskBuilder.Representation.from_dict(data), global_config, file_manager) + + @classmethod + def known_task_dict(cls) -> Dict[str, Type[BuildTask]]: + raise NotImplemented() + + @classmethod + def read_tasks(cls, tasks: list) -> list: + known_task_dict = cls.known_task_dict() + build_tasks = [] + + # Tasks + for task in tasks: + task_keys = list(task.keys()) + if len(task_keys) != 1: + raise KeyError(f"Task should only contain 1 entry, but contains {task_keys} instead. " + f"Missing list separator '-'?", task) + + task_name = str_to_lowercase_no_separators(task_keys[0]) + task_arguments = task[task_keys[0]] + if task_name in known_task_dict: + task_instance = known_task_dict[task_name].from_dict(task_arguments) + build_tasks.append(task_instance) + else: + raise KeyError(f"Unknown task '{task_name}'") # TODO: check this first on all and return all errors + + # Verify tasks + for task in build_tasks: + if isinstance(task, Verifiable): + task.verify_contents() + + return build_tasks + + def execute(self): + for task in self.tasks: + task.execute() + + self.file_manager.write_to_all() diff --git a/brain_brew/representation/build_config/top_level_task_builder.py b/brain_brew/representation/build_config/top_level_task_builder.py new file mode 100644 index 0000000..e6c2bc5 --- /dev/null +++ b/brain_brew/representation/build_config/top_level_task_builder.py @@ -0,0 +1,10 @@ +from typing import Dict, Type + +from brain_brew.representation.build_config.build_task import TopLevelBuildTask, BuildTask +from brain_brew.representation.build_config.task_builder import TaskBuilder + + +class TopLevelTaskBuilder(TaskBuilder): + @classmethod + def known_task_dict(cls) -> Dict[str, Type[BuildTask]]: + return TopLevelBuildTask.get_all_build_tasks() diff --git a/brain_brew/representation/deck_part_transformers/tr_generic.py b/brain_brew/representation/deck_part_transformers/tr_generic.py deleted file mode 100644 index d26cc0b..0000000 --- a/brain_brew/representation/deck_part_transformers/tr_generic.py +++ /dev/null @@ -1,4 +0,0 @@ - -class TrGeneric: - def execute(self): - raise NotImplemented() diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py index 108d46a..ff909e1 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py @@ -1,6 +1,7 @@ from dataclasses import dataclass from typing import List, Dict +from brain_brew.representation.build_config.build_task import TopLevelBuildTask, GenerateDeckPartBuildTask from brain_brew.representation.configuration.csv_file_mapping import CsvFileMapping from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping from brain_brew.representation.deck_part_transformers.tr_notes_generic import TrNotesToGeneric, TrGenericToNotes @@ -67,7 +68,9 @@ def verify_contents(self): @dataclass -class TrCsvCollectionToNotes(TrCsvCollectionShared, TrGenericToNotes): +class TrCsvCollectionToNotes(GenerateDeckPartBuildTask, TrCsvCollectionShared, TrGenericToNotes): + task_names = ["Notes From Csv Collection", "Notes From Csv", "Notes From Csvs"] + @dataclass(init=False) class Representation(TrCsvCollectionShared.Representation, TrGenericToNotes.Representation): def __init__(self, name, file_mappings, note_model_mappings, save_to_file=None): @@ -119,7 +122,9 @@ def execute(self): @dataclass -class TrNotesToCsvCollection(TrCsvCollectionShared, TrNotesToGeneric): +class TrNotesToCsvCollection(TopLevelBuildTask, TrCsvCollectionShared, TrNotesToGeneric): + task_names = ["Generate Csv Collection", "Generate Csv Collections", "Generate Csv", "Generate Csvs"] + @dataclass(init=False) class Representation(TrCsvCollectionShared.Representation, TrNotesToGeneric.Representation): def __init__(self, name, file_mappings, note_model_mappings): diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_generic.py b/brain_brew/representation/deck_part_transformers/tr_notes_generic.py index 34bcf1b..e20af95 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_generic.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_generic.py @@ -2,12 +2,11 @@ from typing import Optional import re -from brain_brew.representation.deck_part_transformers.tr_generic import TrGeneric from brain_brew.representation.yaml.note_repr import DeckPartNotes from brain_brew.representation.configuration.global_config import GlobalConfig -class TrNotes(TrGeneric): +class TrNotes: @staticmethod def split_tags(tags_value: str) -> list: split = [entry.strip() for entry in re.split(';\s*|,\s*|\s+', tags_value)] diff --git a/brain_brew/representation/yaml/my_yaml.py b/brain_brew/representation/yaml/my_yaml.py index 309200a..6679b11 100644 --- a/brain_brew/representation/yaml/my_yaml.py +++ b/brain_brew/representation/yaml/my_yaml.py @@ -3,7 +3,6 @@ from ruamel.yaml import YAML -from brain_brew.file_manager import FileManager yaml_load = YAML(typ='safe') @@ -22,6 +21,7 @@ class YamlRepresentation: @classmethod def from_deck_part_pool(cls, name: str) -> 'YamlRepresentation': + from brain_brew.file_manager import FileManager return FileManager.get_instance().deck_part_from_pool(name) def write_to_file(self): diff --git a/brain_brew/utils.py b/brain_brew/utils.py index 9d5787e..243cbce 100644 --- a/brain_brew/utils.py +++ b/brain_brew/utils.py @@ -22,6 +22,10 @@ def single_item_to_list(item): return [item] +def str_to_lowercase_no_separators(str_to_tidy: str): + return re.sub(r'[\s+_-]+', '', str_to_tidy.lower()) + + def filename_from_full_path(full_path): return re.findall('[^\\/:*?"<>|\r\n]+$', full_path)[0] diff --git a/tests/test_builder.py b/tests/test_builder.py index 2a68e36..b6df7e9 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -1,22 +1,21 @@ from unittest.mock import patch -from brain_brew.build_tasks.source_crowd_anki import SourceCrowdAnki -from brain_brew.build_tasks.source_csv import SourceCsv -from brain_brew.representation.build_config.builder import Builder +from brain_brew.representation.build_config.top_level_task_builder import TopLevelTaskBuilder +from brain_brew.representation.deck_part_transformers.tr_notes_csv_collection import TrNotesToCsvCollection from brain_brew.representation.generic.yaml_file import YamlFile +from tests.test_file_manager import get_new_file_manager from tests.test_files import TestFiles +from tests.representation.configuration.test_global_config import global_config class TestConstructor: def test_runs(self, global_config): + fm = get_new_file_manager() - with patch.object(SourceCsv, "__init__", return_value=None) as mock_csv, \ - patch.object(SourceCsv, "verify_contents", return_value=None), \ - patch.object(SourceCrowdAnki, "__init__", return_value=None) as mock_ca: + with patch.object(TrNotesToCsvCollection, "__init__", return_value=None) as mock_csv_tr: data = YamlFile.read_file(TestFiles.BuildConfig.ONE_OF_EACH_TYPE) - builder = Builder(data, global_config, read_now=False) + builder = TopLevelTaskBuilder.from_dict(data, global_config, fm) - assert len(builder.build_tasks) == 2 - assert mock_csv.call_count == 1 - assert mock_ca.call_count == 1 + assert len(builder.build_tasks) == 1 + assert mock_csv_tr.call_count == 1 diff --git a/tests/test_files/build_files/builder1.yaml b/tests/test_files/build_files/builder1.yaml index 5c13909..525757f 100644 --- a/tests/test_files/build_files/builder1.yaml +++ b/tests/test_files/build_files/builder1.yaml @@ -1,32 +1,67 @@ -reverse: no tasks: - - csv_collection_to_deck_parts: - notes: csv_first_attempt.json + - generate_csv_collection: + notes: test_from_CA note_model_mappings: - note_model: LL Word csv_columns_to_fields: guid: guid tags: tags + english: Word danish: X Word - personal_fields: [] + danish audio: X Pronunciation (Recording and/or IPA) + esperanto: Y Word + esperanto audio: Y Pronunciation (Recording and/or IPA) + personal_fields: + - picture + - extra + - morphman_focusmorph + - note_model: LL Verb + csv_columns_to_fields: + guid: guid + tags: tags - csv_file_mappings: - - csv: source/vocab/main.csv - note_model: LL Word - sort_by_columns: [english] - reverse_sort: no + english: Word + danish: X Word + danish audio: X Pronunciation (Recording and/or IPA) + esperanto: Y Word + esperanto audio: Y Pronunciation (Recording and/or IPA) - derivatives: - - csv: source/vocab/derivatives/danish/danish_verbs.csv - note_model: LL Verb - - csv: source/vocab/derivatives/danish/danish_nouns.csv - note_model: LL Noun + present: Form Present + past: Form Past + present perfect: Form Perfect Present + personal_fields: + - picture + - extra + - morphman_focusmorph + - note_model: LL Noun + csv_columns_to_fields: + guid: guid + tags: tags + english: Word + danish: X Word + danish audio: X Pronunciation (Recording and/or IPA) + esperanto: Y Word + esperanto audio: Y Pronunciation (Recording and/or IPA) + + plural: Plural + indefinite plural: Indefinite Plural + definite plural: Definite Plural + personal_fields: + - picture + - extra + - morphman_focusmorph - - crowdanki_to_deck_parts: - subconfig: build_tasks/crowdanki/Danish_Esperanto.yaml + csv_file_mappings: + - file: source/vocab/main.csv + note_model: LL Word + sort_by_columns: [english] + reverse_sort: false - headers: default - notes: dan_esp_vocab.json \ No newline at end of file + derivatives: + - file: source/vocab/derivatives/danish/danish_verbs.csv + note_model: LL Verb + - file: source/vocab/derivatives/danish/danish_nouns.csv + note_model: LL Noun \ No newline at end of file diff --git a/tests/test_utils.py b/tests/test_utils.py index 489dc22..030654f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,6 @@ import pytest -from brain_brew.utils import find_media_in_field +from brain_brew.utils import find_media_in_field, str_to_lowercase_no_separators class TestFindMedia: @@ -23,3 +23,14 @@ class TestFindMedia: ]) def test_find_media_in_field(self, field_value, expected_results): assert find_media_in_field(field_value) == expected_results + + +class TestHelperFunctions: + @pytest.mark.parametrize("str_to_tidy", [ + 'Generate Csv Blah Blah', + 'Generate__Csv_Blah-Blah', + 'Generate Csv Blah Blah', + 'generateCsvBlahBlah' + ]) + def test_remove_spacers_from_str(self, str_to_tidy): + assert str_to_lowercase_no_separators(str_to_tidy) == "generatecsvblahblah" From 50b0a7c86019cbe3e790667e6fc61382d5fabb9f Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Tue, 28 Jul 2020 18:21:06 +0200 Subject: [PATCH 16/39] Builder Test Working --- .../configuration/note_model_mapping.py | 3 + .../tr_notes_csv_collection.py | 17 ++++-- .../tr_notes_generic.py | 7 ++- .../json/test_deck_part_note_model.py | 8 +++ tests/test_builder.py | 11 +++- tests/test_files/build_files/builder1.yaml | 57 ++++++------------- 6 files changed, 51 insertions(+), 52 deletions(-) diff --git a/brain_brew/representation/configuration/note_model_mapping.py b/brain_brew/representation/configuration/note_model_mapping.py index 4920d52..d86f9ef 100644 --- a/brain_brew/representation/configuration/note_model_mapping.py +++ b/brain_brew/representation/configuration/note_model_mapping.py @@ -71,6 +71,9 @@ def from_repr(cls, data: Representation): note_models=dict(map(lambda nm: (nm.name, nm), note_models)) # TODO: Use deck part pool ) + def get_note_model_mapping_dict(self): + return {model: self for model in self.note_models} + def verify_contents(self): errors = [] diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py index ff909e1..7ee0bde 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py @@ -23,7 +23,11 @@ def get_file_mappings(self) -> List[CsvFileMapping]: return list(map(CsvFileMapping.from_repr, self.file_mappings)) def get_note_model_mappings(self) -> Dict[str, NoteModelMapping]: - return dict(map(lambda nmm: (nmm.note_model, NoteModelMapping.from_repr(nmm)), self.note_model_mappings)) + def map_nmm(nmm_to_map: str): + nmm = NoteModelMapping.from_repr(nmm_to_map) + return nmm.get_note_model_mapping_dict() + + return dict(*map(map_nmm, self.note_model_mappings)) file_mappings: List[CsvFileMapping] note_model_mappings: Dict[str, NoteModelMapping] @@ -90,6 +94,9 @@ def from_repr(cls, data: Representation): note_model_mappings=data.get_note_model_mappings() ) + def __repr__(self): + return f'TrCsvCollectionToNotes({self.name!r}, {self.save_to_file!r}, {self.file!r}, {self.note_model_mappings!r}, ' + @classmethod def from_dict(cls, data: dict): return cls.from_repr(TrCsvCollectionToNotes.Representation.from_dict(data)) @@ -127,9 +134,9 @@ class TrNotesToCsvCollection(TopLevelBuildTask, TrCsvCollectionShared, TrNotesTo @dataclass(init=False) class Representation(TrCsvCollectionShared.Representation, TrNotesToGeneric.Representation): - def __init__(self, name, file_mappings, note_model_mappings): + def __init__(self, notes, file_mappings, note_model_mappings): TrCsvCollectionShared.Representation.__init__(self, file_mappings, note_model_mappings) - TrNotesToGeneric.Representation.__init__(self, name) + TrNotesToGeneric.Representation.__init__(self, notes) @classmethod def from_dict(cls, data: dict): @@ -138,14 +145,14 @@ def from_dict(cls, data: dict): @classmethod def from_repr(cls, data: Representation): return cls( - notes=DeckPartNotes.from_deck_part_pool(data.name), + notes=DeckPartNotes.from_deck_part_pool(data.notes), file_mappings=data.get_file_mappings(), note_model_mappings=data.get_note_model_mappings() ) @classmethod def from_dict(cls, data: dict): - return cls.from_repr(TrCsvCollectionToNotes.Representation.from_dict(data)) + return cls.from_repr(TrNotesToCsvCollection.Representation.from_dict(data)) def execute(self): notes_data = self.notes.get_notes() diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_generic.py b/brain_brew/representation/deck_part_transformers/tr_notes_generic.py index e20af95..a04017e 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_generic.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_generic.py @@ -6,6 +6,7 @@ from brain_brew.representation.configuration.global_config import GlobalConfig +@dataclass class TrNotes: @staticmethod def split_tags(tags_value: str) -> list: @@ -40,9 +41,9 @@ def __init__(self, name, save_to_file=None): class TrNotesToGeneric(TrNotes): @dataclass class Representation: - name: str + notes: str - def __init__(self, name): - self.name = name + def __init__(self, notes): + self.notes = notes notes: DeckPartNotes diff --git a/tests/representation/json/test_deck_part_note_model.py b/tests/representation/json/test_deck_part_note_model.py index 37ce37c..382acc5 100644 --- a/tests/representation/json/test_deck_part_note_model.py +++ b/tests/representation/json/test_deck_part_note_model.py @@ -1,3 +1,5 @@ +from unittest.mock import Mock + import pytest from brain_brew.representation.json.deck_part_notemodel import DeckPartNoteModel, CANoteModelKeys @@ -6,6 +8,12 @@ from tests.representation.configuration.test_global_config import global_config +def mock_dp_nm(name, read_now): + mock = Mock() + mock.name = name + return mock + + class TestConstructor: @pytest.mark.parametrize("note_model_name", [ TestFiles.NoteModels.TEST_COMPLETE, diff --git a/tests/test_builder.py b/tests/test_builder.py index b6df7e9..2bd85d0 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -1,8 +1,11 @@ -from unittest.mock import patch +from unittest.mock import patch, Mock from brain_brew.representation.build_config.top_level_task_builder import TopLevelTaskBuilder from brain_brew.representation.deck_part_transformers.tr_notes_csv_collection import TrNotesToCsvCollection from brain_brew.representation.generic.yaml_file import YamlFile +from brain_brew.representation.json.deck_part_notemodel import DeckPartNoteModel +from brain_brew.representation.yaml.note_repr import DeckPartNotes +from tests.representation.json.test_deck_part_note_model import mock_dp_nm from tests.test_file_manager import get_new_file_manager from tests.test_files import TestFiles from tests.representation.configuration.test_global_config import global_config @@ -12,10 +15,12 @@ class TestConstructor: def test_runs(self, global_config): fm = get_new_file_manager() - with patch.object(TrNotesToCsvCollection, "__init__", return_value=None) as mock_csv_tr: + with patch.object(TrNotesToCsvCollection, "__init__", return_value=None) as mock_csv_tr, \ + patch.object(DeckPartNotes, "from_deck_part_pool", return_value=Mock()), \ + patch.object(DeckPartNoteModel, "create", side_effect=mock_dp_nm): data = YamlFile.read_file(TestFiles.BuildConfig.ONE_OF_EACH_TYPE) builder = TopLevelTaskBuilder.from_dict(data, global_config, fm) - assert len(builder.build_tasks) == 1 + assert len(builder.tasks) == 1 assert mock_csv_tr.call_count == 1 diff --git a/tests/test_files/build_files/builder1.yaml b/tests/test_files/build_files/builder1.yaml index 525757f..92706da 100644 --- a/tests/test_files/build_files/builder1.yaml +++ b/tests/test_files/build_files/builder1.yaml @@ -3,22 +3,11 @@ tasks: notes: test_from_CA note_model_mappings: - - note_model: LL Word - csv_columns_to_fields: - guid: guid - tags: tags - - english: Word - danish: X Word - danish audio: X Pronunciation (Recording and/or IPA) - esperanto: Y Word - esperanto audio: Y Pronunciation (Recording and/or IPA) - personal_fields: - - picture - - extra - - morphman_focusmorph - - note_model: LL Verb - csv_columns_to_fields: + - note_models: + - LL Word + - LL Verb + - LL Noun + columns_to_fields: guid: guid tags: tags @@ -31,20 +20,6 @@ tasks: present: Form Present past: Form Past present perfect: Form Perfect Present - personal_fields: - - picture - - extra - - morphman_focusmorph - - note_model: LL Noun - csv_columns_to_fields: - guid: guid - tags: tags - - english: Word - danish: X Word - danish audio: X Pronunciation (Recording and/or IPA) - esperanto: Y Word - esperanto audio: Y Pronunciation (Recording and/or IPA) plural: Plural indefinite plural: Indefinite Plural @@ -54,14 +29,14 @@ tasks: - extra - morphman_focusmorph - csv_file_mappings: - - file: source/vocab/main.csv - note_model: LL Word - sort_by_columns: [english] - reverse_sort: false - - derivatives: - - file: source/vocab/derivatives/danish/danish_verbs.csv - note_model: LL Verb - - file: source/vocab/derivatives/danish/danish_nouns.csv - note_model: LL Noun \ No newline at end of file + file_mappings: + - file: source/vocab/main.csv + note_model: LL Word + sort_by_columns: [english] + reverse_sort: false + + derivatives: + - file: source/vocab/derivatives/danish/danish_verbs.csv + note_model: LL Verb + - file: source/vocab/derivatives/danish/danish_nouns.csv + note_model: LL Noun \ No newline at end of file From 6ee2ccd23f680ef92eef931848775b7b8c8d1f70 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sat, 1 Aug 2020 11:57:02 +0200 Subject: [PATCH 17/39] Ongoing Refactor: CrowdAnkiNoteModel repr DeckPartNoteModel repr TrCsv Tests --- brain_brew/build_tasks/source_crowd_anki.py | 2 +- brain_brew/build_tasks/source_csv.py | 30 ----------- .../configuration/note_model_mapping.py | 11 ++-- .../crowdanki/crowdanki_note_model.py | 44 +++++++++++++++ .../json/deck_part_notemodel.py | 54 ------------------- brain_brew/representation/json/json_file.py | 46 ++++------------ brain_brew/representation/yaml/my_yaml.py | 16 ++++-- brain_brew/representation/yaml/note_model.py | 32 +++++++++++ .../yaml/note_model_card_template.py | 0 .../representation/yaml/note_model_field.py | 0 brain_brew/representation/yaml/note_repr.py | 2 +- .../test_source_crowd_anki_json.py | 9 +--- tests/build_tasks/test_source_csv.py | 14 ++--- .../configuration/test_note_model_mapping.py | 42 +++++++-------- .../crowdanki/test_crowdanki_note_model.py | 12 +++++ .../json/test_deck_part_note_model.py | 4 +- tests/test_builder.py | 7 ++- tests/test_files.py | 4 +- 18 files changed, 148 insertions(+), 181 deletions(-) delete mode 100644 brain_brew/build_tasks/source_csv.py create mode 100644 brain_brew/representation/crowdanki/crowdanki_note_model.py delete mode 100644 brain_brew/representation/json/deck_part_notemodel.py create mode 100644 brain_brew/representation/yaml/note_model.py create mode 100644 brain_brew/representation/yaml/note_model_card_template.py create mode 100644 brain_brew/representation/yaml/note_model_field.py create mode 100644 tests/representation/crowdanki/test_crowdanki_note_model.py diff --git a/brain_brew/build_tasks/source_crowd_anki.py b/brain_brew/build_tasks/source_crowd_anki.py index a8826d4..df44694 100644 --- a/brain_brew/build_tasks/source_crowd_anki.py +++ b/brain_brew/build_tasks/source_crowd_anki.py @@ -11,7 +11,7 @@ from brain_brew.representation.generic.yaml_file import ConfigKey, YamlFile from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport from brain_brew.representation.json.deck_part_header import DeckPartHeader -from brain_brew.representation.json.deck_part_notemodel import CANoteModelKeys, DeckPartNoteModel +from brain_brew.representation.yaml.note_model import CANoteModelKeys, DeckPartNoteModel from brain_brew.representation.json.deck_part_notes import CANoteKeys, DeckPartNotes diff --git a/brain_brew/build_tasks/source_csv.py b/brain_brew/build_tasks/source_csv.py deleted file mode 100644 index feb2ac6..0000000 --- a/brain_brew/build_tasks/source_csv.py +++ /dev/null @@ -1,30 +0,0 @@ -from enum import Enum -from typing import List, Dict - -from brain_brew.build_tasks.build_task_generic import BuildTaskGeneric -from brain_brew.constants.build_config_keys import BuildTaskEnum, BuildConfigKeys -from brain_brew.constants.deckpart_keys import DeckPartNoteKeys -from brain_brew.interfaces.verifiable import Verifiable -from brain_brew.representation.configuration.csv_file_mapping import CsvFileMapping -from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping -from brain_brew.utils import single_item_to_list -from brain_brew.representation.generic.yaml_file import ConfigKey, YamlFile -from brain_brew.representation.generic.csv_file import CsvFile, CsvKeys -from brain_brew.representation.json.deck_part_notemodel import DeckPartNoteModel -from brain_brew.representation.json.deck_part_notes import DeckPartNotes - - -class SourceCsvKeys(Enum): - NOTES = "notes" - NOTE_MODEL_MAPPINGS = "note_model_mappings" - CSV_MAPPINGS = "csv_file_mappings" - - -class SourceCsv(YamlFile, BuildTaskGeneric, Verifiable): - @staticmethod - def get_build_keys(): - return [ - BuildTaskEnum("deck_parts_to_csv_collection", SourceCsv, "deck_parts_to_source", "source_to_deck_parts"), - BuildTaskEnum("csv_collection_to_deck_parts", SourceCsv, "source_to_deck_parts", "deck_parts_to_source"), - ] - diff --git a/brain_brew/representation/configuration/note_model_mapping.py b/brain_brew/representation/configuration/note_model_mapping.py index d86f9ef..2d82920 100644 --- a/brain_brew/representation/configuration/note_model_mapping.py +++ b/brain_brew/representation/configuration/note_model_mapping.py @@ -4,13 +4,8 @@ from brain_brew.constants.deckpart_keys import DeckPartNoteKeys from brain_brew.interfaces.verifiable import Verifiable -from brain_brew.representation.generic.yaml_file import YamlFile, ConfigKey -from brain_brew.representation.json.deck_part_notemodel import DeckPartNoteModel -from brain_brew.utils import list_of_str_to_lowercase, single_item_to_list - -NOTE_MODEL = "note_model" -COLUMNS = "csv_columns_to_fields" -PERSONAL_FIELDS = "personal_fields" +from brain_brew.representation.yaml.note_model import DeckPartNoteModel +from brain_brew.utils import single_item_to_list class FieldMapping: @@ -57,7 +52,7 @@ def from_dict(cls, data: dict): @classmethod def from_repr(cls, data: Representation): - note_models = [DeckPartNoteModel.create(model, read_now=True) for model in single_item_to_list(data.note_models)] + note_models = [DeckPartNoteModel.from_deck_part_pool(model) for model in single_item_to_list(data.note_models)] return cls( columns=[FieldMapping( diff --git a/brain_brew/representation/crowdanki/crowdanki_note_model.py b/brain_brew/representation/crowdanki/crowdanki_note_model.py new file mode 100644 index 0000000..3526b68 --- /dev/null +++ b/brain_brew/representation/crowdanki/crowdanki_note_model.py @@ -0,0 +1,44 @@ +from dataclasses import dataclass, field +from typing import List, Optional + + +@dataclass +class CrowdAnkiNoteModel: + @dataclass + class Field: + name: str + ord: int + font: str = field(default="Liberation Sans") + media: List[str] = field(default_factory=lambda: []) + rtl: bool = field(default=False) + size: int = field(default=20) + sticky: bool = field(default=False) + + @dataclass + class Template: + name: str + ord: int + afmt: str + qfmt: str + bafmt: str = field(default="") + bqfmt: str = field(default="") + did: Optional[int] = field(default=None) + + name: str + crowdanki_uuid: str + css: str + flds: List[Field] + tmpls: List[Template] + latexPost: str + latexPre: str + req: List[list] + sortf: int = field(default=0) + tags: List[str] = field(default_factory=lambda: []) + __type__: str = field(default="NoteModel") + type: int = field(default=0) + vers: list = field(default_factory=lambda: []) + + @classmethod + def from_dict(cls, data): + return cls(**data) + diff --git a/brain_brew/representation/json/deck_part_notemodel.py b/brain_brew/representation/json/deck_part_notemodel.py deleted file mode 100644 index 65ffecb..0000000 --- a/brain_brew/representation/json/deck_part_notemodel.py +++ /dev/null @@ -1,54 +0,0 @@ -from enum import Enum -from typing import List - -from brain_brew.utils import list_of_str_to_lowercase -from brain_brew.representation.configuration.global_config import GlobalConfig -from brain_brew.representation.json.json_file import JsonFile - - -class CANoteModelKeys(Enum): - ID = "crowdanki_uuid" - NAME = "name" - FIELDS = "flds" - - -class DeckPartNoteModel(JsonFile): - @property - def name(self) -> str: - return self._data[CANoteModelKeys.NAME.value] - - @property - def id(self) -> str: - return self._data[CANoteModelKeys.ID.value] - - @property - def fields(self) -> List[str]: - return [field['name'] for field in self._data[CANoteModelKeys.FIELDS.value]] - - @property - def fields_lowercase(self): - return list_of_str_to_lowercase(self.fields) - - @classmethod - def formatted_file_location(cls, location): - return cls.get_json_file_location(GlobalConfig.get_instance().deck_parts.note_models, location) - - def __init__(self, location, read_now=True, data_override=None): - super().__init__( - self.formatted_file_location(location), - read_now=read_now, data_override=data_override - ) - - def check_field_overlap(self, fields_to_check: List[str]): - fields_to_check = list_of_str_to_lowercase(fields_to_check) - lower_fields = self.fields_lowercase - - missing = [field for field in lower_fields if field not in fields_to_check] - extra = [field for field in fields_to_check if field not in lower_fields] - - return missing, extra - - def zip_field_to_data(self, data: List[str]) -> dict: - if len(self.fields) != len(data): - raise Exception(f"Data of length {len(data)} cannot map to fields of length {len(self.fields_lowercase)}") - return dict(zip(self.fields_lowercase, data)) diff --git a/brain_brew/representation/json/json_file.py b/brain_brew/representation/json/json_file.py index 4ff08f0..bf96eed 100644 --- a/brain_brew/representation/json/json_file.py +++ b/brain_brew/representation/json/json_file.py @@ -1,45 +1,19 @@ import json -import re - -from brain_brew.representation.generic.generic_file import GenericFile _encoding = "utf-8" -class JsonFile(GenericFile): - _data: dict = {} - - def __init__(self, file, read_now=True, data_override=None): - super(JsonFile, self).__init__(file, read_now=read_now, data_override=data_override) - - def set_data(self, data_override): - super().set_data(data_override) - - def get_data(self, deep_copy: bool = False) -> dict: - return super().get_data(deep_copy=deep_copy) - - def pretty_print(self): - return json.dumps(self._data, indent=4) - - def read_file(self): - with open(self.file_location, "r", encoding=_encoding) as read_file: - self._data = json.load(read_file) - - def write_file(self, data_override=None): - with open(self.file_location, "w", encoding=_encoding) as write_file: - json.dump(data_override or self._data, write_file, indent=4, sort_keys=False, ensure_ascii=False) - - self.file_exists = True - +class JsonFile: @staticmethod - def to_filename_json(filename: str) -> str: - converted = re.sub(r'\s+', '-', filename).strip() - return converted + ".json" if not converted.endswith(".json") else converted + def pretty_print(data): + return json.dumps(data, indent=4) @staticmethod - def get_json_file_location(prepend, location): - return JsonFile.to_filename_json(prepend + location) + def read_file(file_location): + with open(file_location, "r", encoding=_encoding) as read_file: + return json.load(read_file) - @classmethod - def formatted_file_location(cls, location): - return cls.to_filename_json(location) + @staticmethod + def write_file(file_location, data): + with open(file_location, "w", encoding=_encoding) as write_file: + json.dump(data, write_file, indent=4, sort_keys=False, ensure_ascii=False) diff --git a/brain_brew/representation/yaml/my_yaml.py b/brain_brew/representation/yaml/my_yaml.py index 6679b11..b05a66b 100644 --- a/brain_brew/representation/yaml/my_yaml.py +++ b/brain_brew/representation/yaml/my_yaml.py @@ -1,8 +1,7 @@ from dataclasses import dataclass from typing import Optional -from ruamel.yaml import YAML - +from ruamel.yaml import YAML, Path yaml_load = YAML(typ='safe') @@ -20,9 +19,20 @@ class YamlRepresentation: save_to_file: Optional[str] @classmethod - def from_deck_part_pool(cls, name: str) -> 'YamlRepresentation': + def from_deck_part_pool(cls, name: str): from brain_brew.file_manager import FileManager return FileManager.get_instance().deck_part_from_pool(name) + @staticmethod + def filename_to_dict(filename: str): + if filename[-5:] not in [".yaml", ".yml"]: + filename += ".yaml" + + if not Path(filename).is_file(): + raise FileNotFoundError(filename) + + with open(filename) as file: + return yaml_load.load(file) + def write_to_file(self): raise NotImplemented diff --git a/brain_brew/representation/yaml/note_model.py b/brain_brew/representation/yaml/note_model.py new file mode 100644 index 0000000..f12673d --- /dev/null +++ b/brain_brew/representation/yaml/note_model.py @@ -0,0 +1,32 @@ +from dataclasses import dataclass, field +from typing import List + +from brain_brew.representation.yaml.my_yaml import YamlRepresentation +from brain_brew.utils import list_of_str_to_lowercase + + +@dataclass +class DeckPartNoteModel(YamlRepresentation): + name: str + id: str + fields: List[str] + + @property + def fields_lowercase(self): + return list_of_str_to_lowercase(self.fields) + + def check_field_overlap(self, fields_to_check: List[str]): + fields_to_check = list_of_str_to_lowercase(fields_to_check) + lower_fields = self.fields_lowercase + + missing = [field for field in lower_fields if field not in fields_to_check] + extra = [field for field in fields_to_check if field not in lower_fields] + + return missing, extra + + def zip_field_to_data(self, data: List[str]) -> dict: + if len(self.fields) != len(data): + raise Exception(f"Data of length {len(data)} cannot map to fields of length {len(self.fields_lowercase)}") + return dict(zip(self.fields_lowercase, data)) + + diff --git a/brain_brew/representation/yaml/note_model_card_template.py b/brain_brew/representation/yaml/note_model_card_template.py new file mode 100644 index 0000000..e69de29 diff --git a/brain_brew/representation/yaml/note_model_field.py b/brain_brew/representation/yaml/note_model_field.py new file mode 100644 index 0000000..e69de29 diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index 4bdd3c0..0474efb 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -121,7 +121,7 @@ class DeckPartNotes(YamlRepresentation): @classmethod def from_deck_part_pool(cls, name: str) -> 'DeckPartNotes': - return FileManager.get_instance().deck_part_from_pool(name) + return super(DeckPartNotes, cls).from_deck_part_pool(name) @classmethod def from_dict(cls, name: str, save_to_file: Optional[str], data: dict): diff --git a/tests/build_tasks/test_source_crowd_anki_json.py b/tests/build_tasks/test_source_crowd_anki_json.py index 975f446..84f634c 100644 --- a/tests/build_tasks/test_source_crowd_anki_json.py +++ b/tests/build_tasks/test_source_crowd_anki_json.py @@ -6,15 +6,8 @@ from brain_brew.build_tasks.source_crowd_anki import SourceCrowdAnki, CrowdAnkiKeys from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport from brain_brew.representation.json.deck_part_header import DeckPartHeader -from brain_brew.representation.json.deck_part_notemodel import DeckPartNoteModel +from brain_brew.representation.yaml.note_model import DeckPartNoteModel from brain_brew.representation.json.deck_part_notes import DeckPartNotes -from tests.test_files import TestFiles -from tests.representation.json.test_crowd_anki_export import ca_export_test1, temp_ca_export_file -from tests.representation.json.test_deck_part_header import dp_headers_test1, temp_dp_headers_file -from tests.representation.json.test_deck_part_notes import dp_notes_test1, temp_dp_notes_file -from tests.representation.json.test_deck_part_note_model import dp_note_model_test1, temp_dp_note_model_file -from tests.representation.json.test_json_file import temp_json_file -from tests.representation.configuration.test_global_config import global_config def setup_ca_config(file, media, useless_note_keys, notes, headers): diff --git a/tests/build_tasks/test_source_csv.py b/tests/build_tasks/test_source_csv.py index f6ce957..22860d0 100644 --- a/tests/build_tasks/test_source_csv.py +++ b/tests/build_tasks/test_source_csv.py @@ -1,5 +1,5 @@ from typing import List -from unittest.mock import patch, MagicMock +from unittest.mock import patch import pytest @@ -9,16 +9,10 @@ from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping from brain_brew.representation.generic.csv_file import CsvFile from brain_brew.representation.generic.generic_file import GenericFile -from brain_brew.representation.json.deck_part_notemodel import DeckPartNoteModel from brain_brew.representation.json.deck_part_notes import DeckPartNotes -from tests.test_files import TestFiles -from tests.representation.configuration.test_global_config import global_config -from tests.representation.generic.test_csv_file import csv_test1, csv_test2, csv_test1_split1, csv_test1_split2 -from tests.representation.json.test_deck_part_notes import dp_notes_test1, dp_notes_test2 -from tests.representation.json.test_deck_part_note_model import dp_note_model_test1 -from tests.representation.configuration.test_note_model_mapping import nmm_test1, setup_nmm_config -from tests.representation.configuration.test_csv_file_mapping import csv_file_mapping1, csv_file_mapping2, \ - setup_csv_fm_config +from tests.representation.json.test_deck_part_notes import dp_notes_test1 +from tests.representation.configuration.test_note_model_mapping import setup_nmm_config +from tests.representation.configuration.test_csv_file_mapping import setup_csv_fm_config def setup_source_csv_config(notes: str, nmm: list, csv_mappings: list): diff --git a/tests/representation/configuration/test_note_model_mapping.py b/tests/representation/configuration/test_note_model_mapping.py index 313b88a..2f51133 100644 --- a/tests/representation/configuration/test_note_model_mapping.py +++ b/tests/representation/configuration/test_note_model_mapping.py @@ -1,29 +1,22 @@ -from typing import Dict, List from unittest.mock import patch import pytest -from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping, FieldMapping, \ - NOTE_MODEL, COLUMNS, PERSONAL_FIELDS +from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping, FieldMapping from brain_brew.representation.generic.csv_file import CsvFile -from brain_brew.representation.json.deck_part_notemodel import DeckPartNoteModel -from tests.representation.configuration.test_global_config import global_config +from brain_brew.representation.yaml.note_model import DeckPartNoteModel from tests.test_file_manager import get_new_file_manager -from tests.representation.generic.test_csv_file import csv_test1 -def setup_nmm_config(note_model: str, field_mappings: Dict[str, str], personal_fields: List[str]): - return { - NOTE_MODEL: note_model, - COLUMNS: field_mappings, - PERSONAL_FIELDS: personal_fields - } +@pytest.fixture(autouse=True) +def run_around_tests(global_config): + get_new_file_manager() + yield @pytest.fixture() -def nmm_test1(global_config) -> NoteModelMapping: - get_new_file_manager() - config = setup_nmm_config( +def nmm_test1_repr() -> NoteModelMapping.Representation: + return NoteModelMapping.Representation( "Test Model", { "guid": "guid", @@ -34,13 +27,11 @@ def nmm_test1(global_config) -> NoteModelMapping: }, [] ) - return NoteModelMapping(config, True) @pytest.fixture() -def nmm_test_with_personal_fields1(global_config) -> NoteModelMapping: - get_new_file_manager() - config = setup_nmm_config( +def nmm_test2_repr() -> NoteModelMapping.Representation: + return NoteModelMapping.Representation( "Test Model", { "guid": "guid", @@ -51,12 +42,21 @@ def nmm_test_with_personal_fields1(global_config) -> NoteModelMapping: }, ["extra", "morph_focus"] ) - return NoteModelMapping(config, True) + + +@pytest.fixture() +def nmm_test1(nmm_test1_repr) -> NoteModelMapping: + return NoteModelMapping.from_repr(nmm_test1_repr) + + +@pytest.fixture() +def nmm_test2(nmm_test2_repr) -> NoteModelMapping: + return NoteModelMapping.from_repr(nmm_test2_repr) class TestInit: def test_runs(self): - nmm = NoteModelMapping(setup_nmm_config("test", {}, []), read_now=False) + nmm = NoteModelMapping.from_repr(NoteModelMapping.Representation("test", {}, [])) assert isinstance(nmm, NoteModelMapping) @pytest.mark.parametrize("read_file_now, note_model, personal_fields, columns", [ diff --git a/tests/representation/crowdanki/test_crowdanki_note_model.py b/tests/representation/crowdanki/test_crowdanki_note_model.py new file mode 100644 index 0000000..d4f7044 --- /dev/null +++ b/tests/representation/crowdanki/test_crowdanki_note_model.py @@ -0,0 +1,12 @@ +import pytest + +from brain_brew.representation.crowdanki.crowdanki_note_model import CrowdAnkiNoteModel +from brain_brew.representation.json.json_file import JsonFile +from tests.test_files import TestFiles + + +class TestCrowdAnkiNoteModel: + def test_constructor(self): + json_data = JsonFile.read_file(TestFiles.NoteModels.LL_WORD_COMPLETE) + model = CrowdAnkiNoteModel.from_dict(json_data) + assert isinstance(model, CrowdAnkiNoteModel) diff --git a/tests/representation/json/test_deck_part_note_model.py b/tests/representation/json/test_deck_part_note_model.py index 382acc5..297185a 100644 --- a/tests/representation/json/test_deck_part_note_model.py +++ b/tests/representation/json/test_deck_part_note_model.py @@ -2,10 +2,8 @@ import pytest -from brain_brew.representation.json.deck_part_notemodel import DeckPartNoteModel, CANoteModelKeys -from brain_brew.representation.json.json_file import JsonFile +from brain_brew.representation.yaml.note_model import DeckPartNoteModel, CANoteModelKeys from tests.test_files import TestFiles -from tests.representation.configuration.test_global_config import global_config def mock_dp_nm(name, read_now): diff --git a/tests/test_builder.py b/tests/test_builder.py index 2bd85d0..75909f6 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -2,13 +2,12 @@ from brain_brew.representation.build_config.top_level_task_builder import TopLevelTaskBuilder from brain_brew.representation.deck_part_transformers.tr_notes_csv_collection import TrNotesToCsvCollection -from brain_brew.representation.generic.yaml_file import YamlFile -from brain_brew.representation.json.deck_part_notemodel import DeckPartNoteModel +from brain_brew.representation.yaml.my_yaml import YamlRepresentation +from brain_brew.representation.yaml.note_model import DeckPartNoteModel from brain_brew.representation.yaml.note_repr import DeckPartNotes from tests.representation.json.test_deck_part_note_model import mock_dp_nm from tests.test_file_manager import get_new_file_manager from tests.test_files import TestFiles -from tests.representation.configuration.test_global_config import global_config class TestConstructor: @@ -19,7 +18,7 @@ def test_runs(self, global_config): patch.object(DeckPartNotes, "from_deck_part_pool", return_value=Mock()), \ patch.object(DeckPartNoteModel, "create", side_effect=mock_dp_nm): - data = YamlFile.read_file(TestFiles.BuildConfig.ONE_OF_EACH_TYPE) + data = YamlRepresentation.filename_to_dict(TestFiles.BuildConfig.ONE_OF_EACH_TYPE) builder = TopLevelTaskBuilder.from_dict(data, global_config, fm) assert len(builder.tasks) == 1 diff --git a/tests/test_files.py b/tests/test_files.py index beda8a4..8f9b46c 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -20,10 +20,10 @@ class NoteModels: LOC = "tests/test_files/deck_parts/note_models/" TEST = "Test Model" - TEST_COMPLETE = "Test-Model.json" + TEST_COMPLETE = LOC + "Test-Model.json" LL_WORD = "LL Word" - LL_WORD_COMPLETE = "LL_Word.json" + LL_WORD_COMPLETE = LOC + "LL-Word.json" class CsvFiles: LOC = "tests/test_files/csv/" From fc008b1273a9fd407b86940c236c31bc8e4ab40f Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sun, 2 Aug 2020 12:51:53 +0200 Subject: [PATCH 18/39] CrowdAnki Note Model Representation --- .../crowdanki/crowdanki_note_model.py | 175 +++++++++++++--- .../tr_notes_csv_collection.py | 5 +- .../crowdanki/test_crowdanki_note_model.py | 23 ++- tests/test_files.py | 2 + .../note_models/LL-Word-Only-Required.json | 194 ++++++++++++++++++ 5 files changed, 368 insertions(+), 31 deletions(-) create mode 100644 tests/test_files/deck_parts/note_models/LL-Word-Only-Required.json diff --git a/brain_brew/representation/crowdanki/crowdanki_note_model.py b/brain_brew/representation/crowdanki/crowdanki_note_model.py index 3526b68..17331e3 100644 --- a/brain_brew/representation/crowdanki/crowdanki_note_model.py +++ b/brain_brew/representation/crowdanki/crowdanki_note_model.py @@ -1,44 +1,169 @@ from dataclasses import dataclass, field from typing import List, Optional +# CrowdAnki +CROWDANKI_ID = "crowdanki_uuid" +CROWDANKI_TYPE = "__type__" + +# Shared +NAME = "name" +ORDINAL = "ord" + +# Note Model +CSS = "css" +LATEX_POST = "latexPost" +LATEX_PRE = "latexPre" +REQUIRED_FIELDS_PER_TEMPLATE = "req" +FIELDS = "flds" +TEMPLATES = "tmpls" +TAGS = "tags" +SORT_FIELD_NUM = "sortf" +IS_CLOZE = "type" +VERSION = "vers" + +# Field +FONT = "font" +MEDIA = "media" # Unused in Anki +IS_RIGHT_TO_LEFT = "rtl" +FONT_SIZE = "size" +IS_STICKY = "sticky" + +# Template +QUESTION_FORMAT = "qfmt" +ANSWER_FORMAT = "afmt" +BROWSER_ANSWER_FORMAT = "bafmt" +BROWSER_QUESTION_FORMAT = "bqfmt" +DECK_OVERRIDE_ID = "did" + @dataclass class CrowdAnkiNoteModel: @dataclass class Field: + @dataclass + class Representation: + name: str + ord: int + font: str = field(default="Liberation Sans") + media: List[str] = field(default_factory=lambda: []) + rtl: bool = field(default=False) + size: int = field(default=20) + sticky: bool = field(default=False) + + @classmethod + def from_dict(cls, data: dict): + return cls(**data) + name: str - ord: int - font: str = field(default="Liberation Sans") - media: List[str] = field(default_factory=lambda: []) - rtl: bool = field(default=False) - size: int = field(default=20) - sticky: bool = field(default=False) + ordinal: int + font: str + media: List[str] + is_right_to_left: bool + font_size: int + is_sticky: bool + + @classmethod + def from_repr(cls, data: Representation): + return cls( + name=data.name, ordinal=data.ord, font=data.font, media=data.media, + is_right_to_left=data.rtl, font_size=data.size, is_sticky=data.sticky + ) + + @classmethod + def from_dict(cls, data: dict): + return cls.from_repr(cls.Representation.from_dict(data)) @dataclass class Template: + @dataclass + class Representation: + name: str + ord: int + qfmt: str + afmt: str + bqfmt: str = field(default="") + bafmt: str = field(default="") + did: Optional[int] = field(default=None) + + @classmethod + def from_dict(cls, data: dict): + return cls(**data) + name: str - ord: int - afmt: str - qfmt: str - bafmt: str = field(default="") - bqfmt: str = field(default="") - did: Optional[int] = field(default=None) + ordinal: int + question_format: str + answer_format: str + question_format_in_browser: str + answer_format_in_browser: str + deck_override_id: Optional[int] + + @classmethod + def from_repr(cls, data: Representation): + return cls( + name=data.name, ordinal=data.ord, question_format=data.qfmt, answer_format=data.afmt, + question_format_in_browser=data.bqfmt, answer_format_in_browser=data.bafmt, deck_override_id=data.did + ) + + @classmethod + def from_dict(cls, data: dict): + return cls.from_repr(cls.Representation.from_dict(data)) + + @dataclass + class Representation: + name: str + crowdanki_uuid: str + css: str + latexPost: str + latexPre: str + req: List[list] + flds: List[dict] + tmpls: List[dict] + __type__: str = field(default="NoteModel") + tags: List[str] = field(default_factory=lambda: []) + sortf: int = field(default=0) + type: int = field(default=0) + vers: list = field(default_factory=lambda: []) + + @classmethod + def from_dict(cls, data: dict): + return cls(**data) + + crowdanki_id: str + crowdanki_type: str name: str - crowdanki_uuid: str css: str - flds: List[Field] - tmpls: List[Template] - latexPost: str - latexPre: str - req: List[list] - sortf: int = field(default=0) - tags: List[str] = field(default_factory=lambda: []) - __type__: str = field(default="NoteModel") - type: int = field(default=0) - vers: list = field(default_factory=lambda: []) + latex_post: str + latex_pre: str + required_fields_per_template: List[list] + fields: List[Field] + templates: List[Template] + tags: List[str] + sort_field_num: int + is_cloze: bool + version: list # Deprecated in Anki @classmethod - def from_dict(cls, data): - return cls(**data) + def from_repr(cls, data: Representation): + return cls( + fields=[CrowdAnkiNoteModel.Field.from_dict(f) for f in data.flds], + templates=[CrowdAnkiNoteModel.Template.from_dict(t) for t in data.tmpls], + is_cloze=bool(data.type), + name=data.name, css=data.css, latex_pre=data.latexPre, latex_post=data.latexPost, + required_fields_per_template=data.req, tags=data.tags, sort_field_num=data.sortf, version=data.vers, + crowdanki_id=data.crowdanki_uuid, crowdanki_type=data.__type__ + ) + + @classmethod + def from_dict(cls, data: dict): + return cls.from_repr(cls.Representation.from_dict(data)) + + # def encode(self) -> dict: + # data_dict = {} + # super().encode_groupable(data_dict) + # data_dict.setdefault(NOTES, [note.encode() for note in self.notes]) + # return data_dict + def find_media(self): + pass + # Look in templates (and css?) diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py index 7ee0bde..373d8bf 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py @@ -1,6 +1,7 @@ from dataclasses import dataclass from typing import List, Dict +from brain_brew.file_manager import FileManager from brain_brew.representation.build_config.build_task import TopLevelBuildTask, GenerateDeckPartBuildTask from brain_brew.representation.configuration.csv_file_mapping import CsvFileMapping from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping @@ -124,8 +125,8 @@ def execute(self): deck_part_notes.append(Note(guid=guid, tags=tags, note_model=note_model_name, fields=fields)) - DeckPartNotes.from_list_of_notes(self.name, self.save_to_file, deck_part_notes) - # TODO: Save to the singleton holder + dpn = DeckPartNotes.from_list_of_notes(self.name, self.save_to_file, deck_part_notes) + FileManager.get_instance().new_deck_part(dpn) @dataclass diff --git a/tests/representation/crowdanki/test_crowdanki_note_model.py b/tests/representation/crowdanki/test_crowdanki_note_model.py index d4f7044..e4b27d2 100644 --- a/tests/representation/crowdanki/test_crowdanki_note_model.py +++ b/tests/representation/crowdanki/test_crowdanki_note_model.py @@ -6,7 +6,22 @@ class TestCrowdAnkiNoteModel: - def test_constructor(self): - json_data = JsonFile.read_file(TestFiles.NoteModels.LL_WORD_COMPLETE) - model = CrowdAnkiNoteModel.from_dict(json_data) - assert isinstance(model, CrowdAnkiNoteModel) + class TestConstructor: + def test_normal(self): + json_data = JsonFile.read_file(TestFiles.NoteModels.LL_WORD_COMPLETE) + model = CrowdAnkiNoteModel.from_dict(json_data) + assert isinstance(model, CrowdAnkiNoteModel) + + assert model.name == "LL Word" + assert isinstance(model.fields, list) + assert len(model.fields) == 7 + assert all([isinstance(field, CrowdAnkiNoteModel.Field) for field in model.fields]) + + assert isinstance(model.templates, list) + assert len(model.templates) == 7 + assert all([isinstance(template, CrowdAnkiNoteModel.Template) for template in model.templates]) + + def test_only_required(self): + json_data = JsonFile.read_file(TestFiles.NoteModels.LL_WORD_COMPLETE_ONLY_REQUIRED) + model = CrowdAnkiNoteModel.from_dict(json_data) + assert isinstance(model, CrowdAnkiNoteModel) diff --git a/tests/test_files.py b/tests/test_files.py index 8f9b46c..c9704b9 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -25,6 +25,8 @@ class NoteModels: LL_WORD = "LL Word" LL_WORD_COMPLETE = LOC + "LL-Word.json" + LL_WORD_COMPLETE_ONLY_REQUIRED = LOC + "LL-Word-Only-Required.json" + class CsvFiles: LOC = "tests/test_files/csv/" diff --git a/tests/test_files/deck_parts/note_models/LL-Word-Only-Required.json b/tests/test_files/deck_parts/note_models/LL-Word-Only-Required.json new file mode 100644 index 0000000..336c13c --- /dev/null +++ b/tests/test_files/deck_parts/note_models/LL-Word-Only-Required.json @@ -0,0 +1,194 @@ +{ + "crowdanki_uuid": "057a8d66-bc4e-11e9-9822-d8cb8ac9abf0", + "css": ".card {\n font-family: arial;\n font-size: 20px;\n text-align: center;\n color: black;\n background-color: white;\n}\n\n.card1,.card3, .card5 { background-color: #B60F2D; }\n.card2,.card4, .card6 { background-color: #2E9017; }\n.card7 { background: linear-gradient(90deg, #B60F2D 49.9%, #2E9017 50.1%); }\n\n.word {\n font-size:1.5em;\n}\n\n.pronunciation{\n color:blue;\n}\n\n.extrainfo{\n color:lightgrey;\n}", + "flds": [ + { + "font": "Liberation Sans", + "media": [], + "name": "Word", + "ord": 0, + "rtl": false, + "size": 12, + "sticky": false + }, + { + "font": "Arial", + "media": [], + "name": "X Word", + "ord": 1, + "rtl": false, + "size": 20, + "sticky": false + }, + { + "font": "Arial", + "media": [], + "name": "Y Word", + "ord": 2, + "rtl": false, + "size": 20, + "sticky": false + }, + { + "font": "Arial", + "media": [], + "name": "Picture", + "ord": 3, + "rtl": false, + "size": 6, + "sticky": false + }, + { + "font": "Arial", + "media": [], + "name": "Extra", + "ord": 4, + "rtl": false, + "size": 20, + "sticky": false + }, + { + "font": "Arial", + "media": [], + "name": "X Pronunciation (Recording and/or IPA)", + "ord": 5, + "rtl": false, + "size": 20, + "sticky": false + }, + { + "font": "Arial", + "media": [], + "name": "Y Pronunciation (Recording and/or IPA)", + "ord": 6, + "rtl": false, + "size": 20, + "sticky": false + } + ], + "latexPost": "\\end{document}", + "latexPre": "\\documentclass[12pt]{article}\n\\special{papersize=3in,5in}\n\\usepackage{amssymb,amsmath}\n\\pagestyle{empty}\n\\setlength{\\parindent}{0in}\n\\begin{document}\n", + "name": "LL Word", + "req": [ + [ + 0, + "all", + [ + 1 + ] + ], + [ + 1, + "all", + [ + 2 + ] + ], + [ + 2, + "all", + [ + 1, + 3 + ] + ], + [ + 3, + "all", + [ + 2, + 3 + ] + ], + [ + 4, + "all", + [ + 1, + 3 + ] + ], + [ + 5, + "all", + [ + 2, + 3 + ] + ], + [ + 6, + "all", + [ + 1, + 2, + 3 + ] + ] + ], + "tmpls": [ + { + "afmt": "{{#X Word}}\n\t{{X Word}}\n{{/X Word}}\n\n
\n\n{{Picture}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\t
{{X Pronunciation (Recording and/or IPA)}}\n{{/X Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", + "bafmt": "", + "bqfmt": "", + "did": null, + "name": "X Comprehension", + "ord": 0, + "qfmt": "{{#X Word}}\n\t{{text:X Word}}\n{{/X Word}}" + }, + { + "afmt": "{{#Y Word}}\n\t{{Y Word}}\n{{/Y Word}}\n\n
\n\n{{Picture}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\t
{{Y Pronunciation (Recording and/or IPA)}}\n{{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", + "bafmt": "", + "bqfmt": "", + "did": null, + "name": "Y Comprehension", + "ord": 1, + "qfmt": "{{#Y Word}}\n\t{{text:Y Word}}\n{{/Y Word}}" + }, + { + "afmt": "{{FrontSide}}\n\n
\n\n{{X Word}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\t
{{X Pronunciation (Recording and/or IPA)}}\n{{/X Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", + "bafmt": "", + "bqfmt": "", + "did": null, + "name": "X Production", + "ord": 2, + "qfmt": "{{#X Word}}{{#Picture}}\n\t{{Picture}}\n{{/Picture}}{{/X Word}}" + }, + { + "afmt": "{{FrontSide}}\n\n
\n\n{{Y Word}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\t
{{Y Pronunciation (Recording and/or IPA)}}\n{{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", + "bafmt": "", + "bqfmt": "", + "did": null, + "name": "Y Production", + "ord": 3, + "qfmt": "{{#Y Word}}{{#Picture}}\n\t{{Picture}}\n{{/Picture}}{{/Y Word}}" + }, + { + "afmt": "{{FrontSide}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\t
{{X Pronunciation (Recording and/or IPA)}}\n{{/X Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", + "bafmt": "", + "bqfmt": "", + "did": null, + "name": "X Spelling", + "ord": 4, + "qfmt": "{{#X Word}}\n\t
Spell this word:
\n\n\t
{{type:X Word}}
\n\n\t
{{Picture}}\n{{/X Word}}" + }, + { + "afmt": "{{FrontSide}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\t
{{Y Pronunciation (Recording and/or IPA)}}\n{{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", + "bafmt": "", + "bqfmt": "", + "did": null, + "name": "Y Spelling", + "ord": 5, + "qfmt": "{{#Y Word}}\n\t
Spell this word:
\n\n\t
{{type:Y Word}}
\n\n\t
{{Picture}}\n{{/Y Word}}" + }, + { + "afmt": "{{FrontSide}}\n\n
\n\n
{{text:X Word}}
\n
{{text:Y Word}}
\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\t
{{X Pronunciation (Recording and/or IPA)}}\n{{/X Pronunciation (Recording and/or IPA)}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\t
{{Y Pronunciation (Recording and/or IPA)}}\n{{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", + "bafmt": "", + "bqfmt": "", + "did": null, + "name": "X and Y Production", + "ord": 6, + "qfmt": "{{#X Word}}\n{{#Y Word}}\n\t{{Picture}}\n{{/Y Word}}\n{{/X Word}}" + } + ] +} \ No newline at end of file From 22e77d62e448b454e65687ba390d68ccf0507499 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sat, 8 Aug 2020 09:05:46 +0200 Subject: [PATCH 19/39] NoteModel updates --- .../crowdanki/crowdanki_note_model.py | 102 +++++++++++++----- .../tr_notes_csv_collection.py | 23 ++-- .../crowdanki/test_crowdanki_note_model.py | 56 ++++++++-- .../deck_parts/note_models/LL-Word.json | 5 +- 4 files changed, 138 insertions(+), 48 deletions(-) diff --git a/brain_brew/representation/crowdanki/crowdanki_note_model.py b/brain_brew/representation/crowdanki/crowdanki_note_model.py index 17331e3..229d4b2 100644 --- a/brain_brew/representation/crowdanki/crowdanki_note_model.py +++ b/brain_brew/representation/crowdanki/crowdanki_note_model.py @@ -1,3 +1,4 @@ +from collections import OrderedDict from dataclasses import dataclass, field from typing import List, Optional @@ -23,7 +24,7 @@ # Field FONT = "font" -MEDIA = "media" # Unused in Anki +MEDIA = "media" IS_RIGHT_TO_LEFT = "rtl" FONT_SIZE = "size" IS_STICKY = "sticky" @@ -36,6 +37,13 @@ DECK_OVERRIDE_ID = "did" +# Defaults +DEFAULT_LATEX_PRE = "\\documentclass[12pt]{article}\n\\special{papersize=3in,5in}\n\\usepackage{amssymb,amsmath}\n\\pagestyle{empty}\n\\setlength{\\parindent}{0in}\n\\begin{document}\n" +DEFAULT_LATEX_POST = "\\end{document}" +DEFAULT_CROWDANKI_TYPE = "NoteModel" +DEFAULT_FONT = "Liberation Sans" + + @dataclass class CrowdAnkiNoteModel: @dataclass @@ -44,7 +52,7 @@ class Field: class Representation: name: str ord: int - font: str = field(default="Liberation Sans") + font: str = field(default=DEFAULT_FONT) media: List[str] = field(default_factory=lambda: []) rtl: bool = field(default=False) size: int = field(default=20) @@ -56,11 +64,11 @@ def from_dict(cls, data: dict): name: str ordinal: int - font: str - media: List[str] - is_right_to_left: bool - font_size: int - is_sticky: bool + font: str = field(default=DEFAULT_FONT) + media: List[str] = field(default_factory=lambda: []) # Unused in Anki + is_right_to_left: bool = field(default=False) + font_size: int = field(default=20) + is_sticky: bool = field(default=False) @classmethod def from_repr(cls, data: Representation): @@ -73,6 +81,19 @@ def from_repr(cls, data: Representation): def from_dict(cls, data: dict): return cls.from_repr(cls.Representation.from_dict(data)) + def encode(self) -> dict: + data_dict = { + NAME: self.name, + ORDINAL: self.ordinal, + FONT: self.font, + MEDIA: self.media, + IS_RIGHT_TO_LEFT: self.is_right_to_left, + FONT_SIZE: self.font_size, + IS_STICKY: self.is_sticky + } + + return data_dict + @dataclass class Template: @dataclass @@ -93,9 +114,9 @@ def from_dict(cls, data: dict): ordinal: int question_format: str answer_format: str - question_format_in_browser: str - answer_format_in_browser: str - deck_override_id: Optional[int] + question_format_in_browser: str = field(default="") + answer_format_in_browser: str = field(default="") + deck_override_id: Optional[int] = field(default=None) @classmethod def from_repr(cls, data: Representation): @@ -108,17 +129,30 @@ def from_repr(cls, data: Representation): def from_dict(cls, data: dict): return cls.from_repr(cls.Representation.from_dict(data)) + def encode(self) -> dict: + data_dict = { + NAME: self.name, + ORDINAL: self.ordinal, + QUESTION_FORMAT: self.question_format, + ANSWER_FORMAT: self.answer_format, + BROWSER_QUESTION_FORMAT: self.question_format_in_browser, + BROWSER_ANSWER_FORMAT: self.answer_format_in_browser, + DECK_OVERRIDE_ID: self.deck_override_id + } + + return data_dict + @dataclass class Representation: name: str crowdanki_uuid: str css: str - latexPost: str - latexPre: str req: List[list] flds: List[dict] tmpls: List[dict] - __type__: str = field(default="NoteModel") + latexPre: str = field(default=DEFAULT_LATEX_PRE) + latexPost: str = field(default=DEFAULT_LATEX_POST) + __type__: str = field(default=DEFAULT_CROWDANKI_TYPE) tags: List[str] = field(default_factory=lambda: []) sortf: int = field(default=0) type: int = field(default=0) @@ -128,20 +162,20 @@ class Representation: def from_dict(cls, data: dict): return cls(**data) - crowdanki_id: str - crowdanki_type: str - name: str + crowdanki_id: str css: str - latex_post: str - latex_pre: str required_fields_per_template: List[list] fields: List[Field] templates: List[Template] - tags: List[str] - sort_field_num: int - is_cloze: bool - version: list # Deprecated in Anki + + latex_post: str = field(default=DEFAULT_LATEX_PRE) + latex_pre: str = field(default=DEFAULT_LATEX_POST) + sort_field_num: int = field(default=0) + is_cloze: bool = field(default=False) + crowdanki_type: str = field(default=DEFAULT_CROWDANKI_TYPE) # Should always be "NoteModel" + tags: List[str] = field(default_factory=lambda: []) # Tags of the last added note + version: list = field(default_factory=lambda: []) # Legacy version number. Deprecated in Anki @classmethod def from_repr(cls, data: Representation): @@ -158,11 +192,25 @@ def from_repr(cls, data: Representation): def from_dict(cls, data: dict): return cls.from_repr(cls.Representation.from_dict(data)) - # def encode(self) -> dict: - # data_dict = {} - # super().encode_groupable(data_dict) - # data_dict.setdefault(NOTES, [note.encode() for note in self.notes]) - # return data_dict + def encode(self) -> dict: + data_dict = { + NAME: self.name, + CROWDANKI_ID: self.crowdanki_id, + CSS: self.css, + REQUIRED_FIELDS_PER_TEMPLATE: self.required_fields_per_template, + LATEX_PRE: self.latex_pre, + LATEX_POST: self.latex_post, + SORT_FIELD_NUM: self.sort_field_num, + CROWDANKI_TYPE: self.crowdanki_type, + TAGS: self.tags, + VERSION: self.version, + IS_CLOZE: 1 if self.is_cloze else 0 + } + + data_dict.setdefault(FIELDS, [f.encode() for f in self.fields]) + data_dict.setdefault(TEMPLATES, [t.encode() for t in self.templates]) + + return OrderedDict(sorted(data_dict.items())) def find_media(self): pass diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py index 373d8bf..49df5f8 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py @@ -56,17 +56,18 @@ def verify_contents(self): errors.append(f"Missing Note Model Map for {nm}") # Check each of the Csvs (or their derivatives) contain all the necessary columns for their stated note model - # for cfm in self.file_mappings: - # note_model_names = cfm.get_used_note_model_names() - # available_columns = cfm.get_available_columns() - # - # referenced_note_models_maps = [value for key, value in self.note_model_mappings.items() if - # key in note_model_names] - # for nm_map in referenced_note_models_maps: - # missing_columns = [col for col in nm_map.note_model.fields_lowercase if - # col not in nm_map.csv_headers_map_to_note_fields(available_columns)] - # if missing_columns: - # errors.append(KeyError(f"Csvs are missing columns from {nm_map.note_model.name}", missing_columns)) + for cfm in self.file_mappings: + note_model_names = cfm.get_used_note_model_names() + available_columns = cfm.get_available_columns() + + referenced_note_models_maps = [value for key, value in self.note_model_mappings.items() if + key in note_model_names] + for nm_map in referenced_note_models_maps: + for model in nm_map.note_models.values(): + missing_columns = [col for col in model.fields_lowercase if + col not in nm_map.csv_headers_map_to_note_fields(available_columns)] + if missing_columns: + errors.append(KeyError(f"Csvs are missing columns from {model.name}", missing_columns)) if errors: raise Exception(errors) diff --git a/tests/representation/crowdanki/test_crowdanki_note_model.py b/tests/representation/crowdanki/test_crowdanki_note_model.py index e4b27d2..b867857 100644 --- a/tests/representation/crowdanki/test_crowdanki_note_model.py +++ b/tests/representation/crowdanki/test_crowdanki_note_model.py @@ -5,11 +5,20 @@ from tests.test_files import TestFiles +@pytest.fixture +def ca_nm_data_word(): + return JsonFile.read_file(TestFiles.NoteModels.LL_WORD_COMPLETE) + + +@pytest.fixture +def ca_nm_data_word_required_only(): + return JsonFile.read_file(TestFiles.NoteModels.LL_WORD_COMPLETE_ONLY_REQUIRED) + + class TestCrowdAnkiNoteModel: class TestConstructor: - def test_normal(self): - json_data = JsonFile.read_file(TestFiles.NoteModels.LL_WORD_COMPLETE) - model = CrowdAnkiNoteModel.from_dict(json_data) + def test_normal(self, ca_nm_data_word): + model = CrowdAnkiNoteModel.from_dict(ca_nm_data_word) assert isinstance(model, CrowdAnkiNoteModel) assert model.name == "LL Word" @@ -21,7 +30,42 @@ def test_normal(self): assert len(model.templates) == 7 assert all([isinstance(template, CrowdAnkiNoteModel.Template) for template in model.templates]) - def test_only_required(self): - json_data = JsonFile.read_file(TestFiles.NoteModels.LL_WORD_COMPLETE_ONLY_REQUIRED) - model = CrowdAnkiNoteModel.from_dict(json_data) + def test_only_required(self, ca_nm_data_word_required_only): + model = CrowdAnkiNoteModel.from_dict(ca_nm_data_word_required_only) + assert isinstance(model, CrowdAnkiNoteModel) + + def test_manual_construction(self): + model = CrowdAnkiNoteModel( + "name", + "23094149+8124+91284+12984", + "css is garbage", + [], + [CrowdAnkiNoteModel.Field( + "field1", + 0 + )], + [CrowdAnkiNoteModel.Template( + "template1", + 0, + "{{Question}}", + "{{Answer}}" + )] + ) + assert isinstance(model, CrowdAnkiNoteModel) + + class TestEncode: + def test_normal(self, ca_nm_data_word): + model = CrowdAnkiNoteModel.from_dict(ca_nm_data_word) + + encoded = model.encode() + + assert encoded == ca_nm_data_word + + def test_only_required_uses_defaults(self, ca_nm_data_word, ca_nm_data_word_required_only): + model = CrowdAnkiNoteModel.from_dict(ca_nm_data_word_required_only) + + encoded = model.encode() + + assert encoded != ca_nm_data_word_required_only + assert encoded == ca_nm_data_word diff --git a/tests/test_files/deck_parts/note_models/LL-Word.json b/tests/test_files/deck_parts/note_models/LL-Word.json index 9860ec1..6133160 100644 --- a/tests/test_files/deck_parts/note_models/LL-Word.json +++ b/tests/test_files/deck_parts/note_models/LL-Word.json @@ -128,10 +128,7 @@ ] ], "sortf": 0, - "tags": [ - "LL::Grammar::Preposition", - "Meta::InProgress" - ], + "tags": [], "tmpls": [ { "afmt": "{{#X Word}}\n\t{{X Word}}\n{{/X Word}}\n\n
\n\n{{Picture}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\t
{{X Pronunciation (Recording and/or IPA)}}\n{{/X Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", From 4f44169cddc26138d4c48b6f659ba0a5aed65e79 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sat, 8 Aug 2020 11:00:06 +0200 Subject: [PATCH 20/39] NoteModel and CrowdAnkiNoteModel combined --- .../crowdanki/crowdanki_note_model.py | 363 ++++++++++-------- brain_brew/representation/yaml/my_yaml.py | 2 +- .../crowdanki/test_crowdanki_note_model.py | 24 +- tests/test_builder.py | 2 +- 4 files changed, 227 insertions(+), 164 deletions(-) diff --git a/brain_brew/representation/crowdanki/crowdanki_note_model.py b/brain_brew/representation/crowdanki/crowdanki_note_model.py index 229d4b2..bae3a8c 100644 --- a/brain_brew/representation/crowdanki/crowdanki_note_model.py +++ b/brain_brew/representation/crowdanki/crowdanki_note_model.py @@ -1,162 +1,203 @@ from collections import OrderedDict from dataclasses import dataclass, field -from typing import List, Optional +from typing import List, Optional, Union, Dict + + +class AnkiField: + name: str + anki_name: str + default_value: any + + def __init__(self, anki_name, name=None, default_value=None): + self.anki_name = anki_name + self.name = name if name is not None else anki_name + self.default_value = default_value + + def append_name_if_differs(self, dict_to_add_to: dict, value): + if value != self.default_value: + dict_to_add_to.setdefault(self.name, value) + # CrowdAnki -CROWDANKI_ID = "crowdanki_uuid" -CROWDANKI_TYPE = "__type__" +CROWDANKI_ID = AnkiField("crowdanki_uuid", "id") +CROWDANKI_TYPE = AnkiField("__type__", default_value="NoteModel") # Shared -NAME = "name" -ORDINAL = "ord" +NAME = AnkiField("name") +ORDINAL = AnkiField("ord", "ordinal") # Note Model -CSS = "css" -LATEX_POST = "latexPost" -LATEX_PRE = "latexPre" -REQUIRED_FIELDS_PER_TEMPLATE = "req" -FIELDS = "flds" -TEMPLATES = "tmpls" -TAGS = "tags" -SORT_FIELD_NUM = "sortf" -IS_CLOZE = "type" -VERSION = "vers" +CSS = AnkiField("css") +LATEX_PRE = AnkiField("latexPre", "latex_pre", default_value="\\documentclass[12pt]{article}\n\\special{papersize=3in,5in}\n\\usepackage{amssymb,amsmath}\n\\pagestyle{empty}\n\\setlength{\\parindent}{0in}\n\\begin{document}\n") +LATEX_POST = AnkiField("latexPost", "latex_post", default_value="\\end{document}") +REQUIRED_FIELDS_PER_TEMPLATE = AnkiField("req", "required_fields_per_template") +FIELDS = AnkiField("flds", "fields") +TEMPLATES = AnkiField("tmpls", "templates") +TAGS = AnkiField("tags", default_value=[]) +SORT_FIELD_NUM = AnkiField("sortf", "sort_field_num", default_value=0) +IS_CLOZE = AnkiField("type", "is_cloze", default_value=False) +VERSION = AnkiField("vers", "version", default_value=[]) # Field -FONT = "font" -MEDIA = "media" -IS_RIGHT_TO_LEFT = "rtl" -FONT_SIZE = "size" -IS_STICKY = "sticky" +FONT = AnkiField("font", default_value="Liberation Sans") +MEDIA = AnkiField("media", default_value=[]) +IS_RIGHT_TO_LEFT = AnkiField("rtl", "is_right_to_left", default_value=False) +FONT_SIZE = AnkiField("size", default_value=20) +IS_STICKY = AnkiField("sticky", "is_sticky", default_value=False) # Template -QUESTION_FORMAT = "qfmt" -ANSWER_FORMAT = "afmt" -BROWSER_ANSWER_FORMAT = "bafmt" -BROWSER_QUESTION_FORMAT = "bqfmt" -DECK_OVERRIDE_ID = "did" - - -# Defaults -DEFAULT_LATEX_PRE = "\\documentclass[12pt]{article}\n\\special{papersize=3in,5in}\n\\usepackage{amssymb,amsmath}\n\\pagestyle{empty}\n\\setlength{\\parindent}{0in}\n\\begin{document}\n" -DEFAULT_LATEX_POST = "\\end{document}" -DEFAULT_CROWDANKI_TYPE = "NoteModel" -DEFAULT_FONT = "Liberation Sans" +QUESTION_FORMAT = AnkiField("qfmt", "question_format") +ANSWER_FORMAT = AnkiField("afmt", "answer_format") +BROWSER_ANSWER_FORMAT = AnkiField("bafmt", "browser_answer_format", default_value="") +BROWSER_QUESTION_FORMAT = AnkiField("bqfmt", "browser_question_format", default_value="") +DECK_OVERRIDE_ID = AnkiField("did", "deck_override_id", default_value=None) @dataclass -class CrowdAnkiNoteModel: +class Template: @dataclass - class Field: - @dataclass - class Representation: - name: str - ord: int - font: str = field(default=DEFAULT_FONT) - media: List[str] = field(default_factory=lambda: []) - rtl: bool = field(default=False) - size: int = field(default=20) - sticky: bool = field(default=False) - - @classmethod - def from_dict(cls, data: dict): - return cls(**data) - + class CrowdAnki: name: str - ordinal: int - font: str = field(default=DEFAULT_FONT) - media: List[str] = field(default_factory=lambda: []) # Unused in Anki - is_right_to_left: bool = field(default=False) - font_size: int = field(default=20) - is_sticky: bool = field(default=False) - - @classmethod - def from_repr(cls, data: Representation): - return cls( - name=data.name, ordinal=data.ord, font=data.font, media=data.media, - is_right_to_left=data.rtl, font_size=data.size, is_sticky=data.sticky - ) + ord: int + qfmt: str + afmt: str + bqfmt: str = field(default=BROWSER_QUESTION_FORMAT.default_value) + bafmt: str = field(default=BROWSER_ANSWER_FORMAT.default_value) + did: Optional[int] = field(default=None) @classmethod def from_dict(cls, data: dict): - return cls.from_repr(cls.Representation.from_dict(data)) + return cls(**data) - def encode(self) -> dict: - data_dict = { - NAME: self.name, - ORDINAL: self.ordinal, - FONT: self.font, - MEDIA: self.media, - IS_RIGHT_TO_LEFT: self.is_right_to_left, - FONT_SIZE: self.font_size, - IS_STICKY: self.is_sticky - } + name: str + ordinal: int + question_format: str + answer_format: str + question_format_in_browser: str = field(default=BROWSER_QUESTION_FORMAT.default_value) + answer_format_in_browser: str = field(default=BROWSER_ANSWER_FORMAT.default_value) + deck_override_id: Optional[int] = field(default=DECK_OVERRIDE_ID.default_value) - return data_dict + @classmethod + def from_crowdanki(cls, data: Union[CrowdAnki, dict]): + ca: cls.CrowdAnki = data if isinstance(data, cls.CrowdAnki) else cls.CrowdAnki.from_dict(data) + return cls( + name=ca.name, ordinal=ca.ord, question_format=ca.qfmt, answer_format=ca.afmt, + question_format_in_browser=ca.bqfmt, answer_format_in_browser=ca.bafmt, deck_override_id=ca.did + ) - @dataclass - class Template: - @dataclass - class Representation: - name: str - ord: int - qfmt: str - afmt: str - bqfmt: str = field(default="") - bafmt: str = field(default="") - did: Optional[int] = field(default=None) - - @classmethod - def from_dict(cls, data: dict): - return cls(**data) + @classmethod + def from_dict(cls, data: dict): + return cls(**data) - name: str - ordinal: int - question_format: str - answer_format: str - question_format_in_browser: str = field(default="") - answer_format_in_browser: str = field(default="") - deck_override_id: Optional[int] = field(default=None) + def encode_as_crowdanki(self) -> dict: + data_dict = { + NAME.anki_name: self.name, + ORDINAL.anki_name: self.ordinal, + QUESTION_FORMAT.anki_name: self.question_format, + ANSWER_FORMAT.anki_name: self.answer_format, + BROWSER_QUESTION_FORMAT.anki_name: self.question_format_in_browser, + BROWSER_ANSWER_FORMAT.anki_name: self.answer_format_in_browser, + DECK_OVERRIDE_ID.anki_name: self.deck_override_id + } - @classmethod - def from_repr(cls, data: Representation): - return cls( - name=data.name, ordinal=data.ord, question_format=data.qfmt, answer_format=data.afmt, - question_format_in_browser=data.bqfmt, answer_format_in_browser=data.bafmt, deck_override_id=data.did - ) + return data_dict + + def encode_as_deck_part(self) -> dict: + data_dict = { + NAME.name: self.name, + ORDINAL.name: self.ordinal, + QUESTION_FORMAT.name: self.question_format, + ANSWER_FORMAT.name: self.answer_format + } + + BROWSER_QUESTION_FORMAT.append_name_if_differs(data_dict, self.question_format_in_browser) + BROWSER_ANSWER_FORMAT.append_name_if_differs(data_dict, self.answer_format_in_browser) + DECK_OVERRIDE_ID.append_name_if_differs(data_dict, self.deck_override_id) + + return data_dict + + +@dataclass +class Field: + @dataclass + class CrowdAnki: + name: str + ord: int + font: str = field(default=FONT.default_value) + media: List[str] = field(default_factory=lambda: MEDIA.default_value) + rtl: bool = field(default=IS_RIGHT_TO_LEFT.default_value) + size: int = field(default=FONT_SIZE.default_value) + sticky: bool = field(default=IS_STICKY.default_value) @classmethod def from_dict(cls, data: dict): - return cls.from_repr(cls.Representation.from_dict(data)) + return cls(**data) + + name: str + ordinal: int + font: str = field(default=FONT.default_value) + media: List[str] = field(default_factory=lambda: MEDIA.default_value) # Unused in Anki + is_right_to_left: bool = field(default=IS_RIGHT_TO_LEFT.default_value) + font_size: int = field(default=FONT_SIZE.default_value) + is_sticky: bool = field(default=IS_STICKY.default_value) + + @classmethod + def from_crowdanki(cls, data: Union[CrowdAnki, dict]): + ca: cls.CrowdAnki = data if isinstance(data, cls.CrowdAnki) else cls.CrowdAnki.from_dict(data) + return cls( + name=ca.name, ordinal=ca.ord, font=ca.font, media=ca.media, + is_right_to_left=ca.rtl, font_size=ca.size, is_sticky=ca.sticky + ) + + @classmethod + def from_dict(cls, data: dict): + return cls(**data) + + def encode_as_crowdanki(self) -> dict: + data_dict = { + NAME.anki_name: self.name, + ORDINAL.anki_name: self.ordinal, + FONT.anki_name: self.font, + MEDIA.anki_name: self.media, + IS_RIGHT_TO_LEFT.anki_name: self.is_right_to_left, + FONT_SIZE.anki_name: self.font_size, + IS_STICKY.anki_name: self.is_sticky + } + + return data_dict + + def encode_as_deck_part(self) -> dict: + data_dict = { + NAME.anki_name: self.name, + ORDINAL.anki_name: self.ordinal + } + + FONT.append_name_if_differs(data_dict, self.font) + MEDIA.append_name_if_differs(data_dict, self.media) + IS_RIGHT_TO_LEFT.append_name_if_differs(data_dict, self.is_right_to_left) + FONT_SIZE.append_name_if_differs(data_dict, self.font_size) + IS_STICKY.append_name_if_differs(data_dict, self.is_sticky) - def encode(self) -> dict: - data_dict = { - NAME: self.name, - ORDINAL: self.ordinal, - QUESTION_FORMAT: self.question_format, - ANSWER_FORMAT: self.answer_format, - BROWSER_QUESTION_FORMAT: self.question_format_in_browser, - BROWSER_ANSWER_FORMAT: self.answer_format_in_browser, - DECK_OVERRIDE_ID: self.deck_override_id - } + return data_dict - return data_dict +@dataclass +class CrowdAnkiNoteModel: @dataclass - class Representation: + class CrowdAnki: name: str crowdanki_uuid: str css: str req: List[list] flds: List[dict] tmpls: List[dict] - latexPre: str = field(default=DEFAULT_LATEX_PRE) - latexPost: str = field(default=DEFAULT_LATEX_POST) - __type__: str = field(default=DEFAULT_CROWDANKI_TYPE) - tags: List[str] = field(default_factory=lambda: []) - sortf: int = field(default=0) - type: int = field(default=0) - vers: list = field(default_factory=lambda: []) + latexPre: str = field(default=LATEX_PRE.default_value) + latexPost: str = field(default=LATEX_POST.default_value) + __type__: str = field(default=CROWDANKI_TYPE.default_value) + tags: List[str] = field(default_factory=lambda: TAGS.default_value) + sortf: int = field(default=SORT_FIELD_NUM.default_value) + type: int = field(default=0) # Is_Cloze Manually set to 0 + vers: list = field(default_factory=lambda: VERSION.default_value) @classmethod def from_dict(cls, data: dict): @@ -169,46 +210,68 @@ def from_dict(cls, data: dict): fields: List[Field] templates: List[Template] - latex_post: str = field(default=DEFAULT_LATEX_PRE) - latex_pre: str = field(default=DEFAULT_LATEX_POST) - sort_field_num: int = field(default=0) - is_cloze: bool = field(default=False) - crowdanki_type: str = field(default=DEFAULT_CROWDANKI_TYPE) # Should always be "NoteModel" - tags: List[str] = field(default_factory=lambda: []) # Tags of the last added note - version: list = field(default_factory=lambda: []) # Legacy version number. Deprecated in Anki + latex_post: str = field(default=LATEX_PRE.default_value) + latex_pre: str = field(default=LATEX_POST.default_value) + sort_field_num: int = field(default=SORT_FIELD_NUM.default_value) + is_cloze: bool = field(default=IS_CLOZE.default_value) + crowdanki_type: str = field(default=CROWDANKI_TYPE.default_value) # Should always be "NoteModel" + tags: List[str] = field(default_factory=lambda: TAGS.default_value) # Tags of the last added note + version: list = field(default_factory=lambda: VERSION.default_value) # Legacy version number. Deprecated in Anki @classmethod - def from_repr(cls, data: Representation): + def from_crowdanki(cls, data: Union[CrowdAnki, dict]): + ca: cls.CrowdAnki = data if isinstance(data, cls.CrowdAnki) else cls.CrowdAnki.from_dict(data) return cls( - fields=[CrowdAnkiNoteModel.Field.from_dict(f) for f in data.flds], - templates=[CrowdAnkiNoteModel.Template.from_dict(t) for t in data.tmpls], - is_cloze=bool(data.type), - name=data.name, css=data.css, latex_pre=data.latexPre, latex_post=data.latexPost, - required_fields_per_template=data.req, tags=data.tags, sort_field_num=data.sortf, version=data.vers, - crowdanki_id=data.crowdanki_uuid, crowdanki_type=data.__type__ + fields=[Field.from_crowdanki(f) for f in ca.flds], + templates=[Template.from_crowdanki(t) for t in ca.tmpls], + is_cloze=bool(ca.type), + name=ca.name, css=ca.css, latex_pre=ca.latexPre, latex_post=ca.latexPost, + required_fields_per_template=ca.req, tags=ca.tags, sort_field_num=ca.sortf, version=ca.vers, + crowdanki_id=ca.crowdanki_uuid, crowdanki_type=ca.__type__ ) @classmethod def from_dict(cls, data: dict): - return cls.from_repr(cls.Representation.from_dict(data)) + return cls(**data) - def encode(self) -> dict: + def encode_as_crowdanki(self) -> dict: data_dict = { - NAME: self.name, - CROWDANKI_ID: self.crowdanki_id, - CSS: self.css, - REQUIRED_FIELDS_PER_TEMPLATE: self.required_fields_per_template, - LATEX_PRE: self.latex_pre, - LATEX_POST: self.latex_post, - SORT_FIELD_NUM: self.sort_field_num, - CROWDANKI_TYPE: self.crowdanki_type, - TAGS: self.tags, - VERSION: self.version, - IS_CLOZE: 1 if self.is_cloze else 0 + NAME.anki_name: self.name, + CROWDANKI_ID.anki_name: self.crowdanki_id, + CSS.anki_name: self.css, + REQUIRED_FIELDS_PER_TEMPLATE.anki_name: self.required_fields_per_template, + LATEX_PRE.anki_name: self.latex_pre, + LATEX_POST.anki_name: self.latex_post, + SORT_FIELD_NUM.anki_name: self.sort_field_num, + CROWDANKI_TYPE.anki_name: self.crowdanki_type, + TAGS.anki_name: self.tags, + VERSION.anki_name: self.version, + IS_CLOZE.anki_name: 1 if self.is_cloze else 0 } - data_dict.setdefault(FIELDS, [f.encode() for f in self.fields]) - data_dict.setdefault(TEMPLATES, [t.encode() for t in self.templates]) + data_dict.setdefault(FIELDS.anki_name, [f.encode_as_crowdanki() for f in self.fields]) + data_dict.setdefault(TEMPLATES.anki_name, [t.encode_as_crowdanki() for t in self.templates]) + + return OrderedDict(sorted(data_dict.items())) + + def encode_as_deck_part(self) -> dict: + data_dict = { + NAME.name: self.name, + CROWDANKI_ID.name: self.crowdanki_id, + CSS.name: self.css, + REQUIRED_FIELDS_PER_TEMPLATE.name: self.required_fields_per_template, + } + + LATEX_PRE.append_name_if_differs(data_dict, self.latex_pre) + LATEX_POST.append_name_if_differs(data_dict, self.latex_post) + SORT_FIELD_NUM.append_name_if_differs(data_dict, self.sort_field_num) + CROWDANKI_TYPE.append_name_if_differs(data_dict, self.crowdanki_type) + TAGS.append_name_if_differs(data_dict, self.tags) + VERSION.append_name_if_differs(data_dict, self.version) + IS_CLOZE.append_name_if_differs(data_dict, self.is_cloze) + + data_dict.setdefault(FIELDS.name, [f.encode_as_deck_part() for f in self.fields]) + data_dict.setdefault(TEMPLATES.name, [t.encode_as_deck_part() for t in self.templates]) return OrderedDict(sorted(data_dict.items())) diff --git a/brain_brew/representation/yaml/my_yaml.py b/brain_brew/representation/yaml/my_yaml.py index b05a66b..71e3f74 100644 --- a/brain_brew/representation/yaml/my_yaml.py +++ b/brain_brew/representation/yaml/my_yaml.py @@ -24,7 +24,7 @@ def from_deck_part_pool(cls, name: str): return FileManager.get_instance().deck_part_from_pool(name) @staticmethod - def filename_to_dict(filename: str): + def read_to_dict(filename: str): if filename[-5:] not in [".yaml", ".yml"]: filename += ".yaml" diff --git a/tests/representation/crowdanki/test_crowdanki_note_model.py b/tests/representation/crowdanki/test_crowdanki_note_model.py index b867857..c7b1fdd 100644 --- a/tests/representation/crowdanki/test_crowdanki_note_model.py +++ b/tests/representation/crowdanki/test_crowdanki_note_model.py @@ -1,6 +1,6 @@ import pytest -from brain_brew.representation.crowdanki.crowdanki_note_model import CrowdAnkiNoteModel +from brain_brew.representation.crowdanki.crowdanki_note_model import CrowdAnkiNoteModel, Template, Field from brain_brew.representation.json.json_file import JsonFile from tests.test_files import TestFiles @@ -18,20 +18,20 @@ def ca_nm_data_word_required_only(): class TestCrowdAnkiNoteModel: class TestConstructor: def test_normal(self, ca_nm_data_word): - model = CrowdAnkiNoteModel.from_dict(ca_nm_data_word) + model = CrowdAnkiNoteModel.from_crowdanki(ca_nm_data_word) assert isinstance(model, CrowdAnkiNoteModel) assert model.name == "LL Word" assert isinstance(model.fields, list) assert len(model.fields) == 7 - assert all([isinstance(field, CrowdAnkiNoteModel.Field) for field in model.fields]) + assert all([isinstance(field, Field) for field in model.fields]) assert isinstance(model.templates, list) assert len(model.templates) == 7 - assert all([isinstance(template, CrowdAnkiNoteModel.Template) for template in model.templates]) + assert all([isinstance(template, Template) for template in model.templates]) def test_only_required(self, ca_nm_data_word_required_only): - model = CrowdAnkiNoteModel.from_dict(ca_nm_data_word_required_only) + model = CrowdAnkiNoteModel.from_crowdanki(ca_nm_data_word_required_only) assert isinstance(model, CrowdAnkiNoteModel) def test_manual_construction(self): @@ -40,11 +40,11 @@ def test_manual_construction(self): "23094149+8124+91284+12984", "css is garbage", [], - [CrowdAnkiNoteModel.Field( + [Field( "field1", 0 )], - [CrowdAnkiNoteModel.Template( + [Template( "template1", 0, "{{Question}}", @@ -54,18 +54,18 @@ def test_manual_construction(self): assert isinstance(model, CrowdAnkiNoteModel) - class TestEncode: + class TestEncodeAsCrowdAnki: def test_normal(self, ca_nm_data_word): - model = CrowdAnkiNoteModel.from_dict(ca_nm_data_word) + model = CrowdAnkiNoteModel.from_crowdanki(ca_nm_data_word) - encoded = model.encode() + encoded = model.encode_as_crowdanki() assert encoded == ca_nm_data_word def test_only_required_uses_defaults(self, ca_nm_data_word, ca_nm_data_word_required_only): - model = CrowdAnkiNoteModel.from_dict(ca_nm_data_word_required_only) + model = CrowdAnkiNoteModel.from_crowdanki(ca_nm_data_word_required_only) - encoded = model.encode() + encoded = model.encode_as_crowdanki() assert encoded != ca_nm_data_word_required_only assert encoded == ca_nm_data_word diff --git a/tests/test_builder.py b/tests/test_builder.py index 75909f6..e89642a 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -18,7 +18,7 @@ def test_runs(self, global_config): patch.object(DeckPartNotes, "from_deck_part_pool", return_value=Mock()), \ patch.object(DeckPartNoteModel, "create", side_effect=mock_dp_nm): - data = YamlRepresentation.filename_to_dict(TestFiles.BuildConfig.ONE_OF_EACH_TYPE) + data = YamlRepresentation.read_to_dict(TestFiles.BuildConfig.ONE_OF_EACH_TYPE) builder = TopLevelTaskBuilder.from_dict(data, global_config, fm) assert len(builder.tasks) == 1 From 4db7fbe72d29e64f7e4538e06116486dfc1b8c41 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sun, 9 Aug 2020 12:01:29 +0200 Subject: [PATCH 21/39] Separated Dependencies with YamlRepr and DeckPartHolder All DeckParts now inherit from YamlRepr, and are held by DeckPartHolder DeckPartHolder is a generically typed NoteModel removed Ordinals, they are autogenerated when converting to Crowdanki --- brain_brew/build_tasks/source_crowd_anki.py | 2 +- brain_brew/file_manager.py | 11 +- .../configuration/note_model_mapping.py | 19 +- .../tr_notes_csv_collection.py | 22 +- .../tr_notes_generic.py | 7 +- .../representation/yaml/deck_part_holder.py | 41 ++++ brain_brew/representation/yaml/my_yaml.py | 22 +- brain_brew/representation/yaml/note_model.py | 32 --- .../yaml/note_model_card_template.py | 0 .../representation/yaml/note_model_field.py | 0 .../note_model_repr.py} | 71 ++++--- brain_brew/representation/yaml/note_repr.py | 46 +--- .../test_source_crowd_anki_json.py | 2 +- .../configuration/test_global_config.py | 2 +- .../configuration/test_note_model_mapping.py | 2 +- .../crowdanki/test_crowdanki_note_model.py | 71 ------- .../test_tr_notes_csv_collection.py | 51 ++--- .../json/test_deck_part_note_model.py | 10 +- .../yaml/test_note_model_repr.py | 122 +++++++++++ tests/representation/yaml/test_note_repr.py | 16 +- tests/test_builder.py | 12 +- tests/test_files.py | 15 +- .../note_models/LL-Word-No-Defaults.json | 199 ++++++++++++++++++ .../note_models/LL-Word-Only-Required.json | 64 +----- .../yaml/note_models/LL-Word-No-Defaults.yaml | 171 +++++++++++++++ .../note_models/LL-Word-Only-Required.yaml | 106 ++++++++++ .../yaml/{note => notes}/note1.yaml | 0 tests/test_helpers.py | 8 +- 28 files changed, 796 insertions(+), 328 deletions(-) create mode 100644 brain_brew/representation/yaml/deck_part_holder.py delete mode 100644 brain_brew/representation/yaml/note_model.py delete mode 100644 brain_brew/representation/yaml/note_model_card_template.py delete mode 100644 brain_brew/representation/yaml/note_model_field.py rename brain_brew/representation/{crowdanki/crowdanki_note_model.py => yaml/note_model_repr.py} (85%) delete mode 100644 tests/representation/crowdanki/test_crowdanki_note_model.py create mode 100644 tests/representation/yaml/test_note_model_repr.py create mode 100644 tests/test_files/deck_parts/note_models/LL-Word-No-Defaults.json create mode 100644 tests/test_files/deck_parts/yaml/note_models/LL-Word-No-Defaults.yaml create mode 100644 tests/test_files/deck_parts/yaml/note_models/LL-Word-Only-Required.yaml rename tests/test_files/deck_parts/yaml/{note => notes}/note1.yaml (100%) diff --git a/brain_brew/build_tasks/source_crowd_anki.py b/brain_brew/build_tasks/source_crowd_anki.py index df44694..6e53999 100644 --- a/brain_brew/build_tasks/source_crowd_anki.py +++ b/brain_brew/build_tasks/source_crowd_anki.py @@ -11,7 +11,7 @@ from brain_brew.representation.generic.yaml_file import ConfigKey, YamlFile from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport from brain_brew.representation.json.deck_part_header import DeckPartHeader -from brain_brew.representation.yaml.note_model import CANoteModelKeys, DeckPartNoteModel +from brain_brew.representation.yaml.note_model_repr import CANoteModelKeys, DeckPartNoteModel from brain_brew.representation.json.deck_part_notes import CANoteKeys, DeckPartNotes diff --git a/brain_brew/file_manager.py b/brain_brew/file_manager.py index fa5d946..2c673f0 100644 --- a/brain_brew/file_manager.py +++ b/brain_brew/file_manager.py @@ -7,7 +7,8 @@ from brain_brew.representation.configuration.global_config import GlobalConfig from brain_brew.representation.generic.generic_file import GenericFile from brain_brew.representation.generic.media_file import MediaFile -from brain_brew.representation.yaml.my_yaml import YamlRepresentation +from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder +from brain_brew.representation.yaml.my_yaml import YamlRepr from brain_brew.utils import filename_from_full_path, find_all_files_in_directory @@ -17,7 +18,7 @@ class FileManager: known_files_dict: Dict[str, GenericFile] known_media_files_dict: Dict[str, MediaFile] - deck_part_pool: Dict[str, YamlRepresentation] + deck_part_pool: Dict[str, DeckPartHolder[YamlRepr]] write_files_at_end: List[WritesFile] @@ -49,6 +50,9 @@ def file_if_exists(self, file_location) -> Union[GenericFile, None]: return self.known_files_dict[file_location] return None + def deck_part_if_exists(self, dp_name) -> Union[DeckPartHolder[YamlRepr], None]: + return self.deck_part_pool.get(dp_name) + def register_file(self, full_path, file): if full_path in self.known_files_dict: raise FileExistsError(f"File already known to FileManager, cannot be registered twice: {full_path}") @@ -82,10 +86,11 @@ def find_all_deck_part_media_files(self): logging.debug(f"Media files found: {len(self.known_media_files_dict)}") - def new_deck_part(self, dp: YamlRepresentation): + def new_deck_part(self, dp: DeckPartHolder) -> DeckPartHolder: if dp.name in self.deck_part_pool: raise KeyError(f"Cannot use same name '{dp.name}' for multiple Deck Parts") self.deck_part_pool.setdefault(dp.name, dp) + return dp def deck_part_from_pool(self, name: str): if name not in self.deck_part_pool: diff --git a/brain_brew/representation/configuration/note_model_mapping.py b/brain_brew/representation/configuration/note_model_mapping.py index 2d82920..c86ae06 100644 --- a/brain_brew/representation/configuration/note_model_mapping.py +++ b/brain_brew/representation/configuration/note_model_mapping.py @@ -4,7 +4,8 @@ from brain_brew.constants.deckpart_keys import DeckPartNoteKeys from brain_brew.interfaces.verifiable import Verifiable -from brain_brew.representation.yaml.note_model import DeckPartNoteModel +from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder +from brain_brew.representation.yaml.note_model_repr import NoteModel from brain_brew.utils import single_item_to_list @@ -44,7 +45,7 @@ class Representation: def from_dict(cls, data: dict): return cls(**data) - note_models: Dict[str, DeckPartNoteModel] + note_models: Dict[str, DeckPartHolder[NoteModel]] columns: List[FieldMapping] personal_fields: List[FieldMapping] @@ -52,7 +53,7 @@ def from_dict(cls, data: dict): @classmethod def from_repr(cls, data: Representation): - note_models = [DeckPartNoteModel.from_deck_part_pool(model) for model in single_item_to_list(data.note_models)] + note_models = [DeckPartHolder.from_deck_part_pool(model) for model in single_item_to_list(data.note_models)] return cls( columns=[FieldMapping( @@ -63,7 +64,7 @@ def from_repr(cls, data: Representation): field_type=FieldMapping.FieldMappingType.PERSONAL_FIELD, field_name=field, value="") for field in data.personal_fields], - note_models=dict(map(lambda nm: (nm.name, nm), note_models)) # TODO: Use deck part pool + note_models=dict(map(lambda nm: (nm.name, nm), note_models)) ) def get_note_model_mapping_dict(self): @@ -72,7 +73,9 @@ def get_note_model_mapping_dict(self): def verify_contents(self): errors = [] - for model in self.note_models: + for holder in self.note_models.values(): + model: NoteModel = holder.deck_part + # Check for Required Fields missing = [] for req in self.required_fields_definitions: @@ -80,7 +83,7 @@ def verify_contents(self): missing.append(req) if missing: - errors.append(KeyError(f"""Note model(s) "{model.name}" to Csv config error: \ + errors.append(KeyError(f"""Note model(s) "{holder.name}" to Csv config error: \ Definitions for fields {missing} are required.""")) # TODO: Note Model Mappings are allowed extra fields on a specific note model now, since multiple @@ -93,7 +96,7 @@ def verify_contents(self): if missing or extra: errors.append(KeyError( - f"""Note model "{model.name}" to Csv config error. It expected {model.fields} \ + f"""Note model "{holder.name}" to Csv config error. It expected {model.fields} \ but was missing: {missing}, and got extra: {extra} """)) # TODO: Make sure the same note_model is not defined in multiple NMMs @@ -146,4 +149,4 @@ def get_relevant_data(self, row): return relevant_data def field_values_in_note_model_order(self, note_model_name, fields_from_csv): - return [fields_from_csv[field] for field in self.note_models[note_model_name].fields_lowercase] + return [fields_from_csv[f] for f in self.note_models[note_model_name].deck_part.field_names_lowercase] diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py index 49df5f8..cf766b9 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py @@ -6,7 +6,8 @@ from brain_brew.representation.configuration.csv_file_mapping import CsvFileMapping from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping from brain_brew.representation.deck_part_transformers.tr_notes_generic import TrNotesToGeneric, TrGenericToNotes -from brain_brew.representation.yaml.note_repr import DeckPartNotes, Note +from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder +from brain_brew.representation.yaml.note_repr import Notes, Note @dataclass @@ -63,11 +64,11 @@ def verify_contents(self): referenced_note_models_maps = [value for key, value in self.note_model_mappings.items() if key in note_model_names] for nm_map in referenced_note_models_maps: - for model in nm_map.note_models.values(): - missing_columns = [col for col in model.fields_lowercase if + for holder in nm_map.note_models.values(): + missing_columns = [col for col in holder.deck_part.field_names_lowercase if col not in nm_map.csv_headers_map_to_note_fields(available_columns)] if missing_columns: - errors.append(KeyError(f"Csvs are missing columns from {model.name}", missing_columns)) + errors.append(KeyError(f"Csvs are missing columns from {holder.name}", missing_columns)) if errors: raise Exception(errors) @@ -96,9 +97,6 @@ def from_repr(cls, data: Representation): note_model_mappings=data.get_note_model_mappings() ) - def __repr__(self): - return f'TrCsvCollectionToNotes({self.name!r}, {self.save_to_file!r}, {self.file!r}, {self.note_model_mappings!r}, ' - @classmethod def from_dict(cls, data: dict): return cls.from_repr(TrCsvCollectionToNotes.Representation.from_dict(data)) @@ -126,8 +124,8 @@ def execute(self): deck_part_notes.append(Note(guid=guid, tags=tags, note_model=note_model_name, fields=fields)) - dpn = DeckPartNotes.from_list_of_notes(self.name, self.save_to_file, deck_part_notes) - FileManager.get_instance().new_deck_part(dpn) + notes = Notes.from_list_of_notes(deck_part_notes) + DeckPartHolder.override_or_create(self.name, self.save_to_file, notes) @dataclass @@ -147,7 +145,7 @@ def from_dict(cls, data: dict): @classmethod def from_repr(cls, data: Representation): return cls( - notes=DeckPartNotes.from_deck_part_pool(data.notes), + notes=DeckPartHolder.from_deck_part_pool(data.notes), file_mappings=data.get_file_mappings(), note_model_mappings=data.get_note_model_mappings() ) @@ -157,13 +155,13 @@ def from_dict(cls, data: dict): return cls.from_repr(TrNotesToCsvCollection.Representation.from_dict(data)) def execute(self): - notes_data = self.notes.get_notes() + notes_data = self.notes.deck_part.get_notes() self.verify_notes_match_note_model_mappings(notes_data) csv_data: Dict[str, dict] = {} for note in notes_data: nm_name = note.note_model - row = self.note_model_mappings[nm_name].note_models[nm_name].zip_field_to_data(note.fields) + row = self.note_model_mappings[nm_name].note_models[nm_name].deck_part.zip_field_to_data(note.fields) row["guid"] = note.guid row["tags"] = self.join_tags(note.tags) diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_generic.py b/brain_brew/representation/deck_part_transformers/tr_notes_generic.py index a04017e..b951979 100644 --- a/brain_brew/representation/deck_part_transformers/tr_notes_generic.py +++ b/brain_brew/representation/deck_part_transformers/tr_notes_generic.py @@ -2,7 +2,8 @@ from typing import Optional import re -from brain_brew.representation.yaml.note_repr import DeckPartNotes +from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder +from brain_brew.representation.yaml.note_repr import Notes from brain_brew.representation.configuration.global_config import GlobalConfig @@ -34,7 +35,7 @@ def __init__(self, name, save_to_file=None): name: str save_to_file: Optional[str] - data: DeckPartNotes = field(init=False) + data: DeckPartHolder[Notes] = field(init=False) @dataclass @@ -46,4 +47,4 @@ class Representation: def __init__(self, notes): self.notes = notes - notes: DeckPartNotes + notes: DeckPartHolder[Notes] diff --git a/brain_brew/representation/yaml/deck_part_holder.py b/brain_brew/representation/yaml/deck_part_holder.py new file mode 100644 index 0000000..0236a81 --- /dev/null +++ b/brain_brew/representation/yaml/deck_part_holder.py @@ -0,0 +1,41 @@ +from dataclasses import dataclass +from typing import Optional, TypeVar, Generic + +T = TypeVar('T') + + +@dataclass +class DeckPartHolder(Generic[T]): + name: str + save_to_file: Optional[str] + deck_part: T + + file_manager = None + + @classmethod + def get_file_manager(cls): + if not cls.file_manager: + from brain_brew.file_manager import FileManager + cls.file_manager = FileManager.get_instance() + return cls.file_manager + + @classmethod + def from_deck_part_pool(cls, name: str) -> T: + return cls.get_file_manager().deck_part_from_pool(name) + + @classmethod + def override_or_create(cls, name: str, save_to_file: Optional[str], deck_part: T): + fm = cls.get_file_manager() + + dp = fm.deck_part_if_exists(name) + if dp is None: + dp = fm.new_deck_part(DeckPartHolder(name, save_to_file, deck_part)) + else: + dp.deck_part = deck_part + dp.save_to_file = save_to_file # ? + + return dp + + def write_to_file(self): + if self.save_to_file is not None: + self.deck_part.dump_to_yaml(self.save_to_file) diff --git a/brain_brew/representation/yaml/my_yaml.py b/brain_brew/representation/yaml/my_yaml.py index 71e3f74..a1240fb 100644 --- a/brain_brew/representation/yaml/my_yaml.py +++ b/brain_brew/representation/yaml/my_yaml.py @@ -1,7 +1,6 @@ -from dataclasses import dataclass -from typing import Optional +from pathlib import Path -from ruamel.yaml import YAML, Path +from ruamel.yaml import YAML yaml_load = YAML(typ='safe') @@ -13,16 +12,7 @@ # yaml.sort_base_mapping_type_on_output = False -@dataclass -class YamlRepresentation: - name: str - save_to_file: Optional[str] - - @classmethod - def from_deck_part_pool(cls, name: str): - from brain_brew.file_manager import FileManager - return FileManager.get_instance().deck_part_from_pool(name) - +class YamlRepr: @staticmethod def read_to_dict(filename: str): if filename[-5:] not in [".yaml", ".yml"]: @@ -34,5 +24,9 @@ def read_to_dict(filename: str): with open(filename) as file: return yaml_load.load(file) - def write_to_file(self): + def encode(self) -> dict: raise NotImplemented + + def dump_to_yaml(self, filepath): + with open(filepath, 'w+') as fp: # TODO: raise warning/log if file not exists + yaml_dump.dump(self.encode(), fp) diff --git a/brain_brew/representation/yaml/note_model.py b/brain_brew/representation/yaml/note_model.py deleted file mode 100644 index f12673d..0000000 --- a/brain_brew/representation/yaml/note_model.py +++ /dev/null @@ -1,32 +0,0 @@ -from dataclasses import dataclass, field -from typing import List - -from brain_brew.representation.yaml.my_yaml import YamlRepresentation -from brain_brew.utils import list_of_str_to_lowercase - - -@dataclass -class DeckPartNoteModel(YamlRepresentation): - name: str - id: str - fields: List[str] - - @property - def fields_lowercase(self): - return list_of_str_to_lowercase(self.fields) - - def check_field_overlap(self, fields_to_check: List[str]): - fields_to_check = list_of_str_to_lowercase(fields_to_check) - lower_fields = self.fields_lowercase - - missing = [field for field in lower_fields if field not in fields_to_check] - extra = [field for field in fields_to_check if field not in lower_fields] - - return missing, extra - - def zip_field_to_data(self, data: List[str]) -> dict: - if len(self.fields) != len(data): - raise Exception(f"Data of length {len(data)} cannot map to fields of length {len(self.fields_lowercase)}") - return dict(zip(self.fields_lowercase, data)) - - diff --git a/brain_brew/representation/yaml/note_model_card_template.py b/brain_brew/representation/yaml/note_model_card_template.py deleted file mode 100644 index e69de29..0000000 diff --git a/brain_brew/representation/yaml/note_model_field.py b/brain_brew/representation/yaml/note_model_field.py deleted file mode 100644 index e69de29..0000000 diff --git a/brain_brew/representation/crowdanki/crowdanki_note_model.py b/brain_brew/representation/yaml/note_model_repr.py similarity index 85% rename from brain_brew/representation/crowdanki/crowdanki_note_model.py rename to brain_brew/representation/yaml/note_model_repr.py index bae3a8c..253766c 100644 --- a/brain_brew/representation/crowdanki/crowdanki_note_model.py +++ b/brain_brew/representation/yaml/note_model_repr.py @@ -2,6 +2,9 @@ from dataclasses import dataclass, field from typing import List, Optional, Union, Dict +from brain_brew.representation.yaml.my_yaml import YamlRepr +from brain_brew.utils import list_of_str_to_lowercase + class AnkiField: name: str @@ -70,7 +73,6 @@ def from_dict(cls, data: dict): return cls(**data) name: str - ordinal: int question_format: str answer_format: str question_format_in_browser: str = field(default=BROWSER_QUESTION_FORMAT.default_value) @@ -81,7 +83,7 @@ def from_dict(cls, data: dict): def from_crowdanki(cls, data: Union[CrowdAnki, dict]): ca: cls.CrowdAnki = data if isinstance(data, cls.CrowdAnki) else cls.CrowdAnki.from_dict(data) return cls( - name=ca.name, ordinal=ca.ord, question_format=ca.qfmt, answer_format=ca.afmt, + name=ca.name, question_format=ca.qfmt, answer_format=ca.afmt, question_format_in_browser=ca.bqfmt, answer_format_in_browser=ca.bafmt, deck_override_id=ca.did ) @@ -89,10 +91,10 @@ def from_crowdanki(cls, data: Union[CrowdAnki, dict]): def from_dict(cls, data: dict): return cls(**data) - def encode_as_crowdanki(self) -> dict: + def encode_as_crowdanki(self, ordinal: int) -> dict: data_dict = { NAME.anki_name: self.name, - ORDINAL.anki_name: self.ordinal, + ORDINAL.anki_name: ordinal, QUESTION_FORMAT.anki_name: self.question_format, ANSWER_FORMAT.anki_name: self.answer_format, BROWSER_QUESTION_FORMAT.anki_name: self.question_format_in_browser, @@ -105,7 +107,6 @@ def encode_as_crowdanki(self) -> dict: def encode_as_deck_part(self) -> dict: data_dict = { NAME.name: self.name, - ORDINAL.name: self.ordinal, QUESTION_FORMAT.name: self.question_format, ANSWER_FORMAT.name: self.answer_format } @@ -134,7 +135,6 @@ def from_dict(cls, data: dict): return cls(**data) name: str - ordinal: int font: str = field(default=FONT.default_value) media: List[str] = field(default_factory=lambda: MEDIA.default_value) # Unused in Anki is_right_to_left: bool = field(default=IS_RIGHT_TO_LEFT.default_value) @@ -145,7 +145,7 @@ def from_dict(cls, data: dict): def from_crowdanki(cls, data: Union[CrowdAnki, dict]): ca: cls.CrowdAnki = data if isinstance(data, cls.CrowdAnki) else cls.CrowdAnki.from_dict(data) return cls( - name=ca.name, ordinal=ca.ord, font=ca.font, media=ca.media, + name=ca.name, font=ca.font, media=ca.media, is_right_to_left=ca.rtl, font_size=ca.size, is_sticky=ca.sticky ) @@ -153,10 +153,10 @@ def from_crowdanki(cls, data: Union[CrowdAnki, dict]): def from_dict(cls, data: dict): return cls(**data) - def encode_as_crowdanki(self) -> dict: + def encode_as_crowdanki(self, ordinal: int) -> dict: data_dict = { NAME.anki_name: self.name, - ORDINAL.anki_name: self.ordinal, + ORDINAL.anki_name: ordinal, FONT.anki_name: self.font, MEDIA.anki_name: self.media, IS_RIGHT_TO_LEFT.anki_name: self.is_right_to_left, @@ -168,8 +168,7 @@ def encode_as_crowdanki(self) -> dict: def encode_as_deck_part(self) -> dict: data_dict = { - NAME.anki_name: self.name, - ORDINAL.anki_name: self.ordinal + NAME.name: self.name } FONT.append_name_if_differs(data_dict, self.font) @@ -182,7 +181,7 @@ def encode_as_deck_part(self) -> dict: @dataclass -class CrowdAnkiNoteModel: +class NoteModel(YamlRepr): @dataclass class CrowdAnki: name: str @@ -206,7 +205,7 @@ def from_dict(cls, data: dict): name: str crowdanki_id: str css: str - required_fields_per_template: List[list] + required_fields_per_template: List[list] # TODO: Get rid of this as requirement fields: List[Field] templates: List[Template] @@ -249,32 +248,54 @@ def encode_as_crowdanki(self) -> dict: IS_CLOZE.anki_name: 1 if self.is_cloze else 0 } - data_dict.setdefault(FIELDS.anki_name, [f.encode_as_crowdanki() for f in self.fields]) - data_dict.setdefault(TEMPLATES.anki_name, [t.encode_as_crowdanki() for t in self.templates]) + data_dict.setdefault(FIELDS.anki_name, [f.encode_as_crowdanki(num) for num, f in enumerate(self.fields)]) + data_dict.setdefault(TEMPLATES.anki_name, [t.encode_as_crowdanki(num) for num, t in enumerate(self.templates)]) return OrderedDict(sorted(data_dict.items())) - def encode_as_deck_part(self) -> dict: - data_dict = { + def encode(self) -> dict: + data_dict: Dict[str, Union[str, list]] = { NAME.name: self.name, CROWDANKI_ID.name: self.crowdanki_id, - CSS.name: self.css, - REQUIRED_FIELDS_PER_TEMPLATE.name: self.required_fields_per_template, + CSS.name: self.css } - LATEX_PRE.append_name_if_differs(data_dict, self.latex_pre) - LATEX_POST.append_name_if_differs(data_dict, self.latex_post) SORT_FIELD_NUM.append_name_if_differs(data_dict, self.sort_field_num) - CROWDANKI_TYPE.append_name_if_differs(data_dict, self.crowdanki_type) - TAGS.append_name_if_differs(data_dict, self.tags) - VERSION.append_name_if_differs(data_dict, self.version) IS_CLOZE.append_name_if_differs(data_dict, self.is_cloze) + LATEX_PRE.append_name_if_differs(data_dict, self.latex_pre) + LATEX_POST.append_name_if_differs(data_dict, self.latex_post) data_dict.setdefault(FIELDS.name, [f.encode_as_deck_part() for f in self.fields]) data_dict.setdefault(TEMPLATES.name, [t.encode_as_deck_part() for t in self.templates]) - return OrderedDict(sorted(data_dict.items())) + # Useless + TAGS.append_name_if_differs(data_dict, self.tags) + VERSION.append_name_if_differs(data_dict, self.version) + CROWDANKI_TYPE.append_name_if_differs(data_dict, self.crowdanki_type) + data_dict.setdefault(REQUIRED_FIELDS_PER_TEMPLATE.name, self.required_fields_per_template) + + return data_dict def find_media(self): pass # Look in templates (and css?) + + @property + def field_names_lowercase(self): + return list_of_str_to_lowercase(f.name for f in self.fields) + + def check_field_overlap(self, fields_to_check: List[str]): + fields_to_check = list_of_str_to_lowercase(fields_to_check) + lower_fields = self.field_names_lowercase + + missing = [f for f in lower_fields if f not in fields_to_check] + extra = [f for f in fields_to_check if f not in lower_fields] # TODO: Remove? + + return missing, extra + + def zip_field_to_data(self, data: List[str]) -> dict: + if len(self.fields) != len(data): + raise Exception(f"Data of length {len(data)} cannot map to fields of length {len(self.field_names_lowercase)}") + return dict(zip(self.field_names_lowercase, data)) + + diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index 0474efb..de0e645 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -1,5 +1,4 @@ -from brain_brew.file_manager import FileManager -from brain_brew.representation.yaml.my_yaml import yaml_dump, yaml_load, YamlRepresentation +from brain_brew.representation.yaml.my_yaml import YamlRepr from dataclasses import dataclass from typing import List, Optional, Dict, Set @@ -15,7 +14,7 @@ @dataclass -class GroupableNoteData: +class GroupableNoteData(YamlRepr): note_model: Optional[str] tags: Optional[List[str]] @@ -47,10 +46,6 @@ def encode(self) -> dict: super().encode_groupable(data_dict) return data_dict - def dump_to_yaml(self, file): - with open(file, 'w') as fp: - yaml_dump.dump(self.encode(), fp) - def get_media_references(self) -> Set[str]: return {entry for field in self.fields for entry in find_media_in_field(field)} @@ -73,10 +68,6 @@ def encode(self) -> dict: data_dict.setdefault(NOTES, [note.encode() for note in self.notes]) return data_dict - def dump_to_yaml(self, file): - with open(file, 'w') as fp: - yaml_dump.dump(self.encode(), fp) - # TODO: Extract Shared Tags and Note Models # TODO: Sort notes # TODO: Set data @@ -113,44 +104,21 @@ def join_tags(n_tags): @dataclass -class DeckPartNotes(YamlRepresentation): - name: str - save_to_file: Optional[str] +class Notes(YamlRepr): note_groupings: List[NoteGrouping] - # TODO: File location and saving - - @classmethod - def from_deck_part_pool(cls, name: str) -> 'DeckPartNotes': - return super(DeckPartNotes, cls).from_deck_part_pool(name) @classmethod - def from_dict(cls, name: str, save_to_file: Optional[str], data: dict): - return cls( - name=name, - save_to_file=save_to_file, - note_groupings=list(map(NoteGrouping.from_dict, data.get(NOTE_GROUPINGS))) - ) + def from_dict(cls, data: dict): + return cls(note_groupings=list(map(NoteGrouping.from_dict, data.get(NOTE_GROUPINGS)))) @classmethod - def from_list_of_notes(cls, name: str, save_to_file: Optional[str], notes: List[Note]): - return cls( - name=name, - save_to_file=save_to_file, - note_groupings=[NoteGrouping(note_model=None, tags=None, notes=notes)] - ) - - def write_to_file(self): - if self.save_to_file is not None: - self.dump_to_yaml(self.save_to_file) + def from_list_of_notes(cls, notes: List[Note]): + return cls(note_groupings=[NoteGrouping(note_model=None, tags=None, notes=notes)]) def encode(self) -> dict: data_dict = {NOTE_GROUPINGS: [note_grouping.encode() for note_grouping in self.note_groupings]} return data_dict - def dump_to_yaml(self, file): - with open(file, 'w') as fp: - yaml_dump.dump(self.encode(), fp) - def get_all_known_note_model_names(self): return {nms for group in self.note_groupings for nms in group.get_all_known_note_model_names()} diff --git a/tests/build_tasks/test_source_crowd_anki_json.py b/tests/build_tasks/test_source_crowd_anki_json.py index 84f634c..5d8b501 100644 --- a/tests/build_tasks/test_source_crowd_anki_json.py +++ b/tests/build_tasks/test_source_crowd_anki_json.py @@ -6,7 +6,7 @@ from brain_brew.build_tasks.source_crowd_anki import SourceCrowdAnki, CrowdAnkiKeys from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport from brain_brew.representation.json.deck_part_header import DeckPartHeader -from brain_brew.representation.yaml.note_model import DeckPartNoteModel +from brain_brew.representation.yaml.note_model_repr import DeckPartNoteModel from brain_brew.representation.json.deck_part_notes import DeckPartNotes diff --git a/tests/representation/configuration/test_global_config.py b/tests/representation/configuration/test_global_config.py index a08fc4f..e722b68 100644 --- a/tests/representation/configuration/test_global_config.py +++ b/tests/representation/configuration/test_global_config.py @@ -34,7 +34,7 @@ def global_config(): return GlobalConfig({ ConfigKeys.DECK_PARTS.value: { ConfigKeys.HEADERS.value: TestFiles.Headers.LOC, - ConfigKeys.NOTE_MODELS.value: TestFiles.NoteModels.LOC, + ConfigKeys.NOTE_MODELS.value: TestFiles.CrowdAnkiNoteModels.LOC, ConfigKeys.NOTES.value: TestFiles.NoteFiles.LOC, ConfigKeys.MEDIA_FILES.value: TestFiles.MediaFiles.LOC, diff --git a/tests/representation/configuration/test_note_model_mapping.py b/tests/representation/configuration/test_note_model_mapping.py index 2f51133..6dfbf8c 100644 --- a/tests/representation/configuration/test_note_model_mapping.py +++ b/tests/representation/configuration/test_note_model_mapping.py @@ -4,7 +4,7 @@ from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping, FieldMapping from brain_brew.representation.generic.csv_file import CsvFile -from brain_brew.representation.yaml.note_model import DeckPartNoteModel +from brain_brew.representation.yaml.note_model_repr import NoteModel from tests.test_file_manager import get_new_file_manager diff --git a/tests/representation/crowdanki/test_crowdanki_note_model.py b/tests/representation/crowdanki/test_crowdanki_note_model.py deleted file mode 100644 index c7b1fdd..0000000 --- a/tests/representation/crowdanki/test_crowdanki_note_model.py +++ /dev/null @@ -1,71 +0,0 @@ -import pytest - -from brain_brew.representation.crowdanki.crowdanki_note_model import CrowdAnkiNoteModel, Template, Field -from brain_brew.representation.json.json_file import JsonFile -from tests.test_files import TestFiles - - -@pytest.fixture -def ca_nm_data_word(): - return JsonFile.read_file(TestFiles.NoteModels.LL_WORD_COMPLETE) - - -@pytest.fixture -def ca_nm_data_word_required_only(): - return JsonFile.read_file(TestFiles.NoteModels.LL_WORD_COMPLETE_ONLY_REQUIRED) - - -class TestCrowdAnkiNoteModel: - class TestConstructor: - def test_normal(self, ca_nm_data_word): - model = CrowdAnkiNoteModel.from_crowdanki(ca_nm_data_word) - assert isinstance(model, CrowdAnkiNoteModel) - - assert model.name == "LL Word" - assert isinstance(model.fields, list) - assert len(model.fields) == 7 - assert all([isinstance(field, Field) for field in model.fields]) - - assert isinstance(model.templates, list) - assert len(model.templates) == 7 - assert all([isinstance(template, Template) for template in model.templates]) - - def test_only_required(self, ca_nm_data_word_required_only): - model = CrowdAnkiNoteModel.from_crowdanki(ca_nm_data_word_required_only) - assert isinstance(model, CrowdAnkiNoteModel) - - def test_manual_construction(self): - model = CrowdAnkiNoteModel( - "name", - "23094149+8124+91284+12984", - "css is garbage", - [], - [Field( - "field1", - 0 - )], - [Template( - "template1", - 0, - "{{Question}}", - "{{Answer}}" - )] - ) - - assert isinstance(model, CrowdAnkiNoteModel) - - class TestEncodeAsCrowdAnki: - def test_normal(self, ca_nm_data_word): - model = CrowdAnkiNoteModel.from_crowdanki(ca_nm_data_word) - - encoded = model.encode_as_crowdanki() - - assert encoded == ca_nm_data_word - - def test_only_required_uses_defaults(self, ca_nm_data_word, ca_nm_data_word_required_only): - model = CrowdAnkiNoteModel.from_crowdanki(ca_nm_data_word_required_only) - - encoded = model.encode_as_crowdanki() - - assert encoded != ca_nm_data_word_required_only - assert encoded == ca_nm_data_word diff --git a/tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py b/tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py index 50ac84e..3c5eb9c 100644 --- a/tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py +++ b/tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py @@ -3,6 +3,7 @@ from brain_brew.file_manager import FileManager from brain_brew.representation.deck_part_transformers.tr_notes_csv_collection import TrCsvCollectionToNotes +from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder from brain_brew.representation.yaml.my_yaml import yaml_dump, yaml_load from tests.test_file_manager import get_new_file_manager from tests.representation.configuration.test_global_config import global_config @@ -91,49 +92,24 @@ class TestConstructor: # save_to_file: deckparts/notes/csv_first_attempt.yaml note_model_mappings: - - note_model: LL Word + - note_models: + - LL Word + - LL Verb + - LL Noun columns_to_fields: guid: guid tags: tags - - english: Word - danish: X Word - danish audio: X Pronunciation (Recording and/or IPA) - esperanto: Y Word - esperanto audio: Y Pronunciation (Recording and/or IPA) - personal_fields: - - picture - - extra - - morphman_focusmorph - - note_model: LL Verb - columns_to_fields: - guid: guid - tags: tags - + english: Word danish: X Word danish audio: X Pronunciation (Recording and/or IPA) esperanto: Y Word esperanto audio: Y Pronunciation (Recording and/or IPA) - + present: Form Present past: Form Past present perfect: Form Perfect Present - personal_fields: - - picture - - extra - - morphman_focusmorph - - note_model: LL Noun - columns_to_fields: - guid: guid - tags: tags - - english: Word - danish: X Word - danish audio: X Pronunciation (Recording and/or IPA) - esperanto: Y Word - esperanto audio: Y Pronunciation (Recording and/or IPA) - + plural: Plural indefinite plural: Indefinite Plural definite plural: Definite Plural @@ -141,7 +117,7 @@ class TestConstructor: - picture - extra - morphman_focusmorph - + file_mappings: - file: source/vocab/main.csv note_model: LL Word @@ -159,6 +135,11 @@ def test_runs(self, global_config): fm = get_new_file_manager() data = yaml_load.load(self.test_tr_notes) - tr_notes = TrCsvCollectionToNotes.from_dict(data) + def mock_dp_holder(name: str): + return DeckPartHolder(name, None, None) + + with patch.object(DeckPartHolder, "from_deck_part_pool", side_effect=mock_dp_holder): + + tr_notes = TrCsvCollectionToNotes.from_dict(data) - assert isinstance(tr_notes, TrCsvCollectionToNotes) + assert isinstance(tr_notes, TrCsvCollectionToNotes) diff --git a/tests/representation/json/test_deck_part_note_model.py b/tests/representation/json/test_deck_part_note_model.py index 297185a..4d07a0f 100644 --- a/tests/representation/json/test_deck_part_note_model.py +++ b/tests/representation/json/test_deck_part_note_model.py @@ -2,7 +2,7 @@ import pytest -from brain_brew.representation.yaml.note_model import DeckPartNoteModel, CANoteModelKeys +from brain_brew.representation.yaml.note_model_repr import DeckPartNoteModel, CANoteModelKeys from tests.test_files import TestFiles @@ -14,14 +14,14 @@ def mock_dp_nm(name, read_now): class TestConstructor: @pytest.mark.parametrize("note_model_name", [ - TestFiles.NoteModels.TEST_COMPLETE, - TestFiles.NoteModels.TEST, + TestFiles.CrowdAnkiNoteModels.TEST_COMPLETE, + TestFiles.CrowdAnkiNoteModels.TEST, ]) def test_run(self, global_config, note_model_name): file = DeckPartNoteModel(note_model_name) assert isinstance(file, DeckPartNoteModel) - assert file.file_location == TestFiles.NoteModels.LOC + TestFiles.NoteModels.TEST_COMPLETE + assert file.file_location == TestFiles.CrowdAnkiNoteModels.LOC + TestFiles.CrowdAnkiNoteModels.TEST_COMPLETE assert len(file.get_data().keys()) == 13 assert file.name == "Test Model" @@ -45,7 +45,7 @@ def test_config_location_override(self, global_config): @pytest.fixture() def dp_note_model_test1(global_config) -> DeckPartNoteModel: - return DeckPartNoteModel.create(TestFiles.NoteModels.TEST_COMPLETE) + return DeckPartNoteModel.create(TestFiles.CrowdAnkiNoteModels.TEST_COMPLETE) def test_read_fields(dp_note_model_test1): diff --git a/tests/representation/yaml/test_note_model_repr.py b/tests/representation/yaml/test_note_model_repr.py new file mode 100644 index 0000000..adb4c68 --- /dev/null +++ b/tests/representation/yaml/test_note_model_repr.py @@ -0,0 +1,122 @@ +import pytest + +from brain_brew.representation.yaml.note_model_repr import NoteModel, Template, Field +from brain_brew.representation.json.json_file import JsonFile +from brain_brew.representation.yaml.my_yaml import YamlRepr +from tests.test_files import TestFiles +from tests.test_helpers import debug_write_deck_part_to_file + + +# CrowdAnki Files -------------------------------------------------------------------------- +@pytest.fixture +def ca_nm_data_word(): + return JsonFile.read_file(TestFiles.CrowdAnkiNoteModels.LL_WORD_COMPLETE) + + +@pytest.fixture +def ca_nm_word(ca_nm_data_word) -> NoteModel: + return NoteModel.from_crowdanki(ca_nm_data_word) + + +@pytest.fixture +def ca_nm_data_word_required_only(): + return JsonFile.read_file(TestFiles.CrowdAnkiNoteModels.LL_WORD_COMPLETE_ONLY_REQUIRED) + + +@pytest.fixture +def ca_nm_word_required_only(ca_nm_data_word_required_only) -> NoteModel: + return NoteModel.from_crowdanki(ca_nm_data_word_required_only) + + +@pytest.fixture +def ca_nm_data_word_no_defaults(): + return JsonFile.read_file(TestFiles.CrowdAnkiNoteModels.LL_WORD_COMPLETE_NO_DEFAULTS) + + +@pytest.fixture +def ca_nm_word_no_defaults(ca_nm_data_word_no_defaults) -> NoteModel: + return NoteModel.from_crowdanki(ca_nm_data_word_no_defaults) + + +# Yaml Files -------------------------------------------------------------------------- +@pytest.fixture +def nm_data_word_required_only(): + return YamlRepr.read_to_dict(TestFiles.NoteModels.LL_WORD_ONLY_REQUIRED) + + +@pytest.fixture +def nm_data_word_no_defaults(): + return YamlRepr.read_to_dict(TestFiles.NoteModels.LL_WORD_NO_DEFAULTS) + + +class TestCrowdAnkiNoteModel: + class TestConstructor: + def test_normal(self, ca_nm_word): + model = ca_nm_word + assert isinstance(model, NoteModel) + + assert model.name == "LL Word" + assert isinstance(model.fields, list) + assert len(model.fields) == 7 + assert all([isinstance(field, Field) for field in model.fields]) + + assert isinstance(model.templates, list) + assert len(model.templates) == 7 + assert all([isinstance(template, Template) for template in model.templates]) + + def test_only_required(self, ca_nm_word_required_only): + model = ca_nm_word_required_only + assert isinstance(model, NoteModel) + + def test_manual_construction(self): + model = NoteModel( + "name", + "23094149+8124+91284+12984", + "css is garbage", + [], + [Field( + "field1" + )], + [Template( + "template1", + "{{Question}}", + "{{Answer}}" + )] + ) + + assert isinstance(model, NoteModel) + + class TestEncodeAsCrowdAnki: + def test_normal(self, ca_nm_word, ca_nm_data_word): + model = ca_nm_word + + encoded = model.encode_as_crowdanki() + + assert encoded == ca_nm_data_word + + def test_only_required_uses_defaults(self, ca_nm_word_required_only, + ca_nm_data_word, ca_nm_data_word_required_only): + model = ca_nm_word_required_only + + encoded = model.encode_as_crowdanki() + + assert encoded != ca_nm_data_word_required_only + assert encoded == ca_nm_data_word + + class TestEncodeAsDeckPart: + def test_normal(self, ca_nm_word, ca_nm_data_word, ca_nm_data_word_required_only, nm_data_word_required_only): + model = ca_nm_word + + encoded = model.encode() + + assert encoded != ca_nm_data_word + assert encoded != ca_nm_data_word_required_only + assert encoded == nm_data_word_required_only + + def test_only_required_uses_defaults(self, ca_nm_word_no_defaults, ca_nm_data_word_no_defaults, nm_data_word_no_defaults): + model = ca_nm_word_no_defaults + + encoded = model.encode() + + assert encoded != ca_nm_data_word_no_defaults + assert encoded == nm_data_word_no_defaults diff --git a/tests/representation/yaml/test_note_repr.py b/tests/representation/yaml/test_note_repr.py index eb91850..87ae0ee 100644 --- a/tests/representation/yaml/test_note_repr.py +++ b/tests/representation/yaml/test_note_repr.py @@ -7,7 +7,7 @@ import pytest -from brain_brew.representation.yaml.note_repr import Note, NoteGrouping, DeckPartNotes, \ +from brain_brew.representation.yaml.note_repr import Note, NoteGrouping, Notes, \ NOTES, NOTE_GROUPINGS, FIELDS, GUID, NOTE_MODEL, TAGS working_notes = { @@ -79,12 +79,12 @@ def test_from_dict(self, note_grouping_fixtures): class TestDeckPartNote: def test_constructor(self): - dpn = DeckPartNotes(note_groupings=[NoteGrouping.from_dict(working_note_groupings["nothing_grouped"])]) - assert isinstance(dpn, DeckPartNotes) + dpn = Notes(note_groupings=[NoteGrouping.from_dict(working_note_groupings["nothing_grouped"])]) + assert isinstance(dpn, Notes) def test_from_dict(self): - dpn = DeckPartNotes.from_dict({NOTE_GROUPINGS: [working_note_groupings["nothing_grouped"]]}) - assert isinstance(dpn, DeckPartNotes) + dpn = Notes.from_dict({NOTE_GROUPINGS: [working_note_groupings["nothing_grouped"]]}) + assert isinstance(dpn, Notes) class TestDumpToYaml: @@ -245,7 +245,7 @@ class TestDeckPartNotes: def _assert_dump_to_yaml(tmpdir, ystring, groups: list): file = TestDumpToYaml._make_temp_file(tmpdir) - note = DeckPartNotes.from_dict({NOTE_GROUPINGS: [working_note_groupings[name] for name in groups]}) + note = Notes.from_dict({NOTE_GROUPINGS: [working_note_groupings[name] for name in groups]}) note.dump_to_yaml(str(file)) assert file.read() == ystring @@ -315,12 +315,12 @@ def test_grouped(self): class TestDeckPartNotes: def test_two_groups_two_models(self): - dpn = DeckPartNotes.from_dict(working_dpns["two_groups_two_models"]) + dpn = Notes.from_dict(working_dpns["two_groups_two_models"]) models = dpn.get_all_known_note_model_names() assert models == {'LL Test', 'model_name'} def test_two_groups_three_models(self): - dpn = DeckPartNotes.from_dict(working_dpns["two_groups_three_models"]) + dpn = Notes.from_dict(working_dpns["two_groups_three_models"]) models = dpn.get_all_known_note_model_names() assert models == {'LL Test', 'model_name', 'different_model'} diff --git a/tests/test_builder.py b/tests/test_builder.py index e89642a..f6e6c10 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -2,9 +2,9 @@ from brain_brew.representation.build_config.top_level_task_builder import TopLevelTaskBuilder from brain_brew.representation.deck_part_transformers.tr_notes_csv_collection import TrNotesToCsvCollection -from brain_brew.representation.yaml.my_yaml import YamlRepresentation -from brain_brew.representation.yaml.note_model import DeckPartNoteModel -from brain_brew.representation.yaml.note_repr import DeckPartNotes +from brain_brew.representation.yaml.my_yaml import YamlRepr +from brain_brew.representation.yaml.note_model_repr import NoteModel +from brain_brew.representation.yaml.note_repr import Notes from tests.representation.json.test_deck_part_note_model import mock_dp_nm from tests.test_file_manager import get_new_file_manager from tests.test_files import TestFiles @@ -15,10 +15,10 @@ def test_runs(self, global_config): fm = get_new_file_manager() with patch.object(TrNotesToCsvCollection, "__init__", return_value=None) as mock_csv_tr, \ - patch.object(DeckPartNotes, "from_deck_part_pool", return_value=Mock()), \ - patch.object(DeckPartNoteModel, "create", side_effect=mock_dp_nm): + patch.object(Notes, "from_deck_part_pool", return_value=Mock()), \ + patch.object(NoteModel, "from_deck_part_pool", side_effect=mock_dp_nm): - data = YamlRepresentation.read_to_dict(TestFiles.BuildConfig.ONE_OF_EACH_TYPE) + data = YamlRepr.read_to_dict(TestFiles.BuildConfig.ONE_OF_EACH_TYPE) builder = TopLevelTaskBuilder.from_dict(data, global_config, fm) assert len(builder.tasks) == 1 diff --git a/tests/test_files.py b/tests/test_files.py index c9704b9..2e811b5 100644 --- a/tests/test_files.py +++ b/tests/test_files.py @@ -16,7 +16,7 @@ class NoteFiles: TEST1_WITH_SHARED_TAGS_EMPTY_AND_GROUPING = "csvtonotes1_withsharedtagsandgrouping_butnothingtogroup.json" TEST2_WITH_SHARED_TAGS_AND_GROUPING = "csvtonotes2_withsharedtagsandgrouping.json" - class NoteModels: + class CrowdAnkiNoteModels: LOC = "tests/test_files/deck_parts/note_models/" TEST = "Test Model" @@ -27,6 +27,17 @@ class NoteModels: LL_WORD_COMPLETE_ONLY_REQUIRED = LOC + "LL-Word-Only-Required.json" + LL_WORD_COMPLETE_NO_DEFAULTS = LOC + "LL-Word-No-Defaults.json" + + class NoteModels: + LOC = "tests/test_files/deck_parts/yaml/note_models/" + + LL_WORD = LOC + "LL-Word.yaml" + + LL_WORD_ONLY_REQUIRED = LOC + "LL-Word-Only-Required.yaml" + + LL_WORD_NO_DEFAULTS = LOC + "LL-Word-No-Defaults.yaml" + class CsvFiles: LOC = "tests/test_files/csv/" @@ -53,6 +64,6 @@ class MediaFiles: LOC = "tests/test_files/media_files/" class YamlNotes: - LOC = "tests/test_files/yaml/note/" + LOC = "tests/test_files/yaml/notes/" TEST1 = LOC + "note1.yaml" diff --git a/tests/test_files/deck_parts/note_models/LL-Word-No-Defaults.json b/tests/test_files/deck_parts/note_models/LL-Word-No-Defaults.json new file mode 100644 index 0000000..432be61 --- /dev/null +++ b/tests/test_files/deck_parts/note_models/LL-Word-No-Defaults.json @@ -0,0 +1,199 @@ +{ + "__type__": "NoteModelTEST", + "crowdanki_uuid": "057a8d66-bc4e-11e9-9822-d8cb8ac9abf0", + "css": ".card {\n font-family: arial;\n font-size: 20px;\n text-align: center;\n color: black;\n background-color: white;\n}\n\n.card1,.card3, .card5 { background-color: #B60F2D; }\n.card2,.card4, .card6 { background-color: #2E9017; }\n.card7 { background: linear-gradient(90deg, #B60F2D 49.9%, #2E9017 50.1%); }\n\n.word {\n font-size:1.5em;\n}\n\n.pronunciation{\n color:blue;\n}\n\n.extrainfo{\n color:lightgrey;\n}", + "flds": [ + { + "font": "Liberation SansTEST", + "media": ["TEST"], + "name": "Word", + "ord": 0, + "rtl": true, + "size": 10, + "sticky": true + }, + { + "font": "Arial", + "media": ["TEST"], + "name": "X Word", + "ord": 1, + "rtl": true, + "size": 10, + "sticky": true + }, + { + "font": "Arial", + "media": ["TEST"], + "name": "Y Word", + "ord": 2, + "rtl": true, + "size": 10, + "sticky": true + }, + { + "font": "Arial", + "media": ["TEST"], + "name": "Picture", + "ord": 3, + "rtl": true, + "size": 10, + "sticky": true + }, + { + "font": "Arial", + "media": ["TEST"], + "name": "Extra", + "ord": 4, + "rtl": true, + "size": 10, + "sticky": true + }, + { + "font": "Arial", + "media": ["TEST"], + "name": "X Pronunciation (Recording and/or IPA)", + "ord": 5, + "rtl": true, + "size": 10, + "sticky": true + }, + { + "font": "Arial", + "media": ["TEST"], + "name": "Y Pronunciation (Recording and/or IPA)", + "ord": 6, + "rtl": true, + "size": 10, + "sticky": true + } + ], + "latexPost": "\\end{document}TEST", + "latexPre": "\\documentclass[12pt]{article}\n\\special{papersize=3in,5in}\n\\usepackage{amssymb,amsmath}\n\\pagestyle{empty}\n\\setlength{\\parindent}{0in}\n\\begin{document}\nTEST", + "name": "LL Word", + "req": [ + [ + 0, + "all", + [ + 1 + ] + ], + [ + 1, + "all", + [ + 2 + ] + ], + [ + 2, + "all", + [ + 1, + 3 + ] + ], + [ + 3, + "all", + [ + 2, + 3 + ] + ], + [ + 4, + "all", + [ + 1, + 3 + ] + ], + [ + 5, + "all", + [ + 2, + 3 + ] + ], + [ + 6, + "all", + [ + 1, + 2, + 3 + ] + ] + ], + "sortf": 1, + "tags": ["TEST"], + "tmpls": [ + { + "afmt": "{{#X Word}}\n\t{{X Word}}\n{{/X Word}}\n\n
\n\n{{Picture}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\t
{{X Pronunciation (Recording and/or IPA)}}\n{{/X Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", + "bafmt": "TEST", + "bqfmt": "TEST", + "did": 1, + "name": "X Comprehension", + "ord": 0, + "qfmt": "{{#X Word}}\n\t{{text:X Word}}\n{{/X Word}}" + }, + { + "afmt": "{{#Y Word}}\n\t{{Y Word}}\n{{/Y Word}}\n\n
\n\n{{Picture}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\t
{{Y Pronunciation (Recording and/or IPA)}}\n{{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", + "bafmt": "TEST", + "bqfmt": "TEST", + "did": 1, + "name": "Y Comprehension", + "ord": 1, + "qfmt": "{{#Y Word}}\n\t{{text:Y Word}}\n{{/Y Word}}" + }, + { + "afmt": "{{FrontSide}}\n\n
\n\n{{X Word}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\t
{{X Pronunciation (Recording and/or IPA)}}\n{{/X Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", + "bafmt": "TEST", + "bqfmt": "TEST", + "did": 1, + "name": "X Production", + "ord": 2, + "qfmt": "{{#X Word}}{{#Picture}}\n\t{{Picture}}\n{{/Picture}}{{/X Word}}" + }, + { + "afmt": "{{FrontSide}}\n\n
\n\n{{Y Word}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\t
{{Y Pronunciation (Recording and/or IPA)}}\n{{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", + "bafmt": "TEST", + "bqfmt": "TEST", + "did": 1, + "name": "Y Production", + "ord": 3, + "qfmt": "{{#Y Word}}{{#Picture}}\n\t{{Picture}}\n{{/Picture}}{{/Y Word}}" + }, + { + "afmt": "{{FrontSide}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\t
{{X Pronunciation (Recording and/or IPA)}}\n{{/X Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", + "bafmt": "TEST", + "bqfmt": "TEST", + "did": 1, + "name": "X Spelling", + "ord": 4, + "qfmt": "{{#X Word}}\n\t
Spell this word:
\n\n\t
{{type:X Word}}
\n\n\t
{{Picture}}\n{{/X Word}}" + }, + { + "afmt": "{{FrontSide}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\t
{{Y Pronunciation (Recording and/or IPA)}}\n{{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", + "bafmt": "TEST", + "bqfmt": "TEST", + "did": 2, + "name": "Y Spelling", + "ord": 5, + "qfmt": "{{#Y Word}}\n\t
Spell this word:
\n\n\t
{{type:Y Word}}
\n\n\t
{{Picture}}\n{{/Y Word}}" + }, + { + "afmt": "{{FrontSide}}\n\n
\n\n
{{text:X Word}}
\n
{{text:Y Word}}
\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\t
{{X Pronunciation (Recording and/or IPA)}}\n{{/X Pronunciation (Recording and/or IPA)}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\t
{{Y Pronunciation (Recording and/or IPA)}}\n{{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", + "bafmt": "TEST", + "bqfmt": "TEST", + "did": 1, + "name": "X and Y Production", + "ord": 6, + "qfmt": "{{#X Word}}\n{{#Y Word}}\n\t{{Picture}}\n{{/Y Word}}\n{{/X Word}}" + } + ], + "type": 1, + "vers": ["TEST"] +} \ No newline at end of file diff --git a/tests/test_files/deck_parts/note_models/LL-Word-Only-Required.json b/tests/test_files/deck_parts/note_models/LL-Word-Only-Required.json index 336c13c..a9ed2b3 100644 --- a/tests/test_files/deck_parts/note_models/LL-Word-Only-Required.json +++ b/tests/test_files/deck_parts/note_models/LL-Word-Only-Required.json @@ -3,71 +3,42 @@ "css": ".card {\n font-family: arial;\n font-size: 20px;\n text-align: center;\n color: black;\n background-color: white;\n}\n\n.card1,.card3, .card5 { background-color: #B60F2D; }\n.card2,.card4, .card6 { background-color: #2E9017; }\n.card7 { background: linear-gradient(90deg, #B60F2D 49.9%, #2E9017 50.1%); }\n\n.word {\n font-size:1.5em;\n}\n\n.pronunciation{\n color:blue;\n}\n\n.extrainfo{\n color:lightgrey;\n}", "flds": [ { - "font": "Liberation Sans", - "media": [], "name": "Word", "ord": 0, - "rtl": false, - "size": 12, - "sticky": false + "size": 12 }, { "font": "Arial", - "media": [], "name": "X Word", - "ord": 1, - "rtl": false, - "size": 20, - "sticky": false + "ord": 1 }, { "font": "Arial", - "media": [], "name": "Y Word", - "ord": 2, - "rtl": false, - "size": 20, - "sticky": false + "ord": 2 }, { "font": "Arial", - "media": [], "name": "Picture", "ord": 3, - "rtl": false, - "size": 6, - "sticky": false + "size": 6 }, { "font": "Arial", - "media": [], "name": "Extra", - "ord": 4, - "rtl": false, - "size": 20, - "sticky": false + "ord": 4 }, { "font": "Arial", - "media": [], "name": "X Pronunciation (Recording and/or IPA)", - "ord": 5, - "rtl": false, - "size": 20, - "sticky": false + "ord": 5 }, { "font": "Arial", - "media": [], "name": "Y Pronunciation (Recording and/or IPA)", - "ord": 6, - "rtl": false, - "size": 20, - "sticky": false + "ord": 6 } ], - "latexPost": "\\end{document}", - "latexPre": "\\documentclass[12pt]{article}\n\\special{papersize=3in,5in}\n\\usepackage{amssymb,amsmath}\n\\pagestyle{empty}\n\\setlength{\\parindent}{0in}\n\\begin{document}\n", "name": "LL Word", "req": [ [ @@ -129,63 +100,42 @@ "tmpls": [ { "afmt": "{{#X Word}}\n\t{{X Word}}\n{{/X Word}}\n\n
\n\n{{Picture}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\t
{{X Pronunciation (Recording and/or IPA)}}\n{{/X Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", - "bafmt": "", - "bqfmt": "", - "did": null, "name": "X Comprehension", "ord": 0, "qfmt": "{{#X Word}}\n\t{{text:X Word}}\n{{/X Word}}" }, { "afmt": "{{#Y Word}}\n\t{{Y Word}}\n{{/Y Word}}\n\n
\n\n{{Picture}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\t
{{Y Pronunciation (Recording and/or IPA)}}\n{{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", - "bafmt": "", - "bqfmt": "", - "did": null, "name": "Y Comprehension", "ord": 1, "qfmt": "{{#Y Word}}\n\t{{text:Y Word}}\n{{/Y Word}}" }, { "afmt": "{{FrontSide}}\n\n
\n\n{{X Word}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\t
{{X Pronunciation (Recording and/or IPA)}}\n{{/X Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", - "bafmt": "", - "bqfmt": "", - "did": null, "name": "X Production", "ord": 2, "qfmt": "{{#X Word}}{{#Picture}}\n\t{{Picture}}\n{{/Picture}}{{/X Word}}" }, { "afmt": "{{FrontSide}}\n\n
\n\n{{Y Word}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\t
{{Y Pronunciation (Recording and/or IPA)}}\n{{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", - "bafmt": "", - "bqfmt": "", - "did": null, "name": "Y Production", "ord": 3, "qfmt": "{{#Y Word}}{{#Picture}}\n\t{{Picture}}\n{{/Picture}}{{/Y Word}}" }, { "afmt": "{{FrontSide}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\t
{{X Pronunciation (Recording and/or IPA)}}\n{{/X Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", - "bafmt": "", - "bqfmt": "", - "did": null, "name": "X Spelling", "ord": 4, "qfmt": "{{#X Word}}\n\t
Spell this word:
\n\n\t
{{type:X Word}}
\n\n\t
{{Picture}}\n{{/X Word}}" }, { "afmt": "{{FrontSide}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\t
{{Y Pronunciation (Recording and/or IPA)}}\n{{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", - "bafmt": "", - "bqfmt": "", - "did": null, "name": "Y Spelling", "ord": 5, "qfmt": "{{#Y Word}}\n\t
Spell this word:
\n\n\t
{{type:Y Word}}
\n\n\t
{{Picture}}\n{{/Y Word}}" }, { "afmt": "{{FrontSide}}\n\n
\n\n
{{text:X Word}}
\n
{{text:Y Word}}
\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\t
{{X Pronunciation (Recording and/or IPA)}}\n{{/X Pronunciation (Recording and/or IPA)}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\t
{{Y Pronunciation (Recording and/or IPA)}}\n{{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}", - "bafmt": "", - "bqfmt": "", - "did": null, "name": "X and Y Production", "ord": 6, "qfmt": "{{#X Word}}\n{{#Y Word}}\n\t{{Picture}}\n{{/Y Word}}\n{{/X Word}}" diff --git a/tests/test_files/deck_parts/yaml/note_models/LL-Word-No-Defaults.yaml b/tests/test_files/deck_parts/yaml/note_models/LL-Word-No-Defaults.yaml new file mode 100644 index 0000000..9b909c6 --- /dev/null +++ b/tests/test_files/deck_parts/yaml/note_models/LL-Word-No-Defaults.yaml @@ -0,0 +1,171 @@ +name: LL Word +id: 057a8d66-bc4e-11e9-9822-d8cb8ac9abf0 +css: ".card {\n font-family: arial;\n font-size: 20px;\n text-align: center;\n color:\ +\ black;\n background-color: white;\n}\n\n.card1,.card3, .card5 { background-color:\ +\ #B60F2D; }\n.card2,.card4, .card6 { background-color: #2E9017; }\n.card7 { background:\ +\ linear-gradient(90deg, #B60F2D 49.9%, #2E9017 50.1%); }\n\n.word {\n font-size:1.5em;\n\ +}\n\n.pronunciation{\n color:blue;\n}\n\n.extrainfo{\n color:lightgrey;\n}" +sort_field_num: 1 +is_cloze: true +latex_pre: "\\documentclass[12pt]{article}\n\\special{papersize=3in,5in}\n\\usepackage{amssymb,amsmath}\n\ +\\pagestyle{empty}\n\\setlength{\\parindent}{0in}\n\\begin{document}\nTEST" +latex_post: \end{document}TEST +fields: + - name: Word + font: Liberation SansTEST + media: + - TEST + is_right_to_left: true + size: 10 + is_sticky: true + - name: X Word + font: Arial + media: + - TEST + is_right_to_left: true + size: 10 + is_sticky: true + - name: Y Word + font: Arial + media: + - TEST + is_right_to_left: true + size: 10 + is_sticky: true + - name: Picture + font: Arial + media: + - TEST + is_right_to_left: true + size: 10 + is_sticky: true + - name: Extra + font: Arial + media: + - TEST + is_right_to_left: true + size: 10 + is_sticky: true + - name: X Pronunciation (Recording and/or IPA) + font: Arial + media: + - TEST + is_right_to_left: true + size: 10 + is_sticky: true + - name: Y Pronunciation (Recording and/or IPA) + font: Arial + media: + - TEST + is_right_to_left: true + size: 10 + is_sticky: true +templates: + - name: X Comprehension + question_format: "{{#X Word}}\n\t{{text:X Word}}\n\ + {{/X Word}}" + answer_format: "{{#X Word}}\n\t{{X Word}}\n{{/X Word}}\n\ + \n
\n\n{{Picture}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\ + \t
{{X Pronunciation (Recording and/or IPA)}}\n\ + {{/X Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}" + browser_question_format: TEST + browser_answer_format: TEST + deck_override_id: 1 + - name: Y Comprehension + question_format: "{{#Y Word}}\n\t{{text:Y Word}}\n\ + {{/Y Word}}" + answer_format: "{{#Y Word}}\n\t{{Y Word}}\n{{/Y Word}}\n\ + \n
\n\n{{Picture}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\ + \t
{{Y Pronunciation (Recording and/or IPA)}}\n\ + {{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}" + browser_question_format: TEST + browser_answer_format: TEST + deck_override_id: 1 + - name: X Production + question_format: "{{#X Word}}{{#Picture}}\n\t{{Picture}}\n{{/Picture}}{{/X Word}}" + answer_format: "{{FrontSide}}\n\n
\n\n{{X Word}}\n\ + \n{{#X Pronunciation (Recording and/or IPA)}}\n\t
{{X Pronunciation (Recording and/or IPA)}}\n{{/X Pronunciation (Recording\ + \ and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n\ + {{/Extra}}" + browser_question_format: TEST + browser_answer_format: TEST + deck_override_id: 1 + - name: Y Production + question_format: "{{#Y Word}}{{#Picture}}\n\t{{Picture}}\n{{/Picture}}{{/Y Word}}" + answer_format: "{{FrontSide}}\n\n
\n\n{{Y Word}}\n\ + \n{{#Y Pronunciation (Recording and/or IPA)}}\n\t
{{Y Pronunciation (Recording and/or IPA)}}\n{{/Y Pronunciation (Recording\ + \ and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n\ + {{/Extra}}" + browser_question_format: TEST + browser_answer_format: TEST + deck_override_id: 1 + - name: X Spelling + question_format: "{{#X Word}}\n\t
Spell this word:
\n\n\t
{{type:X Word}}
\n\n\t
{{Picture}}\n{{/X Word}}" + answer_format: "{{FrontSide}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\ + \t
{{X Pronunciation (Recording and/or IPA)}}\n\ + {{/X Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}" + browser_question_format: TEST + browser_answer_format: TEST + deck_override_id: 1 + - name: Y Spelling + question_format: "{{#Y Word}}\n\t
Spell this word:
\n\n\t
{{type:Y Word}}
\n\n\t
{{Picture}}\n{{/Y Word}}" + answer_format: "{{FrontSide}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\ + \t
{{Y Pronunciation (Recording and/or IPA)}}\n\ + {{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}" + browser_question_format: TEST + browser_answer_format: TEST + deck_override_id: 2 + - name: X and Y Production + question_format: "{{#X Word}}\n{{#Y Word}}\n\t{{Picture}}\n{{/Y Word}}\n{{/X Word}}" + answer_format: "{{FrontSide}}\n\n
\n\n
{{text:X\ + \ Word}}
\n
{{text:Y Word}}
\n\n{{#X Pronunciation\ + \ (Recording and/or IPA)}}\n\t
{{X Pronunciation\ + \ (Recording and/or IPA)}}\n{{/X Pronunciation (Recording and/or IPA)}}\n\ + \n{{#Y Pronunciation (Recording and/or IPA)}}\n\t
{{Y Pronunciation (Recording and/or IPA)}}\n{{/Y Pronunciation (Recording\ + \ and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n\ + {{/Extra}}" + browser_question_format: TEST + browser_answer_format: TEST + deck_override_id: 1 +tags: + - TEST +version: + - TEST +__type__: NoteModelTEST +required_fields_per_template: + - - 0 + - all + - - 1 + - - 1 + - all + - - 2 + - - 2 + - all + - - 1 + - 3 + - - 3 + - all + - - 2 + - 3 + - - 4 + - all + - - 1 + - 3 + - - 5 + - all + - - 2 + - 3 + - - 6 + - all + - - 1 + - 2 + - 3 diff --git a/tests/test_files/deck_parts/yaml/note_models/LL-Word-Only-Required.yaml b/tests/test_files/deck_parts/yaml/note_models/LL-Word-Only-Required.yaml new file mode 100644 index 0000000..055097a --- /dev/null +++ b/tests/test_files/deck_parts/yaml/note_models/LL-Word-Only-Required.yaml @@ -0,0 +1,106 @@ +name: LL Word +id: 057a8d66-bc4e-11e9-9822-d8cb8ac9abf0 +css: ".card {\n font-family: arial;\n font-size: 20px;\n text-align: center;\n color:\ +\ black;\n background-color: white;\n}\n\n.card1,.card3, .card5 { background-color:\ +\ #B60F2D; }\n.card2,.card4, .card6 { background-color: #2E9017; }\n.card7 { background:\ +\ linear-gradient(90deg, #B60F2D 49.9%, #2E9017 50.1%); }\n\n.word {\n font-size:1.5em;\n\ +}\n\n.pronunciation{\n color:blue;\n}\n\n.extrainfo{\n color:lightgrey;\n}" +fields: + - name: Word + size: 12 + - name: X Word + font: Arial + - name: Y Word + font: Arial + - name: Picture + font: Arial + size: 6 + - name: Extra + font: Arial + - name: X Pronunciation (Recording and/or IPA) + font: Arial + - name: Y Pronunciation (Recording and/or IPA) + font: Arial +templates: + - name: X Comprehension + question_format: "{{#X Word}}\n\t{{text:X Word}}\n\ + {{/X Word}}" + answer_format: "{{#X Word}}\n\t{{X Word}}\n{{/X Word}}\n\ + \n
\n\n{{Picture}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\ + \t
{{X Pronunciation (Recording and/or IPA)}}\n\ + {{/X Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}" + - name: Y Comprehension + question_format: "{{#Y Word}}\n\t{{text:Y Word}}\n\ + {{/Y Word}}" + answer_format: "{{#Y Word}}\n\t{{Y Word}}\n{{/Y Word}}\n\ + \n
\n\n{{Picture}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\ + \t
{{Y Pronunciation (Recording and/or IPA)}}\n\ + {{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}" + - name: X Production + question_format: "{{#X Word}}{{#Picture}}\n\t{{Picture}}\n{{/Picture}}{{/X Word}}" + answer_format: "{{FrontSide}}\n\n
\n\n{{X Word}}\n\ + \n{{#X Pronunciation (Recording and/or IPA)}}\n\t
{{X Pronunciation (Recording and/or IPA)}}\n{{/X Pronunciation (Recording\ + \ and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n\ + {{/Extra}}" + - name: Y Production + question_format: "{{#Y Word}}{{#Picture}}\n\t{{Picture}}\n{{/Picture}}{{/Y Word}}" + answer_format: "{{FrontSide}}\n\n
\n\n{{Y Word}}\n\ + \n{{#Y Pronunciation (Recording and/or IPA)}}\n\t
{{Y Pronunciation (Recording and/or IPA)}}\n{{/Y Pronunciation (Recording\ + \ and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n\ + {{/Extra}}" + - name: X Spelling + question_format: "{{#X Word}}\n\t
Spell this word:
\n\n\t
{{type:X Word}}
\n\n\t
{{Picture}}\n{{/X Word}}" + answer_format: "{{FrontSide}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\ + \t
{{X Pronunciation (Recording and/or IPA)}}\n\ + {{/X Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}" + - name: Y Spelling + question_format: "{{#Y Word}}\n\t
Spell this word:
\n\n\t
{{type:Y Word}}
\n\n\t
{{Picture}}\n{{/Y Word}}" + answer_format: "{{FrontSide}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\ + \t
{{Y Pronunciation (Recording and/or IPA)}}\n\ + {{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}" + - name: X and Y Production + question_format: "{{#X Word}}\n{{#Y Word}}\n\t{{Picture}}\n{{/Y Word}}\n{{/X Word}}" + answer_format: "{{FrontSide}}\n\n
\n\n
{{text:X\ + \ Word}}
\n
{{text:Y Word}}
\n\n{{#X Pronunciation\ + \ (Recording and/or IPA)}}\n\t
{{X Pronunciation\ + \ (Recording and/or IPA)}}\n{{/X Pronunciation (Recording and/or IPA)}}\n\ + \n{{#Y Pronunciation (Recording and/or IPA)}}\n\t
{{Y Pronunciation (Recording and/or IPA)}}\n{{/Y Pronunciation (Recording\ + \ and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n\ + {{/Extra}}" +required_fields_per_template: + - - 0 + - all + - - 1 + - - 1 + - all + - - 2 + - - 2 + - all + - - 1 + - 3 + - - 3 + - all + - - 2 + - 3 + - - 4 + - all + - - 1 + - 3 + - - 5 + - all + - - 2 + - 3 + - - 6 + - all + - - 1 + - 2 + - 3 diff --git a/tests/test_files/deck_parts/yaml/note/note1.yaml b/tests/test_files/deck_parts/yaml/notes/note1.yaml similarity index 100% rename from tests/test_files/deck_parts/yaml/note/note1.yaml rename to tests/test_files/deck_parts/yaml/notes/note1.yaml diff --git a/tests/test_helpers.py b/tests/test_helpers.py index fa8ea97..719d009 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -1,6 +1,6 @@ -from brain_brew.representation.json.json_file import JsonFile +from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder -def debug_write_to_target_json(data, json: JsonFile): - json.set_data(data) - json.write_file() +def debug_write_deck_part_to_file(deck_part, filepath: str): + dp = DeckPartHolder("Blah", filepath, deck_part) + dp.write_to_file() From be976167a6bf88c89f026e78766beead323faf64 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sat, 15 Aug 2020 12:30:19 +0200 Subject: [PATCH 22/39] Separation of Transformers and Build Configs --- .../csv_collection}/__init__.py | 0 .../csv_collection/config/__init__.py | 0 .../config/csv_collection_shared.py | 69 +++++++ .../config/csv_collection_to_notes.py | 43 ++++ .../csv_collection_to_deck_parts.py | 33 ++++ .../csv_collection/generate_csv_collection.py | 61 ++++++ .../generate_deck_parts.py | 4 +- brain_brew/main.py | 6 +- .../representation/build_config/build_task.py | 2 +- .../build_config/task_builder.py | 21 +- .../build_config/top_level_task_builder.py | 2 +- .../configuration/csv_file_mapping.py | 1 - .../tr_notes_crowdanki.py | 35 ---- .../tr_notes_csv_collection.py | 183 ------------------ .../tr_notes_generic.py | 50 ----- .../representation/transformers/__init__.py | 0 .../transformers/generic_to_deck_part.py | 21 ++ .../representation/yaml/note_model_repr.py | 2 +- brain_brew/transformers/__init__.py | 0 brain_brew/transformers/tr_notes_generic.py | 16 ++ .../transformers/transform_crowdanki.py | 35 ++++ .../transformers/transform_csv_collection.py | 43 ++++ brain_brew/utils.py | 6 + .../test_tr_notes_csv_collection.py | 2 +- tests/test_builder.py | 2 +- 25 files changed, 342 insertions(+), 295 deletions(-) rename brain_brew/{representation/deck_part_transformers => build_tasks/csv_collection}/__init__.py (100%) create mode 100644 brain_brew/build_tasks/csv_collection/config/__init__.py create mode 100644 brain_brew/build_tasks/csv_collection/config/csv_collection_shared.py create mode 100644 brain_brew/build_tasks/csv_collection/config/csv_collection_to_notes.py create mode 100644 brain_brew/build_tasks/csv_collection/csv_collection_to_deck_parts.py create mode 100644 brain_brew/build_tasks/csv_collection/generate_csv_collection.py rename brain_brew/{representation/build_config => build_tasks}/generate_deck_parts.py (64%) delete mode 100644 brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py delete mode 100644 brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py delete mode 100644 brain_brew/representation/deck_part_transformers/tr_notes_generic.py create mode 100644 brain_brew/representation/transformers/__init__.py create mode 100644 brain_brew/representation/transformers/generic_to_deck_part.py create mode 100644 brain_brew/transformers/__init__.py create mode 100644 brain_brew/transformers/tr_notes_generic.py create mode 100644 brain_brew/transformers/transform_crowdanki.py create mode 100644 brain_brew/transformers/transform_csv_collection.py diff --git a/brain_brew/representation/deck_part_transformers/__init__.py b/brain_brew/build_tasks/csv_collection/__init__.py similarity index 100% rename from brain_brew/representation/deck_part_transformers/__init__.py rename to brain_brew/build_tasks/csv_collection/__init__.py diff --git a/brain_brew/build_tasks/csv_collection/config/__init__.py b/brain_brew/build_tasks/csv_collection/config/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/brain_brew/build_tasks/csv_collection/config/csv_collection_shared.py b/brain_brew/build_tasks/csv_collection/config/csv_collection_shared.py new file mode 100644 index 0000000..32c3d91 --- /dev/null +++ b/brain_brew/build_tasks/csv_collection/config/csv_collection_shared.py @@ -0,0 +1,69 @@ +from dataclasses import dataclass +from typing import List, Dict + +from brain_brew.representation.configuration.csv_file_mapping import CsvFileMapping +from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping + + +@dataclass +class CsvCollectionShared: + @dataclass(init=False) + class Representation: + file_mappings: List[CsvFileMapping.Representation] + note_model_mappings: List[NoteModelMapping.Representation] + + def __init__(self, file_mappings, note_model_mappings): + self.file_mappings = list(map(CsvFileMapping.Representation.from_dict, file_mappings)) + self.note_model_mappings = list(map(NoteModelMapping.Representation.from_dict, note_model_mappings)) + + def get_file_mappings(self) -> List[CsvFileMapping]: + return list(map(CsvFileMapping.from_repr, self.file_mappings)) + + def get_note_model_mappings(self) -> Dict[str, NoteModelMapping]: + def map_nmm(nmm_to_map: str): + nmm = NoteModelMapping.from_repr(nmm_to_map) + return nmm.get_note_model_mapping_dict() + + return dict(*map(map_nmm, self.note_model_mappings)) + + file_mappings: List[CsvFileMapping] + note_model_mappings: Dict[str, NoteModelMapping] + + def verify_contents(self): + errors = [] + + for nm in self.note_model_mappings.values(): + try: + nm.verify_contents() + except KeyError as e: + errors.append(e) + + for fm in self.file_mappings: + # Check all necessary key values are present + try: + fm.verify_contents() + except KeyError as e: + errors.append(e) + + # Check all referenced note models have a mapping + for csv_map in self.file_mappings: + for nm in csv_map.get_used_note_model_names(): + if nm not in self.note_model_mappings.keys(): + errors.append(f"Missing Note Model Map for {nm}") + + # Check each of the Csvs (or their derivatives) contain all the necessary columns for their stated note model + for cfm in self.file_mappings: + note_model_names = cfm.get_used_note_model_names() + available_columns = cfm.get_available_columns() + + referenced_note_models_maps = [value for key, value in self.note_model_mappings.items() if + key in note_model_names] + for nm_map in referenced_note_models_maps: + for holder in nm_map.note_models.values(): + missing_columns = [col for col in holder.deck_part.field_names_lowercase if + col not in nm_map.csv_headers_map_to_note_fields(available_columns)] + if missing_columns: + errors.append(KeyError(f"Csvs are missing columns from {holder.name}", missing_columns)) + + if errors: + raise Exception(errors) \ No newline at end of file diff --git a/brain_brew/build_tasks/csv_collection/config/csv_collection_to_notes.py b/brain_brew/build_tasks/csv_collection/config/csv_collection_to_notes.py new file mode 100644 index 0000000..65a947e --- /dev/null +++ b/brain_brew/build_tasks/csv_collection/config/csv_collection_to_notes.py @@ -0,0 +1,43 @@ +from dataclasses import dataclass +from typing import Dict, List + +from brain_brew.build_tasks.csv_collection.config.csv_collection_shared import CsvCollectionShared +from brain_brew.representation.transformers.generic_to_deck_part import TrGenericToDeckPart +from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder +from brain_brew.representation.yaml.note_repr import Note, Notes +from brain_brew.transformers.transform_csv_collection import TransformCsvCollection + + +@dataclass +class CsvCollectionToNotes(CsvCollectionShared, TrGenericToDeckPart): + @dataclass(init=False) + class Representation(CsvCollectionShared.Representation, TrGenericToDeckPart.Representation): + def __init__(self, name, file_mappings, note_model_mappings, save_to_file=None): + CsvCollectionShared.Representation.__init__(self, file_mappings, note_model_mappings) + TrGenericToDeckPart.Representation.__init__(self, name, save_to_file) + + @classmethod + def from_repr(cls, data: Representation): + return cls( + name=data.name, + save_to_file=data.save_to_file, + file_mappings=data.get_file_mappings(), + note_model_mappings=data.get_note_model_mappings() + ) + + @classmethod + def from_dict(cls, data: dict): + return cls.from_repr(cls.Representation.from_dict(data)) + + def execute(self): + csv_data_by_guid: Dict[str, dict] = {} + for csv_map in self.file_mappings: + csv_map.compile_data() + csv_data_by_guid = {**csv_data_by_guid, **csv_map.compiled_data} + csv_rows: List[dict] = list(csv_data_by_guid.values()) + + deck_part_notes: List[Note] = TransformCsvCollection.csv_collection_to_notes( + csv_rows, self.note_model_mappings) + + notes = Notes.from_list_of_notes(deck_part_notes) + DeckPartHolder.override_or_create(self.name, self.save_to_file, notes) \ No newline at end of file diff --git a/brain_brew/build_tasks/csv_collection/csv_collection_to_deck_parts.py b/brain_brew/build_tasks/csv_collection/csv_collection_to_deck_parts.py new file mode 100644 index 0000000..7e48fbb --- /dev/null +++ b/brain_brew/build_tasks/csv_collection/csv_collection_to_deck_parts.py @@ -0,0 +1,33 @@ +from dataclasses import dataclass + +from brain_brew.build_tasks.csv_collection.config.csv_collection_to_notes import CsvCollectionToNotes +from brain_brew.representation.build_config.build_task import DeckPartBuildTask +from brain_brew.utils import all_combos_prepend_append + + +@dataclass +class CsvCollectionToDeckParts(DeckPartBuildTask): + task_names = all_combos_prepend_append(["Csv Collection", "Csv"], "From ", "s") + + @dataclass + class Representation: + notes: dict + + @classmethod + def from_dict(cls, data: dict): + return cls(**data) + + notes_transform: CsvCollectionToNotes + + @classmethod + def from_repr(cls, data: Representation): + return cls( + notes_transform=CsvCollectionToNotes.from_dict(data.notes) + ) + + @classmethod + def from_dict(cls, data: dict): + return cls.from_repr(cls.Representation.from_dict(data)) + + def execute(self): + self.notes_transform.execute() diff --git a/brain_brew/build_tasks/csv_collection/generate_csv_collection.py b/brain_brew/build_tasks/csv_collection/generate_csv_collection.py new file mode 100644 index 0000000..741cc6c --- /dev/null +++ b/brain_brew/build_tasks/csv_collection/generate_csv_collection.py @@ -0,0 +1,61 @@ +from dataclasses import dataclass +from typing import List, Dict + +from brain_brew.build_tasks.csv_collection.config.csv_collection_shared import CsvCollectionShared +from brain_brew.representation.build_config.build_task import TopLevelBuildTask +from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder +from brain_brew.representation.yaml.note_repr import Notes, Note +from brain_brew.transformers.transform_csv_collection import TransformCsvCollection +from brain_brew.utils import all_combos_prepend_append + + +@dataclass +class GenerateCsvCollection(CsvCollectionShared, TopLevelBuildTask): + task_names = all_combos_prepend_append(["Csv Collection", "Csv"], "Generate ", "s") + + notes: DeckPartHolder[Notes] + + @dataclass(init=False) + class Representation(CsvCollectionShared.Representation): + notes: str + + def __init__(self, notes, file_mappings, note_model_mappings): + CsvCollectionShared.Representation.__init__(self, file_mappings, note_model_mappings) + self.notes = notes + + @classmethod + def from_dict(cls, data: dict): + return cls(**data) + + @classmethod + def from_repr(cls, data: Representation): + return cls( + notes=DeckPartHolder.from_deck_part_pool(data.notes), + file_mappings=data.get_file_mappings(), + note_model_mappings=data.get_note_model_mappings() + ) + + @classmethod + def from_dict(cls, data: dict): + return cls.from_repr(cls.Representation.from_dict(data)) + + def execute(self): + notes: List[Note] = self.notes.deck_part.get_notes() + self.verify_notes_match_note_model_mappings(notes) + + csv_data: Dict[str, dict] = TransformCsvCollection.notes_to_csv_collection(notes, self.note_model_mappings) + + # TODO: Dry run option, to not save anything at this stage + + for fm in self.file_mappings: + fm.compile_data() + fm.set_relevant_data(csv_data) + + def verify_notes_match_note_model_mappings(self, notes: List[Note]): + note_models_used = {note.note_model for note in notes} + errors = [TypeError(f"Unknown note model type '{model}' in deck part '{self.notes.name}'. " + f"Add mapping for that model.") + for model in note_models_used if model not in self.note_model_mappings.keys()] + + if errors: + raise Exception(errors) \ No newline at end of file diff --git a/brain_brew/representation/build_config/generate_deck_parts.py b/brain_brew/build_tasks/generate_deck_parts.py similarity index 64% rename from brain_brew/representation/build_config/generate_deck_parts.py rename to brain_brew/build_tasks/generate_deck_parts.py index cfe4b6e..4614aa6 100644 --- a/brain_brew/representation/build_config/generate_deck_parts.py +++ b/brain_brew/build_tasks/generate_deck_parts.py @@ -1,6 +1,6 @@ from typing import Dict, Type -from brain_brew.representation.build_config.build_task import TopLevelBuildTask, GenerateDeckPartBuildTask, BuildTask +from brain_brew.representation.build_config.build_task import BuildTask, TopLevelBuildTask, DeckPartBuildTask from brain_brew.representation.build_config.task_builder import TaskBuilder @@ -9,4 +9,4 @@ class GenerateDeckParts(TaskBuilder, TopLevelBuildTask): @classmethod def known_task_dict(cls) -> Dict[str, Type[BuildTask]]: - return GenerateDeckPartBuildTask.get_all_build_tasks() + return DeckPartBuildTask.get_all_build_tasks() diff --git a/brain_brew/main.py b/brain_brew/main.py index ef28fd5..aa7d2cd 100644 --- a/brain_brew/main.py +++ b/brain_brew/main.py @@ -4,7 +4,7 @@ from brain_brew.representation.build_config.top_level_task_builder import TopLevelTaskBuilder from brain_brew.file_manager import FileManager from brain_brew.representation.configuration.global_config import GlobalConfig -from brain_brew.representation.generic.yaml_file import YamlFile +from brain_brew.representation.yaml.my_yaml import yaml_load # sys.path.append(os.path.join(os.path.dirname(__file__), "dist")) @@ -17,14 +17,14 @@ def main(): # Read in Arguments argument_reader = BBArgumentReader() builder_file_name, global_config_file = argument_reader.get_parsed() - builder_config = YamlFile.read_file(builder_file_name) # Read in Global Config File global_config = GlobalConfig.from_yaml(global_config_file) if global_config_file else GlobalConfig.get_default() file_manager = FileManager() # Parse Build Config File - builder = TopLevelTaskBuilder.from_dict(builder_config, global_config, file_manager) + builder_data = TopLevelTaskBuilder.read_to_dict(builder_file_name) + builder = TopLevelTaskBuilder.from_list(builder_data, global_config, file_manager) # If all good, execute it builder.execute() diff --git a/brain_brew/representation/build_config/build_task.py b/brain_brew/representation/build_config/build_task.py index 9d52099..d469938 100644 --- a/brain_brew/representation/build_config/build_task.py +++ b/brain_brew/representation/build_config/build_task.py @@ -36,5 +36,5 @@ class TopLevelBuildTask(BuildTask): pass -class GenerateDeckPartBuildTask(BuildTask): +class DeckPartBuildTask(BuildTask): pass diff --git a/brain_brew/representation/build_config/task_builder.py b/brain_brew/representation/build_config/task_builder.py index 4dfb759..5af0ec7 100644 --- a/brain_brew/representation/build_config/task_builder.py +++ b/brain_brew/representation/build_config/task_builder.py @@ -5,42 +5,31 @@ from brain_brew.interfaces.verifiable import Verifiable from brain_brew.representation.build_config.build_task import BuildTask from brain_brew.representation.configuration.global_config import GlobalConfig +from brain_brew.representation.yaml.my_yaml import YamlRepr from brain_brew.utils import str_to_lowercase_no_separators @dataclass -class TaskBuilder: - @dataclass - class Representation: - tasks: list - - @classmethod - def from_dict(cls, data: dict): - return cls(**data) - +class TaskBuilder(YamlRepr): tasks: List[BuildTask] global_config: GlobalConfig file_manager: FileManager @classmethod - def from_repr(cls, data: Representation, global_config, file_manager): - tasks = cls.read_tasks(data.tasks) + def from_list(cls, data: List[dict], global_config, file_manager): + tasks = cls.read_tasks(data) return cls( tasks=tasks, global_config=global_config, file_manager=file_manager ) - @classmethod - def from_dict(cls, data: dict, global_config, file_manager): - return cls.from_repr(TaskBuilder.Representation.from_dict(data), global_config, file_manager) - @classmethod def known_task_dict(cls) -> Dict[str, Type[BuildTask]]: raise NotImplemented() @classmethod - def read_tasks(cls, tasks: list) -> list: + def read_tasks(cls, tasks: List[dict]) -> list: known_task_dict = cls.known_task_dict() build_tasks = [] diff --git a/brain_brew/representation/build_config/top_level_task_builder.py b/brain_brew/representation/build_config/top_level_task_builder.py index e6c2bc5..4370dd7 100644 --- a/brain_brew/representation/build_config/top_level_task_builder.py +++ b/brain_brew/representation/build_config/top_level_task_builder.py @@ -1,6 +1,6 @@ from typing import Dict, Type -from brain_brew.representation.build_config.build_task import TopLevelBuildTask, BuildTask +from brain_brew.representation.build_config.build_task import BuildTask, TopLevelBuildTask from brain_brew.representation.build_config.task_builder import TaskBuilder diff --git a/brain_brew/representation/configuration/csv_file_mapping.py b/brain_brew/representation/configuration/csv_file_mapping.py index 7598018..d08e27c 100644 --- a/brain_brew/representation/configuration/csv_file_mapping.py +++ b/brain_brew/representation/configuration/csv_file_mapping.py @@ -7,7 +7,6 @@ from brain_brew.interfaces.verifiable import Verifiable from brain_brew.interfaces.writes_file import WritesFile from brain_brew.representation.generic.csv_file import CsvFile, CsvKeys -from brain_brew.representation.generic.yaml_file import YamlFile, ConfigKey from brain_brew.utils import single_item_to_list, generate_anki_guid diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py b/brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py deleted file mode 100644 index 3c71206..0000000 --- a/brain_brew/representation/deck_part_transformers/tr_notes_crowdanki.py +++ /dev/null @@ -1,35 +0,0 @@ -from dataclasses import dataclass -from typing import List, Optional, Union - -from brain_brew.representation.deck_part_transformers.tr_notes_generic import TrNotesToGeneric, TrGenericToNotes -from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport - - -@dataclass -class TrCrowdAnkiToNotes(TrGenericToNotes): - @dataclass - class Representation: - file: str - sort_order: Optional[Union[str, List[str]]] - media: Optional[bool] - useless_note_keys: Optional[Union[dict, list]] - - crowdanki_file: CrowdAnkiExport - sort_order: Optional[List[str]] - media: bool - useless_note_keys: list - - -@dataclass -class TrNotesToCrowdAnki(TrNotesToGeneric): - @dataclass - class Representation: - file: str - sort_order: Optional[Union[str, List[str]]] - media: Optional[bool] - useless_note_keys: Optional[dict] # TODO: use default value - - sort_order: Optional[List[str]] - media: bool - useless_note_keys: dict - diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py b/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py deleted file mode 100644 index cf766b9..0000000 --- a/brain_brew/representation/deck_part_transformers/tr_notes_csv_collection.py +++ /dev/null @@ -1,183 +0,0 @@ -from dataclasses import dataclass -from typing import List, Dict - -from brain_brew.file_manager import FileManager -from brain_brew.representation.build_config.build_task import TopLevelBuildTask, GenerateDeckPartBuildTask -from brain_brew.representation.configuration.csv_file_mapping import CsvFileMapping -from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping -from brain_brew.representation.deck_part_transformers.tr_notes_generic import TrNotesToGeneric, TrGenericToNotes -from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder -from brain_brew.representation.yaml.note_repr import Notes, Note - - -@dataclass -class TrCsvCollectionShared: - @dataclass(init=False) - class Representation: - file_mappings: List[CsvFileMapping.Representation] - note_model_mappings: List[NoteModelMapping.Representation] - - def __init__(self, file_mappings, note_model_mappings): - self.file_mappings = list(map(CsvFileMapping.Representation.from_dict, file_mappings)) - self.note_model_mappings = list(map(NoteModelMapping.Representation.from_dict, note_model_mappings)) - - def get_file_mappings(self) -> List[CsvFileMapping]: - return list(map(CsvFileMapping.from_repr, self.file_mappings)) - - def get_note_model_mappings(self) -> Dict[str, NoteModelMapping]: - def map_nmm(nmm_to_map: str): - nmm = NoteModelMapping.from_repr(nmm_to_map) - return nmm.get_note_model_mapping_dict() - - return dict(*map(map_nmm, self.note_model_mappings)) - - file_mappings: List[CsvFileMapping] - note_model_mappings: Dict[str, NoteModelMapping] - - def verify_contents(self): - errors = [] - - for nm in self.note_model_mappings.values(): - try: - nm.verify_contents() - except KeyError as e: - errors.append(e) - - for fm in self.file_mappings: - # Check all necessary key values are present - try: - fm.verify_contents() - except KeyError as e: - errors.append(e) - - # Check all referenced note models have a mapping - for csv_map in self.file_mappings: - for nm in csv_map.get_used_note_model_names(): - if nm not in self.note_model_mappings.keys(): - errors.append(f"Missing Note Model Map for {nm}") - - # Check each of the Csvs (or their derivatives) contain all the necessary columns for their stated note model - for cfm in self.file_mappings: - note_model_names = cfm.get_used_note_model_names() - available_columns = cfm.get_available_columns() - - referenced_note_models_maps = [value for key, value in self.note_model_mappings.items() if - key in note_model_names] - for nm_map in referenced_note_models_maps: - for holder in nm_map.note_models.values(): - missing_columns = [col for col in holder.deck_part.field_names_lowercase if - col not in nm_map.csv_headers_map_to_note_fields(available_columns)] - if missing_columns: - errors.append(KeyError(f"Csvs are missing columns from {holder.name}", missing_columns)) - - if errors: - raise Exception(errors) - - -@dataclass -class TrCsvCollectionToNotes(GenerateDeckPartBuildTask, TrCsvCollectionShared, TrGenericToNotes): - task_names = ["Notes From Csv Collection", "Notes From Csv", "Notes From Csvs"] - - @dataclass(init=False) - class Representation(TrCsvCollectionShared.Representation, TrGenericToNotes.Representation): - def __init__(self, name, file_mappings, note_model_mappings, save_to_file=None): - TrCsvCollectionShared.Representation.__init__(self, file_mappings, note_model_mappings) - TrGenericToNotes.Representation.__init__(self, name, save_to_file) - - @classmethod - def from_dict(cls, data: dict): - return cls(**data) - - @classmethod - def from_repr(cls, data: Representation): - return cls( - name=data.name, - save_to_file=data.save_to_file, - file_mappings=data.get_file_mappings(), - note_model_mappings=data.get_note_model_mappings() - ) - - @classmethod - def from_dict(cls, data: dict): - return cls.from_repr(TrCsvCollectionToNotes.Representation.from_dict(data)) - - def execute(self): - csv_data_by_guid: Dict[str, dict] = {} - for csv_map in self.file_mappings: - csv_map.compile_data() - csv_data_by_guid = {**csv_data_by_guid, **csv_map.compiled_data} - csv_rows: List[dict] = list(csv_data_by_guid.values()) - - deck_part_notes: List[Note] = [] - - # Get Guid, Tags, NoteTypeName, Fields - for row in csv_rows: - note_model_name = row["note_model"] # TODO: Use object - row_nm: NoteModelMapping = self.note_model_mappings[note_model_name] - - filtered_fields = row_nm.csv_row_map_to_note_fields(row) - - guid = filtered_fields.pop("guid") - tags = self.split_tags(filtered_fields.pop("tags")) - - fields = row_nm.field_values_in_note_model_order(note_model_name, filtered_fields) - - deck_part_notes.append(Note(guid=guid, tags=tags, note_model=note_model_name, fields=fields)) - - notes = Notes.from_list_of_notes(deck_part_notes) - DeckPartHolder.override_or_create(self.name, self.save_to_file, notes) - - -@dataclass -class TrNotesToCsvCollection(TopLevelBuildTask, TrCsvCollectionShared, TrNotesToGeneric): - task_names = ["Generate Csv Collection", "Generate Csv Collections", "Generate Csv", "Generate Csvs"] - - @dataclass(init=False) - class Representation(TrCsvCollectionShared.Representation, TrNotesToGeneric.Representation): - def __init__(self, notes, file_mappings, note_model_mappings): - TrCsvCollectionShared.Representation.__init__(self, file_mappings, note_model_mappings) - TrNotesToGeneric.Representation.__init__(self, notes) - - @classmethod - def from_dict(cls, data: dict): - return cls(**data) - - @classmethod - def from_repr(cls, data: Representation): - return cls( - notes=DeckPartHolder.from_deck_part_pool(data.notes), - file_mappings=data.get_file_mappings(), - note_model_mappings=data.get_note_model_mappings() - ) - - @classmethod - def from_dict(cls, data: dict): - return cls.from_repr(TrNotesToCsvCollection.Representation.from_dict(data)) - - def execute(self): - notes_data = self.notes.deck_part.get_notes() - self.verify_notes_match_note_model_mappings(notes_data) - - csv_data: Dict[str, dict] = {} - for note in notes_data: - nm_name = note.note_model - row = self.note_model_mappings[nm_name].note_models[nm_name].deck_part.zip_field_to_data(note.fields) - row["guid"] = note.guid - row["tags"] = self.join_tags(note.tags) - - formatted_row = self.note_model_mappings[nm_name].note_fields_map_to_csv_row(row) # TODO: Do not edit data, make copy - - csv_data.setdefault(row["guid"], formatted_row) - - for fm in self.file_mappings: - fm.compile_data() - fm.set_relevant_data(csv_data) - - def verify_notes_match_note_model_mappings(self, notes: List[Note]): - note_models_used = {note.note_model for note in notes} - errors = [TypeError(f"Unknown note model type '{model}' in deck part '{self.notes.name}'. " - f"Add mapping for that model.") - for model in note_models_used if model not in self.note_model_mappings.keys()] - - if errors: - raise Exception(errors) diff --git a/brain_brew/representation/deck_part_transformers/tr_notes_generic.py b/brain_brew/representation/deck_part_transformers/tr_notes_generic.py deleted file mode 100644 index b951979..0000000 --- a/brain_brew/representation/deck_part_transformers/tr_notes_generic.py +++ /dev/null @@ -1,50 +0,0 @@ -from dataclasses import dataclass, field -from typing import Optional -import re - -from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder -from brain_brew.representation.yaml.note_repr import Notes -from brain_brew.representation.configuration.global_config import GlobalConfig - - -@dataclass -class TrNotes: - @staticmethod - def split_tags(tags_value: str) -> list: - split = [entry.strip() for entry in re.split(';\s*|,\s*|\s+', tags_value)] - while "" in split: - split.remove("") - return split - - @staticmethod - def join_tags(tags_list: list) -> str: - return GlobalConfig.get_instance().flags.join_values_with.join(tags_list) - - -@dataclass -class TrGenericToNotes(TrNotes): - @dataclass - class Representation: - name: str - save_to_file: Optional[str] - - def __init__(self, name, save_to_file=None): - self.name = name - self.save_to_file = save_to_file - - name: str - save_to_file: Optional[str] - - data: DeckPartHolder[Notes] = field(init=False) - - -@dataclass -class TrNotesToGeneric(TrNotes): - @dataclass - class Representation: - notes: str - - def __init__(self, notes): - self.notes = notes - - notes: DeckPartHolder[Notes] diff --git a/brain_brew/representation/transformers/__init__.py b/brain_brew/representation/transformers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/brain_brew/representation/transformers/generic_to_deck_part.py b/brain_brew/representation/transformers/generic_to_deck_part.py new file mode 100644 index 0000000..dbf8f75 --- /dev/null +++ b/brain_brew/representation/transformers/generic_to_deck_part.py @@ -0,0 +1,21 @@ +from dataclasses import dataclass +from typing import Optional + + +@dataclass +class TrGenericToDeckPart: + @dataclass + class Representation: + name: str + save_to_file: Optional[str] + + def __init__(self, name, save_to_file=None): + self.name = name + self.save_to_file = save_to_file + + @classmethod + def from_dict(cls, data: dict): + return cls(**data) + + name: str + save_to_file: Optional[str] diff --git a/brain_brew/representation/yaml/note_model_repr.py b/brain_brew/representation/yaml/note_model_repr.py index 253766c..b1ab64d 100644 --- a/brain_brew/representation/yaml/note_model_repr.py +++ b/brain_brew/representation/yaml/note_model_repr.py @@ -276,7 +276,7 @@ def encode(self) -> dict: return data_dict - def find_media(self): + def find_media(self): # TODO pass # Look in templates (and css?) diff --git a/brain_brew/transformers/__init__.py b/brain_brew/transformers/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/brain_brew/transformers/tr_notes_generic.py b/brain_brew/transformers/tr_notes_generic.py new file mode 100644 index 0000000..c3b2bf1 --- /dev/null +++ b/brain_brew/transformers/tr_notes_generic.py @@ -0,0 +1,16 @@ +import re + +from brain_brew.representation.configuration.global_config import GlobalConfig + + +class TrNotes: + @staticmethod + def split_tags(tags_value: str) -> list: + split = [entry.strip() for entry in re.split(';\s*|,\s*|\s+', tags_value)] + while "" in split: + split.remove("") + return split + + @staticmethod + def join_tags(tags_list: list) -> str: + return GlobalConfig.get_instance().flags.join_values_with.join(tags_list) diff --git a/brain_brew/transformers/transform_crowdanki.py b/brain_brew/transformers/transform_crowdanki.py new file mode 100644 index 0000000..6ab5598 --- /dev/null +++ b/brain_brew/transformers/transform_crowdanki.py @@ -0,0 +1,35 @@ +from dataclasses import dataclass +from typing import List, Optional, Union + +# from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport +# from brain_brew.representation.transformers.generic_to_deck_part import TrGenericToDeckPart +# +# +# @dataclass +# class TrCrowdAnkiToNotes(TrGenericToDeckPart): +# @dataclass +# class Representation: +# file: str +# sort_order: Optional[Union[str, List[str]]] +# media: Optional[bool] +# useless_note_keys: Optional[Union[dict, list]] +# +# crowdanki_file: CrowdAnkiExport +# sort_order: Optional[List[str]] +# media: bool +# useless_note_keys: list +# +# +# @dataclass +# class TrNotesToCrowdAnki(TrNotesToGeneric): +# @dataclass +# class Representation: +# file: str +# sort_order: Optional[Union[str, List[str]]] +# media: Optional[bool] +# useless_note_keys: Optional[dict] # TODO: use default value +# +# sort_order: Optional[List[str]] +# media: bool +# useless_note_keys: dict +# diff --git a/brain_brew/transformers/transform_csv_collection.py b/brain_brew/transformers/transform_csv_collection.py new file mode 100644 index 0000000..6a3fb23 --- /dev/null +++ b/brain_brew/transformers/transform_csv_collection.py @@ -0,0 +1,43 @@ +from typing import List, Dict + + +from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping +from brain_brew.representation.yaml.note_repr import Note +from brain_brew.transformers.tr_notes_generic import TrNotes + + +class TransformCsvCollection(TrNotes): + @classmethod + def notes_to_csv_collection(cls, notes: List[Note], note_model_mappings: Dict[str, NoteModelMapping]) -> Dict[str, dict]: + csv_data: Dict[str, dict] = {} + for note in notes: + nm_name = note.note_model + row = note_model_mappings[nm_name].note_models[nm_name].deck_part.zip_field_to_data(note.fields) + row["guid"] = note.guid + row["tags"] = cls.join_tags(note.tags) + + formatted_row = note_model_mappings[nm_name].note_fields_map_to_csv_row(row) # TODO: Do not edit data, make copy + + csv_data.setdefault(row["guid"], formatted_row) + + return csv_data + + @classmethod + def csv_collection_to_notes(cls, csv_rows: List[dict], note_model_mappings: Dict[str, NoteModelMapping]) -> List[Note]: + deck_part_notes: List[Note] = [] + + # Get Guid, Tags, NoteTypeName, Fields + for row in csv_rows: + note_model_name = row["note_model"] # TODO: Use object + row_nm: NoteModelMapping = note_model_mappings[note_model_name] + + filtered_fields = row_nm.csv_row_map_to_note_fields(row) + + guid = filtered_fields.pop("guid") + tags = cls.split_tags(filtered_fields.pop("tags")) + + fields = row_nm.field_values_in_note_model_order(note_model_name, filtered_fields) + + deck_part_notes.append(Note(guid=guid, tags=tags, note_model=note_model_name, fields=fields)) + + return deck_part_notes diff --git a/brain_brew/utils.py b/brain_brew/utils.py index 243cbce..e2035cc 100644 --- a/brain_brew/utils.py +++ b/brain_brew/utils.py @@ -22,6 +22,12 @@ def single_item_to_list(item): return [item] +def all_combos_prepend_append(original_list: list, prepend_with: str, append_with: str): + return list({append_or_not for normal in original_list + for prepend_or_not in (normal, prepend_with + normal) + for append_or_not in (prepend_or_not, prepend_or_not + append_with)}) + + def str_to_lowercase_no_separators(str_to_tidy: str): return re.sub(r'[\s+_-]+', '', str_to_tidy.lower()) diff --git a/tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py b/tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py index 3c5eb9c..e11a9e8 100644 --- a/tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py +++ b/tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py @@ -2,7 +2,7 @@ from unittest.mock import patch from brain_brew.file_manager import FileManager -from brain_brew.representation.deck_part_transformers.tr_notes_csv_collection import TrCsvCollectionToNotes +from brain_brew.representation.deck_part_transformers.transform_csv_collection import TrCsvCollectionToNotes from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder from brain_brew.representation.yaml.my_yaml import yaml_dump, yaml_load from tests.test_file_manager import get_new_file_manager diff --git a/tests/test_builder.py b/tests/test_builder.py index f6e6c10..2c49dd0 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -1,7 +1,7 @@ from unittest.mock import patch, Mock from brain_brew.representation.build_config.top_level_task_builder import TopLevelTaskBuilder -from brain_brew.representation.deck_part_transformers.tr_notes_csv_collection import TrNotesToCsvCollection +from brain_brew.representation.deck_part_transformers.transform_csv_collection import TrNotesToCsvCollection from brain_brew.representation.yaml.my_yaml import YamlRepr from brain_brew.representation.yaml.note_model_repr import NoteModel from brain_brew.representation.yaml.note_repr import Notes From 983c23f11d0ace6b8c60d0b36f17d1b4f7de0c53 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sun, 16 Aug 2020 13:23:22 +0200 Subject: [PATCH 23/39] Wrapper classes for CrowdAnki json dicts, plus Notes From_dict removed throughout CrowdAnki build tasks start --- .../__init__.py | 0 .../crowd_anki/crowd_anki_generate.py | 11 +++ .../crowd_anki_to_deck_parts.py} | 0 .../crowd_anki/notes_from_crowd_anki.py | 28 ++++++ .../crowd_anki/notes_to_crowd_anki.py | 23 +++++ .../crowd_anki/shared_base_notes.py | 44 +++++++++ .../csv_collection_to_deck_parts.py | 33 ------- brain_brew/build_tasks/csvs/__init__.py | 0 .../csvs_generate.py} | 27 +++--- .../build_tasks/csvs/csvs_to_deck_parts.py | 28 ++++++ .../notes_from_csvs.py} | 29 +++--- .../shared_base_csvs.py} | 5 +- brain_brew/build_tasks/generate_deck_parts.py | 2 +- brain_brew/build_tasks/source_crowd_anki.py | 10 +-- .../task_builder.py | 0 brain_brew/constants/crowdanki_keys.py | 7 -- .../representation/build_config/build_task.py | 2 +- .../build_config/representation_base.py | 6 ++ .../build_config/top_level_task_builder.py | 2 +- .../configuration/csv_file_mapping.py | 20 ++--- .../configuration/note_model_mapping.py | 7 +- .../representation/generic/media_file.py | 2 +- .../representation/json/crowd_anki_export.py | 22 +++-- .../representation/json/deck_part_notes.py | 8 -- brain_brew/representation/json/json_file.py | 2 +- .../json/wrappers_for_crowd_anki.py | 90 +++++++++++++++++++ .../transformers/generic_to_deck_part.py | 10 +-- .../representation/yaml/note_model_repr.py | 37 ++------ ...tes_generic.py => base_transform_notes.py} | 0 .../transformers/transform_crowdanki.py | 69 +++++++------- .../transformers/transform_csv_collection.py | 2 +- 31 files changed, 336 insertions(+), 190 deletions(-) rename brain_brew/build_tasks/{csv_collection => crowd_anki}/__init__.py (100%) create mode 100644 brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py rename brain_brew/build_tasks/{csv_collection/config/__init__.py => crowd_anki/crowd_anki_to_deck_parts.py} (100%) create mode 100644 brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py create mode 100644 brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py create mode 100644 brain_brew/build_tasks/crowd_anki/shared_base_notes.py delete mode 100644 brain_brew/build_tasks/csv_collection/csv_collection_to_deck_parts.py create mode 100644 brain_brew/build_tasks/csvs/__init__.py rename brain_brew/build_tasks/{csv_collection/generate_csv_collection.py => csvs/csvs_generate.py} (67%) create mode 100644 brain_brew/build_tasks/csvs/csvs_to_deck_parts.py rename brain_brew/build_tasks/{csv_collection/config/csv_collection_to_notes.py => csvs/notes_from_csvs.py} (55%) rename brain_brew/build_tasks/{csv_collection/config/csv_collection_shared.py => csvs/shared_base_csvs.py} (94%) rename brain_brew/{representation/build_config => build_tasks}/task_builder.py (100%) delete mode 100644 brain_brew/constants/crowdanki_keys.py create mode 100644 brain_brew/representation/build_config/representation_base.py create mode 100644 brain_brew/representation/json/wrappers_for_crowd_anki.py rename brain_brew/transformers/{tr_notes_generic.py => base_transform_notes.py} (100%) diff --git a/brain_brew/build_tasks/csv_collection/__init__.py b/brain_brew/build_tasks/crowd_anki/__init__.py similarity index 100% rename from brain_brew/build_tasks/csv_collection/__init__.py rename to brain_brew/build_tasks/crowd_anki/__init__.py diff --git a/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py b/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py new file mode 100644 index 0000000..6853731 --- /dev/null +++ b/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass + + +from brain_brew.representation.build_config.build_task import TopLevelBuildTask +from brain_brew.utils import all_combos_prepend_append + + +@dataclass +class CrowdAnkiGenerate(TopLevelBuildTask): + task_names = all_combos_prepend_append(["CrowdAnki", "CrowdAnki Export"], "Generate ", "s") + diff --git a/brain_brew/build_tasks/csv_collection/config/__init__.py b/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py similarity index 100% rename from brain_brew/build_tasks/csv_collection/config/__init__.py rename to brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py diff --git a/brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py new file mode 100644 index 0000000..5d97e23 --- /dev/null +++ b/brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py @@ -0,0 +1,28 @@ +from dataclasses import dataclass +from typing import Optional, Union, List + +from brain_brew.build_tasks.crowd_anki.shared_base_notes import SharedBaseNotes +from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport +from brain_brew.transformers.transform_crowdanki import TransformCrowdAnki + + +@dataclass +class TrCrowdAnkiToNotes(SharedBaseNotes): + @dataclass + class Representation(SharedBaseNotes.Representation): + pass + + @classmethod + def from_repr(cls, data: Union[Representation, dict]): + rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) + return cls( + name=rep.name, + sort_order=SharedBaseNotes._get_sort_order(rep.sort_order), + move_media=SharedBaseNotes._get_move_media(rep.move_media), + useless_note_keys=SharedBaseNotes._get_useless_note_keys(rep.useless_note_keys) + ) + + def execute(self, crowd_anki_export: CrowdAnkiExport): + ca_wrapper = crowd_anki_export.read_json_file() + + TransformCrowdAnki.crowd_anki_to_notes(ca_wrapper.notes, ) \ No newline at end of file diff --git a/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py new file mode 100644 index 0000000..bfbb216 --- /dev/null +++ b/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py @@ -0,0 +1,23 @@ +from dataclasses import dataclass +from typing import Optional, Union, List + +from brain_brew.build_tasks.crowd_anki.shared_base_notes import SharedBaseNotes + + +@dataclass +class NotesToCrowdAnki(SharedBaseNotes): + @dataclass + class Representation(SharedBaseNotes.Representation): + pass + + @classmethod + def from_repr(cls, data: Union[Representation, dict]): + rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) + return cls( + name=rep.name, + sort_order=SharedBaseNotes._get_sort_order(rep.sort_order), + move_media=SharedBaseNotes._get_move_media(rep.move_media), + useless_note_keys=SharedBaseNotes._get_useless_note_keys(rep.useless_note_keys) + ) + + diff --git a/brain_brew/build_tasks/crowd_anki/shared_base_notes.py b/brain_brew/build_tasks/crowd_anki/shared_base_notes.py new file mode 100644 index 0000000..080f181 --- /dev/null +++ b/brain_brew/build_tasks/crowd_anki/shared_base_notes.py @@ -0,0 +1,44 @@ +from dataclasses import dataclass +from typing import Optional, Union, List + +from brain_brew.representation.build_config.representation_base import RepresentationBase + + +@dataclass +class SharedBaseNotes: + @dataclass + class Representation(RepresentationBase): + name: str + sort_order: Optional[Union[str, List[str]]] + move_media: Optional[bool] + useless_note_keys: Optional[dict] + + @staticmethod + def _get_sort_order(sort_order: Optional[Union[str, List[str]]]): + if isinstance(sort_order, list): + return sort_order + elif isinstance(sort_order, str): + return [sort_order] + return [] + + @staticmethod + def _get_move_media(move_media: Optional[bool]): + return move_media or False + + @staticmethod + def _get_useless_note_keys(useless_note_keys: Optional[dict]): + default_useless_keys = { + "__type__": "Note", + "data": None, + "flags": 0 + } + + if useless_note_keys is None: + return default_useless_keys + + return {**useless_note_keys, **default_useless_keys} + + name: str + sort_order: Optional[List[str]] + move_media: bool + useless_note_keys: dict diff --git a/brain_brew/build_tasks/csv_collection/csv_collection_to_deck_parts.py b/brain_brew/build_tasks/csv_collection/csv_collection_to_deck_parts.py deleted file mode 100644 index 7e48fbb..0000000 --- a/brain_brew/build_tasks/csv_collection/csv_collection_to_deck_parts.py +++ /dev/null @@ -1,33 +0,0 @@ -from dataclasses import dataclass - -from brain_brew.build_tasks.csv_collection.config.csv_collection_to_notes import CsvCollectionToNotes -from brain_brew.representation.build_config.build_task import DeckPartBuildTask -from brain_brew.utils import all_combos_prepend_append - - -@dataclass -class CsvCollectionToDeckParts(DeckPartBuildTask): - task_names = all_combos_prepend_append(["Csv Collection", "Csv"], "From ", "s") - - @dataclass - class Representation: - notes: dict - - @classmethod - def from_dict(cls, data: dict): - return cls(**data) - - notes_transform: CsvCollectionToNotes - - @classmethod - def from_repr(cls, data: Representation): - return cls( - notes_transform=CsvCollectionToNotes.from_dict(data.notes) - ) - - @classmethod - def from_dict(cls, data: dict): - return cls.from_repr(cls.Representation.from_dict(data)) - - def execute(self): - self.notes_transform.execute() diff --git a/brain_brew/build_tasks/csvs/__init__.py b/brain_brew/build_tasks/csvs/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/brain_brew/build_tasks/csv_collection/generate_csv_collection.py b/brain_brew/build_tasks/csvs/csvs_generate.py similarity index 67% rename from brain_brew/build_tasks/csv_collection/generate_csv_collection.py rename to brain_brew/build_tasks/csvs/csvs_generate.py index 741cc6c..ab2823e 100644 --- a/brain_brew/build_tasks/csv_collection/generate_csv_collection.py +++ b/brain_brew/build_tasks/csvs/csvs_generate.py @@ -1,7 +1,7 @@ from dataclasses import dataclass -from typing import List, Dict +from typing import List, Dict, Union -from brain_brew.build_tasks.csv_collection.config.csv_collection_shared import CsvCollectionShared +from brain_brew.build_tasks.csvs.shared_base_csvs import SharedBaseCsvs from brain_brew.representation.build_config.build_task import TopLevelBuildTask from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder from brain_brew.representation.yaml.note_repr import Notes, Note @@ -10,35 +10,28 @@ @dataclass -class GenerateCsvCollection(CsvCollectionShared, TopLevelBuildTask): +class CsvsGenerate(SharedBaseCsvs, TopLevelBuildTask): task_names = all_combos_prepend_append(["Csv Collection", "Csv"], "Generate ", "s") notes: DeckPartHolder[Notes] @dataclass(init=False) - class Representation(CsvCollectionShared.Representation): + class Representation(SharedBaseCsvs.Representation): notes: str def __init__(self, notes, file_mappings, note_model_mappings): - CsvCollectionShared.Representation.__init__(self, file_mappings, note_model_mappings) + SharedBaseCsvs.Representation.__init__(self, file_mappings, note_model_mappings) self.notes = notes - @classmethod - def from_dict(cls, data: dict): - return cls(**data) - @classmethod - def from_repr(cls, data: Representation): + def from_repr(cls, data: Union[Representation, dict]): + rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) return cls( - notes=DeckPartHolder.from_deck_part_pool(data.notes), - file_mappings=data.get_file_mappings(), - note_model_mappings=data.get_note_model_mappings() + notes=DeckPartHolder.from_deck_part_pool(rep.notes), + file_mappings=rep.get_file_mappings(), + note_model_mappings=rep.get_note_model_mappings() ) - @classmethod - def from_dict(cls, data: dict): - return cls.from_repr(cls.Representation.from_dict(data)) - def execute(self): notes: List[Note] = self.notes.deck_part.get_notes() self.verify_notes_match_note_model_mappings(notes) diff --git a/brain_brew/build_tasks/csvs/csvs_to_deck_parts.py b/brain_brew/build_tasks/csvs/csvs_to_deck_parts.py new file mode 100644 index 0000000..c0d4389 --- /dev/null +++ b/brain_brew/build_tasks/csvs/csvs_to_deck_parts.py @@ -0,0 +1,28 @@ +from dataclasses import dataclass +from typing import Union + +from brain_brew.build_tasks.csvs.notes_from_csvs import NotesFromCsvs +from brain_brew.representation.build_config.build_task import DeckPartBuildTask +from brain_brew.representation.build_config.representation_base import RepresentationBase +from brain_brew.utils import all_combos_prepend_append + + +@dataclass +class CsvsToDeckParts(DeckPartBuildTask): + task_names = all_combos_prepend_append(["Csv Collection", "Csv"], "From ", "s") + + @dataclass + class Representation(RepresentationBase): + notes: dict + + notes_transform: NotesFromCsvs + + @classmethod + def from_repr(cls, data: Union[Representation, dict]): + rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) + return cls( + notes_transform=NotesFromCsvs.from_dict(rep.notes) + ) + + def execute(self): + self.notes_transform.execute() diff --git a/brain_brew/build_tasks/csv_collection/config/csv_collection_to_notes.py b/brain_brew/build_tasks/csvs/notes_from_csvs.py similarity index 55% rename from brain_brew/build_tasks/csv_collection/config/csv_collection_to_notes.py rename to brain_brew/build_tasks/csvs/notes_from_csvs.py index 65a947e..9b0e728 100644 --- a/brain_brew/build_tasks/csv_collection/config/csv_collection_to_notes.py +++ b/brain_brew/build_tasks/csvs/notes_from_csvs.py @@ -1,34 +1,31 @@ from dataclasses import dataclass -from typing import Dict, List +from typing import Dict, List, Union -from brain_brew.build_tasks.csv_collection.config.csv_collection_shared import CsvCollectionShared -from brain_brew.representation.transformers.generic_to_deck_part import TrGenericToDeckPart +from brain_brew.build_tasks.csvs.shared_base_csvs import SharedBaseCsvs +from brain_brew.representation.transformers.generic_to_deck_part import DeckPartFromBase from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder from brain_brew.representation.yaml.note_repr import Note, Notes from brain_brew.transformers.transform_csv_collection import TransformCsvCollection @dataclass -class CsvCollectionToNotes(CsvCollectionShared, TrGenericToDeckPart): +class NotesFromCsvs(SharedBaseCsvs, DeckPartFromBase): @dataclass(init=False) - class Representation(CsvCollectionShared.Representation, TrGenericToDeckPart.Representation): + class Representation(SharedBaseCsvs.Representation, DeckPartFromBase.Representation): def __init__(self, name, file_mappings, note_model_mappings, save_to_file=None): - CsvCollectionShared.Representation.__init__(self, file_mappings, note_model_mappings) - TrGenericToDeckPart.Representation.__init__(self, name, save_to_file) + SharedBaseCsvs.Representation.__init__(self, file_mappings, note_model_mappings) + DeckPartFromBase.Representation.__init__(self, name, save_to_file) @classmethod - def from_repr(cls, data: Representation): + def from_repr(cls, data: Union[Representation, dict]): + rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) return cls( - name=data.name, - save_to_file=data.save_to_file, - file_mappings=data.get_file_mappings(), - note_model_mappings=data.get_note_model_mappings() + name=rep.name, + save_to_file=rep.save_to_file, + file_mappings=rep.get_file_mappings(), + note_model_mappings=rep.get_note_model_mappings() ) - @classmethod - def from_dict(cls, data: dict): - return cls.from_repr(cls.Representation.from_dict(data)) - def execute(self): csv_data_by_guid: Dict[str, dict] = {} for csv_map in self.file_mappings: diff --git a/brain_brew/build_tasks/csv_collection/config/csv_collection_shared.py b/brain_brew/build_tasks/csvs/shared_base_csvs.py similarity index 94% rename from brain_brew/build_tasks/csv_collection/config/csv_collection_shared.py rename to brain_brew/build_tasks/csvs/shared_base_csvs.py index 32c3d91..d5bddae 100644 --- a/brain_brew/build_tasks/csv_collection/config/csv_collection_shared.py +++ b/brain_brew/build_tasks/csvs/shared_base_csvs.py @@ -1,14 +1,15 @@ from dataclasses import dataclass from typing import List, Dict +from brain_brew.representation.build_config.representation_base import RepresentationBase from brain_brew.representation.configuration.csv_file_mapping import CsvFileMapping from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping @dataclass -class CsvCollectionShared: +class SharedBaseCsvs: @dataclass(init=False) - class Representation: + class Representation(RepresentationBase): file_mappings: List[CsvFileMapping.Representation] note_model_mappings: List[NoteModelMapping.Representation] diff --git a/brain_brew/build_tasks/generate_deck_parts.py b/brain_brew/build_tasks/generate_deck_parts.py index 4614aa6..2615f1a 100644 --- a/brain_brew/build_tasks/generate_deck_parts.py +++ b/brain_brew/build_tasks/generate_deck_parts.py @@ -1,7 +1,7 @@ from typing import Dict, Type from brain_brew.representation.build_config.build_task import BuildTask, TopLevelBuildTask, DeckPartBuildTask -from brain_brew.representation.build_config.task_builder import TaskBuilder +from brain_brew.build_tasks.task_builder import TaskBuilder class GenerateDeckParts(TaskBuilder, TopLevelBuildTask): diff --git a/brain_brew/build_tasks/source_crowd_anki.py b/brain_brew/build_tasks/source_crowd_anki.py index 6e53999..61755c4 100644 --- a/brain_brew/build_tasks/source_crowd_anki.py +++ b/brain_brew/build_tasks/source_crowd_anki.py @@ -3,25 +3,17 @@ from brain_brew.build_tasks.build_task_generic import BuildTaskGeneric from brain_brew.constants.build_config_keys import BuildTaskEnum, BuildConfigKeys -from brain_brew.constants.crowdanki_keys import CAKeys from brain_brew.constants.deckpart_keys import DeckPartNoteKeys from brain_brew.file_manager import FileManager from brain_brew.representation.generic.media_file import MediaFile from brain_brew.utils import blank_str_if_none from brain_brew.representation.generic.yaml_file import ConfigKey, YamlFile -from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport +from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport, CAKeys from brain_brew.representation.json.deck_part_header import DeckPartHeader from brain_brew.representation.yaml.note_model_repr import CANoteModelKeys, DeckPartNoteModel from brain_brew.representation.json.deck_part_notes import CANoteKeys, DeckPartNotes -class CrowdAnkiKeys(Enum): - FILE = "file" - NOTE_SORT_ORDER = "note_sort_order" - MEDIA = "media" - USELESS_NOTE_KEYS = "useless_note_keys" - - class SourceCrowdAnki(YamlFile, BuildTaskGeneric): @staticmethod def get_build_keys(): diff --git a/brain_brew/representation/build_config/task_builder.py b/brain_brew/build_tasks/task_builder.py similarity index 100% rename from brain_brew/representation/build_config/task_builder.py rename to brain_brew/build_tasks/task_builder.py diff --git a/brain_brew/constants/crowdanki_keys.py b/brain_brew/constants/crowdanki_keys.py deleted file mode 100644 index b9849f9..0000000 --- a/brain_brew/constants/crowdanki_keys.py +++ /dev/null @@ -1,7 +0,0 @@ -from enum import Enum - - -class CAKeys(Enum): - NOTE_MODELS = "note_models" - NOTES = "notes" - MEDIA_FILES = "media_files" diff --git a/brain_brew/representation/build_config/build_task.py b/brain_brew/representation/build_config/build_task.py index d469938..0c0bbeb 100644 --- a/brain_brew/representation/build_config/build_task.py +++ b/brain_brew/representation/build_config/build_task.py @@ -10,7 +10,7 @@ def execute(self): raise NotImplemented() @classmethod - def from_dict(cls, data: dict): + def from_repr(cls, data: dict): raise NotImplemented() @classmethod diff --git a/brain_brew/representation/build_config/representation_base.py b/brain_brew/representation/build_config/representation_base.py new file mode 100644 index 0000000..5e32181 --- /dev/null +++ b/brain_brew/representation/build_config/representation_base.py @@ -0,0 +1,6 @@ + + +class RepresentationBase: + @classmethod + def from_dict(cls, data: dict): + return cls(**data) diff --git a/brain_brew/representation/build_config/top_level_task_builder.py b/brain_brew/representation/build_config/top_level_task_builder.py index 4370dd7..64845e6 100644 --- a/brain_brew/representation/build_config/top_level_task_builder.py +++ b/brain_brew/representation/build_config/top_level_task_builder.py @@ -1,7 +1,7 @@ from typing import Dict, Type from brain_brew.representation.build_config.build_task import BuildTask, TopLevelBuildTask -from brain_brew.representation.build_config.task_builder import TaskBuilder +from brain_brew.build_tasks.task_builder import TaskBuilder class TopLevelTaskBuilder(TaskBuilder): diff --git a/brain_brew/representation/configuration/csv_file_mapping.py b/brain_brew/representation/configuration/csv_file_mapping.py index d08e27c..f5a8e1b 100644 --- a/brain_brew/representation/configuration/csv_file_mapping.py +++ b/brain_brew/representation/configuration/csv_file_mapping.py @@ -6,6 +6,7 @@ from brain_brew.constants.deckpart_keys import DeckPartNoteKeys from brain_brew.interfaces.verifiable import Verifiable from brain_brew.interfaces.writes_file import WritesFile +from brain_brew.representation.build_config.representation_base import RepresentationBase from brain_brew.representation.generic.csv_file import CsvFile, CsvKeys from brain_brew.utils import single_item_to_list, generate_anki_guid @@ -20,7 +21,7 @@ @dataclass class CsvFileMappingDerivative: @dataclass(init=False) - class Representation: + class Representation(RepresentationBase): file: str note_model: Optional[str] sort_by_columns: Optional[Union[list, str]] @@ -34,10 +35,6 @@ def __init__(self, file, note_model=None, sort_by_columns=None, reverse_sort=Non self.reverse_sort = reverse_sort self.derivatives = list(map(CsvFileMappingDerivative.Representation.from_dict, derivatives)) if derivatives is not None else [] - @classmethod - def from_dict(cls, data: dict): - return cls(**data) - compiled_data: Dict[str, dict] = field(init=False) csv_file: CsvFile @@ -48,13 +45,14 @@ def from_dict(cls, data: dict): derivatives: Optional[List['CsvFileMappingDerivative']] @classmethod - def from_repr(cls, data: Representation): + def from_repr(cls, data: Union[Representation, dict]): + rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) return cls( - csv_file=CsvFile.create(data.file, True), # TODO: Fix Read Now - note_model=None if not data.note_model.strip() else data.note_model.strip(), - sort_by_columns=single_item_to_list(data.sort_by_columns), - reverse_sort=data.reverse_sort or False, - derivatives=list(map(cls.from_repr, data.derivatives)) if data.derivatives is not None else [] + csv_file=CsvFile.create(rep.file, True), # TODO: Fix Read Now + note_model=None if not rep.note_model.strip() else rep.note_model.strip(), + sort_by_columns=single_item_to_list(rep.sort_by_columns), + reverse_sort=rep.reverse_sort or False, + derivatives=list(map(cls.from_repr, rep.derivatives)) if rep.derivatives is not None else [] ) def get_available_columns(self): diff --git a/brain_brew/representation/configuration/note_model_mapping.py b/brain_brew/representation/configuration/note_model_mapping.py index c86ae06..25c289d 100644 --- a/brain_brew/representation/configuration/note_model_mapping.py +++ b/brain_brew/representation/configuration/note_model_mapping.py @@ -4,6 +4,7 @@ from brain_brew.constants.deckpart_keys import DeckPartNoteKeys from brain_brew.interfaces.verifiable import Verifiable +from brain_brew.representation.build_config.representation_base import RepresentationBase from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder from brain_brew.representation.yaml.note_model_repr import NoteModel from brain_brew.utils import single_item_to_list @@ -36,15 +37,11 @@ def __init__(self, field_type: FieldMappingType, field_name: str, value: str): @dataclass class NoteModelMapping(Verifiable): @dataclass - class Representation: + class Representation(RepresentationBase): note_models: Union[str, list] columns_to_fields: Dict[str, str] personal_fields: List[str] - @classmethod - def from_dict(cls, data: dict): - return cls(**data) - note_models: Dict[str, DeckPartHolder[NoteModel]] columns: List[FieldMapping] personal_fields: List[FieldMapping] diff --git a/brain_brew/representation/generic/media_file.py b/brain_brew/representation/generic/media_file.py index 9f70276..e7811a2 100644 --- a/brain_brew/representation/generic/media_file.py +++ b/brain_brew/representation/generic/media_file.py @@ -19,7 +19,7 @@ def __init__(self, target_loc, filename, man_type: ManagementType = ManagementTy self.filename = filename self.man_type = man_type - self.source_loc = target_loc if source_loc is None else source_loc + self.source_loc = source_loc if source_loc is not None else target_loc def set_override(self, source_loc): self.man_type = MediaFile.ManagementType.OVERRIDDEN diff --git a/brain_brew/representation/json/crowd_anki_export.py b/brain_brew/representation/json/crowd_anki_export.py index bb96e79..1ccaf3a 100644 --- a/brain_brew/representation/json/crowd_anki_export.py +++ b/brain_brew/representation/json/crowd_anki_export.py @@ -1,30 +1,33 @@ import glob import logging import pathlib +from enum import Enum from pathlib import Path from typing import List, Dict -from brain_brew.representation.json.json_file import JsonFile from brain_brew.representation.generic.media_file import MediaFile +from brain_brew.representation.json.json_file import JsonFile +from brain_brew.representation.json.wrappers_for_crowd_anki import CrowdAnkiJsonWrapper from brain_brew.utils import filename_from_full_path, find_all_files_in_directory -class CrowdAnkiExport(JsonFile): +class CrowdAnkiExport: folder_location: str + json_file_location: str + # import_config: CrowdAnkiImportConfig # TODO: Make this + contains_media: bool known_media: Dict[str, MediaFile] media_loc: str - def __init__(self, folder_location, read_now=True, data_override=None): + def __init__(self, folder_location): self.folder_location = folder_location if self.folder_location[-1] != "/": self.folder_location = self.folder_location + "/" - json_file_location = self.find_json_file_in_folder() + self.json_file_location = self.find_json_file_in_folder() self.find_all_media() - super().__init__(json_file_location, read_now=read_now, data_override=data_override) - def find_json_file_in_folder(self): files = glob.glob(self.folder_location + "*.json") @@ -53,7 +56,10 @@ def find_all_media(self): logging.info(f"CrowdAnkiExport found {len(self.known_media)} media files in folder") - def write_file(self, data_override=None): - super().write_file(data_override) + def write_to_files(self, json_data): # import_config_data + JsonFile.write_file(self.json_file_location, json_data) for filename, media_file in self.known_media.items(): media_file.copy_source_to_target() + + def read_json_file(self) -> CrowdAnkiJsonWrapper: + return CrowdAnkiJsonWrapper(JsonFile.read_file(self.json_file_location)) diff --git a/brain_brew/representation/json/deck_part_notes.py b/brain_brew/representation/json/deck_part_notes.py index cea5868..e29c5f6 100644 --- a/brain_brew/representation/json/deck_part_notes.py +++ b/brain_brew/representation/json/deck_part_notes.py @@ -10,14 +10,6 @@ from brain_brew.utils import find_media_in_field -class CANoteKeys(Enum): - NOTE_MODEL = "note_model_uuid" - FLAGS = "flags" - GUID = "guid" - TAGS = "tags" - FIELDS = "fields" - - class DeckPartNotes(JsonFile): _data: dict = {} flags: DeckPartNoteFlags diff --git a/brain_brew/representation/json/json_file.py b/brain_brew/representation/json/json_file.py index bf96eed..2fb0540 100644 --- a/brain_brew/representation/json/json_file.py +++ b/brain_brew/representation/json/json_file.py @@ -15,5 +15,5 @@ def read_file(file_location): @staticmethod def write_file(file_location, data): - with open(file_location, "w", encoding=_encoding) as write_file: + with open(file_location, "w+", encoding=_encoding) as write_file: json.dump(data, write_file, indent=4, sort_keys=False, ensure_ascii=False) diff --git a/brain_brew/representation/json/wrappers_for_crowd_anki.py b/brain_brew/representation/json/wrappers_for_crowd_anki.py new file mode 100644 index 0000000..25dc7cc --- /dev/null +++ b/brain_brew/representation/json/wrappers_for_crowd_anki.py @@ -0,0 +1,90 @@ +from typing import List + + +CA_NOTE_MODELS = "note_models" +CA_NOTES = "notes" +CA_MEDIA_FILES = "media_files" + +NOTE_MODEL = "note_model_uuid" +FLAGS = "flags" +GUID = "guid" +TAGS = "tags" +FIELDS = "fields" + + +class CrowdAnkiJsonWrapper: + data: dict + + def __init__(self, data: dict = None): + self.data = data + + @property + def note_models(self) -> str: + return self.data[CA_NOTE_MODELS] + + @note_models.setter + def note_models(self, value: str): + self.data.setdefault(CA_NOTE_MODELS, value) + + @property + def notes(self) -> str: + return self.data[CA_NOTES] + + @notes.setter + def notes(self, value: str): + self.data.setdefault(CA_NOTES, value) + + @property + def media_files(self) -> str: + return self.data[CA_MEDIA_FILES] + + @media_files.setter + def media_files(self, value: str): + self.data.setdefault(CA_MEDIA_FILES, value) + + +class CrowdAnkiNoteWrapper: + data: dict + + def __init__(self, data: dict = None): + self.data = data + + @property + def note_model(self) -> str: + return self.data[NOTE_MODEL] + + @note_model.setter + def note_model(self, value: str): + self.data.setdefault(NOTE_MODEL, value) + + @property + def flags(self) -> int: + return self.data[FLAGS] + + @flags.setter + def flags(self, value: int): + self.data.setdefault(FLAGS, value) + + @property + def guid(self) -> str: + return self.data[GUID] + + @guid.setter + def guid(self, value: str): + self.data.setdefault(GUID, value) + + @property + def tags(self) -> list: + return self.data[TAGS] + + @tags.setter + def tags(self, value: list): + self.data.setdefault(TAGS, value) + + @property + def fields(self) -> List[str]: + return self.data[FIELDS] + + @fields.setter + def fields(self, value: List[str]): + self.data.setdefault(FIELDS, value) diff --git a/brain_brew/representation/transformers/generic_to_deck_part.py b/brain_brew/representation/transformers/generic_to_deck_part.py index dbf8f75..4c088b9 100644 --- a/brain_brew/representation/transformers/generic_to_deck_part.py +++ b/brain_brew/representation/transformers/generic_to_deck_part.py @@ -1,11 +1,13 @@ from dataclasses import dataclass from typing import Optional +from brain_brew.representation.build_config.representation_base import RepresentationBase + @dataclass -class TrGenericToDeckPart: +class DeckPartFromBase: @dataclass - class Representation: + class Representation(RepresentationBase): name: str save_to_file: Optional[str] @@ -13,9 +15,5 @@ def __init__(self, name, save_to_file=None): self.name = name self.save_to_file = save_to_file - @classmethod - def from_dict(cls, data: dict): - return cls(**data) - name: str save_to_file: Optional[str] diff --git a/brain_brew/representation/yaml/note_model_repr.py b/brain_brew/representation/yaml/note_model_repr.py index b1ab64d..a07906f 100644 --- a/brain_brew/representation/yaml/note_model_repr.py +++ b/brain_brew/representation/yaml/note_model_repr.py @@ -2,6 +2,7 @@ from dataclasses import dataclass, field from typing import List, Optional, Union, Dict +from brain_brew.representation.build_config.representation_base import RepresentationBase from brain_brew.representation.yaml.my_yaml import YamlRepr from brain_brew.utils import list_of_str_to_lowercase @@ -57,9 +58,9 @@ def append_name_if_differs(self, dict_to_add_to: dict, value): @dataclass -class Template: +class Template(RepresentationBase): @dataclass - class CrowdAnki: + class CrowdAnki(RepresentationBase): name: str ord: int qfmt: str @@ -68,10 +69,6 @@ class CrowdAnki: bafmt: str = field(default=BROWSER_ANSWER_FORMAT.default_value) did: Optional[int] = field(default=None) - @classmethod - def from_dict(cls, data: dict): - return cls(**data) - name: str question_format: str answer_format: str @@ -87,10 +84,6 @@ def from_crowdanki(cls, data: Union[CrowdAnki, dict]): question_format_in_browser=ca.bqfmt, answer_format_in_browser=ca.bafmt, deck_override_id=ca.did ) - @classmethod - def from_dict(cls, data: dict): - return cls(**data) - def encode_as_crowdanki(self, ordinal: int) -> dict: data_dict = { NAME.anki_name: self.name, @@ -119,9 +112,9 @@ def encode_as_deck_part(self) -> dict: @dataclass -class Field: +class Field(RepresentationBase): @dataclass - class CrowdAnki: + class CrowdAnki(RepresentationBase): name: str ord: int font: str = field(default=FONT.default_value) @@ -130,10 +123,6 @@ class CrowdAnki: size: int = field(default=FONT_SIZE.default_value) sticky: bool = field(default=IS_STICKY.default_value) - @classmethod - def from_dict(cls, data: dict): - return cls(**data) - name: str font: str = field(default=FONT.default_value) media: List[str] = field(default_factory=lambda: MEDIA.default_value) # Unused in Anki @@ -149,10 +138,6 @@ def from_crowdanki(cls, data: Union[CrowdAnki, dict]): is_right_to_left=ca.rtl, font_size=ca.size, is_sticky=ca.sticky ) - @classmethod - def from_dict(cls, data: dict): - return cls(**data) - def encode_as_crowdanki(self, ordinal: int) -> dict: data_dict = { NAME.anki_name: self.name, @@ -181,9 +166,9 @@ def encode_as_deck_part(self) -> dict: @dataclass -class NoteModel(YamlRepr): +class NoteModel(YamlRepr, RepresentationBase): @dataclass - class CrowdAnki: + class CrowdAnki(RepresentationBase): name: str crowdanki_uuid: str css: str @@ -198,10 +183,6 @@ class CrowdAnki: type: int = field(default=0) # Is_Cloze Manually set to 0 vers: list = field(default_factory=lambda: VERSION.default_value) - @classmethod - def from_dict(cls, data: dict): - return cls(**data) - name: str crowdanki_id: str css: str @@ -229,10 +210,6 @@ def from_crowdanki(cls, data: Union[CrowdAnki, dict]): crowdanki_id=ca.crowdanki_uuid, crowdanki_type=ca.__type__ ) - @classmethod - def from_dict(cls, data: dict): - return cls(**data) - def encode_as_crowdanki(self) -> dict: data_dict = { NAME.anki_name: self.name, diff --git a/brain_brew/transformers/tr_notes_generic.py b/brain_brew/transformers/base_transform_notes.py similarity index 100% rename from brain_brew/transformers/tr_notes_generic.py rename to brain_brew/transformers/base_transform_notes.py diff --git a/brain_brew/transformers/transform_crowdanki.py b/brain_brew/transformers/transform_crowdanki.py index 6ab5598..b8b8c68 100644 --- a/brain_brew/transformers/transform_crowdanki.py +++ b/brain_brew/transformers/transform_crowdanki.py @@ -1,35 +1,40 @@ from dataclasses import dataclass from typing import List, Optional, Union -# from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport -# from brain_brew.representation.transformers.generic_to_deck_part import TrGenericToDeckPart -# -# -# @dataclass -# class TrCrowdAnkiToNotes(TrGenericToDeckPart): -# @dataclass -# class Representation: -# file: str -# sort_order: Optional[Union[str, List[str]]] -# media: Optional[bool] -# useless_note_keys: Optional[Union[dict, list]] -# -# crowdanki_file: CrowdAnkiExport -# sort_order: Optional[List[str]] -# media: bool -# useless_note_keys: list -# -# -# @dataclass -# class TrNotesToCrowdAnki(TrNotesToGeneric): -# @dataclass -# class Representation: -# file: str -# sort_order: Optional[Union[str, List[str]]] -# media: Optional[bool] -# useless_note_keys: Optional[dict] # TODO: use default value -# -# sort_order: Optional[List[str]] -# media: bool -# useless_note_keys: dict -# +from brain_brew.representation.json.wrappers_for_crowd_anki import CrowdAnkiNoteWrapper +from brain_brew.representation.yaml.note_repr import Note +from brain_brew.transformers.base_transform_notes import TrNotes +from brain_brew.utils import blank_str_if_none + + +class TransformCrowdAnki(TrNotes): + @classmethod + def crowd_anki_to_notes(cls, notes_json: dict, note_models_id_name_dict) -> List[Note]: + resolved_notes: List[Note] = [] + wrapper: CrowdAnkiNoteWrapper = CrowdAnkiNoteWrapper() + for note in notes_json: + wrapper.data = note + + resolved_notes.append(Note( + note_model=note_models_id_name_dict[wrapper.note_model], + tags=wrapper.tags, + guid=wrapper.guid, + fields=wrapper.fields + )) + return resolved_notes + + @classmethod + def notes_to_crowd_anki(cls, notes: List[Note], note_models_id_name_dict, useless_note_keys) -> List[dict]: + resolved_notes: List[dict] = [] + wrapper: CrowdAnkiNoteWrapper = CrowdAnkiNoteWrapper() + for note in notes: + current = {} + wrapper.data = current + + for key in useless_note_keys: + current[key] = blank_str_if_none(useless_note_keys[key]) + + wrapper.note_model = note_models_id_name_dict[note.note_model] + resolved_notes.append(current) + + return resolved_notes diff --git a/brain_brew/transformers/transform_csv_collection.py b/brain_brew/transformers/transform_csv_collection.py index 6a3fb23..dc92a2c 100644 --- a/brain_brew/transformers/transform_csv_collection.py +++ b/brain_brew/transformers/transform_csv_collection.py @@ -3,7 +3,7 @@ from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping from brain_brew.representation.yaml.note_repr import Note -from brain_brew.transformers.tr_notes_generic import TrNotes +from brain_brew.transformers.base_transform_notes import TrNotes class TransformCsvCollection(TrNotes): From b06a40e65bd37c92dc3ffb1d18d55cd37dadb016 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sat, 22 Aug 2020 14:44:22 +0200 Subject: [PATCH 24/39] Removed GenericFile FileManager no longer writes at end (apart from Media files) Note Models to/from CrowdAnki Headers Flags supported in Notes --- .../crowd_anki/crowd_anki_to_deck_parts.py | 55 +++++++++++++++++ .../crowd_anki/headers_from_crowdanki.py | 30 +++++++++ .../crowd_anki/note_models_from_crowd_anki.py | 61 +++++++++++++++++++ .../crowd_anki/notes_from_crowd_anki.py | 21 ++++--- .../crowd_anki/notes_to_crowd_anki.py | 9 +-- .../crowd_anki/shared_base_notes.py | 23 ------- brain_brew/build_tasks/csvs/csvs_generate.py | 3 +- .../build_tasks/csvs/csvs_to_deck_parts.py | 2 +- .../build_tasks/csvs/notes_from_csvs.py | 10 +-- brain_brew/build_tasks/source_crowd_anki.py | 6 +- brain_brew/file_manager.py | 31 ++-------- brain_brew/interfaces/writes_file.py | 12 ---- .../configuration/csv_file_mapping.py | 5 +- brain_brew/representation/generic/csv_file.py | 26 ++++---- .../representation/generic/generic_file.py | 55 +++++------------ .../representation/json/crowd_anki_export.py | 12 +++- .../json/wrappers_for_crowd_anki.py | 18 ++++-- ...to_deck_part.py => base_deck_part_from.py} | 2 +- .../representation/yaml/deck_part_holder.py | 1 + .../representation/yaml/headers_repr.py | 11 ++++ .../representation/yaml/note_model_repr.py | 2 +- brain_brew/representation/yaml/note_repr.py | 12 +++- .../transformers/transform_crowdanki.py | 59 +++++++++++++----- .../transformers/transform_csv_collection.py | 3 +- tests/build_tasks/test_source_csv.py | 4 +- tests/representation/generic/test_csv_file.py | 8 +-- .../generic/test_generic_file.py | 20 +++--- .../json/test_crowd_anki_export.py | 2 +- .../json/test_deck_part_note_model.py | 2 +- tests/representation/yaml/test_note_repr.py | 50 ++++++++++++--- 30 files changed, 360 insertions(+), 195 deletions(-) create mode 100644 brain_brew/build_tasks/crowd_anki/headers_from_crowdanki.py create mode 100644 brain_brew/build_tasks/crowd_anki/note_models_from_crowd_anki.py delete mode 100644 brain_brew/interfaces/writes_file.py rename brain_brew/representation/transformers/{generic_to_deck_part.py => base_deck_part_from.py} (94%) create mode 100644 brain_brew/representation/yaml/headers_repr.py diff --git a/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py b/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py index e69de29..f7dd3e4 100644 --- a/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py +++ b/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py @@ -0,0 +1,55 @@ +import logging +from dataclasses import dataclass +from typing import Union, Optional + +from brain_brew.build_tasks.crowd_anki.headers_from_crowdanki import HeadersFromCrowdAnki +from brain_brew.build_tasks.crowd_anki.note_models_from_crowd_anki import NoteModelsFromCrowdAnki +from brain_brew.build_tasks.crowd_anki.notes_from_crowd_anki import TrCrowdAnkiToNotes +from brain_brew.representation.build_config.build_task import DeckPartBuildTask +from brain_brew.representation.build_config.representation_base import RepresentationBase +from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport +from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder +from brain_brew.utils import all_combos_prepend_append + + +@dataclass +class CrowdAnkiToDeckParts(DeckPartBuildTask): + task_names = all_combos_prepend_append(["CrowdAnki", "CrowdAnkiExport"], "From ", "s") + + @dataclass + class Representation(RepresentationBase): + folder: str + notes: Optional[dict] + note_models: Optional[list] + headers: Optional[dict] + media: Optional[dict] + + crowd_anki_export: CrowdAnkiExport + notes_transform: TrCrowdAnkiToNotes + note_model_transform: NoteModelsFromCrowdAnki + headers_transform: HeadersFromCrowdAnki + media: int + + @classmethod + def from_repr(cls, data: Union[Representation, dict]): + rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) + return cls( + crowd_anki_export=CrowdAnkiExport.create_or_get(rep.folder), + notes_transform=TrCrowdAnkiToNotes.from_repr(rep.notes), + note_model_transform=NoteModelsFromCrowdAnki.from_list(rep.note_models) + ) + + def execute(self): + ca_wrapper = self.crowd_anki_export.read_json_file() + + if ca_wrapper.children: + logging.warning("Child Decks / Subdecks are not currently supported.") # TODO: Support them + + note_models = self.note_model_transform.execute(ca_wrapper) + + nm_id_to_name: dict = {model.crowdanki_id: model.name for model in note_models} + notes = self.notes_transform.execute(ca_wrapper, nm_id_to_name) + + headers = self.headers_transform.execute(ca_wrapper) + + diff --git a/brain_brew/build_tasks/crowd_anki/headers_from_crowdanki.py b/brain_brew/build_tasks/crowd_anki/headers_from_crowdanki.py new file mode 100644 index 0000000..923b8eb --- /dev/null +++ b/brain_brew/build_tasks/crowd_anki/headers_from_crowdanki.py @@ -0,0 +1,30 @@ +from dataclasses import dataclass +from typing import Optional, Union + +from brain_brew.representation.json.wrappers_for_crowd_anki import CrowdAnkiJsonWrapper +from brain_brew.representation.transformers.base_deck_part_from import BaseDeckPartsFrom +from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder +from brain_brew.representation.yaml.headers_repr import Headers +from brain_brew.transformers.transform_crowdanki import TransformCrowdAnki + + +@dataclass +class HeadersFromCrowdAnki(BaseDeckPartsFrom): + @dataclass + class Representation(BaseDeckPartsFrom.Representation): + pass + + @classmethod + def from_repr(cls, data: Union[Representation, dict]): + rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) + return cls( + name=rep.name, + save_to_file=rep.save_to_file + ) + + def execute(self, ca_wrapper: CrowdAnkiJsonWrapper): + headers = Headers(TransformCrowdAnki.headers_to_crowd_anki(ca_wrapper.data)) + + DeckPartHolder.override_or_create(self.name, self.save_to_file, headers) + + return headers diff --git a/brain_brew/build_tasks/crowd_anki/note_models_from_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/note_models_from_crowd_anki.py new file mode 100644 index 0000000..2ecb081 --- /dev/null +++ b/brain_brew/build_tasks/crowd_anki/note_models_from_crowd_anki.py @@ -0,0 +1,61 @@ +from dataclasses import dataclass +from typing import Optional, Union, List +import logging + +from brain_brew.representation.json.wrappers_for_crowd_anki import CrowdAnkiJsonWrapper +from brain_brew.representation.transformers.base_deck_part_from import BaseDeckPartsFrom +from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder +from brain_brew.representation.yaml.note_model_repr import NoteModel + + +@dataclass +class NoteModelsFromCrowdAnki: + @dataclass + class NoteModelListItem(BaseDeckPartsFrom): + @dataclass + class Representation(BaseDeckPartsFrom.Representation): + model_name: Optional[str] + # TODO: fields: Optional[List[str]] + # TODO: templates: Optional[List[str]] + + @classmethod + def from_repr(cls, data: Union[Representation, dict]): + rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) + return cls( + name=rep.name, + model_name=rep.model_name or rep.name, + save_to_file=rep.save_to_file + ) + + model_name: str + + @classmethod + def from_list(cls, note_model_items: List[dict]): + return cls( + note_model_items=list(map(cls.NoteModelListItem.from_repr, note_model_items)) + ) + + note_model_items: List[NoteModelListItem] + + def execute(self, ca_wrapper: CrowdAnkiJsonWrapper) -> List[NoteModel]: + note_models = {model["name"]: model for model in ca_wrapper.note_models} + + extra_models = list(note_models.keys()) + dp_note_models: List[NoteModel] = [] + + for nm_item in self.note_model_items: + if nm_item.model_name not in note_models: + raise ReferenceError(f"Missing Note Model '{nm_item.model_name}' in CrowdAnki file") + + model = note_models[nm_item.model_name] + extra_models.remove(nm_item.model_name) + + deck_part = NoteModel.from_crowdanki(model) + DeckPartHolder.override_or_create(nm_item.name, nm_item.save_to_file, deck_part) + + dp_note_models.append(deck_part) + + if extra_models: + logging.warning(f"Note Models were converted to Deck Parts, but did you miss some? Possible missing: {extra_models}") + + return dp_note_models diff --git a/brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py index 5d97e23..7137dc7 100644 --- a/brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py +++ b/brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py @@ -3,13 +3,17 @@ from brain_brew.build_tasks.crowd_anki.shared_base_notes import SharedBaseNotes from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport +from brain_brew.representation.json.wrappers_for_crowd_anki import CrowdAnkiJsonWrapper +from brain_brew.representation.transformers.base_deck_part_from import BaseDeckPartsFrom +from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder +from brain_brew.representation.yaml.note_repr import Notes from brain_brew.transformers.transform_crowdanki import TransformCrowdAnki @dataclass -class TrCrowdAnkiToNotes(SharedBaseNotes): +class TrCrowdAnkiToNotes(SharedBaseNotes, BaseDeckPartsFrom): @dataclass - class Representation(SharedBaseNotes.Representation): + class Representation(SharedBaseNotes.Representation, BaseDeckPartsFrom.Representation): pass @classmethod @@ -18,11 +22,14 @@ def from_repr(cls, data: Union[Representation, dict]): return cls( name=rep.name, sort_order=SharedBaseNotes._get_sort_order(rep.sort_order), - move_media=SharedBaseNotes._get_move_media(rep.move_media), - useless_note_keys=SharedBaseNotes._get_useless_note_keys(rep.useless_note_keys) + save_to_file=rep.save_to_file ) - def execute(self, crowd_anki_export: CrowdAnkiExport): - ca_wrapper = crowd_anki_export.read_json_file() + def execute(self, ca_wrapper: CrowdAnkiJsonWrapper, nm_id_to_name: dict) -> Notes: + note_list = TransformCrowdAnki.crowd_anki_to_notes(ca_wrapper.notes, nm_id_to_name) - TransformCrowdAnki.crowd_anki_to_notes(ca_wrapper.notes, ) \ No newline at end of file + notes = Notes.from_list_of_notes(note_list) # TODO: pass in sort method + + DeckPartHolder.override_or_create(self.name, self.save_to_file, notes) + + return notes diff --git a/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py index bfbb216..86e9114 100644 --- a/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py +++ b/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py @@ -8,7 +8,8 @@ class NotesToCrowdAnki(SharedBaseNotes): @dataclass class Representation(SharedBaseNotes.Representation): - pass + name: str + additional_items_to_add: Optional[dict] @classmethod def from_repr(cls, data: Union[Representation, dict]): @@ -16,8 +17,8 @@ def from_repr(cls, data: Union[Representation, dict]): return cls( name=rep.name, sort_order=SharedBaseNotes._get_sort_order(rep.sort_order), - move_media=SharedBaseNotes._get_move_media(rep.move_media), - useless_note_keys=SharedBaseNotes._get_useless_note_keys(rep.useless_note_keys) + additional_items_to_add=rep.additional_items_to_add or {} ) - + name: str + additional_items_to_add: dict diff --git a/brain_brew/build_tasks/crowd_anki/shared_base_notes.py b/brain_brew/build_tasks/crowd_anki/shared_base_notes.py index 080f181..ec64d1c 100644 --- a/brain_brew/build_tasks/crowd_anki/shared_base_notes.py +++ b/brain_brew/build_tasks/crowd_anki/shared_base_notes.py @@ -8,10 +8,7 @@ class SharedBaseNotes: @dataclass class Representation(RepresentationBase): - name: str sort_order: Optional[Union[str, List[str]]] - move_media: Optional[bool] - useless_note_keys: Optional[dict] @staticmethod def _get_sort_order(sort_order: Optional[Union[str, List[str]]]): @@ -21,24 +18,4 @@ def _get_sort_order(sort_order: Optional[Union[str, List[str]]]): return [sort_order] return [] - @staticmethod - def _get_move_media(move_media: Optional[bool]): - return move_media or False - - @staticmethod - def _get_useless_note_keys(useless_note_keys: Optional[dict]): - default_useless_keys = { - "__type__": "Note", - "data": None, - "flags": 0 - } - - if useless_note_keys is None: - return default_useless_keys - - return {**useless_note_keys, **default_useless_keys} - - name: str sort_order: Optional[List[str]] - move_media: bool - useless_note_keys: dict diff --git a/brain_brew/build_tasks/csvs/csvs_generate.py b/brain_brew/build_tasks/csvs/csvs_generate.py index ab2823e..77f8346 100644 --- a/brain_brew/build_tasks/csvs/csvs_generate.py +++ b/brain_brew/build_tasks/csvs/csvs_generate.py @@ -43,6 +43,7 @@ def execute(self): for fm in self.file_mappings: fm.compile_data() fm.set_relevant_data(csv_data) + fm.write_file_on_close() def verify_notes_match_note_model_mappings(self, notes: List[Note]): note_models_used = {note.note_model for note in notes} @@ -51,4 +52,4 @@ def verify_notes_match_note_model_mappings(self, notes: List[Note]): for model in note_models_used if model not in self.note_model_mappings.keys()] if errors: - raise Exception(errors) \ No newline at end of file + raise Exception(errors) diff --git a/brain_brew/build_tasks/csvs/csvs_to_deck_parts.py b/brain_brew/build_tasks/csvs/csvs_to_deck_parts.py index c0d4389..d748243 100644 --- a/brain_brew/build_tasks/csvs/csvs_to_deck_parts.py +++ b/brain_brew/build_tasks/csvs/csvs_to_deck_parts.py @@ -21,7 +21,7 @@ class Representation(RepresentationBase): def from_repr(cls, data: Union[Representation, dict]): rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) return cls( - notes_transform=NotesFromCsvs.from_dict(rep.notes) + notes_transform=NotesFromCsvs.from_repr(rep.notes) ) def execute(self): diff --git a/brain_brew/build_tasks/csvs/notes_from_csvs.py b/brain_brew/build_tasks/csvs/notes_from_csvs.py index 9b0e728..ccc087f 100644 --- a/brain_brew/build_tasks/csvs/notes_from_csvs.py +++ b/brain_brew/build_tasks/csvs/notes_from_csvs.py @@ -2,19 +2,19 @@ from typing import Dict, List, Union from brain_brew.build_tasks.csvs.shared_base_csvs import SharedBaseCsvs -from brain_brew.representation.transformers.generic_to_deck_part import DeckPartFromBase +from brain_brew.representation.transformers.base_deck_part_from import BaseDeckPartsFrom from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder from brain_brew.representation.yaml.note_repr import Note, Notes from brain_brew.transformers.transform_csv_collection import TransformCsvCollection @dataclass -class NotesFromCsvs(SharedBaseCsvs, DeckPartFromBase): +class NotesFromCsvs(SharedBaseCsvs, BaseDeckPartsFrom): @dataclass(init=False) - class Representation(SharedBaseCsvs.Representation, DeckPartFromBase.Representation): + class Representation(SharedBaseCsvs.Representation, BaseDeckPartsFrom.Representation): def __init__(self, name, file_mappings, note_model_mappings, save_to_file=None): SharedBaseCsvs.Representation.__init__(self, file_mappings, note_model_mappings) - DeckPartFromBase.Representation.__init__(self, name, save_to_file) + BaseDeckPartsFrom.Representation.__init__(self, name, save_to_file) @classmethod def from_repr(cls, data: Union[Representation, dict]): @@ -37,4 +37,4 @@ def execute(self): csv_rows, self.note_model_mappings) notes = Notes.from_list_of_notes(deck_part_notes) - DeckPartHolder.override_or_create(self.name, self.save_to_file, notes) \ No newline at end of file + DeckPartHolder.override_or_create(self.name, self.save_to_file, notes) diff --git a/brain_brew/build_tasks/source_crowd_anki.py b/brain_brew/build_tasks/source_crowd_anki.py index 61755c4..307bd68 100644 --- a/brain_brew/build_tasks/source_crowd_anki.py +++ b/brain_brew/build_tasks/source_crowd_anki.py @@ -50,7 +50,7 @@ def __init__(self, config_data: dict, read_now=True): self.headers = DeckPartHeader.create(self.config_entry[BuildConfigKeys.HEADERS.value], read_now=read_now) self.notes = DeckPartNotes.create(self.config_entry[BuildConfigKeys.NOTES.value], read_now=read_now) - self.crowd_anki_export = CrowdAnkiExport.create(self.config_entry[CrowdAnkiKeys.FILE.value], read_now=read_now) + self.crowd_anki_export = CrowdAnkiExport.create_or_get(self.config_entry[CrowdAnkiKeys.FILE.value], read_now=read_now) self.should_handle_media = self.config_entry[CrowdAnkiKeys.MEDIA.value] self.useless_note_keys = self.config_entry[CrowdAnkiKeys.USELESS_NOTE_KEYS.value] @@ -87,7 +87,7 @@ def source_to_deck_parts(self): # Note Models note_models = [ - DeckPartNoteModel.create(model[CANoteModelKeys.NAME.value], data_override=model) + DeckPartNoteModel.create_or_get(model[CANoteModelKeys.NAME.value], data_override=model) for model in source_data[CAKeys.NOTE_MODELS.value] ] @@ -140,7 +140,7 @@ def deck_parts_to_source(self): ) # Note Models - note_models = [DeckPartNoteModel.create(name) for name in self.notes.get_all_known_note_model_names()] + note_models = [DeckPartNoteModel.create_or_get(name) for name in self.notes.get_all_known_note_model_names()] ca_json.setdefault(CAKeys.NOTE_MODELS.value, [model.get_data() for model in note_models]) diff --git a/brain_brew/file_manager.py b/brain_brew/file_manager.py index 2c673f0..63b046e 100644 --- a/brain_brew/file_manager.py +++ b/brain_brew/file_manager.py @@ -3,9 +3,9 @@ import pathlib from typing import Dict, List, Union -from brain_brew.interfaces.writes_file import WritesFile + from brain_brew.representation.configuration.global_config import GlobalConfig -from brain_brew.representation.generic.generic_file import GenericFile +from brain_brew.representation.generic.generic_file import SourceFile from brain_brew.representation.generic.media_file import MediaFile from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder from brain_brew.representation.yaml.my_yaml import YamlRepr @@ -16,12 +16,10 @@ class FileManager: __instance = None global_config: GlobalConfig - known_files_dict: Dict[str, GenericFile] + known_files_dict: Dict[str, SourceFile] known_media_files_dict: Dict[str, MediaFile] deck_part_pool: Dict[str, DeckPartHolder[YamlRepr]] - write_files_at_end: List[WritesFile] - def __init__(self): if FileManager.__instance is None: FileManager.__instance = self @@ -31,7 +29,6 @@ def __init__(self): self.global_config = GlobalConfig.get_instance() self.known_files_dict = {} - self.write_files_at_end = [] self.deck_part_pool = {} self.find_all_deck_part_media_files() @@ -45,7 +42,7 @@ def clear_instance(): if FileManager.__instance: FileManager.__instance = None - def file_if_exists(self, file_location) -> Union[GenericFile, None]: + def file_if_exists(self, file_location) -> Union[SourceFile, None]: if file_location in self.known_files_dict.keys(): return self.known_files_dict[file_location] return None @@ -70,9 +67,6 @@ def _register_media_file(self, file: MediaFile): logging.error(f"Duplicate media file '{file.filename}' in both '{file.source_loc}'" f" and '{self.known_media_files_dict[file.filename].source_loc}'") - def register_write_file_for_end(self, file: WritesFile): - self.write_files_at_end.append(file) - def new_media_file(self, filename, source_loc): self._register_media_file(MediaFile(self.global_config.deck_parts.media_files + filename, filename, MediaFile.ManagementType.TO_BE_CLONED, source_loc)) @@ -98,24 +92,7 @@ def deck_part_from_pool(self, name: str): return self.deck_part_pool[name] def write_to_all(self): - files_to_create = [] - for location, file in self.known_files_dict.items(): - if not file.file_exists: - files_to_create.append(location) - # logging.info(f"Will create {len(files_to_create)} new files: ", files_to_create) - for write_file in self.write_files_at_end: - write_file.write_file_on_close() - - for location, file in self.known_files_dict.items(): - if file.data_state == GenericFile.DataState.DATA_SET: - logging.info(f"Wrote to {file.file_location}") - file.write_file() - file.data_state = GenericFile.DataState.READ_IN_DATA - - for dp in self.deck_part_pool.values(): - dp.write_to_file() - for filename, media_file in self.known_media_files_dict.items(): media_file.copy_source_to_target() diff --git a/brain_brew/interfaces/writes_file.py b/brain_brew/interfaces/writes_file.py deleted file mode 100644 index f5a57c6..0000000 --- a/brain_brew/interfaces/writes_file.py +++ /dev/null @@ -1,12 +0,0 @@ -import abc - - -class WritesFile(abc.ABC): - @abc.abstractmethod - def write_file_on_close(self): - pass - - def __init__(self): - from brain_brew.file_manager import FileManager - fm = FileManager.get_instance() - fm.register_write_file_for_end(self) diff --git a/brain_brew/representation/configuration/csv_file_mapping.py b/brain_brew/representation/configuration/csv_file_mapping.py index f5a8e1b..9eb8da4 100644 --- a/brain_brew/representation/configuration/csv_file_mapping.py +++ b/brain_brew/representation/configuration/csv_file_mapping.py @@ -5,7 +5,6 @@ from brain_brew.constants.deckpart_keys import DeckPartNoteKeys from brain_brew.interfaces.verifiable import Verifiable -from brain_brew.interfaces.writes_file import WritesFile from brain_brew.representation.build_config.representation_base import RepresentationBase from brain_brew.representation.generic.csv_file import CsvFile, CsvKeys from brain_brew.utils import single_item_to_list, generate_anki_guid @@ -48,7 +47,7 @@ def __init__(self, file, note_model=None, sort_by_columns=None, reverse_sort=Non def from_repr(cls, data: Union[Representation, dict]): rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) return cls( - csv_file=CsvFile.create(rep.file, True), # TODO: Fix Read Now + csv_file=CsvFile.create_or_get(rep.file), note_model=None if not rep.note_model.strip() else rep.note_model.strip(), sort_by_columns=single_item_to_list(rep.sort_by_columns), reverse_sort=rep.reverse_sort or False, @@ -110,7 +109,7 @@ def write_to_csv(self, data_to_set): @dataclass -class CsvFileMapping(CsvFileMappingDerivative, Verifiable, WritesFile): +class CsvFileMapping(CsvFileMappingDerivative, Verifiable): note_model: str # Override Optional on Parent data_set_has_changed: bool = field(init=False, default=False) diff --git a/brain_brew/representation/generic/csv_file.py b/brain_brew/representation/generic/csv_file.py index 5d9b8ce..0c50b91 100644 --- a/brain_brew/representation/generic/csv_file.py +++ b/brain_brew/representation/generic/csv_file.py @@ -7,7 +7,7 @@ from brain_brew.representation.configuration.global_config import GlobalConfig from brain_brew.representation.generic.yaml_file import YamlFile, ConfigKey from brain_brew.utils import list_of_str_to_lowercase, generate_anki_guid -from brain_brew.representation.generic.generic_file import GenericFile +from brain_brew.representation.generic.generic_file import SourceFile class CsvKeys(Enum): @@ -15,14 +15,19 @@ class CsvKeys(Enum): TAGS = "tags" -class CsvFile(GenericFile): +class CsvFile(SourceFile): file_location: str = "" _data: List[dict] = [] - column_headers: list = [] - def __init__(self, file, read_now=True, data_override=None): - super(CsvFile, self).__init__(file, read_now=read_now, data_override=data_override) + def __init__(self, file): + self.file_location = file + + self.read_file() + + @classmethod + def from_file_loc(cls, file_loc) -> 'CsvFile': + return cls(file_loc) def read_file(self): self._data = [] @@ -35,9 +40,8 @@ def read_file(self): for row in csv_reader: self._data.append({key.lower(): row[key] for key in row}) - self.data_state = GenericFile.DataState.READ_IN_DATA - def write_file(self): + logging.info(f"Writing to Csv '{self.file_location}'") with open(self.file_location, mode='w') as csv_file: csv_writer = csv.DictWriter(csv_file, fieldnames=self.column_headers) @@ -46,10 +50,8 @@ def write_file(self): for row in self._data: csv_writer.writerow(row) - self.file_exists = True - def set_data(self, data_override): - super().set_data(data_override) + self._data = data_override self.column_headers = list(data_override[0].keys()) if data_override else [] def set_data_from_superset(self, superset: List[dict], column_header_override=None): @@ -65,10 +67,10 @@ def set_data_from_superset(self, superset: List[dict], column_header_override=No new_row[column] = row[column] data_to_set.append(new_row) - super().set_data(data_to_set) + self.set_data(data_to_set) def get_data(self, deep_copy=False) -> List[dict]: - return super().get_data(deep_copy=deep_copy) + return self.get_deep_copy(self._data) if deep_copy else self._data @staticmethod def to_filename_csv(filename: str) -> str: diff --git a/brain_brew/representation/generic/generic_file.py b/brain_brew/representation/generic/generic_file.py index 3881815..0b5e83f 100644 --- a/brain_brew/representation/generic/generic_file.py +++ b/brain_brew/representation/generic/generic_file.py @@ -1,51 +1,28 @@ import copy -from enum import Enum from pathlib import Path from brain_brew.representation.configuration.global_config import GlobalConfig -class GenericFile: - class DataState(Enum): - NOTHING_READ_OR_SET = 0 - READ_IN_DATA = 1 - DATA_SET = 2 - - _data = None - file_location: str - - file_exists: bool - data_state: DataState = DataState.NOTHING_READ_OR_SET - - def __init__(self, file, read_now, data_override): - self.file_location = file - - self.file_exists = Path(file).is_file() - - if data_override: - self.data_state = GenericFile.DataState.DATA_SET - self.set_data(data_override) - elif read_now: - if not self.file_exists: - return # raise FileNotFoundError(file) # TODO: Fix - self.data_state = GenericFile.DataState.READ_IN_DATA - self.read_file() - - def set_data(self, data_override): - self.data_state = GenericFile.DataState.DATA_SET - self._data = data_override +class SourceFile: + @classmethod + def from_file_loc(cls, file_loc) -> 'SourceFile': + pass - def get_data(self, deep_copy: bool = False): - return copy.deepcopy(self._data) if deep_copy else self._data + @classmethod + def is_file(cls, filename: str): + return Path(filename).is_file() - def read_file(self): - raise NotImplemented + @classmethod + def is_dir(cls, folder_name: str): + return Path(folder_name).is_file() - def write_file(self): - raise NotImplemented + @classmethod + def get_deep_copy(cls, data): + return copy.deepcopy(data) @classmethod - def create(cls, location, read_now=True, data_override=None): + def create_or_get(cls, location): from brain_brew.file_manager import FileManager _file_manager = FileManager.get_instance() formatted_location = cls.formatted_file_location(location) @@ -54,7 +31,7 @@ def create(cls, location, read_now=True, data_override=None): if file is not None: return file - file = cls(location, read_now=read_now, data_override=data_override) + file = cls.from_file_loc(location) _file_manager.register_file(formatted_location, file) return file @@ -63,7 +40,7 @@ def formatted_file_location(cls, location): return location @staticmethod - def _sort_data(data, sort_by_keys, reverse_sort, case_insensitive_sort=None): + def _sort_data(data, sort_by_keys, reverse_sort, case_insensitive_sort=None): # TODO: Move to NoteGroupings if case_insensitive_sort is None: case_insensitive_sort = GlobalConfig.get_instance().flags.sort_case_insensitive diff --git a/brain_brew/representation/json/crowd_anki_export.py b/brain_brew/representation/json/crowd_anki_export.py index 1ccaf3a..1e13453 100644 --- a/brain_brew/representation/json/crowd_anki_export.py +++ b/brain_brew/representation/json/crowd_anki_export.py @@ -5,13 +5,14 @@ from pathlib import Path from typing import List, Dict +from brain_brew.representation.generic.generic_file import SourceFile from brain_brew.representation.generic.media_file import MediaFile from brain_brew.representation.json.json_file import JsonFile from brain_brew.representation.json.wrappers_for_crowd_anki import CrowdAnkiJsonWrapper from brain_brew.utils import filename_from_full_path, find_all_files_in_directory -class CrowdAnkiExport: +class CrowdAnkiExport(SourceFile): folder_location: str json_file_location: str # import_config: CrowdAnkiImportConfig # TODO: Make this @@ -25,9 +26,16 @@ def __init__(self, folder_location): if self.folder_location[-1] != "/": self.folder_location = self.folder_location + "/" + if not self.is_dir(self.folder_location): + raise FileNotFoundError(f"Missing CrowdAnkiExport '{self.folder_location}'") + self.json_file_location = self.find_json_file_in_folder() self.find_all_media() + @classmethod + def from_file_loc(cls, file_loc) -> 'CrowdAnkiExport': + return cls(file_loc) + def find_json_file_in_folder(self): files = glob.glob(self.folder_location + "*.json") @@ -43,7 +51,7 @@ def find_json_file_in_folder(self): def find_all_media(self): self.known_media = {} self.media_loc = self.folder_location + "media/" # TODO: Make media folder if not exists - self.contains_media = Path(self.media_loc).is_dir() + self.contains_media = self.is_dir(self.media_loc) if not self.contains_media: return diff --git a/brain_brew/representation/json/wrappers_for_crowd_anki.py b/brain_brew/representation/json/wrappers_for_crowd_anki.py index 25dc7cc..e7d62e6 100644 --- a/brain_brew/representation/json/wrappers_for_crowd_anki.py +++ b/brain_brew/representation/json/wrappers_for_crowd_anki.py @@ -4,6 +4,8 @@ CA_NOTE_MODELS = "note_models" CA_NOTES = "notes" CA_MEDIA_FILES = "media_files" +CA_CHILDREN = "children" +CA_TYPE = "__type__" NOTE_MODEL = "note_model_uuid" FLAGS = "flags" @@ -19,27 +21,31 @@ def __init__(self, data: dict = None): self.data = data @property - def note_models(self) -> str: + def children(self) -> list: + return self.data[CA_CHILDREN] + + @property + def note_models(self) -> list: return self.data[CA_NOTE_MODELS] @note_models.setter - def note_models(self, value: str): + def note_models(self, value: list): self.data.setdefault(CA_NOTE_MODELS, value) @property - def notes(self) -> str: + def notes(self) -> list: return self.data[CA_NOTES] @notes.setter - def notes(self, value: str): + def notes(self, value: list): self.data.setdefault(CA_NOTES, value) @property - def media_files(self) -> str: + def media_files(self) -> list: return self.data[CA_MEDIA_FILES] @media_files.setter - def media_files(self, value: str): + def media_files(self, value: list): self.data.setdefault(CA_MEDIA_FILES, value) diff --git a/brain_brew/representation/transformers/generic_to_deck_part.py b/brain_brew/representation/transformers/base_deck_part_from.py similarity index 94% rename from brain_brew/representation/transformers/generic_to_deck_part.py rename to brain_brew/representation/transformers/base_deck_part_from.py index 4c088b9..281ee9a 100644 --- a/brain_brew/representation/transformers/generic_to_deck_part.py +++ b/brain_brew/representation/transformers/base_deck_part_from.py @@ -5,7 +5,7 @@ @dataclass -class DeckPartFromBase: +class BaseDeckPartsFrom: @dataclass class Representation(RepresentationBase): name: str diff --git a/brain_brew/representation/yaml/deck_part_holder.py b/brain_brew/representation/yaml/deck_part_holder.py index 0236a81..6164e69 100644 --- a/brain_brew/representation/yaml/deck_part_holder.py +++ b/brain_brew/representation/yaml/deck_part_holder.py @@ -33,6 +33,7 @@ def override_or_create(cls, name: str, save_to_file: Optional[str], deck_part: T else: dp.deck_part = deck_part dp.save_to_file = save_to_file # ? + dp.write_to_file() return dp diff --git a/brain_brew/representation/yaml/headers_repr.py b/brain_brew/representation/yaml/headers_repr.py new file mode 100644 index 0000000..2b9fd91 --- /dev/null +++ b/brain_brew/representation/yaml/headers_repr.py @@ -0,0 +1,11 @@ +from dataclasses import dataclass + +from brain_brew.representation.yaml.my_yaml import YamlRepr + + +@dataclass +class Headers(YamlRepr): + data: dict + + def encode(self) -> dict: + return self.data diff --git a/brain_brew/representation/yaml/note_model_repr.py b/brain_brew/representation/yaml/note_model_repr.py index a07906f..ee39a1a 100644 --- a/brain_brew/representation/yaml/note_model_repr.py +++ b/brain_brew/representation/yaml/note_model_repr.py @@ -199,7 +199,7 @@ class CrowdAnki(RepresentationBase): version: list = field(default_factory=lambda: VERSION.default_value) # Legacy version number. Deprecated in Anki @classmethod - def from_crowdanki(cls, data: Union[CrowdAnki, dict]): + def from_crowdanki(cls, data: Union[CrowdAnki, dict]): # TODO: field_whitelist: List[str] = None, note_model_whitelist: List[str] = None): ca: cls.CrowdAnki = data if isinstance(data, cls.CrowdAnki) else cls.CrowdAnki.from_dict(data) return cls( fields=[Field.from_crowdanki(f) for f in ca.flds], diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index de0e645..3cffc4a 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -8,6 +8,7 @@ GUID = 'guid' TAGS = 'tags' NOTE_MODEL = 'note_model' +FLAGS = "flags" NOTES = "notes" NOTE_GROUPINGS = "note_groupings" MEDIA_REFERENCES = "media_references" @@ -30,6 +31,7 @@ def encode_groupable(self, data_dict): class Note(GroupableNoteData): fields: List[str] guid: str + flags: int # media_references: Optional[Set[str]] @classmethod @@ -38,11 +40,14 @@ def from_dict(cls, data: dict): fields=data.get(FIELDS), guid=data.get(GUID), note_model=data.get(NOTE_MODEL, None), - tags=data.get(TAGS, None) + tags=data.get(TAGS, None), + flags=data.get(FLAGS, 0) ) def encode(self) -> dict: - data_dict = {FIELDS: self.fields, GUID: self.guid} + data_dict: Dict[str, any] = {FIELDS: self.fields, GUID: self.guid} + if self.flags is not None and self.flags != 0: + data_dict.setdefault(FLAGS, self.flags) super().encode_groupable(data_dict) return data_dict @@ -99,6 +104,7 @@ def join_tags(n_tags): tags=join_tags(n.tags), fields=n.fields, guid=n.guid, + flags=n.flags # media_references=n.media_references or n.get_media_references() ) for n in self.notes] @@ -113,7 +119,7 @@ def from_dict(cls, data: dict): @classmethod def from_list_of_notes(cls, notes: List[Note]): - return cls(note_groupings=[NoteGrouping(note_model=None, tags=None, notes=notes)]) + return cls(note_groupings=[NoteGrouping(note_model=None, tags=None, notes=notes)]) # TODO: Check grouping here def encode(self) -> dict: data_dict = {NOTE_GROUPINGS: [note_grouping.encode() for note_grouping in self.note_groupings]} diff --git a/brain_brew/transformers/transform_crowdanki.py b/brain_brew/transformers/transform_crowdanki.py index b8b8c68..e337026 100644 --- a/brain_brew/transformers/transform_crowdanki.py +++ b/brain_brew/transformers/transform_crowdanki.py @@ -6,35 +6,62 @@ from brain_brew.transformers.base_transform_notes import TrNotes from brain_brew.utils import blank_str_if_none +from brain_brew.representation.json.wrappers_for_crowd_anki import CA_NOTE_MODELS, CA_NOTES, CA_MEDIA_FILES,\ + CA_CHILDREN, CA_TYPE + class TransformCrowdAnki(TrNotes): + headers_skip_keys = [CA_NOTE_MODELS, CA_NOTES, CA_MEDIA_FILES] + headers_default_values = { + CA_TYPE: "Deck", + CA_CHILDREN: [], + } + @classmethod - def crowd_anki_to_notes(cls, notes_json: dict, note_models_id_name_dict) -> List[Note]: + def notes_to_crowd_anki(cls, notes: List[Note], note_models_name_to_id: dict, additional_items_to_add: dict) -> List[dict]: + resolved_notes: List[dict] = [] + wrapper: CrowdAnkiNoteWrapper = CrowdAnkiNoteWrapper() + for note in notes: + current = { + "__type__": "Note", + "data": None + } + wrapper.data = current + + for key, value in additional_items_to_add.items(): + current[key] = blank_str_if_none(value) + + wrapper.guid = note.guid + wrapper.fields = note.fields + wrapper.tags = note.tags + wrapper.note_model = note_models_name_to_id[note.note_model] + wrapper.flags = note.flags + + resolved_notes.append(current) + + return resolved_notes + + @classmethod + def crowd_anki_to_notes(cls, notes_json: list, nm_id_to_name: dict) -> List[Note]: resolved_notes: List[Note] = [] wrapper: CrowdAnkiNoteWrapper = CrowdAnkiNoteWrapper() for note in notes_json: wrapper.data = note resolved_notes.append(Note( - note_model=note_models_id_name_dict[wrapper.note_model], + note_model=nm_id_to_name[wrapper.note_model], tags=wrapper.tags, guid=wrapper.guid, - fields=wrapper.fields + fields=wrapper.fields, + flags=wrapper.flags )) return resolved_notes @classmethod - def notes_to_crowd_anki(cls, notes: List[Note], note_models_id_name_dict, useless_note_keys) -> List[dict]: - resolved_notes: List[dict] = [] - wrapper: CrowdAnkiNoteWrapper = CrowdAnkiNoteWrapper() - for note in notes: - current = {} - wrapper.data = current + def headers_to_crowd_anki(cls, headers_data: dict): + return {**headers_data, **cls.headers_default_values} - for key in useless_note_keys: - current[key] = blank_str_if_none(useless_note_keys[key]) - - wrapper.note_model = note_models_id_name_dict[note.note_model] - resolved_notes.append(current) - - return resolved_notes + @classmethod + def crowd_anki_to_headers(cls, ca_data: dict): + return {key: value for key, value in ca_data + if key not in cls.headers_skip_keys and key not in cls.headers_default_values.keys()} diff --git a/brain_brew/transformers/transform_csv_collection.py b/brain_brew/transformers/transform_csv_collection.py index dc92a2c..5bf66ca 100644 --- a/brain_brew/transformers/transform_csv_collection.py +++ b/brain_brew/transformers/transform_csv_collection.py @@ -35,9 +35,10 @@ def csv_collection_to_notes(cls, csv_rows: List[dict], note_model_mappings: Dict guid = filtered_fields.pop("guid") tags = cls.split_tags(filtered_fields.pop("tags")) + flags = filtered_fields.pop("flags") if "flags" in filtered_fields else 0 fields = row_nm.field_values_in_note_model_order(note_model_name, filtered_fields) - deck_part_notes.append(Note(guid=guid, tags=tags, note_model=note_model_name, fields=fields)) + deck_part_notes.append(Note(guid=guid, tags=tags, note_model=note_model_name, fields=fields, flags=flags)) return deck_part_notes diff --git a/tests/build_tasks/test_source_csv.py b/tests/build_tasks/test_source_csv.py index 22860d0..4a9a0b6 100644 --- a/tests/build_tasks/test_source_csv.py +++ b/tests/build_tasks/test_source_csv.py @@ -8,7 +8,7 @@ from brain_brew.representation.configuration.csv_file_mapping import CsvFileMapping from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping from brain_brew.representation.generic.csv_file import CsvFile -from brain_brew.representation.generic.generic_file import GenericFile +from brain_brew.representation.generic.generic_file import SourceFile from brain_brew.representation.json.deck_part_notes import DeckPartNotes from tests.representation.json.test_deck_part_notes import dp_notes_test1 from tests.representation.configuration.test_note_model_mapping import setup_nmm_config @@ -130,7 +130,7 @@ def run_dpts(csv_source: SourceCsv, csv_file: CsvFile): def assert_format(source_data): assert source_data == csv_file.get_data() - with patch.object(GenericFile, "set_data", side_effect=assert_format) as mock_set_data: + with patch.object(SourceFile, "set_data", side_effect=assert_format) as mock_set_data: csv_source.deck_parts_to_source() assert csv_source.csv_file_mappings[0].data_set_has_changed is False diff --git a/tests/representation/generic/test_csv_file.py b/tests/representation/generic/test_csv_file.py index dd1da2f..5d7cd75 100644 --- a/tests/representation/generic/test_csv_file.py +++ b/tests/representation/generic/test_csv_file.py @@ -3,7 +3,7 @@ import pytest from brain_brew.representation.generic.csv_file import CsvFile, CsvKeys -from brain_brew.representation.generic.generic_file import GenericFile +from brain_brew.representation.generic.generic_file import SourceFile from tests.test_files import TestFiles @@ -47,7 +47,7 @@ def temp_csv_test1(tmpdir, csv_test1) -> CsvFile: file = tmpdir.mkdir("json").join("file.csv") file.write("blank") - return CsvFile.create(file.strpath, data_override=csv_test1.get_data()) + return CsvFile.create_or_get(file.strpath, data_override=csv_test1.get_data()) class TestConstructor: @@ -82,13 +82,13 @@ def test_runs(self, csv_test1_not_read_initially_test): assert csv_test1_not_read_initially_test.get_data() == [] assert csv_test1_not_read_initially_test.column_headers == [] assert csv_test1_not_read_initially_test.file_location == TestFiles.CsvFiles.TEST1 - assert csv_test1_not_read_initially_test.data_state == GenericFile.DataState.NOTHING_READ_OR_SET + assert csv_test1_not_read_initially_test.data_state == SourceFile.DataState.NOTHING_READ_OR_SET csv_test1_not_read_initially_test.read_file() assert len(csv_test1_not_read_initially_test.get_data()) == 15 assert "guid" in csv_test1_not_read_initially_test.column_headers - assert csv_test1_not_read_initially_test.data_state == GenericFile.DataState.READ_IN_DATA + assert csv_test1_not_read_initially_test.data_state == SourceFile.DataState.READ_IN_DATA class TestWriteFile: diff --git a/tests/representation/generic/test_generic_file.py b/tests/representation/generic/test_generic_file.py index 282a4cb..dc66ac7 100644 --- a/tests/representation/generic/test_generic_file.py +++ b/tests/representation/generic/test_generic_file.py @@ -3,7 +3,7 @@ import pytest from unittest.mock import MagicMock -from brain_brew.representation.generic.generic_file import GenericFile +from brain_brew.representation.generic.generic_file import SourceFile from tests.test_file_manager import get_new_file_manager from tests.test_files import TestFiles @@ -11,9 +11,9 @@ class TestConstructor: def test_runs(self): file_location = TestFiles.CsvFiles.TEST1 - file = GenericFile(file_location, read_now=False, data_override=None) + file = SourceFile(file_location, read_now=False, data_override=None) - assert isinstance(file, GenericFile) + assert isinstance(file, SourceFile) assert file.file_location == file_location assert file._data is None @@ -21,13 +21,13 @@ def test_no_file_found(self): file_location = "sdfsdfgdsfsdfsdsdg/sdfsdf/sdfsdf/sdfsd/" with pytest.raises(FileNotFoundError): - GenericFile(file_location, read_now=True, data_override=None) + SourceFile(file_location, read_now=True, data_override=None) def test_override_data(self): override_data = {"Test": 1} - file = GenericFile("", read_now=True, data_override=override_data) + file = SourceFile("", read_now=True, data_override=override_data) - assert isinstance(file, GenericFile) + assert isinstance(file, SourceFile) assert file._data == override_data @@ -36,9 +36,9 @@ def test_runs(self): fm = get_new_file_manager() assert len(fm.known_files_dict) == 0 - first = GenericFile.create("test1", read_now=False) + first = SourceFile.create_or_get("test1", read_now=False) - assert isinstance(first, GenericFile) + assert isinstance(first, SourceFile) assert len(fm.known_files_dict) == 1 assert fm.known_files_dict["test1"] @@ -46,8 +46,8 @@ def test_returns_existing_object(self): fm = get_new_file_manager() assert len(fm.known_files_dict) == 0 - first = GenericFile.create("test1", read_now=False) - second = GenericFile.create("test1", read_now=False) + first = SourceFile.create_or_get("test1", read_now=False) + second = SourceFile.create_or_get("test1", read_now=False) assert first == second assert len(fm.known_files_dict) == 1 diff --git a/tests/representation/json/test_crowd_anki_export.py b/tests/representation/json/test_crowd_anki_export.py index 4be26ae..1187cb7 100644 --- a/tests/representation/json/test_crowd_anki_export.py +++ b/tests/representation/json/test_crowd_anki_export.py @@ -38,7 +38,7 @@ def test_too_many_json_files(self, tmpdir): @pytest.fixture() def ca_export_test1() -> CrowdAnkiExport: - return CrowdAnkiExport.create(TestFiles.CrowdAnkiExport.TEST1_FOLDER) + return CrowdAnkiExport.create_or_get(TestFiles.CrowdAnkiExport.TEST1_FOLDER) @pytest.fixture() diff --git a/tests/representation/json/test_deck_part_note_model.py b/tests/representation/json/test_deck_part_note_model.py index 4d07a0f..f7f745b 100644 --- a/tests/representation/json/test_deck_part_note_model.py +++ b/tests/representation/json/test_deck_part_note_model.py @@ -45,7 +45,7 @@ def test_config_location_override(self, global_config): @pytest.fixture() def dp_note_model_test1(global_config) -> DeckPartNoteModel: - return DeckPartNoteModel.create(TestFiles.CrowdAnkiNoteModels.TEST_COMPLETE) + return DeckPartNoteModel.create_or_get(TestFiles.CrowdAnkiNoteModels.TEST_COMPLETE) def test_read_fields(dp_note_model_test1): diff --git a/tests/representation/yaml/test_note_repr.py b/tests/representation/yaml/test_note_repr.py index 87ae0ee..e5d08a5 100644 --- a/tests/representation/yaml/test_note_repr.py +++ b/tests/representation/yaml/test_note_repr.py @@ -8,7 +8,7 @@ import pytest from brain_brew.representation.yaml.note_repr import Note, NoteGrouping, Notes, \ - NOTES, NOTE_GROUPINGS, FIELDS, GUID, NOTE_MODEL, TAGS + NOTES, NOTE_GROUPINGS, FIELDS, GUID, NOTE_MODEL, TAGS, FLAGS working_notes = { "test1": {FIELDS: ['first'], GUID: "12345", NOTE_MODEL: "model_name", TAGS: ['noun', 'other']}, @@ -17,7 +17,9 @@ "no_note_model2": {FIELDS: ['second'], GUID: "67890", TAGS: ['noun', 'other']}, "no_tags1": {FIELDS: ['first'], GUID: "12345", NOTE_MODEL: "model_name"}, "no_tags2": {FIELDS: ['first'], GUID: "12345", NOTE_MODEL: "model_name", TAGS: []}, - "no_model_or_tags": {FIELDS: ['first'], GUID: "12345"} + "no_model_or_tags": {FIELDS: ['first'], GUID: "12345"}, + "test1_with_default_flags": {FIELDS: ['first'], GUID: "12345", NOTE_MODEL: "model_name", TAGS: ['noun', 'other'], FLAGS: 0}, + "test1_with_flags": {FIELDS: ['first'], GUID: "12345", NOTE_MODEL: "model_name", TAGS: ['noun', 'other'], FLAGS: 1}, } working_note_groupings = { @@ -48,20 +50,21 @@ def note_grouping_fixtures(request): class TestConstructor: class TestNote: - @pytest.mark.parametrize("fields, guid, note_model, tags, media", [ - ([], "", "", [], {}), - (None, None, None, None, None), - (["test", "blah", "whatever"], "1234567890x", "model_name", ["noun"], {}), - (["test", "blah", ""], "1234567890x", "model_name", ["noun"], {"animal.jpg"}), + @pytest.mark.parametrize("fields, guid, note_model, tags, flags, media", [ + ([], "", "", [], 0, {}), + (None, None, None, None, None, None), + (["test", "blah", "whatever"], "1234567890x", "model_name", ["noun"], 1, {}), + (["test", "blah", ""], "1234567890x", "model_name", ["noun"], 2, {"animal.jpg"}), ]) - def test_constructor(self, fields: List[str], guid: str, note_model: str, tags: List[str], media: Set[str]): - note = Note(fields=fields, guid=guid, note_model=note_model, tags=tags) + def test_constructor(self, fields: List[str], guid: str, note_model: str, tags: List[str], flags: int, media: Set[str]): + note = Note(fields=fields, guid=guid, note_model=note_model, tags=tags, flags=flags) assert isinstance(note, Note) assert note.fields == fields assert note.guid == guid assert note.note_model == note_model assert note.tags == tags + assert note.flags == flags # assert note.media_references == media def test_from_dict(self, note_fixtures): @@ -154,6 +157,33 @@ def test_no_tags(self, tmpdir): self._assert_dump_to_yaml(tmpdir.mkdir(str(num)), ystring, note) + def test_with_flags(self, tmpdir): + ystring = dedent(f'''\ + {FIELDS}: + - first + {GUID}: '12345' + {FLAGS}: 1 + {NOTE_MODEL}: model_name + {TAGS}: + - noun + - other + ''') + + self._assert_dump_to_yaml(tmpdir, ystring, "test1_with_flags") + + def test_with_default_flags(self, tmpdir): + ystring = dedent(f'''\ + {FIELDS}: + - first + {GUID}: '12345' + {NOTE_MODEL}: model_name + {TAGS}: + - noun + - other + ''') + + self._assert_dump_to_yaml(tmpdir, ystring, "test1_with_default_flags") + class TestNoteGrouping: @staticmethod def _assert_dump_to_yaml(tmpdir, ystring, note_grouping_name): @@ -296,7 +326,7 @@ class TestNote: (["", "[sound:test.mp3]", "[sound:test.mp3]"], 2), ]) def test_all(self, fields, expected_count): - note = Note(fields=fields, note_model=None, guid="", tags=None,) + note = Note(fields=fields, note_model=None, guid="", tags=None, flags=0) media_found = note.get_media_references() assert isinstance(media_found, Set) assert len(media_found) == expected_count From cff5b43d676fad928e3d62176a9493764d08043e Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sun, 23 Aug 2020 10:04:51 +0200 Subject: [PATCH 25/39] CrowdAnki Transforms 99% complete! - Headers - Media - Notes - Note Models --- .../crowd_anki/crowd_anki_generate.py | 55 ++++++++++ .../crowd_anki/crowd_anki_to_deck_parts.py | 28 +++-- .../crowd_anki/headers_from_crowdanki.py | 2 +- .../crowd_anki/headers_to_crowd_anki.py | 35 ++++++ .../crowd_anki/media_to_from_crowd_anki.py | 84 ++++++++++++++ .../crowd_anki/note_models_to_crowd_anki.py | 22 ++++ .../crowd_anki/notes_from_crowd_anki.py | 2 +- .../crowd_anki/notes_to_crowd_anki.py | 16 ++- brain_brew/file_manager.py | 2 +- .../representation/generic/media_file.py | 15 +-- .../representation/generic/yaml_file.py | 103 ------------------ .../representation/json/crowd_anki_export.py | 3 - .../representation/json/deck_part_header.py | 15 --- .../representation/yaml/note_model_repr.py | 19 +++- brain_brew/representation/yaml/note_repr.py | 14 +++ .../transformers/transform_crowdanki.py | 4 +- .../representation/generic/test_media_file.py | 8 +- 17 files changed, 272 insertions(+), 155 deletions(-) create mode 100644 brain_brew/build_tasks/crowd_anki/headers_to_crowd_anki.py create mode 100644 brain_brew/build_tasks/crowd_anki/media_to_from_crowd_anki.py create mode 100644 brain_brew/build_tasks/crowd_anki/note_models_to_crowd_anki.py delete mode 100644 brain_brew/representation/generic/yaml_file.py delete mode 100644 brain_brew/representation/json/deck_part_header.py diff --git a/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py b/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py index 6853731..3b882f8 100644 --- a/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py +++ b/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py @@ -1,7 +1,20 @@ from dataclasses import dataclass +import logging +from typing import Union, Optional, List +from brain_brew.build_tasks.crowd_anki.headers_from_crowdanki import HeadersFromCrowdAnki +from brain_brew.build_tasks.crowd_anki.headers_to_crowd_anki import HeadersToCrowdAnki +from brain_brew.build_tasks.crowd_anki.media_to_from_crowd_anki import MediaToFromCrowdAnki +from brain_brew.build_tasks.crowd_anki.note_models_from_crowd_anki import NoteModelsFromCrowdAnki +from brain_brew.build_tasks.crowd_anki.note_models_to_crowd_anki import NoteModelsToCrowdAnki +from brain_brew.build_tasks.crowd_anki.notes_from_crowd_anki import NotesFromCrowdAnki +from brain_brew.build_tasks.crowd_anki.notes_to_crowd_anki import NotesToCrowdAnki +from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport +from brain_brew.representation.json.wrappers_for_crowd_anki import CrowdAnkiJsonWrapper +from brain_brew.representation.yaml.note_model_repr import NoteModel from brain_brew.representation.build_config.build_task import TopLevelBuildTask +from brain_brew.representation.build_config.representation_base import RepresentationBase from brain_brew.utils import all_combos_prepend_append @@ -9,3 +22,45 @@ class CrowdAnkiGenerate(TopLevelBuildTask): task_names = all_combos_prepend_append(["CrowdAnki", "CrowdAnki Export"], "Generate ", "s") + @dataclass + class Representation(RepresentationBase): + folder: str + notes: Optional[dict] + note_models: Optional[list] + headers: Optional[dict] + media: Optional[dict] + + @classmethod + def from_repr(cls, data: Union[Representation, dict]): + rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) + return cls( + crowd_anki_export=CrowdAnkiExport.create_or_get(rep.folder), + notes_transform=NotesToCrowdAnki.from_repr(rep.notes), + note_model_transform=NoteModelsToCrowdAnki.from_list(rep.note_models), + headers_transform=HeadersToCrowdAnki.from_repr(rep.headers), + media_transform=MediaToFromCrowdAnki.from_repr(rep.media) + ) + + crowd_anki_export: CrowdAnkiExport + notes_transform: NotesToCrowdAnki + note_model_transform: NoteModelsToCrowdAnki + headers_transform: HeadersToCrowdAnki + media_transform: MediaToFromCrowdAnki + + def execute(self): + headers = self.headers_transform.execute() + ca_wrapper = CrowdAnkiJsonWrapper(headers) + + note_models: List[dict] = self.note_model_transform.execute() + + nm_name_to_id: dict = {model.name: model.crowdanki_id for model in self.note_model_transform.note_models} + notes = self.notes_transform.execute(nm_name_to_id) + + media_files = self.media_transform.move_to_crowd_anki( + self.notes_transform.notes, self.note_model_transform.note_models, self.crowd_anki_export) + + ca_wrapper.note_models = note_models + ca_wrapper.notes = notes + ca_wrapper.media_files = list(media_files) + + #Set to CrowdAnkiExport \ No newline at end of file diff --git a/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py b/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py index f7dd3e4..3b91daf 100644 --- a/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py +++ b/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py @@ -1,14 +1,16 @@ import logging from dataclasses import dataclass -from typing import Union, Optional +from typing import Union, Optional, List from brain_brew.build_tasks.crowd_anki.headers_from_crowdanki import HeadersFromCrowdAnki +from brain_brew.build_tasks.crowd_anki.media_to_from_crowd_anki import MediaToFromCrowdAnki from brain_brew.build_tasks.crowd_anki.note_models_from_crowd_anki import NoteModelsFromCrowdAnki -from brain_brew.build_tasks.crowd_anki.notes_from_crowd_anki import TrCrowdAnkiToNotes +from brain_brew.build_tasks.crowd_anki.notes_from_crowd_anki import NotesFromCrowdAnki from brain_brew.representation.build_config.build_task import DeckPartBuildTask from brain_brew.representation.build_config.representation_base import RepresentationBase from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder +from brain_brew.representation.yaml.note_model_repr import NoteModel from brain_brew.utils import all_combos_prepend_append @@ -24,32 +26,34 @@ class Representation(RepresentationBase): headers: Optional[dict] media: Optional[dict] - crowd_anki_export: CrowdAnkiExport - notes_transform: TrCrowdAnkiToNotes - note_model_transform: NoteModelsFromCrowdAnki - headers_transform: HeadersFromCrowdAnki - media: int - @classmethod def from_repr(cls, data: Union[Representation, dict]): rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) return cls( crowd_anki_export=CrowdAnkiExport.create_or_get(rep.folder), - notes_transform=TrCrowdAnkiToNotes.from_repr(rep.notes), - note_model_transform=NoteModelsFromCrowdAnki.from_list(rep.note_models) + notes_transform=NotesFromCrowdAnki.from_repr(rep.notes), + note_model_transform=NoteModelsFromCrowdAnki.from_list(rep.note_models), + headers_transform=HeadersFromCrowdAnki.from_repr(rep.headers), + media_transform=MediaToFromCrowdAnki.from_repr(rep.media) ) + crowd_anki_export: CrowdAnkiExport + notes_transform: NotesFromCrowdAnki + note_model_transform: NoteModelsFromCrowdAnki + headers_transform: HeadersFromCrowdAnki + media_transform: MediaToFromCrowdAnki + def execute(self): ca_wrapper = self.crowd_anki_export.read_json_file() if ca_wrapper.children: logging.warning("Child Decks / Subdecks are not currently supported.") # TODO: Support them - note_models = self.note_model_transform.execute(ca_wrapper) + note_models: List[NoteModel] = self.note_model_transform.execute(ca_wrapper) nm_id_to_name: dict = {model.crowdanki_id: model.name for model in note_models} notes = self.notes_transform.execute(ca_wrapper, nm_id_to_name) headers = self.headers_transform.execute(ca_wrapper) - + self.media_transform.move_to_deck_parts(notes, note_models) diff --git a/brain_brew/build_tasks/crowd_anki/headers_from_crowdanki.py b/brain_brew/build_tasks/crowd_anki/headers_from_crowdanki.py index 923b8eb..c479af7 100644 --- a/brain_brew/build_tasks/crowd_anki/headers_from_crowdanki.py +++ b/brain_brew/build_tasks/crowd_anki/headers_from_crowdanki.py @@ -23,7 +23,7 @@ def from_repr(cls, data: Union[Representation, dict]): ) def execute(self, ca_wrapper: CrowdAnkiJsonWrapper): - headers = Headers(TransformCrowdAnki.headers_to_crowd_anki(ca_wrapper.data)) + headers = Headers(TransformCrowdAnki.crowd_anki_to_headers(ca_wrapper.data)) DeckPartHolder.override_or_create(self.name, self.save_to_file, headers) diff --git a/brain_brew/build_tasks/crowd_anki/headers_to_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/headers_to_crowd_anki.py new file mode 100644 index 0000000..4f70d45 --- /dev/null +++ b/brain_brew/build_tasks/crowd_anki/headers_to_crowd_anki.py @@ -0,0 +1,35 @@ +from dataclasses import dataclass +from typing import Optional, Union + +from brain_brew.representation.build_config.representation_base import RepresentationBase +from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder +from brain_brew.representation.yaml.headers_repr import Headers +from brain_brew.transformers.transform_crowdanki import TransformCrowdAnki + + +@dataclass +class HeadersToCrowdAnki: + @dataclass + class Representation(RepresentationBase): + name: str + + @classmethod + def from_repr(cls, data: Union[Representation, dict, str]): + rep: cls.Representation + if isinstance(data, cls.Representation): + rep = data + elif isinstance(data, dict): + rep = cls.Representation.from_dict(data) + else: + rep = cls.Representation(name=data) # Support single string being passed in + + return cls( + headers=DeckPartHolder.from_deck_part_pool(rep.name), + ) + + headers: Headers + + def execute(self): + headers = Headers(TransformCrowdAnki.headers_to_crowd_anki(self.headers.data)) + + return headers diff --git a/brain_brew/build_tasks/crowd_anki/media_to_from_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/media_to_from_crowd_anki.py new file mode 100644 index 0000000..e85c9ca --- /dev/null +++ b/brain_brew/build_tasks/crowd_anki/media_to_from_crowd_anki.py @@ -0,0 +1,84 @@ +from dataclasses import dataclass +from typing import Union, List, Set + +from brain_brew.file_manager import FileManager +from brain_brew.representation.build_config.representation_base import RepresentationBase +from brain_brew.representation.generic.media_file import MediaFile +from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport +from brain_brew.representation.yaml.note_model_repr import NoteModel +from brain_brew.representation.yaml.note_repr import Notes + + +@dataclass +class MediaToFromCrowdAnki: + @dataclass + class Representation(RepresentationBase): + from_notes: bool + from_note_models: bool + + @classmethod + def from_repr(cls, data: Union[Representation, dict, bool]): + rep: cls.Representation + if isinstance(data, cls.Representation): + rep = data + elif isinstance(data, dict): + rep = cls.Representation.from_dict(data) + else: + rep = cls.Representation(from_notes=data, from_note_models=data) # Support single boolean for default + + return cls( + from_notes=rep.from_notes, + from_note_models=rep.from_note_models + ) + + from_notes: bool + from_note_models: bool + + file_manager: FileManager = None + + def __post_init__(self): + self.file_manager = FileManager.get_instance() + + def move_to_crowd_anki(self, notes: Notes, note_models: List[NoteModel], ca_export: CrowdAnkiExport) -> Set[str]: + def move_media(media_files): + for file in media_files: + filename = file.filename + if filename in ca_export.known_media: + ca_export.known_media[filename].set_override(file.source_loc) + else: + ca_export.known_media.setdefault( + filename, MediaFile(ca_export.media_loc + filename, + filename, MediaFile.ManagementType.TO_BE_CLONED, file.source_loc) + ) + + all_media: Set[str] = set() + + if self.from_notes: + notes_media = notes.get_all_media_references() + move_media(notes_media) + all_media = all_media.union(notes_media) + + if self.from_note_models: + for model in note_models: + model_media = model.get_all_media_references() + move_media(model_media) + all_media = all_media.union(model_media) + + return all_media + + def move_to_deck_parts(self, notes: Notes, note_models: List[NoteModel]): + def move_media(media_files): + for file in media_files: + filename = file.filename + dp_media_file = self.file_manager.media_file_if_exists(filename) + if dp_media_file: + dp_media_file.set_override(file.source_loc) + else: + self.file_manager.new_media_file(filename, file.source_loc) + + if self.from_notes: + move_media(notes.get_all_media_references()) + + if self.from_note_models: + for model in note_models: + move_media(model.get_all_media_references()) diff --git a/brain_brew/build_tasks/crowd_anki/note_models_to_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/note_models_to_crowd_anki.py new file mode 100644 index 0000000..f2e4c56 --- /dev/null +++ b/brain_brew/build_tasks/crowd_anki/note_models_to_crowd_anki.py @@ -0,0 +1,22 @@ +from dataclasses import dataclass +from typing import Optional, Union, List +import logging + +from brain_brew.representation.json.wrappers_for_crowd_anki import CrowdAnkiJsonWrapper +from brain_brew.representation.transformers.base_deck_part_from import BaseDeckPartsFrom +from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder +from brain_brew.representation.yaml.note_model_repr import NoteModel + + +@dataclass +class NoteModelsToCrowdAnki: + @classmethod + def from_list(cls, note_model_items: List[str]): + return cls( + note_models=list(map(DeckPartHolder.from_deck_part_pool, note_model_items)) + ) + + note_models: List[NoteModel] + + def execute(self) -> List[dict]: + return [model.encode_as_crowdanki() for model in self.note_models] diff --git a/brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py index 7137dc7..4173801 100644 --- a/brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py +++ b/brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py @@ -11,7 +11,7 @@ @dataclass -class TrCrowdAnkiToNotes(SharedBaseNotes, BaseDeckPartsFrom): +class NotesFromCrowdAnki(SharedBaseNotes, BaseDeckPartsFrom): @dataclass class Representation(SharedBaseNotes.Representation, BaseDeckPartsFrom.Representation): pass diff --git a/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py index 86e9114..4a634c1 100644 --- a/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py +++ b/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py @@ -2,6 +2,9 @@ from typing import Optional, Union, List from brain_brew.build_tasks.crowd_anki.shared_base_notes import SharedBaseNotes +from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder +from brain_brew.representation.yaml.note_repr import Notes +from brain_brew.transformers.transform_crowdanki import TransformCrowdAnki @dataclass @@ -15,10 +18,19 @@ class Representation(SharedBaseNotes.Representation): def from_repr(cls, data: Union[Representation, dict]): rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) return cls( - name=rep.name, + notes=DeckPartHolder.from_deck_part_pool(rep.name), sort_order=SharedBaseNotes._get_sort_order(rep.sort_order), additional_items_to_add=rep.additional_items_to_add or {} ) - name: str + notes: Notes additional_items_to_add: dict + + def execute(self, nm_name_to_id: dict) -> List[dict]: + notes = self.notes.get_notes() + + note_dicts = TransformCrowdAnki.notes_to_crowd_anki(notes, nm_name_to_id, self.additional_items_to_add) + + # TODO: Sort + + return note_dicts diff --git a/brain_brew/file_manager.py b/brain_brew/file_manager.py index 63b046e..a633057 100644 --- a/brain_brew/file_manager.py +++ b/brain_brew/file_manager.py @@ -78,7 +78,7 @@ def find_all_deck_part_media_files(self): filename = filename_from_full_path(full_path) self._register_media_file(MediaFile(full_path, filename)) - logging.debug(f"Media files found: {len(self.known_media_files_dict)}") + logging.debug(f"DeckPart Media files found: {len(self.known_media_files_dict)}") def new_deck_part(self, dp: DeckPartHolder) -> DeckPartHolder: if dp.name in self.deck_part_pool: diff --git a/brain_brew/representation/generic/media_file.py b/brain_brew/representation/generic/media_file.py index e7811a2..966f060 100644 --- a/brain_brew/representation/generic/media_file.py +++ b/brain_brew/representation/generic/media_file.py @@ -8,27 +8,28 @@ class ManagementType(Enum): OVERRIDDEN = 1 TO_BE_CLONED = 2 - target_loc: str + file_location: str filename: str man_type: ManagementType source_loc: str - def __init__(self, target_loc, filename, man_type: ManagementType = ManagementType.EXISTS, source_loc=None): - self.target_loc = target_loc + def __init__(self, file_location, filename, man_type: ManagementType = ManagementType.EXISTS, source_loc=None): + self.file_location = file_location self.filename = filename self.man_type = man_type - self.source_loc = source_loc if source_loc is not None else target_loc + self.source_loc = source_loc if source_loc is not None else file_location def set_override(self, source_loc): - self.man_type = MediaFile.ManagementType.OVERRIDDEN - self.source_loc = source_loc + if source_loc != self.source_loc: + self.man_type = MediaFile.ManagementType.OVERRIDDEN + self.source_loc = source_loc def copy_source_to_target(self): if self.should_write(): # TODO: If ManagementType.OVERRIDDEN check if override necessary - shutil.copy2(self.source_loc, self.target_loc) + shutil.copy2(self.source_loc, self.file_location) def should_write(self): return self.man_type in [MediaFile.ManagementType.OVERRIDDEN, MediaFile.ManagementType.TO_BE_CLONED] diff --git a/brain_brew/representation/generic/yaml_file.py b/brain_brew/representation/generic/yaml_file.py deleted file mode 100644 index b70249a..0000000 --- a/brain_brew/representation/generic/yaml_file.py +++ /dev/null @@ -1,103 +0,0 @@ -from collections import namedtuple -from pathlib import Path - -import yaml - -from brain_brew.constants.build_config_keys import BuildConfigKeys - -ConfigKey = namedtuple("ConfigKey", "required entry_type children") - - -class YamlFile: - config_entry: dict - expected_keys: dict - subconfig_filter: list - - @staticmethod - def read_file(src: str): - if src[-5:] not in [".yaml", ".yml"]: - src += ".yaml" - - if not Path(src).is_file(): - raise FileNotFoundError(src) - - with open(src, 'r') as yml_file: - f = yaml.full_load(yml_file) - - return f - - @staticmethod - def check_config_recursive(expected_keys, keys, parent_key_name=""): - errors = [] - - keys_left_to_check = list(expected_keys.keys()) - - for key in keys: - key_name = f"{parent_key_name}/{key}" - - if key in keys_left_to_check: - keys_left_to_check.remove(key) - - if expected_keys[key].children: - errors = errors + \ - YamlFile.check_config_recursive(expected_keys[key].children, keys[key], key_name) - - if not isinstance(keys[key], expected_keys[key].entry_type): - errors.append(f"Expected '{key_name}' to be of type {expected_keys[key].entry_type}" - f", not {type(keys[key])}") - - else: - errors.append(f"Unexpected key '{key_name}'") - - for key in keys_left_to_check: - if expected_keys[key].required: - errors.append(f"Missing key '{key}'") - - return errors - - def verify_config_entry(self): - errors = YamlFile.check_config_recursive(self.expected_keys, self.config_entry) - if errors: - self.raise_error_in_config(errors) - - def get_config(self, enum_key, otherwise=None): - if enum_key.value in self.config_entry: - return self.config_entry[enum_key.value] - if otherwise is not None: - return otherwise - raise KeyError(f"Expected key {enum_key.value}") - - def raise_error_in_config(self, error_message): - raise KeyError(error_message) - - def setup_config_with_subconfig_replacement(self, config_entry: dict): - if BuildConfigKeys.SUBCONFIG.value not in config_entry: - self.config_entry = config_entry - return - - sub = config_entry[BuildConfigKeys.SUBCONFIG.value] - - clone = config_entry.copy() - clone.pop(BuildConfigKeys.SUBCONFIG.value) - - def verify_and_read_sub(sub_to_verify: str, keep_only_keys): - if not isinstance(sub_to_verify, str): - raise TypeError(f"Unknown type in {BuildConfigKeys.SUBCONFIG.value}") - data = YamlFile.read_file(sub_to_verify) - if keep_only_keys is not None: - return {k: data[k] for k in data if k in keep_only_keys} - return data - - if isinstance(sub, str): - replacement_sub: dict = verify_and_read_sub(sub, self.subconfig_filter) - clone = {**clone, **replacement_sub} - elif isinstance(sub, list): - replacement_list = [] - for s in sub: - replacement_list.append(verify_and_read_sub(s, self.subconfig_filter)) - replacement_sub = {BuildConfigKeys.SUBCONFIG.value: replacement_list} - clone = {**clone, **replacement_sub} - else: - raise TypeError(f"{BuildConfigKeys.SUBCONFIG.value} is the wrong type: {type(sub)}") - - self.config_entry = clone diff --git a/brain_brew/representation/json/crowd_anki_export.py b/brain_brew/representation/json/crowd_anki_export.py index 1e13453..6588cf7 100644 --- a/brain_brew/representation/json/crowd_anki_export.py +++ b/brain_brew/representation/json/crowd_anki_export.py @@ -1,8 +1,5 @@ import glob import logging -import pathlib -from enum import Enum -from pathlib import Path from typing import List, Dict from brain_brew.representation.generic.generic_file import SourceFile diff --git a/brain_brew/representation/json/deck_part_header.py b/brain_brew/representation/json/deck_part_header.py deleted file mode 100644 index 5d4f519..0000000 --- a/brain_brew/representation/json/deck_part_header.py +++ /dev/null @@ -1,15 +0,0 @@ -from brain_brew.representation.configuration.global_config import GlobalConfig -from brain_brew.representation.json.json_file import JsonFile - - -class DeckPartHeader(JsonFile): - - @classmethod - def formatted_file_location(cls, location): - return cls.get_json_file_location(GlobalConfig.get_instance().deck_parts.headers, location) - - def __init__(self, location, read_now=True, data_override=None): - super().__init__( - self.formatted_file_location(location), - read_now=read_now, data_override=data_override - ) diff --git a/brain_brew/representation/yaml/note_model_repr.py b/brain_brew/representation/yaml/note_model_repr.py index ee39a1a..5fec649 100644 --- a/brain_brew/representation/yaml/note_model_repr.py +++ b/brain_brew/representation/yaml/note_model_repr.py @@ -4,7 +4,7 @@ from brain_brew.representation.build_config.representation_base import RepresentationBase from brain_brew.representation.yaml.my_yaml import YamlRepr -from brain_brew.utils import list_of_str_to_lowercase +from brain_brew.utils import list_of_str_to_lowercase, find_media_in_field class AnkiField: @@ -84,6 +84,14 @@ def from_crowdanki(cls, data: Union[CrowdAnki, dict]): question_format_in_browser=ca.bqfmt, answer_format_in_browser=ca.bafmt, deck_override_id=ca.did ) + def get_all_media_references(self) -> set: + all_media = set()\ + .union(find_media_in_field(self.question_format))\ + .union(find_media_in_field(self.answer_format))\ + .union(find_media_in_field(self.question_format_in_browser))\ + .union(find_media_in_field(self.answer_format_in_browser)) + return all_media + def encode_as_crowdanki(self, ordinal: int) -> dict: data_dict = { NAME.anki_name: self.name, @@ -253,9 +261,12 @@ def encode(self) -> dict: return data_dict - def find_media(self): # TODO - pass - # Look in templates (and css?) + def get_all_media_references(self) -> set: + all_media = set() + for template in self.templates: + all_media = all_media.union(template.get_all_media_references()) + + return all_media @property def field_names_lowercase(self): diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index 3cffc4a..1985097 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -88,6 +88,13 @@ def verify_groupings(self): def get_all_known_note_model_names(self) -> set: return {self.note_model} if self.note_model else {note.note_model for note in self.notes} + def get_all_media_references(self) -> set: + all_media = set() + for note in self.notes: + for media in note.get_media_references(): + all_media = all_media.union(media) + return all_media + def get_all_notes_copy(self) -> List[Note]: def join_tags(n_tags): if self.tags is None and n_tags is None: @@ -128,5 +135,12 @@ def encode(self) -> dict: def get_all_known_note_model_names(self): return {nms for group in self.note_groupings for nms in group.get_all_known_note_model_names()} + def get_all_media_references(self) -> set: + all_media = set() + for note in self.note_groupings: + for media in note.get_all_media_references(): + all_media = all_media.union(media) + return all_media + def get_notes(self): return [note for group in self.note_groupings for note in group.get_all_notes_copy()] diff --git a/brain_brew/transformers/transform_crowdanki.py b/brain_brew/transformers/transform_crowdanki.py index e337026..67814ee 100644 --- a/brain_brew/transformers/transform_crowdanki.py +++ b/brain_brew/transformers/transform_crowdanki.py @@ -18,7 +18,7 @@ class TransformCrowdAnki(TrNotes): } @classmethod - def notes_to_crowd_anki(cls, notes: List[Note], note_models_name_to_id: dict, additional_items_to_add: dict) -> List[dict]: + def notes_to_crowd_anki(cls, notes: List[Note], nm_name_to_id: dict, additional_items_to_add: dict) -> List[dict]: resolved_notes: List[dict] = [] wrapper: CrowdAnkiNoteWrapper = CrowdAnkiNoteWrapper() for note in notes: @@ -34,7 +34,7 @@ def notes_to_crowd_anki(cls, notes: List[Note], note_models_name_to_id: dict, ad wrapper.guid = note.guid wrapper.fields = note.fields wrapper.tags = note.tags - wrapper.note_model = note_models_name_to_id[note.note_model] + wrapper.note_model = nm_name_to_id[note.note_model] wrapper.flags = note.flags resolved_notes.append(current) diff --git a/tests/representation/generic/test_media_file.py b/tests/representation/generic/test_media_file.py index 90a5a48..4cb8655 100644 --- a/tests/representation/generic/test_media_file.py +++ b/tests/representation/generic/test_media_file.py @@ -19,7 +19,7 @@ def test_without_override(self): media_file = MediaFile(loc, name) assert isinstance(media_file, MediaFile) - assert media_file.source_loc == media_file.target_loc == loc + assert media_file.source_loc == media_file.file_location == loc assert media_file.filename == name assert media_file.man_type == MediaFile.ManagementType.EXISTS @@ -32,7 +32,7 @@ def test_with_override(self): media_file = MediaFile(target_loc, name, man_type, source_loc) assert isinstance(media_file, MediaFile) - assert media_file.target_loc == target_loc + assert media_file.file_location == target_loc assert media_file.source_loc == source_loc assert media_file.filename == name assert media_file.man_type == man_type @@ -40,12 +40,12 @@ def test_with_override(self): def test_set_override(media_file_test1): assert media_file_test1.man_type == MediaFile.ManagementType.EXISTS - assert media_file_test1.source_loc == media_file_test1.target_loc == "loc" + assert media_file_test1.source_loc == media_file_test1.file_location == "loc" media_file_test1.set_override("new loc") assert media_file_test1.source_loc == "new loc" - assert media_file_test1.target_loc == "loc" + assert media_file_test1.file_location == "loc" assert media_file_test1.man_type == MediaFile.ManagementType.OVERRIDDEN From ef0100063ff0c0902f3a8c9ca99c993782ec0be4 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sat, 29 Aug 2020 11:28:09 +0200 Subject: [PATCH 26/39] Finished Media Move Logic, no longer inside FileManager; Removed Transformers; CrowdAnki Transformers are complete --- .../crowd_anki/crowd_anki_generate.py | 9 +- .../crowd_anki/crowd_anki_to_deck_parts.py | 5 +- .../crowd_anki/headers_from_crowdanki.py | 17 ++- .../crowd_anki/headers_to_crowd_anki.py | 15 ++- .../crowd_anki/media_to_from_crowd_anki.py | 106 ++++++++++++------ .../crowd_anki/note_models_to_crowd_anki.py | 45 +++++++- .../crowd_anki/notes_from_crowd_anki.py | 22 +++- .../crowd_anki/notes_to_crowd_anki.py | 29 ++++- brain_brew/build_tasks/csvs/csvs_generate.py | 19 +++- .../build_tasks/csvs/notes_from_csvs.py | 21 +++- .../build_tasks/csvs/shared_base_csvs.py | 2 +- brain_brew/build_tasks/task_builder.py | 3 +- brain_brew/file_manager.py | 38 +++---- .../representation/json/crowd_anki_export.py | 4 +- .../representation/yaml/note_model_repr.py | 6 +- brain_brew/representation/yaml/note_repr.py | 5 +- brain_brew/transformers/__init__.py | 0 .../transformers/base_transform_notes.py | 16 --- .../transformers/transform_crowdanki.py | 67 ----------- .../transformers/transform_csv_collection.py | 44 -------- brain_brew/utils.py | 13 +++ 21 files changed, 261 insertions(+), 225 deletions(-) delete mode 100644 brain_brew/transformers/__init__.py delete mode 100644 brain_brew/transformers/base_transform_notes.py delete mode 100644 brain_brew/transformers/transform_crowdanki.py delete mode 100644 brain_brew/transformers/transform_csv_collection.py diff --git a/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py b/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py index 3b882f8..a2d1916 100644 --- a/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py +++ b/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py @@ -36,7 +36,7 @@ def from_repr(cls, data: Union[Representation, dict]): return cls( crowd_anki_export=CrowdAnkiExport.create_or_get(rep.folder), notes_transform=NotesToCrowdAnki.from_repr(rep.notes), - note_model_transform=NoteModelsToCrowdAnki.from_list(rep.note_models), + note_model_transform=NoteModelsToCrowdAnki.from_repr(rep.note_models), headers_transform=HeadersToCrowdAnki.from_repr(rep.headers), media_transform=MediaToFromCrowdAnki.from_repr(rep.media) ) @@ -61,6 +61,9 @@ def execute(self): ca_wrapper.note_models = note_models ca_wrapper.notes = notes - ca_wrapper.media_files = list(media_files) + ca_wrapper.media_files = [m.filename for m in media_files] - #Set to CrowdAnkiExport \ No newline at end of file + # Set to CrowdAnkiExport + self.crowd_anki_export.write_to_files(ca_wrapper.data) + for media in media_files: + media.copy_source_to_target() diff --git a/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py b/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py index 3b91daf..e075a36 100644 --- a/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py +++ b/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py @@ -9,7 +9,6 @@ from brain_brew.representation.build_config.build_task import DeckPartBuildTask from brain_brew.representation.build_config.representation_base import RepresentationBase from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport -from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder from brain_brew.representation.yaml.note_model_repr import NoteModel from brain_brew.utils import all_combos_prepend_append @@ -56,4 +55,6 @@ def execute(self): headers = self.headers_transform.execute(ca_wrapper) - self.media_transform.move_to_deck_parts(notes, note_models) + media_files = self.media_transform.move_to_deck_parts(notes, note_models, self.crowd_anki_export) + for media in media_files: + media.copy_source_to_target() diff --git a/brain_brew/build_tasks/crowd_anki/headers_from_crowdanki.py b/brain_brew/build_tasks/crowd_anki/headers_from_crowdanki.py index c479af7..2e2e628 100644 --- a/brain_brew/build_tasks/crowd_anki/headers_from_crowdanki.py +++ b/brain_brew/build_tasks/crowd_anki/headers_from_crowdanki.py @@ -5,7 +5,15 @@ from brain_brew.representation.transformers.base_deck_part_from import BaseDeckPartsFrom from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder from brain_brew.representation.yaml.headers_repr import Headers -from brain_brew.transformers.transform_crowdanki import TransformCrowdAnki +from brain_brew.representation.json.wrappers_for_crowd_anki import CA_NOTE_MODELS, CA_NOTES, CA_MEDIA_FILES,\ + CA_CHILDREN, CA_TYPE + + +headers_skip_keys = [CA_NOTE_MODELS, CA_NOTES, CA_MEDIA_FILES] +headers_default_values = { + CA_TYPE: "Deck", + CA_CHILDREN: [], +} @dataclass @@ -23,8 +31,13 @@ def from_repr(cls, data: Union[Representation, dict]): ) def execute(self, ca_wrapper: CrowdAnkiJsonWrapper): - headers = Headers(TransformCrowdAnki.crowd_anki_to_headers(ca_wrapper.data)) + headers = Headers(self.crowd_anki_to_headers(ca_wrapper.data)) DeckPartHolder.override_or_create(self.name, self.save_to_file, headers) return headers + + @staticmethod + def crowd_anki_to_headers(ca_data: dict): + return {key: value for key, value in ca_data + if key not in headers_skip_keys and key not in headers_default_values.keys()} diff --git a/brain_brew/build_tasks/crowd_anki/headers_to_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/headers_to_crowd_anki.py index 4f70d45..d40f851 100644 --- a/brain_brew/build_tasks/crowd_anki/headers_to_crowd_anki.py +++ b/brain_brew/build_tasks/crowd_anki/headers_to_crowd_anki.py @@ -1,17 +1,17 @@ from dataclasses import dataclass from typing import Optional, Union +from brain_brew.build_tasks.crowd_anki.headers_from_crowdanki import headers_default_values from brain_brew.representation.build_config.representation_base import RepresentationBase from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder from brain_brew.representation.yaml.headers_repr import Headers -from brain_brew.transformers.transform_crowdanki import TransformCrowdAnki @dataclass class HeadersToCrowdAnki: @dataclass class Representation(RepresentationBase): - name: str + deck_part: str @classmethod def from_repr(cls, data: Union[Representation, dict, str]): @@ -21,15 +21,20 @@ def from_repr(cls, data: Union[Representation, dict, str]): elif isinstance(data, dict): rep = cls.Representation.from_dict(data) else: - rep = cls.Representation(name=data) # Support single string being passed in + rep = cls.Representation(deck_part=data) # Support single string being passed in return cls( - headers=DeckPartHolder.from_deck_part_pool(rep.name), + headers=DeckPartHolder.from_deck_part_pool(rep.deck_part), ) headers: Headers def execute(self): - headers = Headers(TransformCrowdAnki.headers_to_crowd_anki(self.headers.data)) + headers = Headers(self.headers_to_crowd_anki(self.headers.data)) return headers + + @staticmethod + def headers_to_crowd_anki(headers_data: dict): + return {**headers_data, **headers_default_values} + diff --git a/brain_brew/build_tasks/crowd_anki/media_to_from_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/media_to_from_crowd_anki.py index e85c9ca..ef3c3fc 100644 --- a/brain_brew/build_tasks/crowd_anki/media_to_from_crowd_anki.py +++ b/brain_brew/build_tasks/crowd_anki/media_to_from_crowd_anki.py @@ -1,3 +1,4 @@ +import logging from dataclasses import dataclass from typing import Union, List, Set @@ -39,46 +40,85 @@ def from_repr(cls, data: Union[Representation, dict, bool]): def __post_init__(self): self.file_manager = FileManager.get_instance() - def move_to_crowd_anki(self, notes: Notes, note_models: List[NoteModel], ca_export: CrowdAnkiExport) -> Set[str]: - def move_media(media_files): - for file in media_files: - filename = file.filename - if filename in ca_export.known_media: - ca_export.known_media[filename].set_override(file.source_loc) - else: - ca_export.known_media.setdefault( - filename, MediaFile(ca_export.media_loc + filename, - filename, MediaFile.ManagementType.TO_BE_CLONED, file.source_loc) - ) - - all_media: Set[str] = set() + def move_to_crowd_anki(self, notes: Notes, note_models: List[NoteModel], ca_export: CrowdAnkiExport) -> Set[MediaFile]: + resolved_media: Set[MediaFile] = set() + missing_media: Set[str] = set() if self.from_notes: - notes_media = notes.get_all_media_references() - move_media(notes_media) - all_media = all_media.union(notes_media) + res, miss = self.resolve_media_references_to_deck_parts(notes.get_all_media_references()) + resolved_media, missing_media = resolved_media.union(res), missing_media.union(miss) + self._move_ca(res, ca_export) if self.from_note_models: for model in note_models: - model_media = model.get_all_media_references() - move_media(model_media) - all_media = all_media.union(model_media) - - return all_media - - def move_to_deck_parts(self, notes: Notes, note_models: List[NoteModel]): - def move_media(media_files): - for file in media_files: - filename = file.filename - dp_media_file = self.file_manager.media_file_if_exists(filename) - if dp_media_file: - dp_media_file.set_override(file.source_loc) - else: - self.file_manager.new_media_file(filename, file.source_loc) + res, miss = self.resolve_media_references_to_deck_parts(model.get_all_media_references()) + resolved_media, missing_media = resolved_media.union(res), missing_media.union(miss) + self._move_ca(res, ca_export) + + if len(missing_media) > 0: + logging.error(f"Unresolved references in DeckParts to {len(missing_media)} files: {missing_media}") + + return resolved_media + + def move_to_deck_parts(self, notes: Notes, note_models: List[NoteModel], ca_export: CrowdAnkiExport) -> Set[MediaFile]: + resolved_media: Set[MediaFile] = set() + missing_media: Set[str] = set() if self.from_notes: - move_media(notes.get_all_media_references()) + res, miss = self.resolve_media_references_to_ca(notes.get_all_media_references(), ca_export) + resolved_media, missing_media = resolved_media.union(res), missing_media.union(miss) + self._move_dps(res) if self.from_note_models: for model in note_models: - move_media(model.get_all_media_references()) + res, miss = self.resolve_media_references_to_ca(model.get_all_media_references(), ca_export) + resolved_media, missing_media = resolved_media.union(res), missing_media.union(miss) + self._move_dps(res) + + if len(missing_media) > 0: + logging.error(f"Unresolved references in CrowdAnki to {len(missing_media)} files: {missing_media}") + + return resolved_media + + @classmethod + def resolve_media_references_to_ca(cls, filenames: Set[str], ca_export: CrowdAnkiExport) -> (List[MediaFile], List[str]): + resolved_media: List[MediaFile] = [] + missing_media: List[str] = [] + for filename in filenames: + if filename in ca_export.known_media.keys(): + resolved_media.append(ca_export.known_media[filename]) + else: + missing_media.append(filename) + return resolved_media, missing_media + + @classmethod + def resolve_media_references_to_deck_parts(cls, filenames: Set[str]) -> (List[MediaFile], List[str]): + resolved_media: List[MediaFile] = [] + missing_media: List[str] = [] + for filename in filenames: + media_file = cls.file_manager.media_file_if_exists(filename) + if media_file: + resolved_media.append(media_file) + else: + missing_media.append(filename) + return resolved_media, missing_media + + @classmethod + def _move_ca(cls, media_files: List[MediaFile], ca_export: CrowdAnkiExport): + for file in media_files: + if file.filename in ca_export.known_media: + ca_export.known_media[file.filename].set_override(file.source_loc) + else: + ca_export.known_media.setdefault( + file.filename, MediaFile(ca_export.media_loc + file.filename, + file.filename, MediaFile.ManagementType.TO_BE_CLONED, file.source_loc) + ) + + @classmethod + def _move_dps(cls, media_files: List[MediaFile]): + for file in media_files: + dp_media_file = cls.file_manager.media_file_if_exists(file.filename) + if dp_media_file: + dp_media_file.set_override(file.source_loc) + else: + cls.file_manager.new_media_file(file.filename, file.source_loc) diff --git a/brain_brew/build_tasks/crowd_anki/note_models_to_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/note_models_to_crowd_anki.py index f2e4c56..d869627 100644 --- a/brain_brew/build_tasks/crowd_anki/note_models_to_crowd_anki.py +++ b/brain_brew/build_tasks/crowd_anki/note_models_to_crowd_anki.py @@ -2,6 +2,7 @@ from typing import Optional, Union, List import logging +from brain_brew.representation.build_config.representation_base import RepresentationBase from brain_brew.representation.json.wrappers_for_crowd_anki import CrowdAnkiJsonWrapper from brain_brew.representation.transformers.base_deck_part_from import BaseDeckPartsFrom from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder @@ -10,13 +11,49 @@ @dataclass class NoteModelsToCrowdAnki: + @dataclass + class NoteModelListItem: + @dataclass + class Representation(RepresentationBase): + deck_part: str + # TODO: fields: Optional[List[str]] + # TODO: templates: Optional[List[str]] + + @classmethod + def from_repr(cls, data: Union[Representation, dict, str]): + rep: cls.Representation + if isinstance(data, cls.Representation): + rep = data + elif isinstance(data, dict): + rep = cls.Representation.from_dict(data) + else: + rep = cls.Representation(deck_part=data) # Support string + + return cls( + deck_part=DeckPartHolder.from_deck_part_pool(rep.deck_part) + ) + + deck_part: NoteModel + + @dataclass + class Representation(RepresentationBase): + deck_parts: List[Union[dict, str]] + @classmethod - def from_list(cls, note_model_items: List[str]): + def from_repr(cls, data: Union[Representation, dict, List[str]]): + rep: cls.Representation + if isinstance(data, cls.Representation): + rep = data + elif isinstance(data, dict): + rep = cls.Representation.from_dict(data) + else: + rep = cls.Representation(deck_parts=data) # Support list of Note Models + return cls( - note_models=list(map(DeckPartHolder.from_deck_part_pool, note_model_items)) + note_models=list(map(cls.NoteModelListItem.from_repr, rep.deck_parts)) ) - note_models: List[NoteModel] + note_models: List[NoteModelListItem] def execute(self) -> List[dict]: - return [model.encode_as_crowdanki() for model in self.note_models] + return [model.deck_part.encode_as_crowdanki() for model in self.note_models] diff --git a/brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py index 4173801..f064912 100644 --- a/brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py +++ b/brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py @@ -1,13 +1,11 @@ from dataclasses import dataclass -from typing import Optional, Union, List +from typing import Union from brain_brew.build_tasks.crowd_anki.shared_base_notes import SharedBaseNotes -from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport -from brain_brew.representation.json.wrappers_for_crowd_anki import CrowdAnkiJsonWrapper +from brain_brew.representation.json.wrappers_for_crowd_anki import CrowdAnkiJsonWrapper, CrowdAnkiNoteWrapper from brain_brew.representation.transformers.base_deck_part_from import BaseDeckPartsFrom from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder -from brain_brew.representation.yaml.note_repr import Notes -from brain_brew.transformers.transform_crowdanki import TransformCrowdAnki +from brain_brew.representation.yaml.note_repr import Notes, Note @dataclass @@ -26,10 +24,22 @@ def from_repr(cls, data: Union[Representation, dict]): ) def execute(self, ca_wrapper: CrowdAnkiJsonWrapper, nm_id_to_name: dict) -> Notes: - note_list = TransformCrowdAnki.crowd_anki_to_notes(ca_wrapper.notes, nm_id_to_name) + note_list = [self.ca_note_to_note(note, nm_id_to_name) for note in ca_wrapper.notes] notes = Notes.from_list_of_notes(note_list) # TODO: pass in sort method DeckPartHolder.override_or_create(self.name, self.save_to_file, notes) return notes + + @staticmethod + def ca_note_to_note(note: dict, nm_id_to_name: dict) -> Note: + wrapper = CrowdAnkiNoteWrapper(note) + + return Note( + note_model=nm_id_to_name[wrapper.note_model], + tags=wrapper.tags, + guid=wrapper.guid, + fields=wrapper.fields, + flags=wrapper.flags + ) diff --git a/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py index 4a634c1..2820bf7 100644 --- a/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py +++ b/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py @@ -2,23 +2,24 @@ from typing import Optional, Union, List from brain_brew.build_tasks.crowd_anki.shared_base_notes import SharedBaseNotes +from brain_brew.representation.json.wrappers_for_crowd_anki import CrowdAnkiNoteWrapper from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder -from brain_brew.representation.yaml.note_repr import Notes -from brain_brew.transformers.transform_crowdanki import TransformCrowdAnki +from brain_brew.representation.yaml.note_repr import Notes, Note +from brain_brew.utils import blank_str_if_none @dataclass class NotesToCrowdAnki(SharedBaseNotes): @dataclass class Representation(SharedBaseNotes.Representation): - name: str + deck_part: str additional_items_to_add: Optional[dict] @classmethod def from_repr(cls, data: Union[Representation, dict]): rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) return cls( - notes=DeckPartHolder.from_deck_part_pool(rep.name), + notes=DeckPartHolder.from_deck_part_pool(rep.deck_part), sort_order=SharedBaseNotes._get_sort_order(rep.sort_order), additional_items_to_add=rep.additional_items_to_add or {} ) @@ -29,8 +30,26 @@ def from_repr(cls, data: Union[Representation, dict]): def execute(self, nm_name_to_id: dict) -> List[dict]: notes = self.notes.get_notes() - note_dicts = TransformCrowdAnki.notes_to_crowd_anki(notes, nm_name_to_id, self.additional_items_to_add) + note_dicts = [self.note_to_ca_note(note, nm_name_to_id, self.additional_items_to_add) for note in notes] # TODO: Sort return note_dicts + + @staticmethod + def note_to_ca_note(note: Note, nm_name_to_id: dict, additional_items_to_add: dict) -> dict: + wrapper = CrowdAnkiNoteWrapper({ + "__type__": "Note", + "data": None + }) + + for key, value in additional_items_to_add.items(): + wrapper.data[key] = blank_str_if_none(value) + + wrapper.guid = note.guid + wrapper.fields = note.fields + wrapper.tags = note.tags + wrapper.note_model = nm_name_to_id[note.note_model] + wrapper.flags = note.flags + + return wrapper.data diff --git a/brain_brew/build_tasks/csvs/csvs_generate.py b/brain_brew/build_tasks/csvs/csvs_generate.py index 77f8346..58c9d50 100644 --- a/brain_brew/build_tasks/csvs/csvs_generate.py +++ b/brain_brew/build_tasks/csvs/csvs_generate.py @@ -3,10 +3,10 @@ from brain_brew.build_tasks.csvs.shared_base_csvs import SharedBaseCsvs from brain_brew.representation.build_config.build_task import TopLevelBuildTask +from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder from brain_brew.representation.yaml.note_repr import Notes, Note -from brain_brew.transformers.transform_csv_collection import TransformCsvCollection -from brain_brew.utils import all_combos_prepend_append +from brain_brew.utils import all_combos_prepend_append, join_tags @dataclass @@ -36,13 +36,14 @@ def execute(self): notes: List[Note] = self.notes.deck_part.get_notes() self.verify_notes_match_note_model_mappings(notes) - csv_data: Dict[str, dict] = TransformCsvCollection.notes_to_csv_collection(notes, self.note_model_mappings) + csv_data: List[dict] = [self.note_to_csv_row(note, self.note_model_mappings) for note in notes] + rows_by_guid = {row["guid"]: row for row in csv_data} # TODO: Dry run option, to not save anything at this stage for fm in self.file_mappings: fm.compile_data() - fm.set_relevant_data(csv_data) + fm.set_relevant_data(rows_by_guid) fm.write_file_on_close() def verify_notes_match_note_model_mappings(self, notes: List[Note]): @@ -53,3 +54,13 @@ def verify_notes_match_note_model_mappings(self, notes: List[Note]): if errors: raise Exception(errors) + + @staticmethod + def note_to_csv_row(note: Note, note_model_mappings: Dict[str, NoteModelMapping]) -> dict: + nm_name = note.note_model + row = note_model_mappings[nm_name].note_models[nm_name].deck_part.zip_field_to_data(note.fields) + row["guid"] = note.guid + row["tags"] = join_tags(note.tags) + # TODO: Flags? + + return note_model_mappings[nm_name].note_fields_map_to_csv_row(row) # TODO: Do not edit data, make copy diff --git a/brain_brew/build_tasks/csvs/notes_from_csvs.py b/brain_brew/build_tasks/csvs/notes_from_csvs.py index ccc087f..0655505 100644 --- a/brain_brew/build_tasks/csvs/notes_from_csvs.py +++ b/brain_brew/build_tasks/csvs/notes_from_csvs.py @@ -2,10 +2,11 @@ from typing import Dict, List, Union from brain_brew.build_tasks.csvs.shared_base_csvs import SharedBaseCsvs +from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping from brain_brew.representation.transformers.base_deck_part_from import BaseDeckPartsFrom from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder from brain_brew.representation.yaml.note_repr import Note, Notes -from brain_brew.transformers.transform_csv_collection import TransformCsvCollection +from brain_brew.utils import split_tags @dataclass @@ -33,8 +34,22 @@ def execute(self): csv_data_by_guid = {**csv_data_by_guid, **csv_map.compiled_data} csv_rows: List[dict] = list(csv_data_by_guid.values()) - deck_part_notes: List[Note] = TransformCsvCollection.csv_collection_to_notes( - csv_rows, self.note_model_mappings) + deck_part_notes: List[Note] = [self.csv_row_to_note(row, self.note_model_mappings) for row in csv_rows] notes = Notes.from_list_of_notes(deck_part_notes) DeckPartHolder.override_or_create(self.name, self.save_to_file, notes) + + @staticmethod + def csv_row_to_note(row: dict, note_model_mappings: Dict[str, NoteModelMapping]) -> Note: + note_model_name = row["note_model"] # TODO: Use object + row_nm: NoteModelMapping = note_model_mappings[note_model_name] + + filtered_fields = row_nm.csv_row_map_to_note_fields(row) + + guid = filtered_fields.pop("guid") + tags = split_tags(filtered_fields.pop("tags")) + flags = filtered_fields.pop("flags") if "flags" in filtered_fields else 0 + + fields = row_nm.field_values_in_note_model_order(note_model_name, filtered_fields) + + return Note(guid=guid, tags=tags, note_model=note_model_name, fields=fields, flags=flags) diff --git a/brain_brew/build_tasks/csvs/shared_base_csvs.py b/brain_brew/build_tasks/csvs/shared_base_csvs.py index d5bddae..8264610 100644 --- a/brain_brew/build_tasks/csvs/shared_base_csvs.py +++ b/brain_brew/build_tasks/csvs/shared_base_csvs.py @@ -67,4 +67,4 @@ def verify_contents(self): errors.append(KeyError(f"Csvs are missing columns from {holder.name}", missing_columns)) if errors: - raise Exception(errors) \ No newline at end of file + raise Exception(errors) diff --git a/brain_brew/build_tasks/task_builder.py b/brain_brew/build_tasks/task_builder.py index 5af0ec7..cde807b 100644 --- a/brain_brew/build_tasks/task_builder.py +++ b/brain_brew/build_tasks/task_builder.py @@ -43,7 +43,7 @@ def read_tasks(cls, tasks: List[dict]) -> list: task_name = str_to_lowercase_no_separators(task_keys[0]) task_arguments = task[task_keys[0]] if task_name in known_task_dict: - task_instance = known_task_dict[task_name].from_dict(task_arguments) + task_instance = known_task_dict[task_name].from_repr(task_arguments) build_tasks.append(task_instance) else: raise KeyError(f"Unknown task '{task_name}'") # TODO: check this first on all and return all errors @@ -59,4 +59,3 @@ def execute(self): for task in self.tasks: task.execute() - self.file_manager.write_to_all() diff --git a/brain_brew/file_manager.py b/brain_brew/file_manager.py index a633057..9513eba 100644 --- a/brain_brew/file_manager.py +++ b/brain_brew/file_manager.py @@ -42,6 +42,7 @@ def clear_instance(): if FileManager.__instance: FileManager.__instance = None + # If Exists def file_if_exists(self, file_location) -> Union[SourceFile, None]: if file_location in self.known_files_dict.keys(): return self.known_files_dict[file_location] @@ -50,16 +51,22 @@ def file_if_exists(self, file_location) -> Union[SourceFile, None]: def deck_part_if_exists(self, dp_name) -> Union[DeckPartHolder[YamlRepr], None]: return self.deck_part_pool.get(dp_name) + def media_file_if_exists(self, filename) -> Union[MediaFile, None]: + if filename in self.known_media_files_dict.keys(): + return self.known_media_files_dict[filename] + return None + + # Registration def register_file(self, full_path, file): if full_path in self.known_files_dict: raise FileExistsError(f"File already known to FileManager, cannot be registered twice: {full_path}") self.known_files_dict.setdefault(full_path, file) - def media_file_if_exists(self, filename) -> Union[MediaFile, None]: - if filename in self.known_media_files_dict.keys(): - return self.known_media_files_dict[filename] - return None - + def new_media_file(self, filename, source_loc) -> MediaFile: + self._register_media_file(MediaFile(self.global_config.deck_parts.media_files + filename, + filename, MediaFile.ManagementType.TO_BE_CLONED, source_loc)) + return self.known_media_files_dict[filename] + def _register_media_file(self, file: MediaFile): if file.filename not in self.known_media_files_dict: self.known_media_files_dict.setdefault(file.filename, file) @@ -67,10 +74,13 @@ def _register_media_file(self, file: MediaFile): logging.error(f"Duplicate media file '{file.filename}' in both '{file.source_loc}'" f" and '{self.known_media_files_dict[file.filename].source_loc}'") - def new_media_file(self, filename, source_loc): - self._register_media_file(MediaFile(self.global_config.deck_parts.media_files + filename, - filename, MediaFile.ManagementType.TO_BE_CLONED, source_loc)) + def new_deck_part(self, dp: DeckPartHolder) -> DeckPartHolder: + if dp.name in self.deck_part_pool: + raise KeyError(f"Cannot use same name '{dp.name}' for multiple Deck Parts") + self.deck_part_pool.setdefault(dp.name, dp) + return dp + # Gets def find_all_deck_part_media_files(self): self.known_media_files_dict = {} @@ -80,19 +90,7 @@ def find_all_deck_part_media_files(self): logging.debug(f"DeckPart Media files found: {len(self.known_media_files_dict)}") - def new_deck_part(self, dp: DeckPartHolder) -> DeckPartHolder: - if dp.name in self.deck_part_pool: - raise KeyError(f"Cannot use same name '{dp.name}' for multiple Deck Parts") - self.deck_part_pool.setdefault(dp.name, dp) - return dp - def deck_part_from_pool(self, name: str): if name not in self.deck_part_pool: raise KeyError(f"Cannot find Deck Part '{name}'") return self.deck_part_pool[name] - - def write_to_all(self): - # logging.info(f"Will create {len(files_to_create)} new files: ", files_to_create) - - for filename, media_file in self.known_media_files_dict.items(): - media_file.copy_source_to_target() diff --git a/brain_brew/representation/json/crowd_anki_export.py b/brain_brew/representation/json/crowd_anki_export.py index 6588cf7..e9059b3 100644 --- a/brain_brew/representation/json/crowd_anki_export.py +++ b/brain_brew/representation/json/crowd_anki_export.py @@ -63,8 +63,8 @@ def find_all_media(self): def write_to_files(self, json_data): # import_config_data JsonFile.write_file(self.json_file_location, json_data) - for filename, media_file in self.known_media.items(): - media_file.copy_source_to_target() + # for filename, media_file in self.known_media.items(): + # media_file.copy_source_to_target() def read_json_file(self) -> CrowdAnkiJsonWrapper: return CrowdAnkiJsonWrapper(JsonFile.read_file(self.json_file_location)) diff --git a/brain_brew/representation/yaml/note_model_repr.py b/brain_brew/representation/yaml/note_model_repr.py index 5fec649..e12dbb5 100644 --- a/brain_brew/representation/yaml/note_model_repr.py +++ b/brain_brew/representation/yaml/note_model_repr.py @@ -1,6 +1,6 @@ from collections import OrderedDict from dataclasses import dataclass, field -from typing import List, Optional, Union, Dict +from typing import List, Optional, Union, Dict, Set from brain_brew.representation.build_config.representation_base import RepresentationBase from brain_brew.representation.yaml.my_yaml import YamlRepr @@ -84,7 +84,7 @@ def from_crowdanki(cls, data: Union[CrowdAnki, dict]): question_format_in_browser=ca.bqfmt, answer_format_in_browser=ca.bafmt, deck_override_id=ca.did ) - def get_all_media_references(self) -> set: + def get_all_media_references(self) -> Set[str]: all_media = set()\ .union(find_media_in_field(self.question_format))\ .union(find_media_in_field(self.answer_format))\ @@ -261,7 +261,7 @@ def encode(self) -> dict: return data_dict - def get_all_media_references(self) -> set: + def get_all_media_references(self) -> Set[str]: all_media = set() for template in self.templates: all_media = all_media.union(template.get_all_media_references()) diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index 1985097..5550870 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -75,7 +75,6 @@ def encode(self) -> dict: # TODO: Extract Shared Tags and Note Models # TODO: Sort notes - # TODO: Set data def verify_groupings(self): errors = [] @@ -88,7 +87,7 @@ def verify_groupings(self): def get_all_known_note_model_names(self) -> set: return {self.note_model} if self.note_model else {note.note_model for note in self.notes} - def get_all_media_references(self) -> set: + def get_all_media_references(self) -> Set[str]: all_media = set() for note in self.notes: for media in note.get_media_references(): @@ -135,7 +134,7 @@ def encode(self) -> dict: def get_all_known_note_model_names(self): return {nms for group in self.note_groupings for nms in group.get_all_known_note_model_names()} - def get_all_media_references(self) -> set: + def get_all_media_references(self) -> Set[str]: all_media = set() for note in self.note_groupings: for media in note.get_all_media_references(): diff --git a/brain_brew/transformers/__init__.py b/brain_brew/transformers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/brain_brew/transformers/base_transform_notes.py b/brain_brew/transformers/base_transform_notes.py deleted file mode 100644 index c3b2bf1..0000000 --- a/brain_brew/transformers/base_transform_notes.py +++ /dev/null @@ -1,16 +0,0 @@ -import re - -from brain_brew.representation.configuration.global_config import GlobalConfig - - -class TrNotes: - @staticmethod - def split_tags(tags_value: str) -> list: - split = [entry.strip() for entry in re.split(';\s*|,\s*|\s+', tags_value)] - while "" in split: - split.remove("") - return split - - @staticmethod - def join_tags(tags_list: list) -> str: - return GlobalConfig.get_instance().flags.join_values_with.join(tags_list) diff --git a/brain_brew/transformers/transform_crowdanki.py b/brain_brew/transformers/transform_crowdanki.py deleted file mode 100644 index 67814ee..0000000 --- a/brain_brew/transformers/transform_crowdanki.py +++ /dev/null @@ -1,67 +0,0 @@ -from dataclasses import dataclass -from typing import List, Optional, Union - -from brain_brew.representation.json.wrappers_for_crowd_anki import CrowdAnkiNoteWrapper -from brain_brew.representation.yaml.note_repr import Note -from brain_brew.transformers.base_transform_notes import TrNotes -from brain_brew.utils import blank_str_if_none - -from brain_brew.representation.json.wrappers_for_crowd_anki import CA_NOTE_MODELS, CA_NOTES, CA_MEDIA_FILES,\ - CA_CHILDREN, CA_TYPE - - -class TransformCrowdAnki(TrNotes): - headers_skip_keys = [CA_NOTE_MODELS, CA_NOTES, CA_MEDIA_FILES] - headers_default_values = { - CA_TYPE: "Deck", - CA_CHILDREN: [], - } - - @classmethod - def notes_to_crowd_anki(cls, notes: List[Note], nm_name_to_id: dict, additional_items_to_add: dict) -> List[dict]: - resolved_notes: List[dict] = [] - wrapper: CrowdAnkiNoteWrapper = CrowdAnkiNoteWrapper() - for note in notes: - current = { - "__type__": "Note", - "data": None - } - wrapper.data = current - - for key, value in additional_items_to_add.items(): - current[key] = blank_str_if_none(value) - - wrapper.guid = note.guid - wrapper.fields = note.fields - wrapper.tags = note.tags - wrapper.note_model = nm_name_to_id[note.note_model] - wrapper.flags = note.flags - - resolved_notes.append(current) - - return resolved_notes - - @classmethod - def crowd_anki_to_notes(cls, notes_json: list, nm_id_to_name: dict) -> List[Note]: - resolved_notes: List[Note] = [] - wrapper: CrowdAnkiNoteWrapper = CrowdAnkiNoteWrapper() - for note in notes_json: - wrapper.data = note - - resolved_notes.append(Note( - note_model=nm_id_to_name[wrapper.note_model], - tags=wrapper.tags, - guid=wrapper.guid, - fields=wrapper.fields, - flags=wrapper.flags - )) - return resolved_notes - - @classmethod - def headers_to_crowd_anki(cls, headers_data: dict): - return {**headers_data, **cls.headers_default_values} - - @classmethod - def crowd_anki_to_headers(cls, ca_data: dict): - return {key: value for key, value in ca_data - if key not in cls.headers_skip_keys and key not in cls.headers_default_values.keys()} diff --git a/brain_brew/transformers/transform_csv_collection.py b/brain_brew/transformers/transform_csv_collection.py deleted file mode 100644 index 5bf66ca..0000000 --- a/brain_brew/transformers/transform_csv_collection.py +++ /dev/null @@ -1,44 +0,0 @@ -from typing import List, Dict - - -from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping -from brain_brew.representation.yaml.note_repr import Note -from brain_brew.transformers.base_transform_notes import TrNotes - - -class TransformCsvCollection(TrNotes): - @classmethod - def notes_to_csv_collection(cls, notes: List[Note], note_model_mappings: Dict[str, NoteModelMapping]) -> Dict[str, dict]: - csv_data: Dict[str, dict] = {} - for note in notes: - nm_name = note.note_model - row = note_model_mappings[nm_name].note_models[nm_name].deck_part.zip_field_to_data(note.fields) - row["guid"] = note.guid - row["tags"] = cls.join_tags(note.tags) - - formatted_row = note_model_mappings[nm_name].note_fields_map_to_csv_row(row) # TODO: Do not edit data, make copy - - csv_data.setdefault(row["guid"], formatted_row) - - return csv_data - - @classmethod - def csv_collection_to_notes(cls, csv_rows: List[dict], note_model_mappings: Dict[str, NoteModelMapping]) -> List[Note]: - deck_part_notes: List[Note] = [] - - # Get Guid, Tags, NoteTypeName, Fields - for row in csv_rows: - note_model_name = row["note_model"] # TODO: Use object - row_nm: NoteModelMapping = note_model_mappings[note_model_name] - - filtered_fields = row_nm.csv_row_map_to_note_fields(row) - - guid = filtered_fields.pop("guid") - tags = cls.split_tags(filtered_fields.pop("tags")) - flags = filtered_fields.pop("flags") if "flags" in filtered_fields else 0 - - fields = row_nm.field_values_in_note_model_order(note_model_name, filtered_fields) - - deck_part_notes.append(Note(guid=guid, tags=tags, note_model=note_model_name, fields=fields, flags=flags)) - - return deck_part_notes diff --git a/brain_brew/utils.py b/brain_brew/utils.py index e2035cc..f045522 100644 --- a/brain_brew/utils.py +++ b/brain_brew/utils.py @@ -5,6 +5,8 @@ import re from typing import List +from brain_brew.representation.configuration.global_config import GlobalConfig + def blank_str_if_none(s): return '' if s is None else s @@ -56,6 +58,17 @@ def find_all_files_in_directory(directory, recursive=False): return found_files +def split_tags(tags_value: str) -> list: + split = [entry.strip() for entry in re.split(';\s*|,\s*|\s+', tags_value)] + while "" in split: + split.remove("") + return split + + +def join_tags(tags_list: list) -> str: + return GlobalConfig.get_instance().flags.join_values_with.join(tags_list) + + def generate_anki_guid() -> str: """Return a base91-encoded 64bit random number.""" From 4c245d1451eb19f2853cba5c626891bd0807240a Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sat, 29 Aug 2020 14:54:06 +0200 Subject: [PATCH 27/39] DeckPart Reader; Global Config; --- .../build_tasks/csvs/shared_base_csvs.py | 12 +- .../deck_parts}/__init__.py | 0 .../build_tasks/deck_parts/from_deck_part.py | 45 +++++ brain_brew/build_tasks/source_crowd_anki.py | 154 ------------------ brain_brew/constants/build_config_keys.py | 12 -- brain_brew/constants/deckpart_keys.py | 36 ---- brain_brew/constants/global_config_keys.py | 19 --- brain_brew/main.py | 4 +- .../build_config}/generate_deck_parts.py | 2 +- .../build_config}/task_builder.py | 8 +- .../build_config/top_level_task_builder.py | 2 +- .../configuration/csv_file_mapping.py | 10 +- .../configuration/global_config.py | 119 ++++++-------- .../representation/generic/generic_file.py | 2 +- .../representation/yaml/headers_repr.py | 4 + .../representation/yaml/note_model_repr.py | 4 + brain_brew/representation/yaml/note_repr.py | 4 + brain_brew/utils.py | 2 +- tests/build_tasks/test_source_csv.py | 6 +- .../configuration/test_csv_file_mapping.py | 22 +-- 20 files changed, 139 insertions(+), 328 deletions(-) rename brain_brew/{constants => build_tasks/deck_parts}/__init__.py (100%) create mode 100644 brain_brew/build_tasks/deck_parts/from_deck_part.py delete mode 100644 brain_brew/build_tasks/source_crowd_anki.py delete mode 100644 brain_brew/constants/build_config_keys.py delete mode 100644 brain_brew/constants/deckpart_keys.py delete mode 100644 brain_brew/constants/global_config_keys.py rename brain_brew/{build_tasks => representation/build_config}/generate_deck_parts.py (84%) rename brain_brew/{build_tasks => representation/build_config}/task_builder.py (88%) diff --git a/brain_brew/build_tasks/csvs/shared_base_csvs.py b/brain_brew/build_tasks/csvs/shared_base_csvs.py index 8264610..3c3f70f 100644 --- a/brain_brew/build_tasks/csvs/shared_base_csvs.py +++ b/brain_brew/build_tasks/csvs/shared_base_csvs.py @@ -2,7 +2,7 @@ from typing import List, Dict from brain_brew.representation.build_config.representation_base import RepresentationBase -from brain_brew.representation.configuration.csv_file_mapping import CsvFileMapping +from brain_brew.representation.configuration.csv_file_mapping import FileMapping from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping @@ -10,15 +10,15 @@ class SharedBaseCsvs: @dataclass(init=False) class Representation(RepresentationBase): - file_mappings: List[CsvFileMapping.Representation] + file_mappings: List[FileMapping.Representation] note_model_mappings: List[NoteModelMapping.Representation] def __init__(self, file_mappings, note_model_mappings): - self.file_mappings = list(map(CsvFileMapping.Representation.from_dict, file_mappings)) + self.file_mappings = list(map(FileMapping.Representation.from_dict, file_mappings)) self.note_model_mappings = list(map(NoteModelMapping.Representation.from_dict, note_model_mappings)) - def get_file_mappings(self) -> List[CsvFileMapping]: - return list(map(CsvFileMapping.from_repr, self.file_mappings)) + def get_file_mappings(self) -> List[FileMapping]: + return list(map(FileMapping.from_repr, self.file_mappings)) def get_note_model_mappings(self) -> Dict[str, NoteModelMapping]: def map_nmm(nmm_to_map: str): @@ -27,7 +27,7 @@ def map_nmm(nmm_to_map: str): return dict(*map(map_nmm, self.note_model_mappings)) - file_mappings: List[CsvFileMapping] + file_mappings: List[FileMapping] note_model_mappings: Dict[str, NoteModelMapping] def verify_contents(self): diff --git a/brain_brew/constants/__init__.py b/brain_brew/build_tasks/deck_parts/__init__.py similarity index 100% rename from brain_brew/constants/__init__.py rename to brain_brew/build_tasks/deck_parts/__init__.py diff --git a/brain_brew/build_tasks/deck_parts/from_deck_part.py b/brain_brew/build_tasks/deck_parts/from_deck_part.py new file mode 100644 index 0000000..138ecf8 --- /dev/null +++ b/brain_brew/build_tasks/deck_parts/from_deck_part.py @@ -0,0 +1,45 @@ +from dataclasses import dataclass +from typing import Union, List, Optional + +from brain_brew.representation.build_config.build_task import DeckPartBuildTask +from brain_brew.representation.build_config.representation_base import RepresentationBase +from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder +from brain_brew.representation.yaml.note_repr import Notes +from brain_brew.utils import all_combos_prepend_append + + +@dataclass +class FromDeckParts(DeckPartBuildTask): + task_names = all_combos_prepend_append(["DeckPart", "ReadDeckPart"], "From ", "s") + + @dataclass + class DeckPartToRead(RepresentationBase): + name: str + file: str + + @dataclass + class Representation(RepresentationBase): + notes: Optional[List[dict]] + note_models: Optional[List[dict]] + headers: Optional[List[dict]] + + notes: List[DeckPartToRead] + note_models: List[DeckPartToRead] + headers: List[DeckPartToRead] + + @classmethod + def from_repr(cls, data: Union[Representation, dict]): + rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) + return cls( + notes=list(map(FromDeckParts.DeckPartToRead.from_dict, rep.notes)), + note_models=list(map(FromDeckParts.DeckPartToRead.from_dict, rep.notes)), + headers=list(map(FromDeckParts.DeckPartToRead.from_dict, rep.notes)) + ) + + def execute(self): + for note in self.notes: + DeckPartHolder.override_or_create(name=note.name, save_to_file=None, deck_part=Notes.from_file(note.file)) + for model in self.note_models: + DeckPartHolder.override_or_create(name=model.name, save_to_file=None, deck_part=Notes.from_file(model.file)) + for header in self.headers: + DeckPartHolder.override_or_create(name=header.name, save_to_file=None, deck_part=Notes.from_file(header.file)) diff --git a/brain_brew/build_tasks/source_crowd_anki.py b/brain_brew/build_tasks/source_crowd_anki.py deleted file mode 100644 index 307bd68..0000000 --- a/brain_brew/build_tasks/source_crowd_anki.py +++ /dev/null @@ -1,154 +0,0 @@ -from collections import OrderedDict -from enum import Enum - -from brain_brew.build_tasks.build_task_generic import BuildTaskGeneric -from brain_brew.constants.build_config_keys import BuildTaskEnum, BuildConfigKeys -from brain_brew.constants.deckpart_keys import DeckPartNoteKeys -from brain_brew.file_manager import FileManager -from brain_brew.representation.generic.media_file import MediaFile -from brain_brew.utils import blank_str_if_none -from brain_brew.representation.generic.yaml_file import ConfigKey, YamlFile -from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport, CAKeys -from brain_brew.representation.json.deck_part_header import DeckPartHeader -from brain_brew.representation.yaml.note_model_repr import CANoteModelKeys, DeckPartNoteModel -from brain_brew.representation.json.deck_part_notes import CANoteKeys, DeckPartNotes - - -class SourceCrowdAnki(YamlFile, BuildTaskGeneric): - @staticmethod - def get_build_keys(): - return [ - BuildTaskEnum("deck_parts_to_crowdanki", SourceCrowdAnki, "deck_parts_to_source", "source_to_deck_parts"), - BuildTaskEnum("crowdanki_to_deck_parts", SourceCrowdAnki, "source_to_deck_parts", "deck_parts_to_source"), - ] - - config_entry = {} - expected_keys = { - BuildConfigKeys.NOTES.value: ConfigKey(True, str, None), - BuildConfigKeys.HEADERS.value: ConfigKey(True, str, None), - - CrowdAnkiKeys.FILE.value: ConfigKey(True, str, None), - CrowdAnkiKeys.NOTE_SORT_ORDER.value: ConfigKey(False, list, None), - CrowdAnkiKeys.MEDIA.value: ConfigKey(True, bool, None), - CrowdAnkiKeys.USELESS_NOTE_KEYS.value: ConfigKey(True, dict, None) - } - subconfig_filter = None - file_manager: FileManager - - headers: DeckPartHeader - notes: DeckPartNotes - - crowd_anki_export: CrowdAnkiExport - should_handle_media: bool - useless_note_keys: dict - - def __init__(self, config_data: dict, read_now=True): - self.setup_config_with_subconfig_replacement(config_data) - self.verify_config_entry() - - self.file_manager = FileManager.get_instance() - - self.headers = DeckPartHeader.create(self.config_entry[BuildConfigKeys.HEADERS.value], read_now=read_now) - self.notes = DeckPartNotes.create(self.config_entry[BuildConfigKeys.NOTES.value], read_now=read_now) - self.crowd_anki_export = CrowdAnkiExport.create_or_get(self.config_entry[CrowdAnkiKeys.FILE.value], read_now=read_now) - - self.should_handle_media = self.config_entry[CrowdAnkiKeys.MEDIA.value] - self.useless_note_keys = self.config_entry[CrowdAnkiKeys.USELESS_NOTE_KEYS.value] - - @classmethod - def from_yaml(cls, yaml_file_name, read_now=True): - config_data = YamlFile.read_file(yaml_file_name) - - return SourceCrowdAnki(config_data, read_now=read_now) - - def notes_to_deck_parts(self, notes_json, note_models_id_name_dict): - for note in notes_json: - for key in self.useless_note_keys: - if key in note: - del note[key] - # TODO: else raise error? - - if note[CANoteKeys.NOTE_MODEL.value] in note_models_id_name_dict: - note[DeckPartNoteKeys.NOTE_MODEL.value] = note_models_id_name_dict[note[CANoteKeys.NOTE_MODEL.value]] - del note[CANoteKeys.NOTE_MODEL.value] - else: - raise KeyError(f"Unknown NoteModel '{note[CANoteKeys.NOTE_MODEL.value]}'") - - return notes_json - - def source_to_deck_parts(self): - source_data = self.crowd_anki_export.get_data(deep_copy=True) - - # Headers - header_keys_to_ignore = {CAKeys.NOTE_MODELS.value, CAKeys.NOTES.value, CAKeys.MEDIA_FILES.value} - - headers_data = {key: source_data[key] for key in source_data if key not in header_keys_to_ignore} - self.headers.set_data(headers_data) - - # Note Models - note_models = [ - DeckPartNoteModel.create_or_get(model[CANoteModelKeys.NAME.value], data_override=model) - for model in source_data[CAKeys.NOTE_MODELS.value] - ] - - note_models_id_name_dict = {model.id: model.name for model in note_models} - - # Media - for filename, file in self.crowd_anki_export.known_media.items(): - dp_media_file = self.file_manager.media_file_if_exists(filename) - if dp_media_file: - dp_media_file.set_override(file.source_loc) - else: - self.file_manager.new_media_file(filename, file.source_loc) - - # Notes - notes_json = source_data[CAKeys.NOTES.value] - notes_data = self.notes_to_deck_parts(notes_json, note_models_id_name_dict) - - self.notes.set_data(notes_data) - - def notes_to_source(self, note_models_dict_id_name): - res_notes = self.notes.get_data(deep_copy=True)[DeckPartNoteKeys.NOTES.value] - - for note in res_notes: - for key in self.useless_note_keys: - note[key] = blank_str_if_none(self.useless_note_keys[key]) - - note[CANoteKeys.NOTE_MODEL.value] = note_models_dict_id_name[note[DeckPartNoteKeys.NOTE_MODEL.value]] - del note[DeckPartNoteKeys.NOTE_MODEL.value] - - return [OrderedDict(sorted(note.items())) for note in res_notes] - - def deck_parts_to_source(self): - ca_json = {} - - # Headers - ca_json.update(self.headers.get_data()) - - # Media - media_files = self.notes.referenced_media_files - ca_json.setdefault(CAKeys.MEDIA_FILES.value, list(sorted([file.filename for file in media_files]))) - - for file in media_files: - filename = file.filename - if filename in self.crowd_anki_export.known_media: - self.crowd_anki_export.known_media[filename].set_override(file.source_loc) - else: - self.crowd_anki_export.known_media.setdefault( - filename, MediaFile(self.crowd_anki_export.media_loc + filename, - filename, MediaFile.ManagementType.TO_BE_CLONED, file.source_loc) - ) - - # Note Models - note_models = [DeckPartNoteModel.create_or_get(name) for name in self.notes.get_all_known_note_model_names()] - - ca_json.setdefault(CAKeys.NOTE_MODELS.value, [model.get_data() for model in note_models]) - - note_models_dict_id_name = {model.name: model.id for model in note_models} - - # Notes - ca_json.setdefault(CAKeys.NOTES.value, self.notes_to_source(note_models_dict_id_name)) - - ordered_keys = OrderedDict(sorted(ca_json.items())) - - self.crowd_anki_export.set_data(ordered_keys) diff --git a/brain_brew/constants/build_config_keys.py b/brain_brew/constants/build_config_keys.py deleted file mode 100644 index a9c82b6..0000000 --- a/brain_brew/constants/build_config_keys.py +++ /dev/null @@ -1,12 +0,0 @@ -from collections import namedtuple -from enum import Enum - -# TODO: Make this a class -BuildTaskEnum = namedtuple("BuildTaskEnum", "key_name source_type task_to_execute reverse_task_to_execute") - - -class BuildConfigKeys(Enum): - SUBCONFIG = "subconfig" - - NOTES = "notes" - HEADERS = "headers" diff --git a/brain_brew/constants/deckpart_keys.py b/brain_brew/constants/deckpart_keys.py deleted file mode 100644 index c54c69f..0000000 --- a/brain_brew/constants/deckpart_keys.py +++ /dev/null @@ -1,36 +0,0 @@ -from dataclasses import dataclass -from enum import Enum - - -class DeckPartNoteKeys(Enum): - NOTES = "notes" - FIELDS = "fields" - GUID = "guid" - NOTE_MODEL = "note_model" - TAGS = "tags" - SHARED_TAGS = "_shared_tags" - FLAGS = "_flags" - - -class NoteFlagKeys(Enum): - GROUP_BY_NOTE_MODEL = "group_by_note_model" - EXTRACT_SHARED_TAGS = "extract_shared_tags" - - -@dataclass -class DeckPartNoteFlags: - group_by_note_model: bool = False - extract_shared_tags: bool = False - - @staticmethod - def as_formatted_dict(group_by_note_model: bool, extract_shared_tags: bool): - return {DeckPartNoteKeys.FLAGS.value: { - NoteFlagKeys.EXTRACT_SHARED_TAGS.value: extract_shared_tags, - NoteFlagKeys.GROUP_BY_NOTE_MODEL.value: group_by_note_model - }} - - def get_formatted_dict(self): - return DeckPartNoteFlags.as_formatted_dict(self.group_by_note_model, self.extract_shared_tags) - - def any_enabled(self): - return self.group_by_note_model or self.extract_shared_tags diff --git a/brain_brew/constants/global_config_keys.py b/brain_brew/constants/global_config_keys.py deleted file mode 100644 index 3279b4d..0000000 --- a/brain_brew/constants/global_config_keys.py +++ /dev/null @@ -1,19 +0,0 @@ -from enum import Enum - - -class ConfigKeys(Enum): - AUTHORS = "authors" - - DECK_PARTS = "deck_parts" - HEADERS = "headers" - NOTE_MODELS = "note_models" - NOTES = "notes" - MEDIA_FILES = "media_files" - - FLAGS = "flags" - SORT_CASE_INSENSITIVE = "sort_case_insensitive" - JOIN_VALUES_WITH = "join_values_with" - - DECK_PARTS_NOTES_STRUCTURE = "deck_part_notes_structure" - NOTE_SORT_ORDER = "note_sort_order" - REVERSE_SORT = "reverse_sort" diff --git a/brain_brew/main.py b/brain_brew/main.py index aa7d2cd..7223394 100644 --- a/brain_brew/main.py +++ b/brain_brew/main.py @@ -19,12 +19,12 @@ def main(): builder_file_name, global_config_file = argument_reader.get_parsed() # Read in Global Config File - global_config = GlobalConfig.from_yaml(global_config_file) if global_config_file else GlobalConfig.get_default() + global_config = GlobalConfig.from_file(global_config_file) if global_config_file else GlobalConfig.from_file() file_manager = FileManager() # Parse Build Config File builder_data = TopLevelTaskBuilder.read_to_dict(builder_file_name) - builder = TopLevelTaskBuilder.from_list(builder_data, global_config, file_manager) + builder = TopLevelTaskBuilder.from_list(builder_data) # If all good, execute it builder.execute() diff --git a/brain_brew/build_tasks/generate_deck_parts.py b/brain_brew/representation/build_config/generate_deck_parts.py similarity index 84% rename from brain_brew/build_tasks/generate_deck_parts.py rename to brain_brew/representation/build_config/generate_deck_parts.py index 2615f1a..4614aa6 100644 --- a/brain_brew/build_tasks/generate_deck_parts.py +++ b/brain_brew/representation/build_config/generate_deck_parts.py @@ -1,7 +1,7 @@ from typing import Dict, Type from brain_brew.representation.build_config.build_task import BuildTask, TopLevelBuildTask, DeckPartBuildTask -from brain_brew.build_tasks.task_builder import TaskBuilder +from brain_brew.representation.build_config.task_builder import TaskBuilder class GenerateDeckParts(TaskBuilder, TopLevelBuildTask): diff --git a/brain_brew/build_tasks/task_builder.py b/brain_brew/representation/build_config/task_builder.py similarity index 88% rename from brain_brew/build_tasks/task_builder.py rename to brain_brew/representation/build_config/task_builder.py index cde807b..65a9d66 100644 --- a/brain_brew/build_tasks/task_builder.py +++ b/brain_brew/representation/build_config/task_builder.py @@ -12,16 +12,12 @@ @dataclass class TaskBuilder(YamlRepr): tasks: List[BuildTask] - global_config: GlobalConfig - file_manager: FileManager @classmethod - def from_list(cls, data: List[dict], global_config, file_manager): + def from_list(cls, data: List[dict]): tasks = cls.read_tasks(data) return cls( - tasks=tasks, - global_config=global_config, - file_manager=file_manager + tasks=tasks ) @classmethod diff --git a/brain_brew/representation/build_config/top_level_task_builder.py b/brain_brew/representation/build_config/top_level_task_builder.py index 64845e6..4370dd7 100644 --- a/brain_brew/representation/build_config/top_level_task_builder.py +++ b/brain_brew/representation/build_config/top_level_task_builder.py @@ -1,7 +1,7 @@ from typing import Dict, Type from brain_brew.representation.build_config.build_task import BuildTask, TopLevelBuildTask -from brain_brew.build_tasks.task_builder import TaskBuilder +from brain_brew.representation.build_config.task_builder import TaskBuilder class TopLevelTaskBuilder(TaskBuilder): diff --git a/brain_brew/representation/configuration/csv_file_mapping.py b/brain_brew/representation/configuration/csv_file_mapping.py index 9eb8da4..8651aa6 100644 --- a/brain_brew/representation/configuration/csv_file_mapping.py +++ b/brain_brew/representation/configuration/csv_file_mapping.py @@ -18,21 +18,21 @@ @dataclass -class CsvFileMappingDerivative: +class FileMappingDerivative: @dataclass(init=False) class Representation(RepresentationBase): file: str note_model: Optional[str] sort_by_columns: Optional[Union[list, str]] reverse_sort: Optional[bool] - derivatives: Optional[List['CsvFileMappingDerivative.Representation']] + derivatives: Optional[List['FileMappingDerivative.Representation']] def __init__(self, file, note_model=None, sort_by_columns=None, reverse_sort=None, derivatives=None): self.file = file self.note_model = note_model self.sort_by_columns = sort_by_columns self.reverse_sort = reverse_sort - self.derivatives = list(map(CsvFileMappingDerivative.Representation.from_dict, derivatives)) if derivatives is not None else [] + self.derivatives = list(map(FileMappingDerivative.Representation.from_dict, derivatives)) if derivatives is not None else [] compiled_data: Dict[str, dict] = field(init=False) @@ -41,7 +41,7 @@ def __init__(self, file, note_model=None, sort_by_columns=None, reverse_sort=Non note_model: Optional[str] sort_by_columns: Optional[list] reverse_sort: Optional[bool] - derivatives: Optional[List['CsvFileMappingDerivative']] + derivatives: Optional[List['FileMappingDerivative']] @classmethod def from_repr(cls, data: Union[Representation, dict]): @@ -109,7 +109,7 @@ def write_to_csv(self, data_to_set): @dataclass -class CsvFileMapping(CsvFileMappingDerivative, Verifiable): +class FileMapping(FileMappingDerivative, Verifiable): note_model: str # Override Optional on Parent data_set_has_changed: bool = field(init=False, default=False) diff --git a/brain_brew/representation/configuration/global_config.py b/brain_brew/representation/configuration/global_config.py index 202c91b..89f7cea 100644 --- a/brain_brew/representation/configuration/global_config.py +++ b/brain_brew/representation/configuration/global_config.py @@ -1,93 +1,72 @@ from dataclasses import dataclass, field +from typing import List, Union, Optional -from brain_brew.constants.deckpart_keys import NoteFlagKeys, DeckPartNoteFlags -from brain_brew.constants.global_config_keys import * -from brain_brew.representation.generic.yaml_file import YamlFile, ConfigKey +from brain_brew.representation.build_config.representation_base import RepresentationBase +from brain_brew.representation.yaml.my_yaml import YamlRepr @dataclass -class DeckPartConfig: - headers: str - note_models: str - notes: str - media_files: str - - -class GlobalConfig(YamlFile): +class GlobalConfig(YamlRepr): __instance = None @dataclass - class ConfigFlags: - note_sort_order: list = field(default_factory=list) - sort_case_insensitive: bool = False - reverse_sort: bool = False - join_values_with: str = " " + class Defaults: + class Representation(RepresentationBase): + note_sort_order: Optional[Union[List[str], str]] + sort_case_insensitive: Optional[bool] + reverse_sort: Optional[bool] + join_values_with: Optional[str] + + note_sort_order: list + sort_case_insensitive: bool + reverse_sort: bool + join_values_with: str + + @classmethod + def from_repr(cls, data: Union[Representation, dict]): + rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) + return cls( + note_sort_order=rep.note_sort_order or [], + sort_case_insensitive=rep.sort_case_insensitive or False, + reverse_sort=rep.reverse_sort or False, + join_values_with=rep.join_values_with or " " + ) + + @dataclass + class DeckPartLocations(RepresentationBase): + headers: str + note_models: str + notes: str + media_files: str - deck_parts: DeckPartConfig - flags: ConfigFlags - deck_part_notes_flags: DeckPartNoteFlags - join_values_with: str + @dataclass + class Representation(RepresentationBase): + deck_parts: dict + defaults: Optional[dict] - config_entry = {} - expected_keys = { - ConfigKeys.DECK_PARTS.value: ConfigKey(True, dict, { - ConfigKeys.HEADERS.value: ConfigKey(True, str, None), - ConfigKeys.NOTE_MODELS.value: ConfigKey(True, str, None), - ConfigKeys.NOTES.value: ConfigKey(True, str, None), - ConfigKeys.MEDIA_FILES.value: ConfigKey(True, str, None), + deck_parts: DeckPartLocations + defaults: Defaults - ConfigKeys.DECK_PARTS_NOTES_STRUCTURE.value: ConfigKey(True, dict, { - flag.value: ConfigKey(False, bool, None) for flag in NoteFlagKeys - }) - }), - ConfigKeys.FLAGS.value: ConfigKey(False, dict, { - ConfigKeys.NOTE_SORT_ORDER.value: ConfigKey(False, list, None), - ConfigKeys.SORT_CASE_INSENSITIVE.value: ConfigKey(False, bool, None), - ConfigKeys.REVERSE_SORT.value: ConfigKey(False, bool, None), - ConfigKeys.JOIN_VALUES_WITH.value: ConfigKey(False, str, None), - }) - } - subconfig_filter = None + @classmethod + def from_repr(cls, data: Union[Representation, dict]): + rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) + return cls( + deck_parts=cls.DeckPartLocations.from_dict(rep.deck_parts), + defaults=cls.Defaults.from_repr(rep.defaults) + ) - def __init__(self, config_data: dict): + def __post_init__(self): if GlobalConfig.__instance is None: GlobalConfig.__instance = self else: raise Exception("Multiple GlobalConfigs created") - self.setup_config_with_subconfig_replacement(config_data) - self.verify_config_entry() - - dp = self.get_config(ConfigKeys.DECK_PARTS) - self.deck_parts = DeckPartConfig( - dp[ConfigKeys.HEADERS.value], dp[ConfigKeys.NOTE_MODELS.value], - dp[ConfigKeys.NOTES.value], dp[ConfigKeys.MEDIA_FILES.value] - ) - - self.flags = GlobalConfig.ConfigFlags() - flag_keys = self.get_config(ConfigKeys.FLAGS, {}) - for flag in self.expected_keys[ConfigKeys.FLAGS.value].children.keys(): - if flag in flag_keys: - setattr(self.flags, flag, flag_keys[flag]) - - self.deck_part_notes_flags = DeckPartNoteFlags() - note_flags = dp[ConfigKeys.DECK_PARTS_NOTES_STRUCTURE.value] - for flag in note_flags.keys(): - setattr(self.deck_part_notes_flags, flag, note_flags[flag]) - @classmethod - def get_default(cls): - global_config_name = "brain_brew_config.yaml" - return cls.from_yaml(global_config_name) - - @classmethod - def from_yaml(cls, yaml_file): - gb_config_yaml = YamlFile.read_file(yaml_file) - - return GlobalConfig(gb_config_yaml) + def from_file(cls, filename: str = "brain_brew_config.yaml"): + return cls.from_repr(cls.read_to_dict(filename)) @classmethod - def get_instance(cls): + def get_instance(cls) -> 'GlobalConfig': return cls.__instance @classmethod diff --git a/brain_brew/representation/generic/generic_file.py b/brain_brew/representation/generic/generic_file.py index 0b5e83f..4a4223c 100644 --- a/brain_brew/representation/generic/generic_file.py +++ b/brain_brew/representation/generic/generic_file.py @@ -42,7 +42,7 @@ def formatted_file_location(cls, location): @staticmethod def _sort_data(data, sort_by_keys, reverse_sort, case_insensitive_sort=None): # TODO: Move to NoteGroupings if case_insensitive_sort is None: - case_insensitive_sort = GlobalConfig.get_instance().flags.sort_case_insensitive + case_insensitive_sort = GlobalConfig.get_instance().defaults.sort_case_insensitive if sort_by_keys: if case_insensitive_sort: diff --git a/brain_brew/representation/yaml/headers_repr.py b/brain_brew/representation/yaml/headers_repr.py index 2b9fd91..c12ebea 100644 --- a/brain_brew/representation/yaml/headers_repr.py +++ b/brain_brew/representation/yaml/headers_repr.py @@ -7,5 +7,9 @@ class Headers(YamlRepr): data: dict + @classmethod + def from_file(cls, filename: str): + return cls(data=cls.read_to_dict(filename)) + def encode(self) -> dict: return self.data diff --git a/brain_brew/representation/yaml/note_model_repr.py b/brain_brew/representation/yaml/note_model_repr.py index e12dbb5..b7b0a32 100644 --- a/brain_brew/representation/yaml/note_model_repr.py +++ b/brain_brew/representation/yaml/note_model_repr.py @@ -206,6 +206,10 @@ class CrowdAnki(RepresentationBase): tags: List[str] = field(default_factory=lambda: TAGS.default_value) # Tags of the last added note version: list = field(default_factory=lambda: VERSION.default_value) # Legacy version number. Deprecated in Anki + @classmethod + def from_file(cls, filename: str): + return cls.from_dict(cls.read_to_dict(filename)) + @classmethod def from_crowdanki(cls, data: Union[CrowdAnki, dict]): # TODO: field_whitelist: List[str] = None, note_model_whitelist: List[str] = None): ca: cls.CrowdAnki = data if isinstance(data, cls.CrowdAnki) else cls.CrowdAnki.from_dict(data) diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index 5550870..d42db28 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -119,6 +119,10 @@ def join_tags(n_tags): class Notes(YamlRepr): note_groupings: List[NoteGrouping] + @classmethod + def from_file(cls, filename: str): + return cls.from_dict(cls.read_to_dict(filename)) + @classmethod def from_dict(cls, data: dict): return cls(note_groupings=list(map(NoteGrouping.from_dict, data.get(NOTE_GROUPINGS)))) diff --git a/brain_brew/utils.py b/brain_brew/utils.py index f045522..b9bab8b 100644 --- a/brain_brew/utils.py +++ b/brain_brew/utils.py @@ -66,7 +66,7 @@ def split_tags(tags_value: str) -> list: def join_tags(tags_list: list) -> str: - return GlobalConfig.get_instance().flags.join_values_with.join(tags_list) + return GlobalConfig.get_instance().defaults.join_values_with.join(tags_list) def generate_anki_guid() -> str: diff --git a/tests/build_tasks/test_source_csv.py b/tests/build_tasks/test_source_csv.py index 4a9a0b6..949e021 100644 --- a/tests/build_tasks/test_source_csv.py +++ b/tests/build_tasks/test_source_csv.py @@ -5,7 +5,7 @@ from brain_brew.build_tasks.source_csv import SourceCsv, SourceCsvKeys from brain_brew.constants.deckpart_keys import DeckPartNoteKeys -from brain_brew.representation.configuration.csv_file_mapping import CsvFileMapping +from brain_brew.representation.configuration.csv_file_mapping import FileMapping from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping from brain_brew.representation.generic.csv_file import CsvFile from brain_brew.representation.generic.generic_file import SourceFile @@ -23,7 +23,7 @@ def setup_source_csv_config(notes: str, nmm: list, csv_mappings: list): } -def get_csv_default(notes: DeckPartNotes, nmm: List[NoteModelMapping], csv_maps: List[CsvFileMapping]) -> SourceCsv: +def get_csv_default(notes: DeckPartNotes, nmm: List[NoteModelMapping], csv_maps: List[FileMapping]) -> SourceCsv: csv_source = SourceCsv(setup_source_csv_config("", [], []), read_now=False) csv_source.notes = notes @@ -87,7 +87,7 @@ def assert_dpn(config, read_now): assert config == notes assert read_now is False - with patch.object(CsvFileMapping, "__init__", side_effect=assert_csv), \ + with patch.object(FileMapping, "__init__", side_effect=assert_csv), \ patch.object(NoteModelMapping, "__init__", side_effect=assert_nmm), \ patch.object(NoteModelMapping, "note_model"), \ patch.object(DeckPartNotes, "create", side_effect=assert_dpn): diff --git a/tests/representation/configuration/test_csv_file_mapping.py b/tests/representation/configuration/test_csv_file_mapping.py index 59cf8fe..faa85c2 100644 --- a/tests/representation/configuration/test_csv_file_mapping.py +++ b/tests/representation/configuration/test_csv_file_mapping.py @@ -4,7 +4,7 @@ import pytest -from brain_brew.representation.configuration.csv_file_mapping import CsvFileMappingDerivative, CsvFileMapping, \ +from brain_brew.representation.configuration.csv_file_mapping import FileMappingDerivative, FileMapping, \ SORT_BY_COLUMNS, REVERSE_SORT, NOTE_MODEL, DERIVATIVES, FILE from brain_brew.representation.generic.csv_file import CsvFile from tests.test_file_manager import get_new_file_manager @@ -44,12 +44,12 @@ def assert_csv(passed_file, read_now): assert passed_file == csv assert read_now == read_file_now - with patch.object(CsvFileMappingDerivative, "create_derivative", return_value=None) as mock_derivatives, \ + with patch.object(FileMappingDerivative, "create_derivative", return_value=None) as mock_derivatives, \ patch.object(CsvFile, "create", side_effect=assert_csv) as mock_csv: - csv_fm = CsvFileMapping(config, read_now=read_file_now) + csv_fm = FileMapping(config, read_now=read_file_now) - assert isinstance(csv_fm, CsvFileMapping) + assert isinstance(csv_fm, FileMapping) assert csv_fm.reverse_sort == reverse_sort assert csv_fm.sort_by_columns == sort_by_columns assert csv_fm.note_model_name == note_model_name @@ -74,17 +74,17 @@ def assert_der(passed_file, read_now): assert passed_file == der assert read_now is False - with patch.object(CsvFileMappingDerivative, "create_derivative", side_effect=assert_der) as mock_derivatives, \ + with patch.object(FileMappingDerivative, "create_derivative", side_effect=assert_der) as mock_derivatives, \ patch.object(CsvFile, "create", return_value=None): - csv_fm = CsvFileMapping(config, read_now=False) + csv_fm = FileMapping(config, read_now=False) assert mock_derivatives.call_count == len(csv_fm.derivatives) == expected_call_count def csv_fixture_gen(csv_fix): with patch.object(CsvFile, "create", return_value=csv_fix): - csv = CsvFileMapping(setup_csv_fm_config("", note_model_name="Test Model")) + csv = FileMapping(setup_csv_fm_config("", note_model_name="Test Model")) csv.compile_data() return csv @@ -120,7 +120,7 @@ def csv_file_mapping2_missing_guids(csv_test2_missing_guids): class TestSetRelevantData: - def test_no_change(self, csv_file_mapping1: CsvFileMapping, csv_file_mapping_split1: CsvFileMapping): + def test_no_change(self, csv_file_mapping1: FileMapping, csv_file_mapping_split1: FileMapping): assert csv_file_mapping1.data_set_has_changed is False previous_data = csv_file_mapping1.compiled_data.copy() @@ -129,7 +129,7 @@ def test_no_change(self, csv_file_mapping1: CsvFileMapping, csv_file_mapping_spl assert previous_data == csv_file_mapping1.compiled_data assert csv_file_mapping1.data_set_has_changed is False - def test_change_but_no_extra(self, csv_file_mapping1: CsvFileMapping, csv_file_mapping2: CsvFileMapping): + def test_change_but_no_extra(self, csv_file_mapping1: FileMapping, csv_file_mapping2: FileMapping): assert csv_file_mapping1.data_set_has_changed is False assert len(csv_file_mapping1.compiled_data) == 15 @@ -140,7 +140,7 @@ def test_change_but_no_extra(self, csv_file_mapping1: CsvFileMapping, csv_file_m assert csv_file_mapping1.data_set_has_changed is True assert len(csv_file_mapping1.compiled_data) == 15 - def test_change_extra_row(self, csv_file_mapping1: CsvFileMapping, csv_file_mapping3: CsvFileMapping): + def test_change_extra_row(self, csv_file_mapping1: FileMapping, csv_file_mapping3: FileMapping): assert csv_file_mapping1.data_set_has_changed is False assert len(csv_file_mapping1.compiled_data) == 15 @@ -159,7 +159,7 @@ def get_num(self): self.num += 1 return self.num - def test_when_missing_guids(self, csv_file_mapping2_missing_guids: CsvFileMapping): + def test_when_missing_guids(self, csv_file_mapping2_missing_guids: FileMapping): with patch("brain_brew.representation.configuration.csv_file_mapping.generate_anki_guid", wraps=self.get_num) as mock_guid: csv_file_mapping2_missing_guids.compile_data() From 5c0690f893f6176187c113a36b31e2159a1b3e5c Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sat, 29 Aug 2020 15:12:07 +0200 Subject: [PATCH 28/39] Fix last import errors --- .../crowd_anki/crowd_anki_to_deck_parts.py | 2 +- .../configuration/csv_file_mapping.py | 6 +- .../configuration/note_model_mapping.py | 4 +- brain_brew/representation/generic/csv_file.py | 6 +- .../representation/json/deck_part_notes.py | 213 ------------------ 5 files changed, 7 insertions(+), 224 deletions(-) delete mode 100644 brain_brew/representation/json/deck_part_notes.py diff --git a/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py b/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py index e075a36..27cf9a5 100644 --- a/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py +++ b/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py @@ -46,7 +46,7 @@ def execute(self): ca_wrapper = self.crowd_anki_export.read_json_file() if ca_wrapper.children: - logging.warning("Child Decks / Subdecks are not currently supported.") # TODO: Support them + logging.warning("Child Decks / Sub-decks are not currently supported.") # TODO: Support them note_models: List[NoteModel] = self.note_model_transform.execute(ca_wrapper) diff --git a/brain_brew/representation/configuration/csv_file_mapping.py b/brain_brew/representation/configuration/csv_file_mapping.py index 8651aa6..97dee67 100644 --- a/brain_brew/representation/configuration/csv_file_mapping.py +++ b/brain_brew/representation/configuration/csv_file_mapping.py @@ -1,9 +1,7 @@ import logging from dataclasses import dataclass, field -from enum import Enum from typing import Dict, List, Optional, Union -from brain_brew.constants.deckpart_keys import DeckPartNoteKeys from brain_brew.interfaces.verifiable import Verifiable from brain_brew.representation.build_config.representation_base import RepresentationBase from brain_brew.representation.generic.csv_file import CsvFile, CsvKeys @@ -90,7 +88,7 @@ def _build_data_recursive(self) -> List[dict]: found_match = True # Set Note Model to matching Derivative Note Model if der.note_model is not None: - row.setdefault(DeckPartNoteKeys.NOTE_MODEL.value, der.note_model) + row.setdefault(NOTE_MODEL, der.note_model) break if not found_match: der_match_errors.append(ValueError(f"Cannot match derivative row {der_row} to parent")) @@ -127,7 +125,7 @@ def compile_data(self): # Set Note Model if not already set if self.note_model is not None: for row in data_in_progress: - row.setdefault(DeckPartNoteKeys.NOTE_MODEL.value, self.note_model) + row.setdefault(NOTE_MODEL, self.note_model) # Fill in Guid if no Guid guids_generated = 0 diff --git a/brain_brew/representation/configuration/note_model_mapping.py b/brain_brew/representation/configuration/note_model_mapping.py index 25c289d..aa5b26e 100644 --- a/brain_brew/representation/configuration/note_model_mapping.py +++ b/brain_brew/representation/configuration/note_model_mapping.py @@ -2,11 +2,11 @@ from enum import Enum from typing import List, Union, Dict -from brain_brew.constants.deckpart_keys import DeckPartNoteKeys from brain_brew.interfaces.verifiable import Verifiable from brain_brew.representation.build_config.representation_base import RepresentationBase from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder from brain_brew.representation.yaml.note_model_repr import NoteModel +from brain_brew.representation.yaml.note_repr import GUID, TAGS from brain_brew.utils import single_item_to_list @@ -46,7 +46,7 @@ class Representation(RepresentationBase): columns: List[FieldMapping] personal_fields: List[FieldMapping] - required_fields_definitions = [DeckPartNoteKeys.GUID.value, DeckPartNoteKeys.TAGS.value] + required_fields_definitions = [GUID, TAGS] @classmethod def from_repr(cls, data: Representation): diff --git a/brain_brew/representation/generic/csv_file.py b/brain_brew/representation/generic/csv_file.py index 0c50b91..d51a416 100644 --- a/brain_brew/representation/generic/csv_file.py +++ b/brain_brew/representation/generic/csv_file.py @@ -2,11 +2,9 @@ import logging import re from enum import Enum -from typing import List, Dict +from typing import List -from brain_brew.representation.configuration.global_config import GlobalConfig -from brain_brew.representation.generic.yaml_file import YamlFile, ConfigKey -from brain_brew.utils import list_of_str_to_lowercase, generate_anki_guid +from brain_brew.utils import list_of_str_to_lowercase from brain_brew.representation.generic.generic_file import SourceFile diff --git a/brain_brew/representation/json/deck_part_notes.py b/brain_brew/representation/json/deck_part_notes.py deleted file mode 100644 index e29c5f6..0000000 --- a/brain_brew/representation/json/deck_part_notes.py +++ /dev/null @@ -1,213 +0,0 @@ -import logging -import re -from typing import List - -from brain_brew.file_manager import FileManager -from brain_brew.representation.configuration.global_config import GlobalConfig -from brain_brew.representation.json.json_file import JsonFile -from brain_brew.constants.deckpart_keys import * -from brain_brew.representation.generic.media_file import MediaFile -from brain_brew.utils import find_media_in_field - - -class DeckPartNotes(JsonFile): - _data: dict = {} - flags: DeckPartNoteFlags - referenced_media_files: List[MediaFile] - - global_config: GlobalConfig - file_manager: FileManager - - @classmethod - def formatted_file_location(cls, location): - return cls.get_json_file_location(GlobalConfig.get_instance().deck_parts.notes, location) - - def __init__(self, location, read_now=True, data_override=None): - self.global_config = GlobalConfig.get_instance() - self.file_manager = FileManager.get_instance() - self.flags = DeckPartNoteFlags() - self.referenced_media_files = [] - super().__init__( - self.formatted_file_location(location), - read_now=read_now, data_override=data_override - ) - - def set_data(self, data_override, deck_part_flags_included=False): - if not deck_part_flags_included: - data_override = {**self.flags.get_formatted_dict(), **{DeckPartNoteKeys.NOTES.value: data_override}} - - super().set_data(data_override) - self.interpret_data() - - def write_file(self, data_override=None): - super().write_file(data_override or self.implement_note_structure()) - - def read_file(self): - super().read_file() - self.interpret_data() - - def interpret_data(self): - self.verify_data() - - self.read_note_config() - self._data = self.remove_notes_structure() - self.read_note_config() - - self.sort_data( - self.global_config.flags.note_sort_order, - self.global_config.flags.reverse_sort - ) - - self.find_all_media_references() - - def read_note_config(self): - self.flags.group_by_note_model = \ - self._data[DeckPartNoteKeys.FLAGS.value][NoteFlagKeys.GROUP_BY_NOTE_MODEL.value] - self.flags.extract_shared_tags = \ - self._data[DeckPartNoteKeys.FLAGS.value][NoteFlagKeys.EXTRACT_SHARED_TAGS.value] - - def verify_data(self): - errors = [] - if DeckPartNoteKeys.FLAGS.value not in self._data: - errors.append(KeyError(f"Missing '{DeckPartNoteKeys.FLAGS.value}' key in '{self.file_location}'")) - else: - for flag in NoteFlagKeys: - if flag.value not in self._data[DeckPartNoteKeys.FLAGS.value]: - errors.append(KeyError(f"Missing '{flag.value}' flag in '{self.file_location}'")) - - if errors: - raise Exception(errors) - - def get_all_known_note_model_names(self): - model_set = {note[DeckPartNoteKeys.NOTE_MODEL.value] for note in self._data[DeckPartNoteKeys.NOTES.value]} - return sorted(list(model_set)) - - def implement_note_structure(self): - """ - :return: the notes structured based on the settings in the Global Config file - """ - if not self.global_config.deck_part_notes_flags.any_enabled(): - return self._data - - def top_level_structure(): - if self.global_config.deck_part_notes_flags.extract_shared_tags: - return {DeckPartNoteKeys.SHARED_TAGS.value: [], DeckPartNoteKeys.NOTES.value: []} - else: - return {DeckPartNoteKeys.NOTES.value: []} - - if self.global_config.deck_part_notes_flags.group_by_note_model: - structured_notes: dict = {name: top_level_structure() for name in self.get_all_known_note_model_names()} - for note in self._data[DeckPartNoteKeys.NOTES.value]: - for name in structured_notes: - if note[DeckPartNoteKeys.NOTE_MODEL.value] == name: - structured_notes[name][DeckPartNoteKeys.NOTES.value].append(note) - break - del note[DeckPartNoteKeys.NOTE_MODEL.value] - else: - structured_notes = top_level_structure() - structured_notes[DeckPartNoteKeys.NOTES.value] = self._data - - if self.global_config.deck_part_notes_flags.extract_shared_tags: - if self.global_config.deck_part_notes_flags.group_by_note_model: - for name in structured_notes: - structured_notes[name][DeckPartNoteKeys.SHARED_TAGS.value] = self.extract_shared_tags( - structured_notes[name][DeckPartNoteKeys.NOTES.value] - ) - else: - structured_notes[DeckPartNoteKeys.SHARED_TAGS.value] = self.extract_shared_tags( - structured_notes[DeckPartNoteKeys.NOTES.value] - ) - - final_structure = self.global_config.deck_part_notes_flags.get_formatted_dict() - for key in structured_notes: - final_structure.setdefault(key, structured_notes[key]) - - return final_structure - - @staticmethod - def extract_shared_tags(notes): - shared_tags = notes[0][DeckPartNoteKeys.TAGS.value] - - for note in notes: - shared_tags = [tag for tag in note[DeckPartNoteKeys.TAGS.value] if tag in shared_tags] - if not shared_tags: - break - - if shared_tags: - for note in notes: - for tag in shared_tags: - note[DeckPartNoteKeys.TAGS.value].remove(tag) - - return shared_tags - - def remove_notes_structure(self): - """ - :return: notes with their global config structure removed - """ - if not self.flags.any_enabled(): - return self._data - - unstructured_notes = self.get_data(deep_copy=True) - - if self.flags.extract_shared_tags: - if self.flags.group_by_note_model: - for group in unstructured_notes: - if group[0] != "_": - self.spread_out_shared_tags(unstructured_notes[group]) - else: - self.spread_out_shared_tags(unstructured_notes) - - unstructured_notes[DeckPartNoteKeys.FLAGS.value][NoteFlagKeys.EXTRACT_SHARED_TAGS.value] = False - - if self.flags.group_by_note_model: - ungrouped_notes = [] - for group in unstructured_notes: - if group[0] != "_": - for note in unstructured_notes[group][DeckPartNoteKeys.NOTES.value]: - note[DeckPartNoteKeys.NOTE_MODEL.value] = group - ungrouped_notes.append(note) - - top_level_ungrouped = {group: unstructured_notes[group] for group in unstructured_notes if group[0] == "_"} - top_level_ungrouped.setdefault(DeckPartNoteKeys.NOTES.value, ungrouped_notes) - unstructured_notes = top_level_ungrouped - - unstructured_notes[DeckPartNoteKeys.FLAGS.value][NoteFlagKeys.GROUP_BY_NOTE_MODEL.value] = False - - return unstructured_notes - - @staticmethod - def spread_out_shared_tags(top_level): - shared_tags = top_level[DeckPartNoteKeys.SHARED_TAGS.value] - if shared_tags: - for note in top_level[DeckPartNoteKeys.NOTES.value]: - note[DeckPartNoteKeys.TAGS.value] += shared_tags - - del top_level[DeckPartNoteKeys.SHARED_TAGS.value] - - def sort_data(self, sort_by_keys, reverse_sort): - sorted_data = self._sort_data(self._data[DeckPartNoteKeys.NOTES.value], sort_by_keys, reverse_sort) - - self._data[DeckPartNoteKeys.NOTES.value] = sorted_data - - def find_all_media_references(self): - unknown_references = [] - self.referenced_media_files.clear() - - for note in self._data[DeckPartNoteKeys.NOTES.value]: - for field in note[DeckPartNoteKeys.FIELDS.value]: - files_found = find_media_in_field(field) - if files_found: - for filename in files_found: - file = self.file_manager.media_file_if_exists(filename) - if file: - if file not in self.referenced_media_files: - self.referenced_media_files.append(file) - else: - unknown_references.append(filename) - - if len(unknown_references) > 0: - logging.error(f"Found {len(unknown_references)} unreferenced media files in {self.file_location}: [" - f"{', '.join(unknown_references)}]") - - logging.info(f"Found {len(self.referenced_media_files)} referenced media files") - From 459741c68ad07f3bcd4ec444222bd2248fa1b76f Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sun, 30 Aug 2020 10:56:20 +0200 Subject: [PATCH 29/39] Final Fixes; Yaml shift Create folder that does not exist --- .../crowd_anki/headers_from_crowdanki.py | 2 +- .../crowd_anki/media_to_from_crowd_anki.py | 5 +++-- brain_brew/main.py | 2 -- .../representation/build_config/build_task.py | 10 +++++++-- .../build_config/generate_deck_parts.py | 15 ++++++++++++- .../build_config/top_level_task_builder.py | 6 +++++ .../configuration/global_config.py | 17 +++++++------- .../representation/generic/generic_file.py | 2 +- .../representation/yaml/deck_part_holder.py | 3 ++- brain_brew/representation/yaml/my_yaml.py | 22 ++++++++++++++----- brain_brew/representation/yaml/note_repr.py | 8 +++---- 11 files changed, 65 insertions(+), 27 deletions(-) diff --git a/brain_brew/build_tasks/crowd_anki/headers_from_crowdanki.py b/brain_brew/build_tasks/crowd_anki/headers_from_crowdanki.py index 2e2e628..ffc2a47 100644 --- a/brain_brew/build_tasks/crowd_anki/headers_from_crowdanki.py +++ b/brain_brew/build_tasks/crowd_anki/headers_from_crowdanki.py @@ -39,5 +39,5 @@ def execute(self, ca_wrapper: CrowdAnkiJsonWrapper): @staticmethod def crowd_anki_to_headers(ca_data: dict): - return {key: value for key, value in ca_data + return {key: value for key, value in ca_data.items() if key not in headers_skip_keys and key not in headers_default_values.keys()} diff --git a/brain_brew/build_tasks/crowd_anki/media_to_from_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/media_to_from_crowd_anki.py index ef3c3fc..385f508 100644 --- a/brain_brew/build_tasks/crowd_anki/media_to_from_crowd_anki.py +++ b/brain_brew/build_tasks/crowd_anki/media_to_from_crowd_anki.py @@ -38,7 +38,8 @@ def from_repr(cls, data: Union[Representation, dict, bool]): file_manager: FileManager = None def __post_init__(self): - self.file_manager = FileManager.get_instance() + if not MediaToFromCrowdAnki.file_manager: + MediaToFromCrowdAnki.file_manager = FileManager.get_instance() def move_to_crowd_anki(self, notes: Notes, note_models: List[NoteModel], ca_export: CrowdAnkiExport) -> Set[MediaFile]: resolved_media: Set[MediaFile] = set() @@ -76,7 +77,7 @@ def move_to_deck_parts(self, notes: Notes, note_models: List[NoteModel], ca_expo self._move_dps(res) if len(missing_media) > 0: - logging.error(f"Unresolved references in CrowdAnki to {len(missing_media)} files: {missing_media}") + logging.error(f"Unresolved media in CrowdAnki to {len(missing_media)} files: {missing_media}") return resolved_media diff --git a/brain_brew/main.py b/brain_brew/main.py index 7223394..1072dce 100644 --- a/brain_brew/main.py +++ b/brain_brew/main.py @@ -4,8 +4,6 @@ from brain_brew.representation.build_config.top_level_task_builder import TopLevelTaskBuilder from brain_brew.file_manager import FileManager from brain_brew.representation.configuration.global_config import GlobalConfig -from brain_brew.representation.yaml.my_yaml import yaml_load - # sys.path.append(os.path.join(os.path.dirname(__file__), "dist")) # sys.path.append(os.path.dirname(__file__)) diff --git a/brain_brew/representation/build_config/build_task.py b/brain_brew/representation/build_config/build_task.py index 0c0bbeb..53fcb44 100644 --- a/brain_brew/representation/build_config/build_task.py +++ b/brain_brew/representation/build_config/build_task.py @@ -1,3 +1,4 @@ +import logging from typing import Dict, List, Type from brain_brew.utils import str_to_lowercase_no_separators @@ -29,12 +30,17 @@ def get_all_build_tasks(cls) -> Dict[str, Type['BuildTask']]: known_build_tasks.setdefault(task_name, sc) + # logging.debug(f"Known build tasks: {known_build_tasks}") return known_build_tasks class TopLevelBuildTask(BuildTask): - pass + @classmethod + def get_all_build_tasks(cls) -> Dict[str, Type['BuildTask']]: + return super(TopLevelBuildTask, cls).get_all_build_tasks() class DeckPartBuildTask(BuildTask): - pass + @classmethod + def get_all_build_tasks(cls) -> Dict[str, Type['BuildTask']]: + return super(DeckPartBuildTask, cls).get_all_build_tasks() diff --git a/brain_brew/representation/build_config/generate_deck_parts.py b/brain_brew/representation/build_config/generate_deck_parts.py index 4614aa6..a0bc88f 100644 --- a/brain_brew/representation/build_config/generate_deck_parts.py +++ b/brain_brew/representation/build_config/generate_deck_parts.py @@ -1,12 +1,25 @@ -from typing import Dict, Type +from dataclasses import dataclass +from typing import Dict, Type, List from brain_brew.representation.build_config.build_task import BuildTask, TopLevelBuildTask, DeckPartBuildTask from brain_brew.representation.build_config.task_builder import TaskBuilder +# Build Tasks +from brain_brew.build_tasks.deck_parts.from_deck_part import FromDeckParts +from brain_brew.build_tasks.csvs.csvs_to_deck_parts import CsvsToDeckParts +from brain_brew.build_tasks.crowd_anki.crowd_anki_to_deck_parts import CrowdAnkiToDeckParts + +@dataclass class GenerateDeckParts(TaskBuilder, TopLevelBuildTask): task_names = ["Generate Deck Parts", "Generate Deck Part", "Deck Part", "Deck Parts"] @classmethod def known_task_dict(cls) -> Dict[str, Type[BuildTask]]: return DeckPartBuildTask.get_all_build_tasks() + + @classmethod + def from_repr(cls, data: List[dict]): + if not isinstance(data, list): + raise TypeError(f"GenerateDeckParts needs a list") + return cls.from_list(data) diff --git a/brain_brew/representation/build_config/top_level_task_builder.py b/brain_brew/representation/build_config/top_level_task_builder.py index 4370dd7..7e7bc14 100644 --- a/brain_brew/representation/build_config/top_level_task_builder.py +++ b/brain_brew/representation/build_config/top_level_task_builder.py @@ -1,8 +1,14 @@ +import logging from typing import Dict, Type from brain_brew.representation.build_config.build_task import BuildTask, TopLevelBuildTask from brain_brew.representation.build_config.task_builder import TaskBuilder +# Build Tasks +from brain_brew.build_tasks.csvs.csvs_generate import CsvsGenerate +from brain_brew.build_tasks.crowd_anki.crowd_anki_generate import CrowdAnkiGenerate +from brain_brew.representation.build_config.generate_deck_parts import GenerateDeckParts + class TopLevelTaskBuilder(TaskBuilder): @classmethod diff --git a/brain_brew/representation/configuration/global_config.py b/brain_brew/representation/configuration/global_config.py index 89f7cea..5b3dc0a 100644 --- a/brain_brew/representation/configuration/global_config.py +++ b/brain_brew/representation/configuration/global_config.py @@ -11,11 +11,12 @@ class GlobalConfig(YamlRepr): @dataclass class Defaults: + @dataclass class Representation(RepresentationBase): - note_sort_order: Optional[Union[List[str], str]] - sort_case_insensitive: Optional[bool] - reverse_sort: Optional[bool] - join_values_with: Optional[str] + note_sort_order: Optional[Union[List[str], str]] = field(default_factory=[]) + sort_case_insensitive: Optional[bool] = field(default=False) + reverse_sort: Optional[bool] = field(default=False) + join_values_with: Optional[str] = field(default=" ") note_sort_order: list sort_case_insensitive: bool @@ -26,10 +27,10 @@ class Representation(RepresentationBase): def from_repr(cls, data: Union[Representation, dict]): rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) return cls( - note_sort_order=rep.note_sort_order or [], - sort_case_insensitive=rep.sort_case_insensitive or False, - reverse_sort=rep.reverse_sort or False, - join_values_with=rep.join_values_with or " " + note_sort_order=rep.note_sort_order, + sort_case_insensitive=rep.sort_case_insensitive, + reverse_sort=rep.reverse_sort, + join_values_with=rep.join_values_with ) @dataclass diff --git a/brain_brew/representation/generic/generic_file.py b/brain_brew/representation/generic/generic_file.py index 4a4223c..0d51e64 100644 --- a/brain_brew/representation/generic/generic_file.py +++ b/brain_brew/representation/generic/generic_file.py @@ -15,7 +15,7 @@ def is_file(cls, filename: str): @classmethod def is_dir(cls, folder_name: str): - return Path(folder_name).is_file() + return Path(folder_name).is_dir() @classmethod def get_deep_copy(cls, data): diff --git a/brain_brew/representation/yaml/deck_part_holder.py b/brain_brew/representation/yaml/deck_part_holder.py index 6164e69..9e9d05d 100644 --- a/brain_brew/representation/yaml/deck_part_holder.py +++ b/brain_brew/representation/yaml/deck_part_holder.py @@ -33,7 +33,8 @@ def override_or_create(cls, name: str, save_to_file: Optional[str], deck_part: T else: dp.deck_part = deck_part dp.save_to_file = save_to_file # ? - dp.write_to_file() + + dp.write_to_file() return dp diff --git a/brain_brew/representation/yaml/my_yaml.py b/brain_brew/representation/yaml/my_yaml.py index a1240fb..4f020a9 100644 --- a/brain_brew/representation/yaml/my_yaml.py +++ b/brain_brew/representation/yaml/my_yaml.py @@ -1,5 +1,6 @@ +import logging from pathlib import Path - +import os from ruamel.yaml import YAML yaml_load = YAML(typ='safe') @@ -7,7 +8,7 @@ yaml_dump = YAML() yaml_dump.preserve_quotes = False -yaml_dump.indent(mapping=0, sequence=4, offset=2) +yaml_dump.indent(mapping=2, sequence=2, offset=0) yaml_dump.representer.ignore_aliases = lambda *data: True # yaml.sort_base_mapping_type_on_output = False @@ -15,8 +16,7 @@ class YamlRepr: @staticmethod def read_to_dict(filename: str): - if filename[-5:] not in [".yaml", ".yml"]: - filename += ".yaml" + filename = YamlRepr.append_yaml_if_needed(filename) if not Path(filename).is_file(): raise FileNotFoundError(filename) @@ -28,5 +28,17 @@ def encode(self) -> dict: raise NotImplemented def dump_to_yaml(self, filepath): - with open(filepath, 'w+') as fp: # TODO: raise warning/log if file not exists + filepath = YamlRepr.append_yaml_if_needed(filepath) + + if not Path(filepath).is_file(): + logging.warning(f"Creating missing filepath '{filepath}'") + os.makedirs(os.path.dirname(filepath), exist_ok=True) + + with open(filepath, 'w') as fp: yaml_dump.dump(self.encode(), fp) + + @staticmethod + def append_yaml_if_needed(filename: str): + if filename[-5:] != ".yaml" and filename[-4:] != ".yml": + return filename + ".yaml" + return filename diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index d42db28..c64a404 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -90,8 +90,8 @@ def get_all_known_note_model_names(self) -> set: def get_all_media_references(self) -> Set[str]: all_media = set() for note in self.notes: - for media in note.get_media_references(): - all_media = all_media.union(media) + media = note.get_media_references() + all_media = all_media.union(media) return all_media def get_all_notes_copy(self) -> List[Note]: @@ -141,8 +141,8 @@ def get_all_known_note_model_names(self): def get_all_media_references(self) -> Set[str]: all_media = set() for note in self.note_groupings: - for media in note.get_all_media_references(): - all_media = all_media.union(media) + media = note.get_all_media_references() + all_media = all_media.union(media) return all_media def get_notes(self): From 606862421659f91a1c8c65f24eac006c5fb507b8 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sun, 30 Aug 2020 12:53:05 +0200 Subject: [PATCH 30/39] Minor fixes: NoteModel fixes NoteModel Mappings --- brain_brew/build_tasks/csvs/csvs_generate.py | 8 +++++--- .../build_tasks/csvs/notes_from_csvs.py | 4 +++- .../build_tasks/csvs/shared_base_csvs.py | 17 +++++++++-------- .../build_tasks/deck_parts/from_deck_part.py | 18 ++++++++++-------- .../configuration/note_model_mapping.py | 13 +++++++------ .../representation/yaml/note_model_repr.py | 19 ++++++++++++------- 6 files changed, 46 insertions(+), 33 deletions(-) diff --git a/brain_brew/build_tasks/csvs/csvs_generate.py b/brain_brew/build_tasks/csvs/csvs_generate.py index 58c9d50..43f80e6 100644 --- a/brain_brew/build_tasks/csvs/csvs_generate.py +++ b/brain_brew/build_tasks/csvs/csvs_generate.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import List, Dict, Union from brain_brew.build_tasks.csvs.shared_base_csvs import SharedBaseCsvs @@ -13,7 +13,7 @@ class CsvsGenerate(SharedBaseCsvs, TopLevelBuildTask): task_names = all_combos_prepend_append(["Csv Collection", "Csv"], "Generate ", "s") - notes: DeckPartHolder[Notes] + notes: DeckPartHolder[Notes] = field(default=None) @dataclass(init=False) class Representation(SharedBaseCsvs.Representation): @@ -29,10 +29,12 @@ def from_repr(cls, data: Union[Representation, dict]): return cls( notes=DeckPartHolder.from_deck_part_pool(rep.notes), file_mappings=rep.get_file_mappings(), - note_model_mappings=rep.get_note_model_mappings() + note_model_mappings_representations=rep.note_model_mappings ) def execute(self): + self.get_note_model_mappings() + notes: List[Note] = self.notes.deck_part.get_notes() self.verify_notes_match_note_model_mappings(notes) diff --git a/brain_brew/build_tasks/csvs/notes_from_csvs.py b/brain_brew/build_tasks/csvs/notes_from_csvs.py index 0655505..68de8f5 100644 --- a/brain_brew/build_tasks/csvs/notes_from_csvs.py +++ b/brain_brew/build_tasks/csvs/notes_from_csvs.py @@ -24,10 +24,12 @@ def from_repr(cls, data: Union[Representation, dict]): name=rep.name, save_to_file=rep.save_to_file, file_mappings=rep.get_file_mappings(), - note_model_mappings=rep.get_note_model_mappings() + note_model_mappings_representations=rep.note_model_mappings ) def execute(self): + self.get_note_model_mappings() + csv_data_by_guid: Dict[str, dict] = {} for csv_map in self.file_mappings: csv_map.compile_data() diff --git a/brain_brew/build_tasks/csvs/shared_base_csvs.py b/brain_brew/build_tasks/csvs/shared_base_csvs.py index 3c3f70f..81f59ce 100644 --- a/brain_brew/build_tasks/csvs/shared_base_csvs.py +++ b/brain_brew/build_tasks/csvs/shared_base_csvs.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import List, Dict from brain_brew.representation.build_config.representation_base import RepresentationBase @@ -20,15 +20,16 @@ def __init__(self, file_mappings, note_model_mappings): def get_file_mappings(self) -> List[FileMapping]: return list(map(FileMapping.from_repr, self.file_mappings)) - def get_note_model_mappings(self) -> Dict[str, NoteModelMapping]: - def map_nmm(nmm_to_map: str): - nmm = NoteModelMapping.from_repr(nmm_to_map) - return nmm.get_note_model_mapping_dict() + file_mappings: List[FileMapping] + note_model_mappings_representations: List[NoteModelMapping.Representation] + note_model_mappings: Dict[str, NoteModelMapping] = field(default_factory=dict) - return dict(*map(map_nmm, self.note_model_mappings)) + def get_note_model_mappings(self): + def map_nmm(nmm_to_map: str): + nmm = NoteModelMapping.from_repr(nmm_to_map) + return nmm.get_note_model_mapping_dict() - file_mappings: List[FileMapping] - note_model_mappings: Dict[str, NoteModelMapping] + self.note_model_mappings = dict(*map(map_nmm, self.note_model_mappings_representations)) def verify_contents(self): errors = [] diff --git a/brain_brew/build_tasks/deck_parts/from_deck_part.py b/brain_brew/build_tasks/deck_parts/from_deck_part.py index 138ecf8..c2149b4 100644 --- a/brain_brew/build_tasks/deck_parts/from_deck_part.py +++ b/brain_brew/build_tasks/deck_parts/from_deck_part.py @@ -1,9 +1,11 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Union, List, Optional from brain_brew.representation.build_config.build_task import DeckPartBuildTask from brain_brew.representation.build_config.representation_base import RepresentationBase from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder +from brain_brew.representation.yaml.headers_repr import Headers +from brain_brew.representation.yaml.note_model_repr import NoteModel from brain_brew.representation.yaml.note_repr import Notes from brain_brew.utils import all_combos_prepend_append @@ -19,9 +21,9 @@ class DeckPartToRead(RepresentationBase): @dataclass class Representation(RepresentationBase): - notes: Optional[List[dict]] - note_models: Optional[List[dict]] - headers: Optional[List[dict]] + notes: List[dict] = field(default_factory=list) + note_models: List[dict] = field(default_factory=list) + headers: List[dict] = field(default_factory=list) notes: List[DeckPartToRead] note_models: List[DeckPartToRead] @@ -32,14 +34,14 @@ def from_repr(cls, data: Union[Representation, dict]): rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) return cls( notes=list(map(FromDeckParts.DeckPartToRead.from_dict, rep.notes)), - note_models=list(map(FromDeckParts.DeckPartToRead.from_dict, rep.notes)), - headers=list(map(FromDeckParts.DeckPartToRead.from_dict, rep.notes)) + note_models=list(map(FromDeckParts.DeckPartToRead.from_dict, rep.note_models)), + headers=list(map(FromDeckParts.DeckPartToRead.from_dict, rep.headers)) ) def execute(self): for note in self.notes: DeckPartHolder.override_or_create(name=note.name, save_to_file=None, deck_part=Notes.from_file(note.file)) for model in self.note_models: - DeckPartHolder.override_or_create(name=model.name, save_to_file=None, deck_part=Notes.from_file(model.file)) + DeckPartHolder.override_or_create(name=model.name, save_to_file=None, deck_part=NoteModel.from_file(model.file)) for header in self.headers: - DeckPartHolder.override_or_create(name=header.name, save_to_file=None, deck_part=Notes.from_file(header.file)) + DeckPartHolder.override_or_create(name=header.name, save_to_file=None, deck_part=Headers.from_file(header.file)) diff --git a/brain_brew/representation/configuration/note_model_mapping.py b/brain_brew/representation/configuration/note_model_mapping.py index aa5b26e..a7f5137 100644 --- a/brain_brew/representation/configuration/note_model_mapping.py +++ b/brain_brew/representation/configuration/note_model_mapping.py @@ -106,7 +106,8 @@ def csv_row_map_to_note_fields(self, row: dict) -> dict: for pf in self.personal_fields: # Add in Personal Fields relevant_row_data.setdefault(pf.field_name, False) for column in self.columns: # Rename from Csv Column to Note Type Field - relevant_row_data[column.value] = relevant_row_data.pop(column.field_name) + if column.value in relevant_row_data: + relevant_row_data[column.field_name] = relevant_row_data.pop(column.value) # TODO: Insert FieldMappings with Default values @@ -127,15 +128,15 @@ def note_fields_map_to_csv_row(self, row): return relevant_row_data def get_relevant_data(self, row): - relevant_columns = [field.field_name for field in self.columns] + relevant_columns = [field.value for field in self.columns] if not relevant_columns: return [] - cols = row.keys() + cols = list(row.keys()) - errors = [KeyError(f"Missing column {rel_col}") for rel_col in relevant_columns if rel_col not in cols] - if errors: - raise Exception(errors) + # errors = [KeyError(f"Missing column {rel_col}") for rel_col in relevant_columns if rel_col not in cols] + # if errors: + # raise Exception(errors) irrelevant_columns = [column for column in cols if column not in relevant_columns] if not irrelevant_columns: diff --git a/brain_brew/representation/yaml/note_model_repr.py b/brain_brew/representation/yaml/note_model_repr.py index b7b0a32..22dfd6f 100644 --- a/brain_brew/representation/yaml/note_model_repr.py +++ b/brain_brew/representation/yaml/note_model_repr.py @@ -46,7 +46,7 @@ def append_name_if_differs(self, dict_to_add_to: dict, value): FONT = AnkiField("font", default_value="Liberation Sans") MEDIA = AnkiField("media", default_value=[]) IS_RIGHT_TO_LEFT = AnkiField("rtl", "is_right_to_left", default_value=False) -FONT_SIZE = AnkiField("size", default_value=20) +FONT_SIZE = AnkiField("size", "font_size", default_value=20) IS_STICKY = AnkiField("sticky", "is_sticky", default_value=False) # Template @@ -192,7 +192,7 @@ class CrowdAnki(RepresentationBase): vers: list = field(default_factory=lambda: VERSION.default_value) name: str - crowdanki_id: str + id: str css: str required_fields_per_template: List[list] # TODO: Get rid of this as requirement fields: List[Field] @@ -208,7 +208,12 @@ class CrowdAnki(RepresentationBase): @classmethod def from_file(cls, filename: str): - return cls.from_dict(cls.read_to_dict(filename)) + data = cls.read_to_dict(filename) + return cls( + fields=[Field(**f) for f in data.pop(FIELDS.name)], + templates=[Template(**t) for t in data.pop(TEMPLATES.name)], + **data + ) @classmethod def from_crowdanki(cls, data: Union[CrowdAnki, dict]): # TODO: field_whitelist: List[str] = None, note_model_whitelist: List[str] = None): @@ -219,13 +224,13 @@ def from_crowdanki(cls, data: Union[CrowdAnki, dict]): # TODO: field_whitelist: is_cloze=bool(ca.type), name=ca.name, css=ca.css, latex_pre=ca.latexPre, latex_post=ca.latexPost, required_fields_per_template=ca.req, tags=ca.tags, sort_field_num=ca.sortf, version=ca.vers, - crowdanki_id=ca.crowdanki_uuid, crowdanki_type=ca.__type__ + id=ca.crowdanki_uuid, crowdanki_type=ca.__type__ ) def encode_as_crowdanki(self) -> dict: data_dict = { NAME.anki_name: self.name, - CROWDANKI_ID.anki_name: self.crowdanki_id, + CROWDANKI_ID.anki_name: self.id, CSS.anki_name: self.css, REQUIRED_FIELDS_PER_TEMPLATE.anki_name: self.required_fields_per_template, LATEX_PRE.anki_name: self.latex_pre, @@ -245,7 +250,7 @@ def encode_as_crowdanki(self) -> dict: def encode(self) -> dict: data_dict: Dict[str, Union[str, list]] = { NAME.name: self.name, - CROWDANKI_ID.name: self.crowdanki_id, + CROWDANKI_ID.name: self.id, CSS.name: self.css } @@ -274,7 +279,7 @@ def get_all_media_references(self) -> Set[str]: @property def field_names_lowercase(self): - return list_of_str_to_lowercase(f.name for f in self.fields) + return list_of_str_to_lowercase([f.name for f in self.fields]) def check_field_overlap(self, fields_to_check: List[str]): fields_to_check = list_of_str_to_lowercase(fields_to_check) From da392694f3f9753d8742d3f5e729090e4a359e9f Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Fri, 4 Sep 2020 09:45:35 +0200 Subject: [PATCH 31/39] Remove some Todos; Create Media folder if not found; --- brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py | 2 +- brain_brew/build_tasks/csvs/csvs_generate.py | 4 +--- brain_brew/representation/build_config/task_builder.py | 2 -- brain_brew/representation/configuration/note_model_mapping.py | 2 -- brain_brew/representation/generic/media_file.py | 2 ++ brain_brew/representation/json/crowd_anki_export.py | 2 +- brain_brew/representation/yaml/note_model_repr.py | 2 +- 7 files changed, 6 insertions(+), 10 deletions(-) diff --git a/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py b/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py index 27cf9a5..9a619fa 100644 --- a/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py +++ b/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py @@ -46,7 +46,7 @@ def execute(self): ca_wrapper = self.crowd_anki_export.read_json_file() if ca_wrapper.children: - logging.warning("Child Decks / Sub-decks are not currently supported.") # TODO: Support them + logging.warning("Child Decks / Sub-decks are not currently supported.") note_models: List[NoteModel] = self.note_model_transform.execute(ca_wrapper) diff --git a/brain_brew/build_tasks/csvs/csvs_generate.py b/brain_brew/build_tasks/csvs/csvs_generate.py index 43f80e6..a8384cd 100644 --- a/brain_brew/build_tasks/csvs/csvs_generate.py +++ b/brain_brew/build_tasks/csvs/csvs_generate.py @@ -41,8 +41,6 @@ def execute(self): csv_data: List[dict] = [self.note_to_csv_row(note, self.note_model_mappings) for note in notes] rows_by_guid = {row["guid"]: row for row in csv_data} - # TODO: Dry run option, to not save anything at this stage - for fm in self.file_mappings: fm.compile_data() fm.set_relevant_data(rows_by_guid) @@ -65,4 +63,4 @@ def note_to_csv_row(note: Note, note_model_mappings: Dict[str, NoteModelMapping] row["tags"] = join_tags(note.tags) # TODO: Flags? - return note_model_mappings[nm_name].note_fields_map_to_csv_row(row) # TODO: Do not edit data, make copy + return note_model_mappings[nm_name].note_fields_map_to_csv_row(row) diff --git a/brain_brew/representation/build_config/task_builder.py b/brain_brew/representation/build_config/task_builder.py index 65a9d66..0f4ddc9 100644 --- a/brain_brew/representation/build_config/task_builder.py +++ b/brain_brew/representation/build_config/task_builder.py @@ -1,10 +1,8 @@ from dataclasses import dataclass from typing import Dict, List, Type -from brain_brew.file_manager import FileManager from brain_brew.interfaces.verifiable import Verifiable from brain_brew.representation.build_config.build_task import BuildTask -from brain_brew.representation.configuration.global_config import GlobalConfig from brain_brew.representation.yaml.my_yaml import YamlRepr from brain_brew.utils import str_to_lowercase_no_separators diff --git a/brain_brew/representation/configuration/note_model_mapping.py b/brain_brew/representation/configuration/note_model_mapping.py index a7f5137..f3fc56b 100644 --- a/brain_brew/representation/configuration/note_model_mapping.py +++ b/brain_brew/representation/configuration/note_model_mapping.py @@ -109,8 +109,6 @@ def csv_row_map_to_note_fields(self, row: dict) -> dict: if column.value in relevant_row_data: relevant_row_data[column.field_name] = relevant_row_data.pop(column.value) - # TODO: Insert FieldMappings with Default values - return relevant_row_data def csv_headers_map_to_note_fields(self, row: list) -> list: diff --git a/brain_brew/representation/generic/media_file.py b/brain_brew/representation/generic/media_file.py index 966f060..e4bbe7a 100644 --- a/brain_brew/representation/generic/media_file.py +++ b/brain_brew/representation/generic/media_file.py @@ -1,3 +1,4 @@ +import os import shutil from enum import Enum @@ -29,6 +30,7 @@ def set_override(self, source_loc): def copy_source_to_target(self): if self.should_write(): # TODO: If ManagementType.OVERRIDDEN check if override necessary + os.makedirs(os.path.dirname(self.file_location), exist_ok=True) shutil.copy2(self.source_loc, self.file_location) def should_write(self): diff --git a/brain_brew/representation/json/crowd_anki_export.py b/brain_brew/representation/json/crowd_anki_export.py index e9059b3..9f6d98d 100644 --- a/brain_brew/representation/json/crowd_anki_export.py +++ b/brain_brew/representation/json/crowd_anki_export.py @@ -47,7 +47,7 @@ def find_json_file_in_folder(self): def find_all_media(self): self.known_media = {} - self.media_loc = self.folder_location + "media/" # TODO: Make media folder if not exists + self.media_loc = self.folder_location + "media/" self.contains_media = self.is_dir(self.media_loc) if not self.contains_media: diff --git a/brain_brew/representation/yaml/note_model_repr.py b/brain_brew/representation/yaml/note_model_repr.py index 22dfd6f..b8d8d63 100644 --- a/brain_brew/representation/yaml/note_model_repr.py +++ b/brain_brew/representation/yaml/note_model_repr.py @@ -194,7 +194,7 @@ class CrowdAnki(RepresentationBase): name: str id: str css: str - required_fields_per_template: List[list] # TODO: Get rid of this as requirement + required_fields_per_template: List[list] fields: List[Field] templates: List[Template] From 82af2502fbca1f2fed9efe28895a26fef2bc5045 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Fri, 4 Sep 2020 11:09:26 +0200 Subject: [PATCH 32/39] Reordering CrowdAnki Keys; Read DeckParts in Init, not execute; --- .../crowd_anki/crowd_anki_generate.py | 5 ++-- .../crowd_anki/headers_to_crowd_anki.py | 8 +++--- .../crowd_anki/note_models_to_crowd_anki.py | 12 ++++++--- .../crowd_anki/notes_to_crowd_anki.py | 14 +++++----- brain_brew/build_tasks/csvs/csvs_generate.py | 4 +-- .../build_tasks/csvs/shared_base_csvs.py | 15 +++++------ .../build_tasks/deck_parts/from_deck_part.py | 27 ++++++++++--------- .../json/wrappers_for_crowd_anki.py | 9 +++++++ .../representation/yaml/headers_repr.py | 9 +++++++ .../representation/yaml/note_model_repr.py | 16 +++++------ 10 files changed, 71 insertions(+), 48 deletions(-) diff --git a/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py b/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py index a2d1916..d825dd8 100644 --- a/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py +++ b/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py @@ -53,15 +53,16 @@ def execute(self): note_models: List[dict] = self.note_model_transform.execute() - nm_name_to_id: dict = {model.name: model.crowdanki_id for model in self.note_model_transform.note_models} + nm_name_to_id: dict = {model.name: model.id for model in self.note_model_transform.note_models} notes = self.notes_transform.execute(nm_name_to_id) media_files = self.media_transform.move_to_crowd_anki( self.notes_transform.notes, self.note_model_transform.note_models, self.crowd_anki_export) + ca_wrapper.media_files = sorted([m.filename for m in media_files]) + ca_wrapper.name = self.headers_transform.headers.name ca_wrapper.note_models = note_models ca_wrapper.notes = notes - ca_wrapper.media_files = [m.filename for m in media_files] # Set to CrowdAnkiExport self.crowd_anki_export.write_to_files(ca_wrapper.data) diff --git a/brain_brew/build_tasks/crowd_anki/headers_to_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/headers_to_crowd_anki.py index d40f851..8a4a626 100644 --- a/brain_brew/build_tasks/crowd_anki/headers_to_crowd_anki.py +++ b/brain_brew/build_tasks/crowd_anki/headers_to_crowd_anki.py @@ -24,17 +24,17 @@ def from_repr(cls, data: Union[Representation, dict, str]): rep = cls.Representation(deck_part=data) # Support single string being passed in return cls( - headers=DeckPartHolder.from_deck_part_pool(rep.deck_part), + headers=DeckPartHolder.from_deck_part_pool(rep.deck_part).deck_part, ) headers: Headers - def execute(self): - headers = Headers(self.headers_to_crowd_anki(self.headers.data)) + def execute(self) -> dict: + headers = self.headers_to_crowd_anki(self.headers.data_without_name) return headers @staticmethod def headers_to_crowd_anki(headers_data: dict): - return {**headers_data, **headers_default_values} + return {**headers_default_values, **headers_data} diff --git a/brain_brew/build_tasks/crowd_anki/note_models_to_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/note_models_to_crowd_anki.py index d869627..14d75b2 100644 --- a/brain_brew/build_tasks/crowd_anki/note_models_to_crowd_anki.py +++ b/brain_brew/build_tasks/crowd_anki/note_models_to_crowd_anki.py @@ -30,9 +30,12 @@ def from_repr(cls, data: Union[Representation, dict, str]): rep = cls.Representation(deck_part=data) # Support string return cls( - deck_part=DeckPartHolder.from_deck_part_pool(rep.deck_part) + deck_part=DeckPartHolder.from_deck_part_pool(rep.deck_part).deck_part ) + def get_note_model(self) -> NoteModel: + return self.deck_part # Todo: add filters in here + deck_part: NoteModel @dataclass @@ -49,11 +52,12 @@ def from_repr(cls, data: Union[Representation, dict, List[str]]): else: rep = cls.Representation(deck_parts=data) # Support list of Note Models + note_model_items = list(map(cls.NoteModelListItem.from_repr, rep.deck_parts)) return cls( - note_models=list(map(cls.NoteModelListItem.from_repr, rep.deck_parts)) + note_models=[nm.get_note_model() for nm in note_model_items] ) - note_models: List[NoteModelListItem] + note_models: List[NoteModel] def execute(self) -> List[dict]: - return [model.deck_part.encode_as_crowdanki() for model in self.note_models] + return [model.encode_as_crowdanki() for model in self.note_models] diff --git a/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py index 2820bf7..02fa18a 100644 --- a/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py +++ b/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Optional, Union, List from brain_brew.build_tasks.crowd_anki.shared_base_notes import SharedBaseNotes @@ -13,13 +13,13 @@ class NotesToCrowdAnki(SharedBaseNotes): @dataclass class Representation(SharedBaseNotes.Representation): deck_part: str - additional_items_to_add: Optional[dict] + additional_items_to_add: Optional[dict] = field(default_factory=lambda: None) @classmethod def from_repr(cls, data: Union[Representation, dict]): rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) return cls( - notes=DeckPartHolder.from_deck_part_pool(rep.deck_part), + notes=DeckPartHolder.from_deck_part_pool(rep.deck_part).deck_part, sort_order=SharedBaseNotes._get_sort_order(rep.sort_order), additional_items_to_add=rep.additional_items_to_add or {} ) @@ -40,16 +40,16 @@ def execute(self, nm_name_to_id: dict) -> List[dict]: def note_to_ca_note(note: Note, nm_name_to_id: dict, additional_items_to_add: dict) -> dict: wrapper = CrowdAnkiNoteWrapper({ "__type__": "Note", - "data": None + "data": "" }) for key, value in additional_items_to_add.items(): wrapper.data[key] = blank_str_if_none(value) - wrapper.guid = note.guid wrapper.fields = note.fields - wrapper.tags = note.tags - wrapper.note_model = nm_name_to_id[note.note_model] wrapper.flags = note.flags + wrapper.guid = note.guid + wrapper.note_model = nm_name_to_id[note.note_model] + wrapper.tags = note.tags return wrapper.data diff --git a/brain_brew/build_tasks/csvs/csvs_generate.py b/brain_brew/build_tasks/csvs/csvs_generate.py index a8384cd..300ad85 100644 --- a/brain_brew/build_tasks/csvs/csvs_generate.py +++ b/brain_brew/build_tasks/csvs/csvs_generate.py @@ -29,12 +29,10 @@ def from_repr(cls, data: Union[Representation, dict]): return cls( notes=DeckPartHolder.from_deck_part_pool(rep.notes), file_mappings=rep.get_file_mappings(), - note_model_mappings_representations=rep.note_model_mappings + note_model_mappings=rep.get_note_model_mappings() ) def execute(self): - self.get_note_model_mappings() - notes: List[Note] = self.notes.deck_part.get_notes() self.verify_notes_match_note_model_mappings(notes) diff --git a/brain_brew/build_tasks/csvs/shared_base_csvs.py b/brain_brew/build_tasks/csvs/shared_base_csvs.py index 81f59ce..ec2a3ac 100644 --- a/brain_brew/build_tasks/csvs/shared_base_csvs.py +++ b/brain_brew/build_tasks/csvs/shared_base_csvs.py @@ -20,16 +20,15 @@ def __init__(self, file_mappings, note_model_mappings): def get_file_mappings(self) -> List[FileMapping]: return list(map(FileMapping.from_repr, self.file_mappings)) - file_mappings: List[FileMapping] - note_model_mappings_representations: List[NoteModelMapping.Representation] - note_model_mappings: Dict[str, NoteModelMapping] = field(default_factory=dict) + def get_note_model_mappings(self): + def map_nmm(nmm_to_map: str): + nmm = NoteModelMapping.from_repr(nmm_to_map) + return nmm.get_note_model_mapping_dict() - def get_note_model_mappings(self): - def map_nmm(nmm_to_map: str): - nmm = NoteModelMapping.from_repr(nmm_to_map) - return nmm.get_note_model_mapping_dict() + return dict(*map(map_nmm, self.note_model_mappings)) - self.note_model_mappings = dict(*map(map_nmm, self.note_model_mappings_representations)) + file_mappings: List[FileMapping] + note_model_mappings: Dict[str, NoteModelMapping] def verify_contents(self): errors = [] diff --git a/brain_brew/build_tasks/deck_parts/from_deck_part.py b/brain_brew/build_tasks/deck_parts/from_deck_part.py index c2149b4..c50a287 100644 --- a/brain_brew/build_tasks/deck_parts/from_deck_part.py +++ b/brain_brew/build_tasks/deck_parts/from_deck_part.py @@ -25,23 +25,26 @@ class Representation(RepresentationBase): note_models: List[dict] = field(default_factory=list) headers: List[dict] = field(default_factory=list) - notes: List[DeckPartToRead] - note_models: List[DeckPartToRead] - headers: List[DeckPartToRead] + notes: List[Notes] + note_models: List[NoteModel] + headers: List[Headers] @classmethod def from_repr(cls, data: Union[Representation, dict]): rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) + + notes: List[cls.DeckPartToRead] = list(map(FromDeckParts.DeckPartToRead.from_dict, rep.notes)) + note_models: List[cls.DeckPartToRead] = list(map(FromDeckParts.DeckPartToRead.from_dict, rep.note_models)) + headers: List[cls.DeckPartToRead] = list(map(FromDeckParts.DeckPartToRead.from_dict, rep.headers)) + return cls( - notes=list(map(FromDeckParts.DeckPartToRead.from_dict, rep.notes)), - note_models=list(map(FromDeckParts.DeckPartToRead.from_dict, rep.note_models)), - headers=list(map(FromDeckParts.DeckPartToRead.from_dict, rep.headers)) + notes=[DeckPartHolder.override_or_create( + name=note.name, save_to_file=None, deck_part=Notes.from_file(note.file)) for note in notes], + note_models=[DeckPartHolder.override_or_create( + name=model.name, save_to_file=None, deck_part=NoteModel.from_file(model.file)) for model in note_models], + headers=[DeckPartHolder.override_or_create( + name=header.name, save_to_file=None, deck_part=Headers.from_file(header.file)) for header in headers] ) def execute(self): - for note in self.notes: - DeckPartHolder.override_or_create(name=note.name, save_to_file=None, deck_part=Notes.from_file(note.file)) - for model in self.note_models: - DeckPartHolder.override_or_create(name=model.name, save_to_file=None, deck_part=NoteModel.from_file(model.file)) - for header in self.headers: - DeckPartHolder.override_or_create(name=header.name, save_to_file=None, deck_part=Headers.from_file(header.file)) + pass diff --git a/brain_brew/representation/json/wrappers_for_crowd_anki.py b/brain_brew/representation/json/wrappers_for_crowd_anki.py index e7d62e6..241d809 100644 --- a/brain_brew/representation/json/wrappers_for_crowd_anki.py +++ b/brain_brew/representation/json/wrappers_for_crowd_anki.py @@ -6,6 +6,7 @@ CA_MEDIA_FILES = "media_files" CA_CHILDREN = "children" CA_TYPE = "__type__" +CA_NAME = "name" NOTE_MODEL = "note_model_uuid" FLAGS = "flags" @@ -48,6 +49,14 @@ def media_files(self) -> list: def media_files(self, value: list): self.data.setdefault(CA_MEDIA_FILES, value) + @property + def name(self) -> list: + return self.data[CA_NAME] + + @name.setter + def name(self, value: list): + self.data.setdefault(CA_NAME, value) + class CrowdAnkiNoteWrapper: data: dict diff --git a/brain_brew/representation/yaml/headers_repr.py b/brain_brew/representation/yaml/headers_repr.py index c12ebea..74162ed 100644 --- a/brain_brew/representation/yaml/headers_repr.py +++ b/brain_brew/representation/yaml/headers_repr.py @@ -1,5 +1,6 @@ from dataclasses import dataclass +from brain_brew.representation.json.wrappers_for_crowd_anki import CA_NAME from brain_brew.representation.yaml.my_yaml import YamlRepr @@ -13,3 +14,11 @@ def from_file(cls, filename: str): def encode(self) -> dict: return self.data + + @property + def name(self) -> str: + return self.data[CA_NAME] + + @property + def data_without_name(self) -> dict: + return {k: v for k, v in self.data.items() if k != CA_NAME} diff --git a/brain_brew/representation/yaml/note_model_repr.py b/brain_brew/representation/yaml/note_model_repr.py index b8d8d63..f2ed648 100644 --- a/brain_brew/representation/yaml/note_model_repr.py +++ b/brain_brew/representation/yaml/note_model_repr.py @@ -94,13 +94,13 @@ def get_all_media_references(self) -> Set[str]: def encode_as_crowdanki(self, ordinal: int) -> dict: data_dict = { + ANSWER_FORMAT.anki_name: self.answer_format, + BROWSER_ANSWER_FORMAT.anki_name: self.answer_format_in_browser, + BROWSER_QUESTION_FORMAT.anki_name: self.question_format_in_browser, + DECK_OVERRIDE_ID.anki_name: self.deck_override_id, NAME.anki_name: self.name, ORDINAL.anki_name: ordinal, QUESTION_FORMAT.anki_name: self.question_format, - ANSWER_FORMAT.anki_name: self.answer_format, - BROWSER_QUESTION_FORMAT.anki_name: self.question_format_in_browser, - BROWSER_ANSWER_FORMAT.anki_name: self.answer_format_in_browser, - DECK_OVERRIDE_ID.anki_name: self.deck_override_id } return data_dict @@ -148,10 +148,10 @@ def from_crowdanki(cls, data: Union[CrowdAnki, dict]): def encode_as_crowdanki(self, ordinal: int) -> dict: data_dict = { - NAME.anki_name: self.name, - ORDINAL.anki_name: ordinal, FONT.anki_name: self.font, MEDIA.anki_name: self.media, + NAME.anki_name: self.name, + ORDINAL.anki_name: ordinal, IS_RIGHT_TO_LEFT.anki_name: self.is_right_to_left, FONT_SIZE.anki_name: self.font_size, IS_STICKY.anki_name: self.is_sticky @@ -198,8 +198,8 @@ class CrowdAnki(RepresentationBase): fields: List[Field] templates: List[Template] - latex_post: str = field(default=LATEX_PRE.default_value) - latex_pre: str = field(default=LATEX_POST.default_value) + latex_post: str = field(default=LATEX_POST.default_value) + latex_pre: str = field(default=LATEX_PRE.default_value) sort_field_num: int = field(default=SORT_FIELD_NUM.default_value) is_cloze: bool = field(default=IS_CLOZE.default_value) crowdanki_type: str = field(default=CROWDANKI_TYPE.default_value) # Should always be "NoteModel" From d970fb392676fea925acc626e8fad6a3a560ac34 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Fri, 4 Sep 2020 16:35:59 +0200 Subject: [PATCH 33/39] Verifications --- brain_brew/build_tasks/csvs/csvs_generate.py | 2 ++ .../build_tasks/csvs/notes_from_csvs.py | 4 +-- .../configuration/csv_file_mapping.py | 6 ++-- .../configuration/note_model_mapping.py | 33 ++++++++++++------- .../representation/yaml/note_model_repr.py | 11 ++++--- 5 files changed, 36 insertions(+), 20 deletions(-) diff --git a/brain_brew/build_tasks/csvs/csvs_generate.py b/brain_brew/build_tasks/csvs/csvs_generate.py index 300ad85..3a9c1f0 100644 --- a/brain_brew/build_tasks/csvs/csvs_generate.py +++ b/brain_brew/build_tasks/csvs/csvs_generate.py @@ -33,6 +33,8 @@ def from_repr(cls, data: Union[Representation, dict]): ) def execute(self): + self.verify_contents() + notes: List[Note] = self.notes.deck_part.get_notes() self.verify_notes_match_note_model_mappings(notes) diff --git a/brain_brew/build_tasks/csvs/notes_from_csvs.py b/brain_brew/build_tasks/csvs/notes_from_csvs.py index 68de8f5..2c85c85 100644 --- a/brain_brew/build_tasks/csvs/notes_from_csvs.py +++ b/brain_brew/build_tasks/csvs/notes_from_csvs.py @@ -24,11 +24,11 @@ def from_repr(cls, data: Union[Representation, dict]): name=rep.name, save_to_file=rep.save_to_file, file_mappings=rep.get_file_mappings(), - note_model_mappings_representations=rep.note_model_mappings + note_model_mappings=rep.get_note_model_mappings() ) def execute(self): - self.get_note_model_mappings() + self.verify_contents() csv_data_by_guid: Dict[str, dict] = {} for csv_map in self.file_mappings: diff --git a/brain_brew/representation/configuration/csv_file_mapping.py b/brain_brew/representation/configuration/csv_file_mapping.py index 97dee67..ac21fa0 100644 --- a/brain_brew/representation/configuration/csv_file_mapping.py +++ b/brain_brew/representation/configuration/csv_file_mapping.py @@ -46,7 +46,7 @@ def from_repr(cls, data: Union[Representation, dict]): rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) return cls( csv_file=CsvFile.create_or_get(rep.file), - note_model=None if not rep.note_model.strip() else rep.note_model.strip(), + note_model=rep.note_model.strip() or None, sort_by_columns=single_item_to_list(rep.sort_by_columns), reverse_sort=rep.reverse_sort or False, derivatives=list(map(cls.from_repr, rep.derivatives)) if rep.derivatives is not None else [] @@ -101,6 +101,7 @@ def _build_data_recursive(self) -> List[dict]: def write_to_csv(self, data_to_set): self.csv_file.set_data_from_superset(data_to_set) self.csv_file.sort_data(self.sort_by_columns, self.reverse_sort) + self.csv_file.write_file() for der in self.derivatives: der.write_to_csv(data_to_set) @@ -113,8 +114,7 @@ class FileMapping(FileMappingDerivative, Verifiable): data_set_has_changed: bool = field(init=False, default=False) def verify_contents(self): - if self.note_model is "": - raise KeyError(f"Top level Csv Mapping requires key {NOTE_MODEL}") + pass def compile_data(self): self.compiled_data = {} diff --git a/brain_brew/representation/configuration/note_model_mapping.py b/brain_brew/representation/configuration/note_model_mapping.py index f3fc56b..97919e5 100644 --- a/brain_brew/representation/configuration/note_model_mapping.py +++ b/brain_brew/representation/configuration/note_model_mapping.py @@ -70,6 +70,9 @@ def get_note_model_mapping_dict(self): def verify_contents(self): errors = [] + extra_fields = [field.field_name for field in self.columns + if field.field_name not in self.required_fields_definitions] + for holder in self.note_models.values(): model: NoteModel = holder.deck_part @@ -83,20 +86,26 @@ def verify_contents(self): errors.append(KeyError(f"""Note model(s) "{holder.name}" to Csv config error: \ Definitions for fields {missing} are required.""")) - # TODO: Note Model Mappings are allowed extra fields on a specific note model now, since multiple - # TODO: can be applied to a single NMM. Check if ANY are missing and/or ALL have Extra instead # Check Fields Align with Note Type - missing, extra = model.check_field_overlap( - [field.value for field in self.columns if field.value not in self.required_fields_definitions] + missing = model.check_field_overlap( + [field.field_name for field in self.columns + if field.field_name not in self.required_fields_definitions] ) missing = [m for m in missing if m not in [field.field_name for field in self.personal_fields]] - if missing or extra: - errors.append(KeyError( - f"""Note model "{holder.name}" to Csv config error. It expected {model.fields} \ - but was missing: {missing}, and got extra: {extra} """)) + if missing: + errors.append( + KeyError(f"Note model '{holder.name}' to Csv config error. " + f"It expected {[field.name for field in model.fields]} but was missing: {missing}") + ) + + # Find mappings which do not exist on any note models + if extra_fields: + extra_fields = model.check_field_extra(extra_fields) + + if extra_fields: + errors.append(KeyError(f"Field(s) '{extra_fields} are defined as mappings, but match no Note Model's field")) - # TODO: Make sure the same note_model is not defined in multiple NMMs if errors: raise Exception(errors) @@ -116,10 +125,12 @@ def csv_headers_map_to_note_fields(self, row: list) -> list: def note_fields_map_to_csv_row(self, row): for column in self.columns: # Rename from Note Type Field to Csv Column - row[column.field_name] = row.pop(column.value) + if column.field_name in row: + row[column.value] = row.pop(column.field_name) for pf in self.personal_fields: # Remove Personal Fields - del row[pf.field_name] + if pf.field_name in row: + del row[pf.field_name] relevant_row_data = self.get_relevant_data(row) diff --git a/brain_brew/representation/yaml/note_model_repr.py b/brain_brew/representation/yaml/note_model_repr.py index f2ed648..5209c0c 100644 --- a/brain_brew/representation/yaml/note_model_repr.py +++ b/brain_brew/representation/yaml/note_model_repr.py @@ -283,12 +283,15 @@ def field_names_lowercase(self): def check_field_overlap(self, fields_to_check: List[str]): fields_to_check = list_of_str_to_lowercase(fields_to_check) - lower_fields = self.field_names_lowercase - missing = [f for f in lower_fields if f not in fields_to_check] - extra = [f for f in fields_to_check if f not in lower_fields] # TODO: Remove? + missing = [f for f in self.field_names_lowercase if f not in fields_to_check] - return missing, extra + return missing + + def check_field_extra(self, fields_to_check: List[str]): + fields_to_check = list_of_str_to_lowercase(fields_to_check) + + return [f for f in fields_to_check if f not in self.field_names_lowercase] def zip_field_to_data(self, data: List[str]) -> dict: if len(self.fields) != len(data): From 06fde9ec5f47cf19521a985e2fb8c6437300cb19 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sat, 5 Sep 2020 09:43:29 +0200 Subject: [PATCH 34/39] Sorting Notes and Csv; Csvs always write; --- .../crowd_anki/crowd_anki_generate.py | 10 ++--- .../crowd_anki/notes_from_crowd_anki.py | 13 ++++-- .../crowd_anki/notes_to_crowd_anki.py | 12 ++++-- .../crowd_anki/shared_base_notes.py | 13 +++--- brain_brew/build_tasks/csvs/csvs_generate.py | 3 +- brain_brew/file_manager.py | 2 +- .../configuration/csv_file_mapping.py | 10 +---- brain_brew/representation/generic/csv_file.py | 6 +-- .../{generic_file.py => source_file.py} | 17 -------- .../representation/json/crowd_anki_export.py | 2 +- brain_brew/representation/yaml/note_repr.py | 41 ++++++++++++++++--- brain_brew/utils.py | 17 ++++++++ tests/build_tasks/test_source_csv.py | 2 +- tests/representation/generic/test_csv_file.py | 2 +- .../generic/test_generic_file.py | 2 +- .../json/test_deck_part_notes.py | 2 +- 16 files changed, 94 insertions(+), 60 deletions(-) rename brain_brew/representation/generic/{generic_file.py => source_file.py} (59%) diff --git a/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py b/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py index d825dd8..019a3f6 100644 --- a/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py +++ b/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field import logging from typing import Union, Optional, List @@ -25,10 +25,10 @@ class CrowdAnkiGenerate(TopLevelBuildTask): @dataclass class Representation(RepresentationBase): folder: str - notes: Optional[dict] - note_models: Optional[list] - headers: Optional[dict] - media: Optional[dict] + notes: dict + note_models: dict + headers: dict + media: Union[dict, bool] = field(default_factory=lambda: False) @classmethod def from_repr(cls, data: Union[Representation, dict]): diff --git a/brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py index f064912..922fecf 100644 --- a/brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py +++ b/brain_brew/build_tasks/crowd_anki/notes_from_crowd_anki.py @@ -1,5 +1,5 @@ -from dataclasses import dataclass -from typing import Union +from dataclasses import dataclass, field +from typing import Union, Optional, List from brain_brew.build_tasks.crowd_anki.shared_base_notes import SharedBaseNotes from brain_brew.representation.json.wrappers_for_crowd_anki import CrowdAnkiJsonWrapper, CrowdAnkiNoteWrapper @@ -11,8 +11,9 @@ @dataclass class NotesFromCrowdAnki(SharedBaseNotes, BaseDeckPartsFrom): @dataclass - class Representation(SharedBaseNotes.Representation, BaseDeckPartsFrom.Representation): - pass + class Representation(BaseDeckPartsFrom.Representation): + sort_order: Optional[List[str]] = field(default_factory=lambda: None) + reverse_sort: Optional[bool] = field(default_factory=lambda: None) @classmethod def from_repr(cls, data: Union[Representation, dict]): @@ -20,9 +21,13 @@ def from_repr(cls, data: Union[Representation, dict]): return cls( name=rep.name, sort_order=SharedBaseNotes._get_sort_order(rep.sort_order), + reverse_sort=SharedBaseNotes._get_reverse_sort(rep.reverse_sort), save_to_file=rep.save_to_file ) + sort_order: Optional[List[str]] = field(default_factory=lambda: None) + reverse_sort: Optional[bool] = field(default_factory=lambda: None) + def execute(self, ca_wrapper: CrowdAnkiJsonWrapper, nm_id_to_name: dict) -> Notes: note_list = [self.ca_note_to_note(note, nm_id_to_name) for note in ca_wrapper.notes] diff --git a/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py b/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py index 02fa18a..5d97cad 100644 --- a/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py +++ b/brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py @@ -2,6 +2,7 @@ from typing import Optional, Union, List from brain_brew.build_tasks.crowd_anki.shared_base_notes import SharedBaseNotes +from brain_brew.representation.build_config.representation_base import RepresentationBase from brain_brew.representation.json.wrappers_for_crowd_anki import CrowdAnkiNoteWrapper from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder from brain_brew.representation.yaml.note_repr import Notes, Note @@ -11,9 +12,11 @@ @dataclass class NotesToCrowdAnki(SharedBaseNotes): @dataclass - class Representation(SharedBaseNotes.Representation): + class Representation(RepresentationBase): deck_part: str additional_items_to_add: Optional[dict] = field(default_factory=lambda: None) + sort_order: Optional[List[str]] = field(default_factory=lambda: None) + reverse_sort: Optional[bool] = field(default_factory=lambda: None) @classmethod def from_repr(cls, data: Union[Representation, dict]): @@ -21,19 +24,20 @@ def from_repr(cls, data: Union[Representation, dict]): return cls( notes=DeckPartHolder.from_deck_part_pool(rep.deck_part).deck_part, sort_order=SharedBaseNotes._get_sort_order(rep.sort_order), + reverse_sort=SharedBaseNotes._get_reverse_sort(rep.reverse_sort), additional_items_to_add=rep.additional_items_to_add or {} ) notes: Notes additional_items_to_add: dict + sort_order: Optional[List[str]] = field(default_factory=lambda: None) + reverse_sort: Optional[bool] = field(default_factory=lambda: None) def execute(self, nm_name_to_id: dict) -> List[dict]: - notes = self.notes.get_notes() + notes = self.notes.get_sorted_notes_copy(sort_by_keys=self.sort_order, reverse_sort=self.reverse_sort) note_dicts = [self.note_to_ca_note(note, nm_name_to_id, self.additional_items_to_add) for note in notes] - # TODO: Sort - return note_dicts @staticmethod diff --git a/brain_brew/build_tasks/crowd_anki/shared_base_notes.py b/brain_brew/build_tasks/crowd_anki/shared_base_notes.py index ec64d1c..48f8308 100644 --- a/brain_brew/build_tasks/crowd_anki/shared_base_notes.py +++ b/brain_brew/build_tasks/crowd_anki/shared_base_notes.py @@ -1,4 +1,4 @@ -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Optional, Union, List from brain_brew.representation.build_config.representation_base import RepresentationBase @@ -6,10 +6,6 @@ @dataclass class SharedBaseNotes: - @dataclass - class Representation(RepresentationBase): - sort_order: Optional[Union[str, List[str]]] - @staticmethod def _get_sort_order(sort_order: Optional[Union[str, List[str]]]): if isinstance(sort_order, list): @@ -18,4 +14,9 @@ def _get_sort_order(sort_order: Optional[Union[str, List[str]]]): return [sort_order] return [] - sort_order: Optional[List[str]] + @staticmethod + def _get_reverse_sort(reverse_sort: Optional[bool]): + return reverse_sort or False + + # sort_order: Optional[List[str]] + # reverse_sort: Optional[bool] diff --git a/brain_brew/build_tasks/csvs/csvs_generate.py b/brain_brew/build_tasks/csvs/csvs_generate.py index 3a9c1f0..1c70341 100644 --- a/brain_brew/build_tasks/csvs/csvs_generate.py +++ b/brain_brew/build_tasks/csvs/csvs_generate.py @@ -35,7 +35,8 @@ def from_repr(cls, data: Union[Representation, dict]): def execute(self): self.verify_contents() - notes: List[Note] = self.notes.deck_part.get_notes() + notes: List[Note] = self.notes.deck_part.get_sorted_notes_copy( + sort_by_keys=[], reverse_sort=False, case_insensitive_sort=False) self.verify_notes_match_note_model_mappings(notes) csv_data: List[dict] = [self.note_to_csv_row(note, self.note_model_mappings) for note in notes] diff --git a/brain_brew/file_manager.py b/brain_brew/file_manager.py index 9513eba..85d9ff7 100644 --- a/brain_brew/file_manager.py +++ b/brain_brew/file_manager.py @@ -5,7 +5,7 @@ from brain_brew.representation.configuration.global_config import GlobalConfig -from brain_brew.representation.generic.generic_file import SourceFile +from brain_brew.representation.generic.source_file import SourceFile from brain_brew.representation.generic.media_file import MediaFile from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder from brain_brew.representation.yaml.my_yaml import YamlRepr diff --git a/brain_brew/representation/configuration/csv_file_mapping.py b/brain_brew/representation/configuration/csv_file_mapping.py index ac21fa0..1b3efed 100644 --- a/brain_brew/representation/configuration/csv_file_mapping.py +++ b/brain_brew/representation/configuration/csv_file_mapping.py @@ -111,14 +111,11 @@ def write_to_csv(self, data_to_set): class FileMapping(FileMappingDerivative, Verifiable): note_model: str # Override Optional on Parent - data_set_has_changed: bool = field(init=False, default=False) - def verify_contents(self): pass def compile_data(self): self.compiled_data = {} - self.data_set_has_changed = False data_in_progress = self._build_data_recursive() @@ -137,7 +134,6 @@ def compile_data(self): self.compiled_data.setdefault(guid, {key.lower(): row[key] for key in row}) if guids_generated > 0: - self.data_set_has_changed = True logging.info(f"Generated {guids_generated} guids in {self.csv_file.file_location}") def set_relevant_data(self, data_set: Dict[str, dict]): @@ -157,12 +153,8 @@ def set_relevant_data(self, data_set: Dict[str, dict]): added += 1 self.compiled_data.setdefault(guid, data_set[guid]) - if changed > 0 or added > 0: - self.data_set_has_changed = True - logging.info(f"Set {self.csv_file.file_location} data; changed {changed}, " f"added {added}, while {unchanged} were identical") def write_file_on_close(self): - if self.data_set_has_changed: - self.write_to_csv(list(self.compiled_data.values())) + self.write_to_csv(list(self.compiled_data.values())) diff --git a/brain_brew/representation/generic/csv_file.py b/brain_brew/representation/generic/csv_file.py index d51a416..ca0e578 100644 --- a/brain_brew/representation/generic/csv_file.py +++ b/brain_brew/representation/generic/csv_file.py @@ -4,8 +4,8 @@ from enum import Enum from typing import List -from brain_brew.utils import list_of_str_to_lowercase -from brain_brew.representation.generic.generic_file import SourceFile +from brain_brew.utils import list_of_str_to_lowercase, sort_dict +from brain_brew.representation.generic.source_file import SourceFile class CsvKeys(Enum): @@ -80,4 +80,4 @@ def formatted_file_location(cls, location): return cls.to_filename_csv(location) def sort_data(self, sort_by_keys, reverse_sort): - self._data = self._sort_data(self._data, sort_by_keys, reverse_sort) + self._data = sort_dict(self._data, sort_by_keys, reverse_sort) diff --git a/brain_brew/representation/generic/generic_file.py b/brain_brew/representation/generic/source_file.py similarity index 59% rename from brain_brew/representation/generic/generic_file.py rename to brain_brew/representation/generic/source_file.py index 0d51e64..c702085 100644 --- a/brain_brew/representation/generic/generic_file.py +++ b/brain_brew/representation/generic/source_file.py @@ -38,20 +38,3 @@ def create_or_get(cls, location): @classmethod def formatted_file_location(cls, location): return location - - @staticmethod - def _sort_data(data, sort_by_keys, reverse_sort, case_insensitive_sort=None): # TODO: Move to NoteGroupings - if case_insensitive_sort is None: - case_insensitive_sort = GlobalConfig.get_instance().defaults.sort_case_insensitive - - if sort_by_keys: - if case_insensitive_sort: - def sort_method(i): return tuple((i[column] == "", i[column].lower()) for column in sort_by_keys) - else: - def sort_method(i): return tuple((i[column] == "", i[column]) for column in sort_by_keys) - - return sorted(data, key=sort_method, reverse=reverse_sort) - elif reverse_sort: - return list(reversed(data)) - - return data diff --git a/brain_brew/representation/json/crowd_anki_export.py b/brain_brew/representation/json/crowd_anki_export.py index 9f6d98d..fffbaf5 100644 --- a/brain_brew/representation/json/crowd_anki_export.py +++ b/brain_brew/representation/json/crowd_anki_export.py @@ -2,7 +2,7 @@ import logging from typing import List, Dict -from brain_brew.representation.generic.generic_file import SourceFile +from brain_brew.representation.generic.source_file import SourceFile from brain_brew.representation.generic.media_file import MediaFile from brain_brew.representation.json.json_file import JsonFile from brain_brew.representation.json.wrappers_for_crowd_anki import CrowdAnkiJsonWrapper diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index c64a404..255e038 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -1,3 +1,6 @@ +import logging + +from brain_brew.representation.configuration.global_config import GlobalConfig from brain_brew.representation.yaml.my_yaml import YamlRepr from dataclasses import dataclass from typing import List, Optional, Dict, Set @@ -94,7 +97,34 @@ def get_all_media_references(self) -> Set[str]: all_media = all_media.union(media) return all_media - def get_all_notes_copy(self) -> List[Note]: + def get_sorted_notes(self, sort_by_keys, reverse_sort, case_insensitive_sort=None): + if case_insensitive_sort is None: + case_insensitive_sort = GlobalConfig.get_instance().defaults.sort_case_insensitive + + if sort_by_keys: + def sort_method(i: Note): + def get_sort_tuple(attr_or_field): + if attr_or_field in [GUID, FLAGS, NOTE_MODEL, TAGS]: + value = getattr(i, attr_or_field) + elif isinstance(attr_or_field, int) and attr_or_field < len(i.fields): + value = i.fields[attr_or_field] + else: + value = "" + logging.warning(f"No known sort value for {attr_or_field}") + + if not isinstance(value, str): + return True, False + return (value == "", value.lower()) if case_insensitive_sort else (value == "", value) + + return tuple(get_sort_tuple(column) for column in sort_by_keys) + + return sorted(self.notes, key=sort_method, reverse=reverse_sort) + elif reverse_sort: + return list(reversed(self.notes)) + + return self.notes + + def get_all_notes_copy(self, sort_by_keys, reverse_sort, case_insensitive_sort=None) -> List[Note]: def join_tags(n_tags): if self.tags is None and n_tags is None: return [] @@ -106,13 +136,13 @@ def join_tags(n_tags): return [*n_tags, *self.tags] return [Note( - note_model=self.note_model if self.note_model is not None else n.note_model, + note_model=self.note_model or n.note_model, tags=join_tags(n.tags), fields=n.fields, guid=n.guid, flags=n.flags # media_references=n.media_references or n.get_media_references() - ) for n in self.notes] + ) for n in self.get_sorted_notes(sort_by_keys, reverse_sort, case_insensitive_sort)] @dataclass @@ -145,5 +175,6 @@ def get_all_media_references(self) -> Set[str]: all_media = all_media.union(media) return all_media - def get_notes(self): - return [note for group in self.note_groupings for note in group.get_all_notes_copy()] + def get_sorted_notes_copy(self, sort_by_keys, reverse_sort, case_insensitive_sort=None): + return [note for group in self.note_groupings + for note in group.get_all_notes_copy(sort_by_keys, reverse_sort, case_insensitive_sort)] diff --git a/brain_brew/utils.py b/brain_brew/utils.py index b9bab8b..a65ee48 100644 --- a/brain_brew/utils.py +++ b/brain_brew/utils.py @@ -88,3 +88,20 @@ def base91(num: int) -> str: return base62(num, _base91_extra_chars) return base91(random.randint(0, 2 ** 64 - 1)) + + +def sort_dict(data, sort_by_keys, reverse_sort, case_insensitive_sort=None): + if case_insensitive_sort is None: + case_insensitive_sort = GlobalConfig.get_instance().defaults.sort_case_insensitive + + if sort_by_keys: + if case_insensitive_sort: + def sort_method(i): return tuple((i[column] == "", i[column].lower()) for column in sort_by_keys) + else: + def sort_method(i): return tuple((i[column] == "", i[column]) for column in sort_by_keys) + + return sorted(data, key=sort_method, reverse=reverse_sort) + elif reverse_sort: + return list(reversed(data)) + + return data diff --git a/tests/build_tasks/test_source_csv.py b/tests/build_tasks/test_source_csv.py index 949e021..ca9fa0b 100644 --- a/tests/build_tasks/test_source_csv.py +++ b/tests/build_tasks/test_source_csv.py @@ -8,7 +8,7 @@ from brain_brew.representation.configuration.csv_file_mapping import FileMapping from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping from brain_brew.representation.generic.csv_file import CsvFile -from brain_brew.representation.generic.generic_file import SourceFile +from brain_brew.representation.generic.source_file import SourceFile from brain_brew.representation.json.deck_part_notes import DeckPartNotes from tests.representation.json.test_deck_part_notes import dp_notes_test1 from tests.representation.configuration.test_note_model_mapping import setup_nmm_config diff --git a/tests/representation/generic/test_csv_file.py b/tests/representation/generic/test_csv_file.py index 5d7cd75..cc5d5e6 100644 --- a/tests/representation/generic/test_csv_file.py +++ b/tests/representation/generic/test_csv_file.py @@ -3,7 +3,7 @@ import pytest from brain_brew.representation.generic.csv_file import CsvFile, CsvKeys -from brain_brew.representation.generic.generic_file import SourceFile +from brain_brew.representation.generic.source_file import SourceFile from tests.test_files import TestFiles diff --git a/tests/representation/generic/test_generic_file.py b/tests/representation/generic/test_generic_file.py index dc66ac7..7dca059 100644 --- a/tests/representation/generic/test_generic_file.py +++ b/tests/representation/generic/test_generic_file.py @@ -3,7 +3,7 @@ import pytest from unittest.mock import MagicMock -from brain_brew.representation.generic.generic_file import SourceFile +from brain_brew.representation.generic.source_file import SourceFile from tests.test_file_manager import get_new_file_manager from tests.test_files import TestFiles diff --git a/tests/representation/json/test_deck_part_notes.py b/tests/representation/json/test_deck_part_notes.py index ed1be21..0597292 100644 --- a/tests/representation/json/test_deck_part_notes.py +++ b/tests/representation/json/test_deck_part_notes.py @@ -48,7 +48,7 @@ class TestSortData: # (["tags"], False, "tags", [(0, "besttag"), (1, "funny"), (2, "tag2, tag3"), (13, ""), (14, "")]), ]) def test_sort(self, dp_notes_test1: DeckPartNotes, keys, reverse, result_column, expected_results): - dp_notes_test1.sort_data( + dp_notes_test1.sort_dict( keys, reverse ) From 120af37671fb35f64a8cf70998cadbc644481f18 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sun, 6 Sep 2020 10:04:39 +0200 Subject: [PATCH 35/39] Yamale Yaml Verifier; Verify Only Run; Task Names to Regex; --- Pipfile | 3 +- Pipfile.lock | 124 ++++++++++-------- brain_brew/argument_reader.py | 10 +- .../crowd_anki/crowd_anki_generate.py | 2 +- .../crowd_anki/crowd_anki_to_deck_parts.py | 2 +- brain_brew/build_tasks/csvs/csvs_generate.py | 2 +- .../build_tasks/csvs/csvs_to_deck_parts.py | 2 +- .../build_tasks/deck_parts/from_deck_part.py | 2 +- brain_brew/main.py | 10 +- .../representation/build_config/build_task.py | 29 ++-- .../build_config/generate_deck_parts.py | 6 +- .../build_config/task_builder.py | 17 ++- .../build_config/top_level_task_builder.py | 13 +- brain_brew/schemas/__init__.py | 0 brain_brew/schemas/builder.yaml | 101 ++++++++++++++ brain_brew/yaml_verifier.py | 41 ++++++ requirements.txt | 20 ++- 17 files changed, 291 insertions(+), 93 deletions(-) create mode 100644 brain_brew/schemas/__init__.py create mode 100644 brain_brew/schemas/builder.yaml create mode 100644 brain_brew/yaml_verifier.py diff --git a/Pipfile b/Pipfile index e4fe427..a0b7c10 100644 --- a/Pipfile +++ b/Pipfile @@ -10,8 +10,9 @@ pytest = "==5.4.1" args = "==0.1.0" clint = "==0.5.1" coverage = "==4.5.4" -PyYAML = "==5.1.2" ruamel-yaml = "==0.16.10" +yamale = "==3.0.4" +PyYaml = "*" [requires] python_version = "3.7" diff --git a/Pipfile.lock b/Pipfile.lock index 09b156b..cf77f63 100644 --- a/Pipfile.lock +++ b/Pipfile.lock @@ -1,7 +1,7 @@ { "_meta": { "hash": { - "sha256": "db6cc2735bbbbd8ba94783e7605faabf6ec983a5703710bcf39f5176a84ca6f1" + "sha256": "9c9d18c53efca3d811a364f0b5207179a9cf9eb8fb47db85c9b8a53fa5b26bda" }, "pipfile-spec": 6, "requires": { @@ -70,84 +70,95 @@ }, "pyyaml": { "hashes": [ - "sha256:0113bc0ec2ad727182326b61326afa3d1d8280ae1122493553fd6f4397f33df9", - "sha256:01adf0b6c6f61bd11af6e10ca52b7d4057dd0be0343eb9283c878cf3af56aee4", - "sha256:5124373960b0b3f4aa7df1707e63e9f109b5263eca5976c66e08b1c552d4eaf8", - "sha256:5ca4f10adbddae56d824b2c09668e91219bb178a1eee1faa56af6f99f11bf696", - "sha256:7907be34ffa3c5a32b60b95f4d95ea25361c951383a894fec31be7252b2b6f34", - "sha256:7ec9b2a4ed5cad025c2278a1e6a19c011c80a3caaac804fd2d329e9cc2c287c9", - "sha256:87ae4c829bb25b9fe99cf71fbb2140c448f534e24c998cc60f39ae4f94396a73", - "sha256:9de9919becc9cc2ff03637872a440195ac4241c80536632fffeb6a1e25a74299", - "sha256:a5a85b10e450c66b49f98846937e8cfca1db3127a9d5d1e31ca45c3d0bef4c5b", - "sha256:b0997827b4f6a7c286c01c5f60384d218dca4ed7d9efa945c3e1aa623d5709ae", - "sha256:b631ef96d3222e62861443cc89d6563ba3eeb816eeb96b2629345ab795e53681", - "sha256:bf47c0607522fdbca6c9e817a6e81b08491de50f3766a7a0e6a5be7905961b41", - "sha256:f81025eddd0327c7d4cfe9b62cf33190e1e736cc6e97502b3ec425f574b3e7a8" + "sha256:06a0d7ba600ce0b2d2fe2e78453a470b5a6e000a985dd4a4e54e436cc36b0e97", + "sha256:240097ff019d7c70a4922b6869d8a86407758333f02203e0fc6ff79c5dcede76", + "sha256:4f4b913ca1a7319b33cfb1369e91e50354d6f07a135f3b901aca02aa95940bd2", + "sha256:69f00dca373f240f842b2931fb2c7e14ddbacd1397d57157a9b005a6a9942648", + "sha256:73f099454b799e05e5ab51423c7bcf361c58d3206fa7b0d555426b1f4d9a3eaf", + "sha256:74809a57b329d6cc0fdccee6318f44b9b8649961fa73144a98735b0aaf029f1f", + "sha256:7739fc0fa8205b3ee8808aea45e968bc90082c10aef6ea95e855e10abf4a37b2", + "sha256:95f71d2af0ff4227885f7a6605c37fd53d3a106fcab511b8860ecca9fcf400ee", + "sha256:b8eac752c5e14d3eca0e6dd9199cd627518cb5ec06add0de9d32baeee6fe645d", + "sha256:cc8955cfbfc7a115fa81d85284ee61147059a753344bc51098f3ccd69b0d7e0c", + "sha256:d13155f591e6fcc1ec3b30685d50bf0711574e2c0dfffd7644babf8b5102ca1a" ], - "index": "Brain Brew", - "version": "==5.1.2" + "version": "==5.3.1" }, "ruamel-yaml": { "hashes": [ - "sha256:0962fd7999e064c4865f96fb1e23079075f4a2a14849bcdc5cdba53a24f9759b" + "sha256:0962fd7999e064c4865f96fb1e23079075f4a2a14849bcdc5cdba53a24f9759b", + "sha256:099c644a778bf72ffa00524f78dd0b6476bca94a1da344130f4bf3381ce5b954" ], "index": "Brain Brew", "version": "==0.16.10" }, "ruamel.yaml.clib": { "hashes": [ - "sha256:1e77424825caba5553bbade750cec2277ef130647d685c2b38f68bc03453bac6", - "sha256:392b7c371312abf27fb549ec2d5e0092f7ef6e6c9f767bfb13e83cb903aca0fd", - "sha256:4d55386129291b96483edcb93b381470f7cd69f97585829b048a3d758d31210a", - "sha256:550168c02d8de52ee58c3d8a8193d5a8a9491a5e7b2462d27ac5bf63717574c9", - "sha256:57933a6986a3036257ad7bf283529e7c19c2810ff24c86f4a0cfeb49d2099919", - "sha256:615b0396a7fad02d1f9a0dcf9f01202bf9caefee6265198f252c865f4227fcc6", - "sha256:77556a7aa190be9a2bd83b7ee075d3df5f3c5016d395613671487e79b082d784", - "sha256:7aee724e1ff424757b5bd8f6c5bbdb033a570b2b4683b17ace4dbe61a99a657b", - "sha256:8073c8b92b06b572e4057b583c3d01674ceaf32167801fe545a087d7a1e8bf52", - "sha256:9c6d040d0396c28d3eaaa6cb20152cb3b2f15adf35a0304f4f40a3cf9f1d2448", - "sha256:a0ff786d2a7dbe55f9544b3f6ebbcc495d7e730df92a08434604f6f470b899c5", - "sha256:b1b7fcee6aedcdc7e62c3a73f238b3d080c7ba6650cd808bce8d7761ec484070", - "sha256:b66832ea8077d9b3f6e311c4a53d06273db5dc2db6e8a908550f3c14d67e718c", - "sha256:be018933c2f4ee7de55e7bd7d0d801b3dfb09d21dad0cce8a97995fd3e44be30", - "sha256:d0d3ac228c9bbab08134b4004d748cf9f8743504875b3603b3afbb97e3472947", - "sha256:d10e9dd744cf85c219bf747c75194b624cc7a94f0c80ead624b06bfa9f61d3bc", - "sha256:ea4362548ee0cbc266949d8a441238d9ad3600ca9910c3fe4e82ee3a50706973", - "sha256:ed5b3698a2bb241b7f5cbbe277eaa7fe48b07a58784fba4f75224fd066d253ad", - "sha256:f9dcc1ae73f36e8059589b601e8e4776b9976effd76c21ad6a855a74318efd6e" - ], - "markers": "platform_python_implementation == 'CPython' and python_version < '3.9'", - "version": "==0.2.0" + "sha256:058a1cc3df2a8aecc12f983a48bda99315cebf55a3b3a5463e37bb599b05727b", + "sha256:2602e91bd5c1b874d6f93d3086f9830f3e907c543c7672cf293a97c3fabdcd91", + "sha256:28116f204103cb3a108dfd37668f20abe6e3cafd0d3fd40dba126c732457b3cc", + "sha256:2d24bd98af676f4990c4d715bcdc2a60b19c56a3fb3a763164d2d8ca0e806ba7", + "sha256:30dca9bbcbb1cc858717438218d11eafb78666759e5094dd767468c0d577a7e7", + "sha256:44c7b0498c39f27795224438f1a6be6c5352f82cb887bc33d962c3a3acc00df6", + "sha256:464e66a04e740d754170be5e740657a3b3b6d2bcc567f0c3437879a6e6087ff6", + "sha256:4df5019e7783d14b79217ad9c56edf1ba7485d614ad5a385d1b3c768635c81c0", + "sha256:4e52c96ca66de04be42ea2278012a2342d89f5e82b4512fb6fb7134e377e2e62", + "sha256:5254af7d8bdf4d5484c089f929cb7f5bafa59b4f01d4f48adda4be41e6d29f99", + "sha256:52ae5739e4b5d6317b52f5b040b1b6639e8af68a5b8fd606a8b08658fbd0cab5", + "sha256:53b9dd1abd70e257a6e32f934ebc482dac5edb8c93e23deb663eac724c30b026", + "sha256:73b3d43e04cc4b228fa6fa5d796409ece6fcb53a6c270eb2048109cbcbc3b9c2", + "sha256:74161d827407f4db9072011adcfb825b5258a5ccb3d2cd518dd6c9edea9e30f1", + "sha256:839dd72545ef7ba78fd2aa1a5dd07b33696adf3e68fae7f31327161c1093001b", + "sha256:8e8fd0a22c9d92af3a34f91e8a2594eeb35cba90ab643c5e0e643567dc8be43e", + "sha256:a873e4d4954f865dcb60bdc4914af7eaae48fb56b60ed6daa1d6251c72f5337c", + "sha256:ab845f1f51f7eb750a78937be9f79baea4a42c7960f5a94dde34e69f3cce1988", + "sha256:b1e981fe1aff1fd11627f531524826a4dcc1f26c726235a52fcb62ded27d150f", + "sha256:daf21aa33ee9b351f66deed30a3d450ab55c14242cfdfcd377798e2c0d25c9f1", + "sha256:e9f7d1d8c26a6a12c23421061f9022bb62704e38211fe375c645485f38df34a2", + "sha256:f6061a31880c1ed6b6ce341215336e2f3d0c1deccd84957b6fa8ca474b41e89f" + ], + "markers": "python_version < '3.9' and platform_python_implementation == 'CPython'", + "version": "==0.2.2" + }, + "yamale": { + "hashes": [ + "sha256:bec53aa08e29c26a9a47c0f5d5cb5632e7ac8aff619a4727acfe490042c83a5f" + ], + "index": "Brain Brew", + "version": "==3.0.4" } }, "develop": { "attrs": { "hashes": [ - "sha256:08a96c641c3a74e44eb59afb61a24f2cb9f4d7188748e76ba4bb5edfa3cb7d1c", - "sha256:f7b7ce16570fe9965acd6d30101a28f62fb4a7f9e926b3bbc9b61f8b04247e72" + "sha256:0ef97238856430dcf9228e07f316aefc17e8939fc8507e18c6501b761ef1a42a", + "sha256:2867b7b9f8326499ab5b0e2d12801fa5c98842d2cbd22b35112ae04bf85b4dff" ], - "version": "==19.3.0" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==20.1.0" }, "importlib-metadata": { "hashes": [ - "sha256:0505dd08068cfec00f53a74a0ad927676d7757da81b7436a6eefe4c7cf75c545", - "sha256:15ec6c0fd909e893e3a08b3a7c76ecb149122fb14b7efe1199ddd4c7c57ea958" + "sha256:90bb658cdbbf6d1735b6341ce708fc7024a3e14e99ffdc5783edea9f9b077f83", + "sha256:dc15b2969b4ce36305c51eebe62d418ac7791e9a157911d58bfb1f9ccd8e2070" ], "markers": "python_version < '3.8'", - "version": "==1.6.1" + "version": "==1.7.0" }, "more-itertools": { "hashes": [ - "sha256:558bb897a2232f5e4f8e2399089e35aecb746e1f9191b6584a151647e89267be", - "sha256:7818f596b1e87be009031c7653d01acc46ed422e6656b394b0f765ce66ed4982" + "sha256:6f83822ae94818eae2612063a5101a7311e68ae8002005b5e05f03fd74a86a20", + "sha256:9b30f12df9393f0d28af9210ff8efe48d10c94f73e5daf886f10c4b0b0b4f03c" ], - "version": "==8.3.0" + "markers": "python_version >= '3.5'", + "version": "==8.5.0" }, "packaging": { "hashes": [ "sha256:4357f74f47b9c12db93624a82154e9b120fa8293699949152b22065d556079f8", "sha256:998416ba6962ae7fbd6596850b80e17859a5753ba17c32284f67bfff33784181" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==20.4" }, "pluggy": { @@ -155,20 +166,23 @@ "sha256:15b2acde666561e1298d71b523007ed7364de07029219b604cf808bfa1c765b0", "sha256:966c145cd83c96502c3c3868f50408687b38434af77734af1e9ca461a4081d2d" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==0.13.1" }, "py": { "hashes": [ - "sha256:5e27081401262157467ad6e7f851b7aa402c5852dbcb3dae06768434de5752aa", - "sha256:c20fdd83a5dbc0af9efd622bee9a5564e278f6380fffcacc43ba6f43db2813b0" + "sha256:366389d1db726cd2fcfc79732e75410e5fe4d31db13692115529d34069a043c2", + "sha256:9ca6883ce56b4e8da7e79ac18787889fa5206c79dcc67fb065376cd2fe03f342" ], - "version": "==1.8.1" + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", + "version": "==1.9.0" }, "pyparsing": { "hashes": [ "sha256:c203ec8783bf771a155b207279b9bccb8dea02d8f0c9e5f8ead507bc3246ecc1", "sha256:ef9d7589ef3c200abe66653d3f1ab1033c3c419ae9b9bdb1240a85b024efc88b" ], + "markers": "python_version >= '2.6' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==2.4.7" }, "pytest": { @@ -184,20 +198,22 @@ "sha256:30639c035cdb23534cd4aa2dd52c3bf48f06e5f4a941509c8bafd8ce11080259", "sha256:8b74bedcbbbaca38ff6d7491d76f2b06b3592611af620f8426e82dddb04a5ced" ], + "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'", "version": "==1.15.0" }, "wcwidth": { "hashes": [ - "sha256:79375666b9954d4a1a10739315816324c3e73110af9d0e102d906fdb0aec009f", - "sha256:8c6b5b6ee1360b842645f336d9e5d68c55817c26d3050f46b235ef2bc650e48f" + "sha256:beb4802a9cebb9144e99086eff703a642a13d6a0052920003a230f3294bbe784", + "sha256:c4d647b99872929fdb7bdcaa4fbe7f01413ed3d98077df798530e5b04f116c83" ], - "version": "==0.2.4" + "version": "==0.2.5" }, "zipp": { "hashes": [ "sha256:aa36550ff0c0b7ef7fa639055d797116ee891440eac1a56f378e2d3179e0320b", "sha256:c599e4d75c98f6798c509911d08a22e6c021d074469042177c8c86fb92eefd96" ], + "markers": "python_version >= '3.6'", "version": "==3.1.0" } } diff --git a/brain_brew/argument_reader.py b/brain_brew/argument_reader.py index 1dec7a1..041dcb6 100644 --- a/brain_brew/argument_reader.py +++ b/brain_brew/argument_reader.py @@ -25,6 +25,13 @@ def _set_parser_arguments(self): type=str, help="Global config file to use" ) + self.add_argument( + "--verify", "-v", + action="store_true", + dest="verify_only", + default=False, + help="Only verify the builder contents, without running it." + ) def get_parsed(self, override_args=None): parsed_args = self.parse_args(args=override_args) @@ -34,8 +41,9 @@ def get_parsed(self, override_args=None): # Optional config_file = parsed_args.config_file + verify_only = parsed_args.verify_only - return builder, config_file + return builder, config_file, verify_only def error_if_blank(self, arg): if arg == "" or arg is None: diff --git a/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py b/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py index 019a3f6..7f902aa 100644 --- a/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py +++ b/brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py @@ -20,7 +20,7 @@ @dataclass class CrowdAnkiGenerate(TopLevelBuildTask): - task_names = all_combos_prepend_append(["CrowdAnki", "CrowdAnki Export"], "Generate ", "s") + task_regex = r'.*crowd[\s_-]+?anki.*' @dataclass class Representation(RepresentationBase): diff --git a/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py b/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py index 9a619fa..5e2b79b 100644 --- a/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py +++ b/brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py @@ -15,7 +15,7 @@ @dataclass class CrowdAnkiToDeckParts(DeckPartBuildTask): - task_names = all_combos_prepend_append(["CrowdAnki", "CrowdAnkiExport"], "From ", "s") + task_regex = r'.*crowd[\s_-]+?anki.*' @dataclass class Representation(RepresentationBase): diff --git a/brain_brew/build_tasks/csvs/csvs_generate.py b/brain_brew/build_tasks/csvs/csvs_generate.py index 1c70341..bb2f4bf 100644 --- a/brain_brew/build_tasks/csvs/csvs_generate.py +++ b/brain_brew/build_tasks/csvs/csvs_generate.py @@ -11,7 +11,7 @@ @dataclass class CsvsGenerate(SharedBaseCsvs, TopLevelBuildTask): - task_names = all_combos_prepend_append(["Csv Collection", "Csv"], "Generate ", "s") + task_regex = r'.*csv.*' notes: DeckPartHolder[Notes] = field(default=None) diff --git a/brain_brew/build_tasks/csvs/csvs_to_deck_parts.py b/brain_brew/build_tasks/csvs/csvs_to_deck_parts.py index d748243..f174b83 100644 --- a/brain_brew/build_tasks/csvs/csvs_to_deck_parts.py +++ b/brain_brew/build_tasks/csvs/csvs_to_deck_parts.py @@ -9,7 +9,7 @@ @dataclass class CsvsToDeckParts(DeckPartBuildTask): - task_names = all_combos_prepend_append(["Csv Collection", "Csv"], "From ", "s") + task_regex = r'.*csv.*' @dataclass class Representation(RepresentationBase): diff --git a/brain_brew/build_tasks/deck_parts/from_deck_part.py b/brain_brew/build_tasks/deck_parts/from_deck_part.py index c50a287..00f7b92 100644 --- a/brain_brew/build_tasks/deck_parts/from_deck_part.py +++ b/brain_brew/build_tasks/deck_parts/from_deck_part.py @@ -12,7 +12,7 @@ @dataclass class FromDeckParts(DeckPartBuildTask): - task_names = all_combos_prepend_append(["DeckPart", "ReadDeckPart"], "From ", "s") + task_regex = r'.*deck[\s_-]+?part.*' @dataclass class DeckPartToRead(RepresentationBase): diff --git a/brain_brew/main.py b/brain_brew/main.py index 1072dce..1bd7eab 100644 --- a/brain_brew/main.py +++ b/brain_brew/main.py @@ -7,6 +7,7 @@ # sys.path.append(os.path.join(os.path.dirname(__file__), "dist")) # sys.path.append(os.path.dirname(__file__)) +from brain_brew.yaml_verifier import YamlVerifier def main(): @@ -14,18 +15,19 @@ def main(): # Read in Arguments argument_reader = BBArgumentReader() - builder_file_name, global_config_file = argument_reader.get_parsed() + builder_file_name, global_config_file, verify_only = argument_reader.get_parsed() # Read in Global Config File global_config = GlobalConfig.from_file(global_config_file) if global_config_file else GlobalConfig.from_file() file_manager = FileManager() # Parse Build Config File - builder_data = TopLevelTaskBuilder.read_to_dict(builder_file_name) - builder = TopLevelTaskBuilder.from_list(builder_data) + YamlVerifier() + builder = TopLevelTaskBuilder.parse_and_read(builder_file_name) # If all good, execute it - builder.execute() + if not verify_only: + builder.execute() if __name__ == "__main__": diff --git a/brain_brew/representation/build_config/build_task.py b/brain_brew/representation/build_config/build_task.py index 53fcb44..5b9c9bc 100644 --- a/brain_brew/representation/build_config/build_task.py +++ b/brain_brew/representation/build_config/build_task.py @@ -5,7 +5,7 @@ class BuildTask(object): - task_names: List[str] + task_regex: str def execute(self): raise NotImplemented() @@ -15,32 +15,29 @@ def from_repr(cls, data: dict): raise NotImplemented() @classmethod - def get_all_build_tasks(cls) -> Dict[str, Type['BuildTask']]: + def get_all_task_regex(cls) -> Dict[str, Type['BuildTask']]: subclasses: List[Type[BuildTask]] = cls.__subclasses__() - known_build_tasks: Dict[str, Type[BuildTask]] = {} + task_regex_matches: Dict[str, Type[BuildTask]] = {} for sc in subclasses: - for original_task_name in sc.task_names: - task_name = str_to_lowercase_no_separators(original_task_name) + if sc.task_regex in task_regex_matches: + raise KeyError(f"Multiple instances of task regex '{sc.task_regex}'") + elif sc.task_regex == "" or sc.task_regex is None: + raise KeyError(f"Unknown task regex {sc.task_regex}") - if task_name in known_build_tasks: - raise KeyError(f"Multiple instances of task name '{task_name}'") - elif task_name == "" or task_name is None: - raise KeyError(f"Unknown task name {original_task_name}") - - known_build_tasks.setdefault(task_name, sc) + task_regex_matches.setdefault(sc.task_regex, sc) # logging.debug(f"Known build tasks: {known_build_tasks}") - return known_build_tasks + return task_regex_matches class TopLevelBuildTask(BuildTask): @classmethod - def get_all_build_tasks(cls) -> Dict[str, Type['BuildTask']]: - return super(TopLevelBuildTask, cls).get_all_build_tasks() + def get_all_task_regex(cls) -> Dict[str, Type['BuildTask']]: + return super(TopLevelBuildTask, cls).get_all_task_regex() class DeckPartBuildTask(BuildTask): @classmethod - def get_all_build_tasks(cls) -> Dict[str, Type['BuildTask']]: - return super(DeckPartBuildTask, cls).get_all_build_tasks() + def get_all_task_regex(cls) -> Dict[str, Type['BuildTask']]: + return super(DeckPartBuildTask, cls).get_all_task_regex() diff --git a/brain_brew/representation/build_config/generate_deck_parts.py b/brain_brew/representation/build_config/generate_deck_parts.py index a0bc88f..ee1f21f 100644 --- a/brain_brew/representation/build_config/generate_deck_parts.py +++ b/brain_brew/representation/build_config/generate_deck_parts.py @@ -8,18 +8,20 @@ from brain_brew.build_tasks.deck_parts.from_deck_part import FromDeckParts from brain_brew.build_tasks.csvs.csvs_to_deck_parts import CsvsToDeckParts from brain_brew.build_tasks.crowd_anki.crowd_anki_to_deck_parts import CrowdAnkiToDeckParts +from brain_brew.utils import str_to_lowercase_no_separators @dataclass class GenerateDeckParts(TaskBuilder, TopLevelBuildTask): - task_names = ["Generate Deck Parts", "Generate Deck Part", "Deck Part", "Deck Parts"] + task_regex = r'.*deck[\s_-]+?part.*' @classmethod def known_task_dict(cls) -> Dict[str, Type[BuildTask]]: - return DeckPartBuildTask.get_all_build_tasks() + return DeckPartBuildTask.get_all_task_regex() @classmethod def from_repr(cls, data: List[dict]): if not isinstance(data, list): raise TypeError(f"GenerateDeckParts needs a list") return cls.from_list(data) + diff --git a/brain_brew/representation/build_config/task_builder.py b/brain_brew/representation/build_config/task_builder.py index 0f4ddc9..553796b 100644 --- a/brain_brew/representation/build_config/task_builder.py +++ b/brain_brew/representation/build_config/task_builder.py @@ -1,5 +1,6 @@ from dataclasses import dataclass from typing import Dict, List, Type +import re from brain_brew.interfaces.verifiable import Verifiable from brain_brew.representation.build_config.build_task import BuildTask @@ -24,9 +25,15 @@ def known_task_dict(cls) -> Dict[str, Type[BuildTask]]: @classmethod def read_tasks(cls, tasks: List[dict]) -> list: - known_task_dict = cls.known_task_dict() + task_regex_matches = cls.known_task_dict() build_tasks = [] + def find_matching_task(task_n): + for regex, task_to_run in task_regex_matches.items(): + if re.match(regex, task_n, re.RegexFlag.IGNORECASE): + return task_to_run + return None + # Tasks for task in tasks: task_keys = list(task.keys()) @@ -34,10 +41,12 @@ def read_tasks(cls, tasks: List[dict]) -> list: raise KeyError(f"Task should only contain 1 entry, but contains {task_keys} instead. " f"Missing list separator '-'?", task) - task_name = str_to_lowercase_no_separators(task_keys[0]) + task_name = task_keys[0] task_arguments = task[task_keys[0]] - if task_name in known_task_dict: - task_instance = known_task_dict[task_name].from_repr(task_arguments) + + matching_task = find_matching_task(task_name) + if matching_task is not None: + task_instance = matching_task.from_repr(task_arguments) build_tasks.append(task_instance) else: raise KeyError(f"Unknown task '{task_name}'") # TODO: check this first on all and return all errors diff --git a/brain_brew/representation/build_config/top_level_task_builder.py b/brain_brew/representation/build_config/top_level_task_builder.py index 7e7bc14..b4aaa1d 100644 --- a/brain_brew/representation/build_config/top_level_task_builder.py +++ b/brain_brew/representation/build_config/top_level_task_builder.py @@ -8,9 +8,20 @@ from brain_brew.build_tasks.csvs.csvs_generate import CsvsGenerate from brain_brew.build_tasks.crowd_anki.crowd_anki_generate import CrowdAnkiGenerate from brain_brew.representation.build_config.generate_deck_parts import GenerateDeckParts +from brain_brew.utils import str_to_lowercase_no_separators class TopLevelTaskBuilder(TaskBuilder): @classmethod def known_task_dict(cls) -> Dict[str, Type[BuildTask]]: - return TopLevelBuildTask.get_all_build_tasks() + values = TopLevelBuildTask.get_all_task_regex() + return values + + @classmethod + def parse_and_read(cls, filename) -> 'TopLevelTaskBuilder': + builder_data = TopLevelTaskBuilder.read_to_dict(filename) + + from brain_brew.yaml_verifier import YamlVerifier + YamlVerifier.get_instance().verify_builder(filename) + + return TopLevelTaskBuilder.from_list(builder_data) diff --git a/brain_brew/schemas/__init__.py b/brain_brew/schemas/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/brain_brew/schemas/builder.yaml b/brain_brew/schemas/builder.yaml new file mode 100644 index 0000000..e138d65 --- /dev/null +++ b/brain_brew/schemas/builder.yaml @@ -0,0 +1,101 @@ +list( + map(include('generate_deck_parts'), key=regex('.*deck[\s_-]+?part.*', ignore_case=True)), + map(include('generate_csv'), key=regex('.*csv.*', ignore_case=True)), + map(include('generate_crowd_anki'), key=regex('.*crowd[\s_-]+?anki.*', ignore_case=True)) +) + +--- + +generate_deck_parts: + list( + map(include('from_deck_parts'), key=regex('.*deck[\s_-]+?part.*', ignore_case=True)), + map(include('from_csv'), key=regex('.*csv.*', ignore_case=True)), + map(include('from_crowd_anki'), key=regex('.*crowd[\s_-]+?anki.*', ignore_case=True)) + ) + + + +from_crowd_anki: + folder: str() + notes: include('from_ca_notes', required=False) + note_models: list(include('from_ca_note_models'), required=False) + headers: include('from_ca_headers', required=False) + media: any(bool(), include('ca_media'), required=False) + + +generate_crowd_anki: + folder: str() + headers: str() + notes: include('notes_to_source') + note_models: include('note_models_to_source') + media: any(bool(), include('ca_media'), required=False) + + + +from_ca_notes: + name: str() + sort_order: list(str(), required=False) + save_to_file: str(required=False) + +from_ca_note_models: + name: str() + model_name: str(required=False) + save_to_file: str(required=False) + +from_ca_headers: + name: str() + save_to_file: str(required=False) + +ca_media: + from_notes: bool() + from_note_models: bool() + +note_models_to_source: + deck_parts: list(include('note_model_to_source')) + +note_model_to_source: + deck_part: str() + +notes_to_source: + deck_part: str() + sort_order: list(str(), required=False) + reverse_sort: bool(required=False) + additional_items_to_add: map(str(), key=str(), required=False) + + + + +from_deck_parts: + notes: list(include('deck_part_to_read'), required=False) + note_models: list(include('deck_part_to_read'), required=False) + headers: list(include('deck_part_to_read'), required=False) + +deck_part_to_read: + name: str() + file: str() + + + +from_csv: + notes: + name: str() + save_to_file: str(required=False) + note_model_mappings: list(include('note_model_mapping')) + file_mappings: list(include('file_mapping')) + +generate_csv: + notes: str() + note_model_mappings: list(include('note_model_mapping')) + file_mappings: list(include('file_mapping')) + +note_model_mapping: + note_models: list(str()) + columns_to_fields: map(str(), key=str()) + personal_fields: list(str()) + +file_mapping: + file: str() + note_model: str(required=False) + sort_by_columns: list(str(), required=False) + reverse_sort: bool(required=False) + derivatives: list(include('file_mapping'), required=False) \ No newline at end of file diff --git a/brain_brew/yaml_verifier.py b/brain_brew/yaml_verifier.py new file mode 100644 index 0000000..6c9603d --- /dev/null +++ b/brain_brew/yaml_verifier.py @@ -0,0 +1,41 @@ +import logging +import os +import sys + +import yamale +from yamale import YamaleError +from yamale.schema import Schema +from yamale.validators import DefaultValidators + +validators = DefaultValidators.copy() + + +class YamlVerifier: + __instance = None + builder_schema: Schema + + def __init__(self): + if YamlVerifier.__instance is None: + YamlVerifier.__instance = self + else: + raise Exception("Multiple YamlVerifiers created") + + path = os.path.join(os.path.dirname(__file__), "schemas/builder.yaml") + self.builder_schema = yamale.make_schema(path, parser='ruamel', validators=validators) + + @staticmethod + def get_instance() -> 'YamlVerifier': + return YamlVerifier.__instance + + def verify_builder(self, filename): + data = yamale.make_data(filename) + try: + yamale.validate(self.builder_schema, data) + except YamaleError as e: + print('Validation failed!\n') + for result in e.results: + print("Error validating data '%s' with '%s'\n\t" % (result.data, result.schema)) + for error in result.errors: + print('\t%s' % error) + exit(1) + logging.info(f"Builder file {filename} is ✔") diff --git a/requirements.txt b/requirements.txt index 72532f1..abe389e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,18 @@ args==0.1.0 +attrs==20.1.0 clint==0.5.1 coverage==4.5.4 -PyYAML==5.1.2 -ruamel-yaml==0.16.10 - -# Testing -pytest==5.4.1 \ No newline at end of file +importlib-metadata==1.7.0 +more-itertools==8.5.0 +packaging==20.4 +pluggy==0.13.1 +py==1.9.0 +pyparsing==2.4.7 +pytest==5.4.1 +PyYAML==5.3.1 +ruamel.yaml==0.16.10 +ruamel.yaml.clib==0.2.2 +six==1.15.0 +wcwidth==0.2.5 +yamale==3.0.4 +zipp==3.1.0 From 76efc839394805ad3ee1aa7dde231e66b9872505 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sun, 6 Sep 2020 15:48:25 +0200 Subject: [PATCH 36/39] Broken Tests Disabled --- brain_brew/file_manager.py | 2 +- .../configuration/csv_file_mapping.py | 9 +- .../configuration/global_config.py | 48 +-- brain_brew/representation/yaml/note_repr.py | 2 +- brain_brew/utils.py | 4 +- tests/build_tasks/test_build_task_generic.py | 34 -- .../test_source_crowd_anki_json.py | 226 ++++++------- tests/build_tasks/test_source_csv.py | 278 ++++++++-------- .../configuration/test_csv_file_mapping.py | 312 +++++++++--------- .../configuration/test_global_config.py | 49 --- .../configuration/test_note_model_mapping.py | 293 ++++++++-------- .../deck_part_transformers/__init__.py | 0 .../test_tr_notes_csv_collection.py | 145 -------- tests/representation/generic/test_csv_file.py | 130 +++----- .../generic/test_generic_file.py | 53 --- .../representation/generic/test_media_file.py | 3 + .../json/test_crowd_anki_export.py | 5 +- .../json/test_deck_part_header.py | 47 --- .../json/test_deck_part_note_model.py | 89 ----- .../json/test_deck_part_notes.py | 89 ----- tests/representation/json/test_json_file.py | 39 --- .../yaml/test_note_model_repr.py | 6 +- tests/representation/yaml/test_note_repr.py | 250 +++++++------- tests/test_argument_reader.py | 12 +- tests/test_builder.py | 37 +-- tests/test_file_manager.py | 49 ++- tests/test_files/build_files/builder1.yaml | 72 ++-- .../csv_collection_config_with_only1.yaml | 13 - .../deck_parts/csvtonotes1_withgrouping.json | 137 -------- ...svtonotes1_withnogroupingorsharedtags.json | 150 --------- .../csvtonotes1_withsharedtags.json | 151 --------- ...aredtagsandgrouping_butnothingtogroup.json | 138 -------- ...csvtonotes2_withsharedtagsandgrouping.json | 101 ------ .../deck_parts/headers/default-header.json | 62 ---- tests/test_helpers.py | 1 + tests/test_utils.py | 32 +- 36 files changed, 870 insertions(+), 2198 deletions(-) delete mode 100644 tests/build_tasks/test_build_task_generic.py delete mode 100644 tests/representation/configuration/test_global_config.py delete mode 100644 tests/representation/deck_part_transformers/__init__.py delete mode 100644 tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py delete mode 100644 tests/representation/generic/test_generic_file.py delete mode 100644 tests/representation/json/test_deck_part_header.py delete mode 100644 tests/representation/json/test_deck_part_note_model.py delete mode 100644 tests/representation/json/test_deck_part_notes.py delete mode 100644 tests/representation/json/test_json_file.py delete mode 100644 tests/test_files/build_files/csv_collection_config_with_only1.yaml delete mode 100644 tests/test_files/deck_parts/csvtonotes1_withgrouping.json delete mode 100644 tests/test_files/deck_parts/csvtonotes1_withnogroupingorsharedtags.json delete mode 100644 tests/test_files/deck_parts/csvtonotes1_withsharedtags.json delete mode 100644 tests/test_files/deck_parts/csvtonotes1_withsharedtagsandgrouping_butnothingtogroup.json delete mode 100644 tests/test_files/deck_parts/csvtonotes2_withsharedtagsandgrouping.json delete mode 100644 tests/test_files/deck_parts/headers/default-header.json diff --git a/brain_brew/file_manager.py b/brain_brew/file_manager.py index 85d9ff7..ceee543 100644 --- a/brain_brew/file_manager.py +++ b/brain_brew/file_manager.py @@ -84,7 +84,7 @@ def new_deck_part(self, dp: DeckPartHolder) -> DeckPartHolder: def find_all_deck_part_media_files(self): self.known_media_files_dict = {} - for full_path in find_all_files_in_directory(self.global_config.deck_parts.media_files, recursive=True): + for full_path in find_all_files_in_directory(self.global_config.media_files_location, recursive=True): filename = filename_from_full_path(full_path) self._register_media_file(MediaFile(full_path, filename)) diff --git a/brain_brew/representation/configuration/csv_file_mapping.py b/brain_brew/representation/configuration/csv_file_mapping.py index 1b3efed..2bdcab8 100644 --- a/brain_brew/representation/configuration/csv_file_mapping.py +++ b/brain_brew/representation/configuration/csv_file_mapping.py @@ -8,7 +8,7 @@ from brain_brew.utils import single_item_to_list, generate_anki_guid -FILE = "csv" +FILE = "csv_file" NOTE_MODEL = "note_model" SORT_BY_COLUMNS = "sort_by_columns" REVERSE_SORT = "reverse_sort" @@ -111,11 +111,14 @@ def write_to_csv(self, data_to_set): class FileMapping(FileMappingDerivative, Verifiable): note_model: str # Override Optional on Parent + data_set_has_changed: bool = field(init=False, default=False) + def verify_contents(self): pass def compile_data(self): self.compiled_data = {} + self.data_set_has_changed = False data_in_progress = self._build_data_recursive() @@ -134,6 +137,7 @@ def compile_data(self): self.compiled_data.setdefault(guid, {key.lower(): row[key] for key in row}) if guids_generated > 0: + self.data_set_has_changed = True logging.info(f"Generated {guids_generated} guids in {self.csv_file.file_location}") def set_relevant_data(self, data_set: Dict[str, dict]): @@ -153,6 +157,9 @@ def set_relevant_data(self, data_set: Dict[str, dict]): added += 1 self.compiled_data.setdefault(guid, data_set[guid]) + if changed > 0 or added > 0: + self.data_set_has_changed = True + logging.info(f"Set {self.csv_file.file_location} data; changed {changed}, " f"added {added}, while {unchanged} were identical") diff --git a/brain_brew/representation/configuration/global_config.py b/brain_brew/representation/configuration/global_config.py index 5b3dc0a..617f60c 100644 --- a/brain_brew/representation/configuration/global_config.py +++ b/brain_brew/representation/configuration/global_config.py @@ -1,5 +1,5 @@ from dataclasses import dataclass, field -from typing import List, Union, Optional +from typing import Union, Optional from brain_brew.representation.build_config.representation_base import RepresentationBase from brain_brew.representation.yaml.my_yaml import YamlRepr @@ -9,51 +9,23 @@ class GlobalConfig(YamlRepr): __instance = None - @dataclass - class Defaults: - @dataclass - class Representation(RepresentationBase): - note_sort_order: Optional[Union[List[str], str]] = field(default_factory=[]) - sort_case_insensitive: Optional[bool] = field(default=False) - reverse_sort: Optional[bool] = field(default=False) - join_values_with: Optional[str] = field(default=" ") - - note_sort_order: list - sort_case_insensitive: bool - reverse_sort: bool - join_values_with: str - - @classmethod - def from_repr(cls, data: Union[Representation, dict]): - rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) - return cls( - note_sort_order=rep.note_sort_order, - sort_case_insensitive=rep.sort_case_insensitive, - reverse_sort=rep.reverse_sort, - join_values_with=rep.join_values_with - ) - - @dataclass - class DeckPartLocations(RepresentationBase): - headers: str - note_models: str - notes: str - media_files: str - @dataclass class Representation(RepresentationBase): - deck_parts: dict - defaults: Optional[dict] + media_files_location: str + sort_case_insensitive: Optional[bool] = field(default=False) + join_values_with: Optional[str] = field(default=" ") - deck_parts: DeckPartLocations - defaults: Defaults + sort_case_insensitive: bool + join_values_with: str + media_files_location: str @classmethod def from_repr(cls, data: Union[Representation, dict]): rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data) return cls( - deck_parts=cls.DeckPartLocations.from_dict(rep.deck_parts), - defaults=cls.Defaults.from_repr(rep.defaults) + media_files_location=rep.media_files_location, + sort_case_insensitive=rep.sort_case_insensitive, + join_values_with=rep.join_values_with ) def __post_init__(self): diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index 255e038..f9c2caa 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -99,7 +99,7 @@ def get_all_media_references(self) -> Set[str]: def get_sorted_notes(self, sort_by_keys, reverse_sort, case_insensitive_sort=None): if case_insensitive_sort is None: - case_insensitive_sort = GlobalConfig.get_instance().defaults.sort_case_insensitive + case_insensitive_sort = GlobalConfig.get_instance().sort_case_insensitive if sort_by_keys: def sort_method(i: Note): diff --git a/brain_brew/utils.py b/brain_brew/utils.py index a65ee48..0a27075 100644 --- a/brain_brew/utils.py +++ b/brain_brew/utils.py @@ -66,7 +66,7 @@ def split_tags(tags_value: str) -> list: def join_tags(tags_list: list) -> str: - return GlobalConfig.get_instance().defaults.join_values_with.join(tags_list) + return GlobalConfig.get_instance().join_values_with.join(tags_list) def generate_anki_guid() -> str: @@ -92,7 +92,7 @@ def base91(num: int) -> str: def sort_dict(data, sort_by_keys, reverse_sort, case_insensitive_sort=None): if case_insensitive_sort is None: - case_insensitive_sort = GlobalConfig.get_instance().defaults.sort_case_insensitive + case_insensitive_sort = GlobalConfig.get_instance().sort_case_insensitive if sort_by_keys: if case_insensitive_sort: diff --git a/tests/build_tasks/test_build_task_generic.py b/tests/build_tasks/test_build_task_generic.py deleted file mode 100644 index acab681..0000000 --- a/tests/build_tasks/test_build_task_generic.py +++ /dev/null @@ -1,34 +0,0 @@ -import pytest - -from brain_brew.build_tasks.build_task_generic import BuildTaskGeneric -from tests.representation.configuration.test_global_config import global_config - - -class TestSplitTags: - @pytest.mark.parametrize("str_to_split, expected_result", [ - ("tags1, tags2", ["tags1", "tags2"]), - ("tags1 tags2", ["tags1", "tags2"]), - ("tags1; tags2", ["tags1", "tags2"]), - ("tags1 tags2", ["tags1", "tags2"]), - ("tags1, tags2, tags3, tags4, tags5, tags6, tags7, tags8, tags9", - ["tags1", "tags2", "tags3", "tags4", "tags5", "tags6", "tags7", "tags8", "tags9"]), - ("tags1, tags2; tags3 tags4 tags5, tags6; tags7 tags8, tags9", - ["tags1", "tags2", "tags3", "tags4", "tags5", "tags6", "tags7", "tags8", "tags9"]), - ("tags1,tags2", ["tags1", "tags2"]), - ("tags1;tags2", ["tags1", "tags2"]), - ("tags1, tags2", ["tags1", "tags2"]), - ("tags1; tags2", ["tags1", "tags2"]), - ]) - def test_runs(self, str_to_split, expected_result): - assert BuildTaskGeneric.split_tags(str_to_split) == expected_result - - -class TestJoinTags: - @pytest.mark.parametrize("join_with, expected_result", [ - (", ", "test, test1, test2") - ]) - def test_joins(self, global_config, join_with, expected_result): - list_to_join = ["test", "test1", "test2"] - global_config.flags.join_values_with = join_with - - assert BuildTaskGeneric.join_tags(list_to_join) == expected_result diff --git a/tests/build_tasks/test_source_crowd_anki_json.py b/tests/build_tasks/test_source_crowd_anki_json.py index 5d8b501..c3abe32 100644 --- a/tests/build_tasks/test_source_crowd_anki_json.py +++ b/tests/build_tasks/test_source_crowd_anki_json.py @@ -1,113 +1,113 @@ -from unittest.mock import patch - -import pytest - -from brain_brew.constants.build_config_keys import BuildConfigKeys -from brain_brew.build_tasks.source_crowd_anki import SourceCrowdAnki, CrowdAnkiKeys -from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport -from brain_brew.representation.json.deck_part_header import DeckPartHeader -from brain_brew.representation.yaml.note_model_repr import DeckPartNoteModel -from brain_brew.representation.json.deck_part_notes import DeckPartNotes - - -def setup_ca_config(file, media, useless_note_keys, notes, headers): - return { - CrowdAnkiKeys.FILE.value: file, - CrowdAnkiKeys.MEDIA.value: media, - CrowdAnkiKeys.USELESS_NOTE_KEYS.value: useless_note_keys, - BuildConfigKeys.NOTES.value: notes, - BuildConfigKeys.HEADERS.value: headers - } - - -class TestConstructor: - @pytest.mark.parametrize("file, media, useless_note_keys, notes, headers, read_file_now", [ - ("test", False, {}, "test.json", "header.json", False), - ("export1", True, {}, "test.json", "header.json", False), - ("json.json", False, {}, "test.json", "", True), - ("", False, {"__type__": "Note", "data": None, "flags": 0}, "test.json", "header.json", False) - ]) - def test_runs(self, file, media, useless_note_keys, notes, headers, read_file_now, global_config): - config = setup_ca_config(file, media, useless_note_keys, notes, headers) - - def assert_dp_header(passed_file, read_now): - assert passed_file == headers - assert read_now == read_file_now - - def assert_dp_notes(passed_file, read_now): - assert passed_file == notes - assert read_now == read_file_now - - def assert_ca_export(passed_file, read_now): - assert passed_file == file - assert read_now == read_file_now - - with patch.object(DeckPartHeader, "create", side_effect=assert_dp_header) as mock_header, \ - patch.object(DeckPartNotes, "create", side_effect=assert_dp_notes) as mock_notes, \ - patch.object(CrowdAnkiExport, "create", side_effect=assert_ca_export) as ca_export: - - source = SourceCrowdAnki(config, read_now=read_file_now) - - assert isinstance(source, SourceCrowdAnki) - assert source.should_handle_media == media - assert source.useless_note_keys == useless_note_keys - - assert mock_header.call_count == 1 - assert mock_notes.call_count == 1 - assert ca_export.call_count == 1 - - -@pytest.fixture() -def source_crowd_anki_test1(global_config) -> SourceCrowdAnki: - with patch.object(DeckPartHeader, "create", return_value=None) as mock_header, \ - patch.object(DeckPartNotes, "create", return_value=None) as mock_notes, \ - patch.object(CrowdAnkiExport, "create", return_value=None) as mock_ca_export: - - source = SourceCrowdAnki( - setup_ca_config("", False, {"__type__": "Note", "data": None, "flags": 0}, "", "") - ) - - # source.notes = dp_notes_test1 - # source.headers = dp_headers_test1 - # source.crowd_anki_export = ca_export_test1 - - return source - - -class TestSourceToDeckParts: - def test_runs(self, source_crowd_anki_test1: SourceCrowdAnki, ca_export_test1, - temp_dp_note_model_file, temp_dp_headers_file, temp_dp_notes_file, - dp_note_model_test1, dp_headers_test1, dp_notes_test1): - - # CrowdAnki Export it will use to write to the DeckParts - source_crowd_anki_test1.crowd_anki_export = ca_export_test1 - - # DeckParts to be written to (+ the NoteModel below) - source_crowd_anki_test1.headers = temp_dp_headers_file - source_crowd_anki_test1.notes = temp_dp_notes_file - - def assert_note_model(name, data_override): - assert data_override == dp_note_model_test1.get_data() - return dp_note_model_test1 - - with patch.object(DeckPartNoteModel, "create", side_effect=assert_note_model) as mock_nm: - source_crowd_anki_test1.source_to_deck_parts() - - assert source_crowd_anki_test1.headers.get_data() == dp_headers_test1.get_data() - assert source_crowd_anki_test1.notes.get_data() == dp_notes_test1.get_data() - - assert mock_nm.call_count == 1 - - -class TestDeckPartsToSource: - def test_runs(self, source_crowd_anki_test1: SourceCrowdAnki, temp_ca_export_file, - ca_export_test1, dp_notes_test1, dp_headers_test1): - source_crowd_anki_test1.crowd_anki_export = temp_ca_export_file # File to write result to - - # DeckParts it will use (+ dp_note_model_test1, but it reads that in as a file) - source_crowd_anki_test1.headers = dp_headers_test1 - source_crowd_anki_test1.notes = dp_notes_test1 - - source_crowd_anki_test1.deck_parts_to_source() # Where the magic happens - - assert temp_ca_export_file.get_data() == ca_export_test1.get_data() +# from unittest.mock import patch +# +# import pytest +# +# from brain_brew.constants.build_config_keys import BuildConfigKeys +# from brain_brew.build_tasks.source_crowd_anki import SourceCrowdAnki, CrowdAnkiKeys +# from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport +# from brain_brew.representation.json.deck_part_header import DeckPartHeader +# from brain_brew.representation.yaml.note_model_repr import DeckPartNoteModel +# from brain_brew.representation.json.deck_part_notes import DeckPartNotes +# +# +# def setup_ca_config(file, media, useless_note_keys, notes, headers): +# return { +# CrowdAnkiKeys.FILE.value: file, +# CrowdAnkiKeys.MEDIA.value: media, +# CrowdAnkiKeys.USELESS_NOTE_KEYS.value: useless_note_keys, +# BuildConfigKeys.NOTES.value: notes, +# BuildConfigKeys.HEADERS.value: headers +# } +# +# +# class TestConstructor: +# @pytest.mark.parametrize("file, media, useless_note_keys, notes, headers, read_file_now", [ +# ("test", False, {}, "test.json", "header.json", False), +# ("export1", True, {}, "test.json", "header.json", False), +# ("json.json", False, {}, "test.json", "", True), +# ("", False, {"__type__": "Note", "data": None, "flags": 0}, "test.json", "header.json", False) +# ]) +# def test_runs(self, file, media, useless_note_keys, notes, headers, read_file_now, global_config): +# config = setup_ca_config(file, media, useless_note_keys, notes, headers) +# +# def assert_dp_header(passed_file, read_now): +# assert passed_file == headers +# assert read_now == read_file_now +# +# def assert_dp_notes(passed_file, read_now): +# assert passed_file == notes +# assert read_now == read_file_now +# +# def assert_ca_export(passed_file, read_now): +# assert passed_file == file +# assert read_now == read_file_now +# +# with patch.object(DeckPartHeader, "create", side_effect=assert_dp_header) as mock_header, \ +# patch.object(DeckPartNotes, "create", side_effect=assert_dp_notes) as mock_notes, \ +# patch.object(CrowdAnkiExport, "create", side_effect=assert_ca_export) as ca_export: +# +# source = SourceCrowdAnki(config, read_now=read_file_now) +# +# assert isinstance(source, SourceCrowdAnki) +# assert source.should_handle_media == media +# assert source.useless_note_keys == useless_note_keys +# +# assert mock_header.call_count == 1 +# assert mock_notes.call_count == 1 +# assert ca_export.call_count == 1 +# +# +# @pytest.fixture() +# def source_crowd_anki_test1(global_config) -> SourceCrowdAnki: +# with patch.object(DeckPartHeader, "create", return_value=None) as mock_header, \ +# patch.object(DeckPartNotes, "create", return_value=None) as mock_notes, \ +# patch.object(CrowdAnkiExport, "create", return_value=None) as mock_ca_export: +# +# source = SourceCrowdAnki( +# setup_ca_config("", False, {"__type__": "Note", "data": None, "flags": 0}, "", "") +# ) +# +# # source.notes = dp_notes_test1 +# # source.headers = dp_headers_test1 +# # source.crowd_anki_export = ca_export_test1 +# +# return source +# +# +# class TestSourceToDeckParts: +# def test_runs(self, source_crowd_anki_test1: SourceCrowdAnki, ca_export_test1, +# temp_dp_note_model_file, temp_dp_headers_file, temp_dp_notes_file, +# dp_note_model_test1, dp_headers_test1, dp_notes_test1): +# +# # CrowdAnki Export it will use to write to the DeckParts +# source_crowd_anki_test1.crowd_anki_export = ca_export_test1 +# +# # DeckParts to be written to (+ the NoteModel below) +# source_crowd_anki_test1.headers = temp_dp_headers_file +# source_crowd_anki_test1.notes = temp_dp_notes_file +# +# def assert_note_model(name, data_override): +# assert data_override == dp_note_model_test1.get_data() +# return dp_note_model_test1 +# +# with patch.object(DeckPartNoteModel, "create", side_effect=assert_note_model) as mock_nm: +# source_crowd_anki_test1.source_to_deck_parts() +# +# assert source_crowd_anki_test1.headers.get_data() == dp_headers_test1.get_data() +# assert source_crowd_anki_test1.notes.get_data() == dp_notes_test1.get_data() +# +# assert mock_nm.call_count == 1 +# +# +# class TestDeckPartsToSource: +# def test_runs(self, source_crowd_anki_test1: SourceCrowdAnki, temp_ca_export_file, +# ca_export_test1, dp_notes_test1, dp_headers_test1): +# source_crowd_anki_test1.crowd_anki_export = temp_ca_export_file # File to write result to +# +# # DeckParts it will use (+ dp_note_model_test1, but it reads that in as a file) +# source_crowd_anki_test1.headers = dp_headers_test1 +# source_crowd_anki_test1.notes = dp_notes_test1 +# +# source_crowd_anki_test1.deck_parts_to_source() # Where the magic happens +# +# assert temp_ca_export_file.get_data() == ca_export_test1.get_data() diff --git a/tests/build_tasks/test_source_csv.py b/tests/build_tasks/test_source_csv.py index ca9fa0b..096d3b6 100644 --- a/tests/build_tasks/test_source_csv.py +++ b/tests/build_tasks/test_source_csv.py @@ -1,140 +1,140 @@ -from typing import List -from unittest.mock import patch - -import pytest - -from brain_brew.build_tasks.source_csv import SourceCsv, SourceCsvKeys -from brain_brew.constants.deckpart_keys import DeckPartNoteKeys -from brain_brew.representation.configuration.csv_file_mapping import FileMapping -from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping -from brain_brew.representation.generic.csv_file import CsvFile -from brain_brew.representation.generic.source_file import SourceFile -from brain_brew.representation.json.deck_part_notes import DeckPartNotes -from tests.representation.json.test_deck_part_notes import dp_notes_test1 -from tests.representation.configuration.test_note_model_mapping import setup_nmm_config -from tests.representation.configuration.test_csv_file_mapping import setup_csv_fm_config - - -def setup_source_csv_config(notes: str, nmm: list, csv_mappings: list): - return { - SourceCsvKeys.NOTES.value: notes, - SourceCsvKeys.NOTE_MODEL_MAPPINGS.value: nmm, - SourceCsvKeys.CSV_MAPPINGS.value: csv_mappings - } - - -def get_csv_default(notes: DeckPartNotes, nmm: List[NoteModelMapping], csv_maps: List[FileMapping]) -> SourceCsv: - csv_source = SourceCsv(setup_source_csv_config("", [], []), read_now=False) - - csv_source.notes = notes - csv_source.note_model_mappings_dict = {nm_map.note_model.name: nm_map for nm_map in nmm} - csv_source.csv_file_mappings = csv_maps - - return csv_source - - -@pytest.fixture() -def csv_source_test1(dp_notes_test1, nmm_test1, csv_file_mapping1) -> SourceCsv: - return get_csv_default(dp_notes_test1, [nmm_test1], [csv_file_mapping1]) - - -@pytest.fixture() -def csv_source_test1_split1(csv_source_default, csv_test1_split1, dp_notes_test1) -> SourceCsv: - csv_source_default.csv_file = csv_test1_split1 - csv_source_default.notes = dp_notes_test1 - return csv_source_default - - -@pytest.fixture() -def csv_source_test1_split2(csv_source_default2, csv_test1_split2, dp_notes_test2) -> SourceCsv: - csv_source_default2.csv_file = csv_test1_split2 - csv_source_default2.notes = dp_notes_test1 - return csv_source_default2 - - -@pytest.fixture() -def csv_source_test2(dp_notes_test2, nmm_test1, csv_file_mapping2) -> SourceCsv: - return get_csv_default(dp_notes_test2, [nmm_test1], [csv_file_mapping2]) - - +# from typing import List +# from unittest.mock import patch +# +# import pytest +# +# from brain_brew.build_tasks.source_csv import SourceCsv, SourceCsvKeys +# from brain_brew.constants.deckpart_keys import DeckPartNoteKeys +# from brain_brew.representation.configuration.csv_file_mapping import FileMapping +# from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping +# from brain_brew.representation.generic.csv_file import CsvFile +# from brain_brew.representation.generic.source_file import SourceFile +# from brain_brew.representation.json.deck_part_notes import DeckPartNotes +# from tests.representation.json.test_deck_part_notes import dp_notes_test1 +# from tests.representation.configuration.test_note_model_mapping import setup_nmm_config +# from tests.representation.configuration.test_csv_file_mapping import setup_csv_fm_config +# +# +# def setup_source_csv_config(notes: str, nmm: list, csv_mappings: list): +# return { +# SourceCsvKeys.NOTES.value: notes, +# SourceCsvKeys.NOTE_MODEL_MAPPINGS.value: nmm, +# SourceCsvKeys.CSV_MAPPINGS.value: csv_mappings +# } +# +# +# def get_csv_default(notes: DeckPartNotes, nmm: List[NoteModelMapping], csv_maps: List[FileMapping]) -> SourceCsv: +# csv_source = SourceCsv(setup_source_csv_config("", [], []), read_now=False) +# +# csv_source.notes = notes +# csv_source.note_model_mappings_dict = {nm_map.note_model.name: nm_map for nm_map in nmm} +# csv_source.csv_file_mappings = csv_maps +# +# return csv_source +# +# # @pytest.fixture() -# def temp_csv_source(global_config, tmpdir) -> SourceCsv: -# file = tmpdir.mkdir("notes").join("file.csv") -# file.write("test,1,2,3") - - -class TestConstructor: - def test_runs(self): - source_csv = get_csv_default(None, [], []) - assert isinstance(source_csv, SourceCsv) - - @pytest.mark.parametrize("notes, model, columns, personal_fields, csv_file", [ - ("notes.json", "Test Model", {"a": "b"}, ["extra"], "file.csv") - ]) - def test_calls_correctly(self, notes, model, columns, personal_fields, csv_file, nmm_test1): - nmm_config = [setup_nmm_config(model, columns, personal_fields)] - csv_config = [setup_csv_fm_config(csv_file, note_model_name=model)] - - def assert_csv(config, read_now): - assert config in csv_config - assert read_now is False - - def assert_nmm(config, read_now): - assert config in nmm_config - assert read_now is False - - def assert_dpn(config, read_now): - assert config == notes - assert read_now is False - - with patch.object(FileMapping, "__init__", side_effect=assert_csv), \ - patch.object(NoteModelMapping, "__init__", side_effect=assert_nmm), \ - patch.object(NoteModelMapping, "note_model"), \ - patch.object(DeckPartNotes, "create", side_effect=assert_dpn): - - #nmm_mock.return_value = False - - source_csv = SourceCsv(setup_source_csv_config( - notes, - nmm_config, - csv_config - ), read_now=False) - - - # def test_missing_non_required_columns - - -class TestSourceToDeckParts: - def test_runs_first(self, csv_source_test1, dp_notes_test1, csv_source_test2, dp_notes_test2): - self.run_s2dp(csv_source_test1, dp_notes_test1) - self.run_s2dp(csv_source_test2, dp_notes_test2) - - @staticmethod - def run_s2dp(csv_source: SourceCsv, dp_notes: DeckPartNotes): - def assert_format(notes_data): - assert notes_data == dp_notes.get_data()[DeckPartNoteKeys.NOTES.value] - - with patch.object(DeckPartNotes, "set_data", side_effect=assert_format) as mock_set_data: - csv_source.source_to_deck_parts() - assert mock_set_data.call_count == 1 - - -class TestDeckPartsToSource: - def test_runs_with_no_change(self, csv_source_test1, csv_test1, csv_source_test2, csv_test2): - - self.run_dpts(csv_source_test1, csv_test1) - self.run_dpts(csv_source_test2, csv_test2) - - @staticmethod - def run_dpts(csv_source: SourceCsv, csv_file: CsvFile): - def assert_format(source_data): - assert source_data == csv_file.get_data() - - with patch.object(SourceFile, "set_data", side_effect=assert_format) as mock_set_data: - csv_source.deck_parts_to_source() - assert csv_source.csv_file_mappings[0].data_set_has_changed is False - - csv_source.csv_file_mappings[0].data_set_has_changed = True - csv_source.csv_file_mappings[0].write_file_on_close() - assert mock_set_data.call_count == 1 - +# def csv_source_test1(dp_notes_test1, nmm_test1, csv_file_mapping1) -> SourceCsv: +# return get_csv_default(dp_notes_test1, [nmm_test1], [csv_file_mapping1]) +# +# +# @pytest.fixture() +# def csv_source_test1_split1(csv_source_default, csv_test1_split1, dp_notes_test1) -> SourceCsv: +# csv_source_default.csv_file = csv_test1_split1 +# csv_source_default.notes = dp_notes_test1 +# return csv_source_default +# +# +# @pytest.fixture() +# def csv_source_test1_split2(csv_source_default2, csv_test1_split2, dp_notes_test2) -> SourceCsv: +# csv_source_default2.csv_file = csv_test1_split2 +# csv_source_default2.notes = dp_notes_test1 +# return csv_source_default2 +# +# +# @pytest.fixture() +# def csv_source_test2(dp_notes_test2, nmm_test1, csv_file_mapping2) -> SourceCsv: +# return get_csv_default(dp_notes_test2, [nmm_test1], [csv_file_mapping2]) +# +# +# # @pytest.fixture() +# # def temp_csv_source(global_config, tmpdir) -> SourceCsv: +# # file = tmpdir.mkdir("notes").join("file.csv") +# # file.write("test,1,2,3") +# +# +# class TestConstructor: +# def test_runs(self): +# source_csv = get_csv_default(None, [], []) +# assert isinstance(source_csv, SourceCsv) +# +# @pytest.mark.parametrize("notes, model, columns, personal_fields, csv_file", [ +# ("notes.json", "Test Model", {"a": "b"}, ["extra"], "file.csv") +# ]) +# def test_calls_correctly(self, notes, model, columns, personal_fields, csv_file, nmm_test1): +# nmm_config = [setup_nmm_config(model, columns, personal_fields)] +# csv_config = [setup_csv_fm_config(csv_file, note_model_name=model)] +# +# def assert_csv(config, read_now): +# assert config in csv_config +# assert read_now is False +# +# def assert_nmm(config, read_now): +# assert config in nmm_config +# assert read_now is False +# +# def assert_dpn(config, read_now): +# assert config == notes +# assert read_now is False +# +# with patch.object(FileMapping, "__init__", side_effect=assert_csv), \ +# patch.object(NoteModelMapping, "__init__", side_effect=assert_nmm), \ +# patch.object(NoteModelMapping, "note_model"), \ +# patch.object(DeckPartNotes, "create", side_effect=assert_dpn): +# +# #nmm_mock.return_value = False +# +# source_csv = SourceCsv(setup_source_csv_config( +# notes, +# nmm_config, +# csv_config +# ), read_now=False) +# +# +# # def test_missing_non_required_columns +# +# +# class TestSourceToDeckParts: +# def test_runs_first(self, csv_source_test1, dp_notes_test1, csv_source_test2, dp_notes_test2): +# self.run_s2dp(csv_source_test1, dp_notes_test1) +# self.run_s2dp(csv_source_test2, dp_notes_test2) +# +# @staticmethod +# def run_s2dp(csv_source: SourceCsv, dp_notes: DeckPartNotes): +# def assert_format(notes_data): +# assert notes_data == dp_notes.get_data()[DeckPartNoteKeys.NOTES.value] +# +# with patch.object(DeckPartNotes, "set_data", side_effect=assert_format) as mock_set_data: +# csv_source.source_to_deck_parts() +# assert mock_set_data.call_count == 1 +# +# +# class TestDeckPartsToSource: +# def test_runs_with_no_change(self, csv_source_test1, csv_test1, csv_source_test2, csv_test2): +# +# self.run_dpts(csv_source_test1, csv_test1) +# self.run_dpts(csv_source_test2, csv_test2) +# +# @staticmethod +# def run_dpts(csv_source: SourceCsv, csv_file: CsvFile): +# def assert_format(source_data): +# assert source_data == csv_file.get_data() +# +# with patch.object(SourceFile, "set_data", side_effect=assert_format) as mock_set_data: +# csv_source.deck_parts_to_source() +# assert csv_source.csv_file_mappings[0].data_set_has_changed is False +# +# csv_source.csv_file_mappings[0].data_set_has_changed = True +# csv_source.csv_file_mappings[0].write_file_on_close() +# assert mock_set_data.call_count == 1 +# diff --git a/tests/representation/configuration/test_csv_file_mapping.py b/tests/representation/configuration/test_csv_file_mapping.py index faa85c2..a2fb4fa 100644 --- a/tests/representation/configuration/test_csv_file_mapping.py +++ b/tests/representation/configuration/test_csv_file_mapping.py @@ -8,165 +8,165 @@ SORT_BY_COLUMNS, REVERSE_SORT, NOTE_MODEL, DERIVATIVES, FILE from brain_brew.representation.generic.csv_file import CsvFile from tests.test_file_manager import get_new_file_manager -from tests.representation.configuration.test_global_config import global_config -from tests.representation.generic.test_csv_file import csv_test1, csv_test2, csv_test3, csv_test1_split1, csv_test1_split2, csv_test2_missing_guids - - -def setup_csv_fm_config(csv: str, sort_by_columns: List[str] = None, reverse_sort: bool = None, - note_model_name: str = None, derivatives: List[dict] = None): - cfm: dict = { - FILE: csv - } - if sort_by_columns is not None: - cfm.setdefault(SORT_BY_COLUMNS, sort_by_columns) - if reverse_sort is not None: - cfm.setdefault(REVERSE_SORT, reverse_sort) - if note_model_name is not None: - cfm.setdefault(NOTE_MODEL, note_model_name) - if derivatives is not None: - cfm.setdefault(DERIVATIVES, derivatives) - - return cfm - - -class TestConstructor: - @pytest.mark.parametrize("read_file_now, note_model_name, csv, sort_by_columns, reverse_sort", [ - (False, "note_model.json", "first.csv", ["guid"], False), - (True, "model_model.json", "second.csv", ["guid", "note_model_name"], True), - (False, "note_model-json", "first.csv", ["guid"], False,) - ]) - def test_runs_without_derivatives(self, global_config, read_file_now, note_model_name, csv, - sort_by_columns, reverse_sort): - get_new_file_manager() - config = setup_csv_fm_config(csv, sort_by_columns, reverse_sort, note_model_name=note_model_name) - - def assert_csv(passed_file, read_now): - assert passed_file == csv - assert read_now == read_file_now - - with patch.object(FileMappingDerivative, "create_derivative", return_value=None) as mock_derivatives, \ - patch.object(CsvFile, "create", side_effect=assert_csv) as mock_csv: - - csv_fm = FileMapping(config, read_now=read_file_now) - - assert isinstance(csv_fm, FileMapping) - assert csv_fm.reverse_sort == reverse_sort - assert csv_fm.sort_by_columns == sort_by_columns - assert csv_fm.note_model_name == note_model_name - - assert mock_csv.call_count == 1 - assert mock_derivatives.call_count == 0 - - @pytest.mark.parametrize("derivatives", [ - [setup_csv_fm_config("test_csv.csv")], - [setup_csv_fm_config("test_csv.csv"), setup_csv_fm_config("second.csv")], - [setup_csv_fm_config("a.csv"), setup_csv_fm_config("b.csv"), setup_csv_fm_config("c.csv")], - [setup_csv_fm_config("a.csv", sort_by_columns=["word", "guid"], reverse_sort=True, note_model_name="d")], - [setup_csv_fm_config("test_csv.csv", derivatives=[setup_csv_fm_config("der_der.csv")])], - ]) - def test_runs_with_derivatives(self, global_config, derivatives: list): - get_new_file_manager() - config = setup_csv_fm_config("test", [], False, note_model_name="nm", derivatives=derivatives.copy()) - expected_call_count = len(derivatives) - - def assert_der(passed_file, read_now): - der = derivatives.pop(0) - assert passed_file == der - assert read_now is False - - with patch.object(FileMappingDerivative, "create_derivative", side_effect=assert_der) as mock_derivatives, \ - patch.object(CsvFile, "create", return_value=None): - - csv_fm = FileMapping(config, read_now=False) - - assert mock_derivatives.call_count == len(csv_fm.derivatives) == expected_call_count - - -def csv_fixture_gen(csv_fix): - with patch.object(CsvFile, "create", return_value=csv_fix): - csv = FileMapping(setup_csv_fm_config("", note_model_name="Test Model")) - csv.compile_data() - return csv - - -@pytest.fixture() -def csv_file_mapping1(csv_test1): - return csv_fixture_gen(csv_test1) - - -@pytest.fixture() -def csv_file_mapping2(csv_test2): - return csv_fixture_gen(csv_test2) - - -@pytest.fixture() -def csv_file_mapping3(csv_test3): - return csv_fixture_gen(csv_test3) - - -@pytest.fixture() -def csv_file_mapping_split1(csv_test1_split1): - return csv_fixture_gen(csv_test1_split1) - - -@pytest.fixture() -def csv_file_mapping_split1(csv_test1_split2): - return csv_fixture_gen(csv_test1_split2) - - -@pytest.fixture() -def csv_file_mapping2_missing_guids(csv_test2_missing_guids): - return csv_fixture_gen(csv_test2_missing_guids) - - -class TestSetRelevantData: - def test_no_change(self, csv_file_mapping1: FileMapping, csv_file_mapping_split1: FileMapping): - assert csv_file_mapping1.data_set_has_changed is False - - previous_data = csv_file_mapping1.compiled_data.copy() - csv_file_mapping1.set_relevant_data(csv_file_mapping_split1.compiled_data) - - assert previous_data == csv_file_mapping1.compiled_data - assert csv_file_mapping1.data_set_has_changed is False - - def test_change_but_no_extra(self, csv_file_mapping1: FileMapping, csv_file_mapping2: FileMapping): - assert csv_file_mapping1.data_set_has_changed is False - assert len(csv_file_mapping1.compiled_data) == 15 - - previous_data = copy.deepcopy(csv_file_mapping1.compiled_data) - csv_file_mapping1.set_relevant_data(csv_file_mapping2.compiled_data) - - assert previous_data != csv_file_mapping1.compiled_data - assert csv_file_mapping1.data_set_has_changed is True - assert len(csv_file_mapping1.compiled_data) == 15 - - def test_change_extra_row(self, csv_file_mapping1: FileMapping, csv_file_mapping3: FileMapping): - assert csv_file_mapping1.data_set_has_changed is False - assert len(csv_file_mapping1.compiled_data) == 15 - - previous_data = copy.deepcopy(csv_file_mapping1.compiled_data.copy()) - csv_file_mapping1.set_relevant_data(csv_file_mapping3.compiled_data) - - assert previous_data != csv_file_mapping1.compiled_data - assert csv_file_mapping1.data_set_has_changed is True - assert len(csv_file_mapping1.compiled_data) == 16 - - -class TestCompileData: - num = 0 - - def get_num(self): - self.num += 1 - return self.num +from tests.representation.generic.test_csv_file import csv_test1, csv_test2, csv_test3, csv_test1_split1,\ + csv_test1_split2, csv_test2_missing_guids + + +# def setup_csv_fm_config(csv: str, sort_by_columns: List[str] = None, reverse_sort: bool = None, +# note_model_name: str = None, derivatives: List[dict] = None): +# cfm: dict = { +# FILE: csv +# } +# if sort_by_columns is not None: +# cfm.setdefault(SORT_BY_COLUMNS, sort_by_columns) +# if reverse_sort is not None: +# cfm.setdefault(REVERSE_SORT, reverse_sort) +# if note_model_name is not None: +# cfm.setdefault(NOTE_MODEL, note_model_name) +# if derivatives is not None: +# cfm.setdefault(DERIVATIVES, derivatives) +# +# return cfm +# - def test_when_missing_guids(self, csv_file_mapping2_missing_guids: FileMapping): - with patch("brain_brew.representation.configuration.csv_file_mapping.generate_anki_guid", wraps=self.get_num) as mock_guid: +# class TestConstructor: +# @pytest.mark.parametrize("read_file_now, note_model_name, csv, sort_by_columns, reverse_sort", [ +# (False, "note_model.json", "first.csv", ["guid"], False), +# (True, "model_model.json", "second.csv", ["guid", "note_model_name"], True), +# (False, "note_model-json", "first.csv", ["guid"], False,) +# ]) +# def test_runs_without_derivatives(self, read_file_now, note_model_name, csv, +# sort_by_columns, reverse_sort): +# get_new_file_manager() +# config = setup_csv_fm_config(csv, sort_by_columns, reverse_sort, note_model_name=note_model_name) +# +# def assert_csv(passed_file, read_now): +# assert passed_file == csv +# assert read_now == read_file_now +# +# with patch.object(FileMappingDerivative, "create_derivative", return_value=None) as mock_derivatives, \ +# patch.object(CsvFile, "create", side_effect=assert_csv) as mock_csv: +# +# csv_fm = FileMapping(config, read_now=read_file_now) +# +# assert isinstance(csv_fm, FileMapping) +# assert csv_fm.reverse_sort == reverse_sort +# assert csv_fm.sort_by_columns == sort_by_columns +# assert csv_fm.note_model_name == note_model_name +# +# assert mock_csv.call_count == 1 +# assert mock_derivatives.call_count == 0 +# +# @pytest.mark.parametrize("derivatives", [ +# [setup_csv_fm_config("test_csv.csv")], +# [setup_csv_fm_config("test_csv.csv"), setup_csv_fm_config("second.csv")], +# [setup_csv_fm_config("a.csv"), setup_csv_fm_config("b.csv"), setup_csv_fm_config("c.csv")], +# [setup_csv_fm_config("a.csv", sort_by_columns=["word", "guid"], reverse_sort=True, note_model_name="d")], +# [setup_csv_fm_config("test_csv.csv", derivatives=[setup_csv_fm_config("der_der.csv")])], +# ]) +# def test_runs_with_derivatives(self, derivatives: list): +# get_new_file_manager() +# config = setup_csv_fm_config("test", [], False, note_model_name="nm", derivatives=derivatives.copy()) +# expected_call_count = len(derivatives) +# +# def assert_der(passed_file, read_now): +# der = derivatives.pop(0) +# assert passed_file == der +# assert read_now is False +# +# with patch.object(FileMappingDerivative, "create_derivative", side_effect=assert_der) as mock_derivatives, \ +# patch.object(CsvFile, "create", return_value=None): +# +# csv_fm = FileMapping(config, read_now=False) +# +# assert mock_derivatives.call_count == len(csv_fm.derivatives) == expected_call_count - csv_file_mapping2_missing_guids.compile_data() - assert csv_file_mapping2_missing_guids.data_set_has_changed is True - assert mock_guid.call_count == 9 - assert list(csv_file_mapping2_missing_guids.compiled_data.keys()) == list(range(1, 10)) +# def csv_fixture_gen(csv_fix): +# with patch.object(CsvFile, "create_or_get", return_value=csv_fix): +# csv = FileMapping(**setup_csv_fm_config("", note_model_name="Test Model")) +# csv.compile_data() +# return csv +# +# +# @pytest.fixture() +# def csv_file_mapping1(csv_test1): +# return csv_fixture_gen(csv_test1) +# +# +# @pytest.fixture() +# def csv_file_mapping2(csv_test2): +# return csv_fixture_gen(csv_test2) +# +# +# @pytest.fixture() +# def csv_file_mapping3(csv_test3): +# return csv_fixture_gen(csv_test3) +# +# +# @pytest.fixture() +# def csv_file_mapping_split1(csv_test1_split1): +# return csv_fixture_gen(csv_test1_split1) +# +# +# @pytest.fixture() +# def csv_file_mapping_split1(csv_test1_split2): +# return csv_fixture_gen(csv_test1_split2) +# +# +# @pytest.fixture() +# def csv_file_mapping2_missing_guids(csv_test2_missing_guids): +# return csv_fixture_gen(csv_test2_missing_guids) +# +# +# class TestSetRelevantData: +# def test_no_change(self, csv_file_mapping1: FileMapping, csv_file_mapping_split1: FileMapping): +# assert csv_file_mapping1.data_set_has_changed is False +# +# previous_data = csv_file_mapping1.compiled_data.copy() +# csv_file_mapping1.set_relevant_data(csv_file_mapping_split1.compiled_data) +# +# assert previous_data == csv_file_mapping1.compiled_data +# assert csv_file_mapping1.data_set_has_changed is False +# +# def test_change_but_no_extra(self, csv_file_mapping1: FileMapping, csv_file_mapping2: FileMapping): +# assert csv_file_mapping1.data_set_has_changed is False +# assert len(csv_file_mapping1.compiled_data) == 15 +# +# previous_data = copy.deepcopy(csv_file_mapping1.compiled_data) +# csv_file_mapping1.set_relevant_data(csv_file_mapping2.compiled_data) +# +# assert previous_data != csv_file_mapping1.compiled_data +# assert csv_file_mapping1.data_set_has_changed is True +# assert len(csv_file_mapping1.compiled_data) == 15 +# +# def test_change_extra_row(self, csv_file_mapping1: FileMapping, csv_file_mapping3: FileMapping): +# assert csv_file_mapping1.data_set_has_changed is False +# assert len(csv_file_mapping1.compiled_data) == 15 +# +# previous_data = copy.deepcopy(csv_file_mapping1.compiled_data.copy()) +# csv_file_mapping1.set_relevant_data(csv_file_mapping3.compiled_data) +# +# assert previous_data != csv_file_mapping1.compiled_data +# assert csv_file_mapping1.data_set_has_changed is True +# assert len(csv_file_mapping1.compiled_data) == 16 +# +# +# class TestCompileData: +# num = 0 +# +# def get_num(self): +# self.num += 1 +# return self.num +# +# def test_when_missing_guids(self, csv_file_mapping2_missing_guids: FileMapping): +# with patch("brain_brew.representation.configuration.csv_file_mapping.generate_anki_guid", wraps=self.get_num) as mock_guid: +# +# csv_file_mapping2_missing_guids.compile_data() +# +# assert csv_file_mapping2_missing_guids.data_set_has_changed is True +# assert mock_guid.call_count == 9 +# assert list(csv_file_mapping2_missing_guids.compiled_data.keys()) == list(range(1, 10)) # Tests still to do: # diff --git a/tests/representation/configuration/test_global_config.py b/tests/representation/configuration/test_global_config.py deleted file mode 100644 index e722b68..0000000 --- a/tests/representation/configuration/test_global_config.py +++ /dev/null @@ -1,49 +0,0 @@ -from unittest.mock import patch - -import pytest - -from brain_brew.constants.deckpart_keys import NoteFlagKeys -from brain_brew.constants.global_config_keys import ConfigKeys - -from brain_brew.representation.configuration.global_config import GlobalConfig -from tests.test_files import TestFiles - - -class TestSingletonConstructor: - def test_runs(self): - fm = GlobalConfig.get_instance() - assert isinstance(fm, GlobalConfig) - - def test_returns_existing_singleton(self): - fm = GlobalConfig.get_instance() - fm.known_files_dict = {'test': None} - fm2 = GlobalConfig.get_instance() - - assert fm2.known_files_dict == {'test': None} - assert fm2 == fm - - def test_raises_error(self): - with pytest.raises(Exception): - GlobalConfig({}) - GlobalConfig({}) - - -@pytest.fixture() -def global_config(): - GlobalConfig.clear_instance() - return GlobalConfig({ - ConfigKeys.DECK_PARTS.value: { - ConfigKeys.HEADERS.value: TestFiles.Headers.LOC, - ConfigKeys.NOTE_MODELS.value: TestFiles.CrowdAnkiNoteModels.LOC, - ConfigKeys.NOTES.value: TestFiles.NoteFiles.LOC, - ConfigKeys.MEDIA_FILES.value: TestFiles.MediaFiles.LOC, - - ConfigKeys.DECK_PARTS_NOTES_STRUCTURE.value: { - NoteFlagKeys.GROUP_BY_NOTE_MODEL.value: False, - NoteFlagKeys.EXTRACT_SHARED_TAGS.value: False - } - }, - ConfigKeys.FLAGS.value: { - - } - }) diff --git a/tests/representation/configuration/test_note_model_mapping.py b/tests/representation/configuration/test_note_model_mapping.py index 6dfbf8c..da4a2d2 100644 --- a/tests/representation/configuration/test_note_model_mapping.py +++ b/tests/representation/configuration/test_note_model_mapping.py @@ -1,147 +1,146 @@ -from unittest.mock import patch - -import pytest - -from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping, FieldMapping -from brain_brew.representation.generic.csv_file import CsvFile -from brain_brew.representation.yaml.note_model_repr import NoteModel -from tests.test_file_manager import get_new_file_manager - - -@pytest.fixture(autouse=True) -def run_around_tests(global_config): - get_new_file_manager() - yield - - -@pytest.fixture() -def nmm_test1_repr() -> NoteModelMapping.Representation: - return NoteModelMapping.Representation( - "Test Model", - { - "guid": "guid", - "tags": "tags", - - "english": "word", - "danish": "otherword" - }, - [] - ) - - -@pytest.fixture() -def nmm_test2_repr() -> NoteModelMapping.Representation: - return NoteModelMapping.Representation( - "Test Model", - { - "guid": "guid", - "tags": "tags", - - "english": "word", - "danish": "otherword" - }, - ["extra", "morph_focus"] - ) - - -@pytest.fixture() -def nmm_test1(nmm_test1_repr) -> NoteModelMapping: - return NoteModelMapping.from_repr(nmm_test1_repr) - - -@pytest.fixture() -def nmm_test2(nmm_test2_repr) -> NoteModelMapping: - return NoteModelMapping.from_repr(nmm_test2_repr) - - -class TestInit: - def test_runs(self): - nmm = NoteModelMapping.from_repr(NoteModelMapping.Representation("test", {}, [])) - assert isinstance(nmm, NoteModelMapping) - - @pytest.mark.parametrize("read_file_now, note_model, personal_fields, columns", [ - (False, "note_model.json", ["x"], {"guid": "guid", "tags": "tags", "english": "word", "danish": "otherword"}), - (True, "model_model", [], {"guid": "guid", "tags": "tags"}), - (False, "note_model-json", ["x", "y", "z"], {"guid": "guid", "tags": "tags", "english": "word", "danish": "otherword"}) - ]) - def test_values(self, read_file_now, note_model, personal_fields, columns): - config = setup_nmm_config(note_model, columns, personal_fields) - - def assert_dp_note_model(passed_file, read_now): - assert passed_file == note_model - assert read_now == read_file_now - - with patch.object(DeckPartNoteModel, "create", side_effect=assert_dp_note_model) as mock_nm: - - nmm = NoteModelMapping(config, read_now=read_file_now) - - assert isinstance(nmm, NoteModelMapping) - assert len(nmm.columns) == len(columns) - assert len(nmm.personal_fields) == len(personal_fields) - - assert mock_nm.call_count == 1 - - -class TestVerifyContents: - pass # TODO - - -class TestCsvRowNoteFieldConversion: - @staticmethod - def get_csv_row(): return { - "guid": "AAAA", - "tags": "nice card", - - "english": "what", - "danish": "hvad" - } - - @staticmethod - def get_note_field(): return{ - "guid": "AAAA", - "tags": "nice card", - - "word": "what", - "otherword": "hvad", - "extra": False, - "morph_focus": False - } - - def test_csv_row_map_to_note_fields(self, nmm_test_with_personal_fields1): - assert nmm_test_with_personal_fields1.csv_row_map_to_note_fields(self.get_csv_row()) == self.get_note_field() - - def test_note_fields_map_to_csv_row(self, nmm_test_with_personal_fields1): - assert nmm_test_with_personal_fields1.note_fields_map_to_csv_row(self.get_note_field()) == self.get_csv_row() - - -class TestGetRelevantData: - def test_data_correct(self, nmm_test_with_personal_fields1: NoteModelMapping, csv_test1: CsvFile): - expected_relevant_columns = ["guid", "english", "danish", "tags"] - data = csv_test1.get_data() - - for row in data: - relevant_data = nmm_test_with_personal_fields1.get_relevant_data(row) - assert len(relevant_data) == 4 - assert list(relevant_data.keys()) == expected_relevant_columns - - def test_data_missing_columns(self, nmm_test_with_personal_fields1: NoteModelMapping, csv_test1: CsvFile): - row_missing = { - "guid": "test", - "english": "test" - } - with pytest.raises(Exception) as e: - relevant_data = nmm_test_with_personal_fields1.get_relevant_data(row_missing) - - errors = e.value.args[0] - assert len(errors) == 2 - assert isinstance(errors[0], KeyError) - assert isinstance(errors[1], KeyError) - assert errors[0].args[0] == "Missing column tags" - assert errors[1].args[0] == "Missing column danish" - - -class TestFieldMapping: - def test_init(self): - fm = FieldMapping(FieldMapping.FieldMappingType.COLUMN, "Csv_Row", "note_model_field") - assert isinstance(fm, FieldMapping) - assert (fm.field_name, fm.value) == ("csv_row", "note_model_field") +# from unittest.mock import patch +# +# import pytest +# +# from brain_brew.representation.configuration.note_model_mapping import NoteModelMapping, FieldMapping +# from brain_brew.representation.generic.csv_file import CsvFile +# from tests.test_file_manager import get_new_file_manager +# +# +# @pytest.fixture(autouse=True) +# def run_around_tests(): +# get_new_file_manager() +# yield +# +# +# @pytest.fixture() +# def nmm_test1_repr() -> NoteModelMapping.Representation: +# return NoteModelMapping.Representation( +# "Test Model", +# { +# "guid": "guid", +# "tags": "tags", +# +# "english": "word", +# "danish": "otherword" +# }, +# [] +# ) +# +# +# @pytest.fixture() +# def nmm_test2_repr() -> NoteModelMapping.Representation: +# return NoteModelMapping.Representation( +# "Test Model", +# { +# "guid": "guid", +# "tags": "tags", +# +# "english": "word", +# "danish": "otherword" +# }, +# ["extra", "morph_focus"] +# ) +# +# +# @pytest.fixture() +# def nmm_test1(nmm_test1_repr) -> NoteModelMapping: +# return NoteModelMapping.from_repr(nmm_test1_repr) +# +# +# @pytest.fixture() +# def nmm_test2(nmm_test2_repr) -> NoteModelMapping: +# return NoteModelMapping.from_repr(nmm_test2_repr) +# +# +# class TestInit: +# def test_runs(self): +# nmm = NoteModelMapping.from_repr(NoteModelMapping.Representation("test", {}, [])) +# assert isinstance(nmm, NoteModelMapping) +# +# @pytest.mark.parametrize("read_file_now, note_model, personal_fields, columns", [ +# (False, "note_model.json", ["x"], {"guid": "guid", "tags": "tags", "english": "word", "danish": "otherword"}), +# (True, "model_model", [], {"guid": "guid", "tags": "tags"}), +# (False, "note_model-json", ["x", "y", "z"], {"guid": "guid", "tags": "tags", "english": "word", "danish": "otherword"}) +# ]) +# def test_values(self, read_file_now, note_model, personal_fields, columns): +# config = setup_nmm_config(note_model, columns, personal_fields) +# +# def assert_dp_note_model(passed_file, read_now): +# assert passed_file == note_model +# assert read_now == read_file_now +# +# with patch.object(DeckPartNoteModel, "create", side_effect=assert_dp_note_model) as mock_nm: +# +# nmm = NoteModelMapping(config, read_now=read_file_now) +# +# assert isinstance(nmm, NoteModelMapping) +# assert len(nmm.columns) == len(columns) +# assert len(nmm.personal_fields) == len(personal_fields) +# +# assert mock_nm.call_count == 1 +# +# +# class TestVerifyContents: +# pass # TODO +# +# +# class TestCsvRowNoteFieldConversion: +# @staticmethod +# def get_csv_row(): return { +# "guid": "AAAA", +# "tags": "nice card", +# +# "english": "what", +# "danish": "hvad" +# } +# +# @staticmethod +# def get_note_field(): return{ +# "guid": "AAAA", +# "tags": "nice card", +# +# "word": "what", +# "otherword": "hvad", +# "extra": False, +# "morph_focus": False +# } +# +# def test_csv_row_map_to_note_fields(self, nmm_test_with_personal_fields1): +# assert nmm_test_with_personal_fields1.csv_row_map_to_note_fields(self.get_csv_row()) == self.get_note_field() +# +# def test_note_fields_map_to_csv_row(self, nmm_test_with_personal_fields1): +# assert nmm_test_with_personal_fields1.note_fields_map_to_csv_row(self.get_note_field()) == self.get_csv_row() +# +# +# class TestGetRelevantData: +# def test_data_correct(self, nmm_test_with_personal_fields1: NoteModelMapping, csv_test1: CsvFile): +# expected_relevant_columns = ["guid", "english", "danish", "tags"] +# data = csv_test1.get_data() +# +# for row in data: +# relevant_data = nmm_test_with_personal_fields1.get_relevant_data(row) +# assert len(relevant_data) == 4 +# assert list(relevant_data.keys()) == expected_relevant_columns +# +# def test_data_missing_columns(self, nmm_test_with_personal_fields1: NoteModelMapping, csv_test1: CsvFile): +# row_missing = { +# "guid": "test", +# "english": "test" +# } +# with pytest.raises(Exception) as e: +# relevant_data = nmm_test_with_personal_fields1.get_relevant_data(row_missing) +# +# errors = e.value.args[0] +# assert len(errors) == 2 +# assert isinstance(errors[0], KeyError) +# assert isinstance(errors[1], KeyError) +# assert errors[0].args[0] == "Missing column tags" +# assert errors[1].args[0] == "Missing column danish" +# +# +# class TestFieldMapping: +# def test_init(self): +# fm = FieldMapping(FieldMapping.FieldMappingType.COLUMN, "Csv_Row", "note_model_field") +# assert isinstance(fm, FieldMapping) +# assert (fm.field_name, fm.value) == ("csv_row", "note_model_field") diff --git a/tests/representation/deck_part_transformers/__init__.py b/tests/representation/deck_part_transformers/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py b/tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py deleted file mode 100644 index e11a9e8..0000000 --- a/tests/representation/deck_part_transformers/test_tr_notes_csv_collection.py +++ /dev/null @@ -1,145 +0,0 @@ -from textwrap import dedent -from unittest.mock import patch - -from brain_brew.file_manager import FileManager -from brain_brew.representation.deck_part_transformers.transform_csv_collection import TrCsvCollectionToNotes -from brain_brew.representation.yaml.deck_part_holder import DeckPartHolder -from brain_brew.representation.yaml.my_yaml import yaml_dump, yaml_load -from tests.test_file_manager import get_new_file_manager -from tests.representation.configuration.test_global_config import global_config - - -nm_mappings = { - "LL Word": dedent(f'''\ - note_model: LL Word - columns_to_fields: - guid: guid - tags: tags - - english: Word - danish: X Word - danish audio: X Pronunciation (Recording and/or IPA) - esperanto: Y Word - esperanto audio: Y Pronunciation (Recording and/or IPA) - personal_fields: - - picture - - extra - - morphman_focusmorph - '''), - "LL Verb": dedent(f'''\ - note_model: LL Verb - csv_columns_to_fields: - guid: guid - tags: tags - - english: Word - danish: X Word - danish audio: X Pronunciation (Recording and/or IPA) - esperanto: Y Word - esperanto audio: Y Pronunciation (Recording and/or IPA) - - present: Form Present - past: Form Past - present perfect: Form Perfect Present - personal_fields: - - picture - - extra - - morphman_focusmorph - '''), - "LL Noun": dedent(f'''\ - note_model: LL Noun - csv_columns_to_fields: - guid: guid - tags: tags - - english: Word - danish: X Word - danish audio: X Pronunciation (Recording and/or IPA) - esperanto: Y Word - esperanto audio: Y Pronunciation (Recording and/or IPA) - - plural: Plural - indefinite plural: Indefinite Plural - definite plural: Definite Plural - personal_fields: - - picture - - extra - - morphman_focusmorph - ''') -} - -file_mappings = { - "Main1": dedent(f'''\ - file: source/vocab/main.csv - note_model: LL Word - sort_by_columns: [english] - reverse_sort: no - '''), - "Der1": dedent(f'''\ - file: source/vocab/derivatives/danish/danish_verbs.csv - note_model: LL Verb - '''), - "Der2": dedent(f'''\ - file: source/vocab/derivatives/danish/danish_nouns.csv - note_model: LL Noun - ''') -} - - -class TestConstructor: - test_tr_notes = dedent(f'''\ - name: csv_first_attempt - # save_to_file: deckparts/notes/csv_first_attempt.yaml - - note_model_mappings: - - note_models: - - LL Word - - LL Verb - - LL Noun - columns_to_fields: - guid: guid - tags: tags - - english: Word - danish: X Word - danish audio: X Pronunciation (Recording and/or IPA) - esperanto: Y Word - esperanto audio: Y Pronunciation (Recording and/or IPA) - - present: Form Present - past: Form Past - present perfect: Form Perfect Present - - plural: Plural - indefinite plural: Indefinite Plural - definite plural: Definite Plural - personal_fields: - - picture - - extra - - morphman_focusmorph - - file_mappings: - - file: source/vocab/main.csv - note_model: LL Word - sort_by_columns: [english] - reverse_sort: no - - derivatives: - - file: source/vocab/derivatives/danish/danish_verbs.csv - note_model: LL Verb - - file: source/vocab/derivatives/danish/danish_nouns.csv - note_model: LL Noun - ''') - - def test_runs(self, global_config): - fm = get_new_file_manager() - data = yaml_load.load(self.test_tr_notes) - - def mock_dp_holder(name: str): - return DeckPartHolder(name, None, None) - - with patch.object(DeckPartHolder, "from_deck_part_pool", side_effect=mock_dp_holder): - - tr_notes = TrCsvCollectionToNotes.from_dict(data) - - assert isinstance(tr_notes, TrCsvCollectionToNotes) diff --git a/tests/representation/generic/test_csv_file.py b/tests/representation/generic/test_csv_file.py index cc5d5e6..bed7d05 100644 --- a/tests/representation/generic/test_csv_file.py +++ b/tests/representation/generic/test_csv_file.py @@ -1,9 +1,7 @@ -from unittest.mock import patch - import pytest -from brain_brew.representation.generic.csv_file import CsvFile, CsvKeys -from brain_brew.representation.generic.source_file import SourceFile +from brain_brew.file_manager import FileManager +from brain_brew.representation.generic.csv_file import CsvFile from tests.test_files import TestFiles @@ -32,11 +30,6 @@ def csv_test3(): return CsvFile(TestFiles.CsvFiles.TEST3) -@pytest.fixture() -def csv_test1_not_read_initially_test(): - return CsvFile(TestFiles.CsvFiles.TEST1, read_now=False) - - @pytest.fixture() def csv_test2_missing_guids(): return CsvFile(TestFiles.CsvFiles.TEST2_MISSING_GUIDS) @@ -47,75 +40,50 @@ def temp_csv_test1(tmpdir, csv_test1) -> CsvFile: file = tmpdir.mkdir("json").join("file.csv") file.write("blank") - return CsvFile.create_or_get(file.strpath, data_override=csv_test1.get_data()) - - -class TestConstructor: - def test_runs(self, csv_test1): - assert isinstance(csv_test1, CsvFile) - assert csv_test1.file_location == TestFiles.CsvFiles.TEST1 - assert "guid" in csv_test1.column_headers - - @pytest.mark.parametrize("column_headers", [ - ["first", "second", "third", "etc"], - ["Word", "OtherWord", "Audio", "Test"], - ["X", "Field name with spaces", "", "MorphMan_FocusMorph"], - ]) - def test_data_override(self, column_headers): - data_override = [{key: "value" for key in column_headers}] - csv = CsvFile("file", data_override=data_override) - - assert csv.column_headers == column_headers - - -def test_to_filename_csv(): - expected = "read-this-file.csv" - - assert expected == CsvFile.to_filename_csv("read this file") - assert expected == CsvFile.to_filename_csv("read-this-file") - assert expected == CsvFile.to_filename_csv("read-this-file.csv") - assert expected == CsvFile.to_filename_csv("read this file") - - -class TestReadFile: - def test_runs(self, csv_test1_not_read_initially_test): - assert csv_test1_not_read_initially_test.get_data() == [] - assert csv_test1_not_read_initially_test.column_headers == [] - assert csv_test1_not_read_initially_test.file_location == TestFiles.CsvFiles.TEST1 - assert csv_test1_not_read_initially_test.data_state == SourceFile.DataState.NOTHING_READ_OR_SET - - csv_test1_not_read_initially_test.read_file() - - assert len(csv_test1_not_read_initially_test.get_data()) == 15 - assert "guid" in csv_test1_not_read_initially_test.column_headers - assert csv_test1_not_read_initially_test.data_state == SourceFile.DataState.READ_IN_DATA - - -class TestWriteFile: - def test_runs(self, temp_csv_test1: CsvFile, csv_test1: CsvFile): - temp_csv_test1.write_file() - temp_csv_test1.read_file() - - assert temp_csv_test1.get_data() == csv_test1.get_data() - - -class TestSortData: - @pytest.mark.parametrize("columns, reverse, result_column, expected_results", [ - (["guid"], False, "guid", [(0, "AAAA"), (1, "BBBB"), (2, "CCCC"), (14, "OOOO")]), - (["guid"], True, "guid", [(14, "AAAA"), (13, "BBBB"), (12, "CCCC"), (0, "OOOO")]), - (["english"], False, "english", [(0, "banana"), (1, "bird"), (2, "cat"), (14, "you")]), - (["english"], True, "english", [(14, "banana"), (13, "bird"), (12, "cat"), (0, "you")]), - (["tags"], False, "tags", [(0, "besttag"), (1, "funny"), (2, "tag2 tag3"), (13, ""), (14, "")]), - (["esperanto", "english"], False, "esperanto", [(0, "banano"), (1, "birdo"), (6, "vi"), (14, "")]), - (["esperanto", "guid"], False, "guid", [(7, "BBBB"), (14, "LLLL")]), - ]) - def test_sort(self, csv_test1: CsvFile, columns, reverse, result_column, expected_results): - csv_test1.sort_data(columns, reverse) - - sorted_data = csv_test1.get_data() - - for result in expected_results: - assert sorted_data[result[0]][result_column] == result[1] - - def test_insensitive(self): - pass + return CsvFile.create_or_get(file.strpath) + + +# class TestConstructor: +# def test_runs(self, csv_test1): +# assert isinstance(csv_test1, CsvFile) +# assert csv_test1.file_location == TestFiles.CsvFiles.TEST1 +# assert "guid" in csv_test1.column_headers +# +# +# def test_to_filename_csv(): +# expected = "read-this-file.csv" +# +# assert expected == CsvFile.to_filename_csv("read this file") +# assert expected == CsvFile.to_filename_csv("read-this-file") +# assert expected == CsvFile.to_filename_csv("read-this-file.csv") +# assert expected == CsvFile.to_filename_csv("read this file") +# +# +# class TestWriteFile: +# def test_runs(self, temp_csv_test1: CsvFile, csv_test1: CsvFile): +# temp_csv_test1.write_file() +# temp_csv_test1.read_file() +# +# assert temp_csv_test1.get_data() == csv_test1.get_data() +# + +# class TestSortData: +# @pytest.mark.parametrize("columns, reverse, result_column, expected_results", [ +# (["guid"], False, "guid", [(0, "AAAA"), (1, "BBBB"), (2, "CCCC"), (14, "OOOO")]), +# (["guid"], True, "guid", [(14, "AAAA"), (13, "BBBB"), (12, "CCCC"), (0, "OOOO")]), +# (["english"], False, "english", [(0, "banana"), (1, "bird"), (2, "cat"), (14, "you")]), +# (["english"], True, "english", [(14, "banana"), (13, "bird"), (12, "cat"), (0, "you")]), +# (["tags"], False, "tags", [(0, "besttag"), (1, "funny"), (2, "tag2 tag3"), (13, ""), (14, "")]), +# (["esperanto", "english"], False, "esperanto", [(0, "banano"), (1, "birdo"), (6, "vi"), (14, "")]), +# (["esperanto", "guid"], False, "guid", [(7, "BBBB"), (14, "LLLL")]), +# ]) +# def test_sort(self, csv_test1: CsvFile, columns, reverse, result_column, expected_results): +# csv_test1.sort_data(columns, reverse) +# +# sorted_data = csv_test1.get_data() +# +# for result in expected_results: +# assert sorted_data[result[0]][result_column] == result[1] +# +# def test_insensitive(self): +# pass diff --git a/tests/representation/generic/test_generic_file.py b/tests/representation/generic/test_generic_file.py deleted file mode 100644 index 7dca059..0000000 --- a/tests/representation/generic/test_generic_file.py +++ /dev/null @@ -1,53 +0,0 @@ -import logging - -import pytest -from unittest.mock import MagicMock - -from brain_brew.representation.generic.source_file import SourceFile -from tests.test_file_manager import get_new_file_manager -from tests.test_files import TestFiles - - -class TestConstructor: - def test_runs(self): - file_location = TestFiles.CsvFiles.TEST1 - file = SourceFile(file_location, read_now=False, data_override=None) - - assert isinstance(file, SourceFile) - assert file.file_location == file_location - assert file._data is None - - def test_no_file_found(self): - file_location = "sdfsdfgdsfsdfsdsdg/sdfsdf/sdfsdf/sdfsd/" - - with pytest.raises(FileNotFoundError): - SourceFile(file_location, read_now=True, data_override=None) - - def test_override_data(self): - override_data = {"Test": 1} - file = SourceFile("", read_now=True, data_override=override_data) - - assert isinstance(file, SourceFile) - assert file._data == override_data - - -class TestCreateFileWithFileManager: - def test_runs(self): - fm = get_new_file_manager() - assert len(fm.known_files_dict) == 0 - - first = SourceFile.create_or_get("test1", read_now=False) - - assert isinstance(first, SourceFile) - assert len(fm.known_files_dict) == 1 - assert fm.known_files_dict["test1"] - - def test_returns_existing_object(self): - fm = get_new_file_manager() - assert len(fm.known_files_dict) == 0 - - first = SourceFile.create_or_get("test1", read_now=False) - second = SourceFile.create_or_get("test1", read_now=False) - - assert first == second - assert len(fm.known_files_dict) == 1 diff --git a/tests/representation/generic/test_media_file.py b/tests/representation/generic/test_media_file.py index 4cb8655..adbf19c 100644 --- a/tests/representation/generic/test_media_file.py +++ b/tests/representation/generic/test_media_file.py @@ -1,3 +1,4 @@ +import os import shutil from unittest.mock import patch @@ -66,9 +67,11 @@ class TestCopy: ]) def test_takes_should_write_into_account(self, media_file_test1, should_write_returns, num_calls_to_copy): with patch.object(MediaFile, "should_write", return_value=should_write_returns), \ + patch.object(os, "makedirs") as mock_dirs, \ patch.object(shutil, "copy2") as mock_copy: media_file_test1.copy_source_to_target() assert mock_copy.call_count == num_calls_to_copy + assert mock_dirs.call_count == num_calls_to_copy def test_copies_file(self, tmpdir): source_dir = tmpdir.mkdir("source") diff --git a/tests/representation/json/test_crowd_anki_export.py b/tests/representation/json/test_crowd_anki_export.py index 1187cb7..c46f915 100644 --- a/tests/representation/json/test_crowd_anki_export.py +++ b/tests/representation/json/test_crowd_anki_export.py @@ -1,7 +1,6 @@ import pytest from brain_brew.representation.json.crowd_anki_export import CrowdAnkiExport -from brain_brew.representation.json.json_file import JsonFile from tests.test_files import TestFiles @@ -15,8 +14,8 @@ def test_runs(self, export_name): assert isinstance(file, CrowdAnkiExport) assert file.folder_location == TestFiles.CrowdAnkiExport.TEST1_FOLDER - assert file.file_location == TestFiles.CrowdAnkiExport.TEST1_JSON - assert len(file.get_data().keys()) == 13 + assert file.json_file_location == TestFiles.CrowdAnkiExport.TEST1_JSON + assert len(file.read_json_file().data.keys()) == 13 class TestFindJsonFileInFolder: diff --git a/tests/representation/json/test_deck_part_header.py b/tests/representation/json/test_deck_part_header.py deleted file mode 100644 index c01bda8..0000000 --- a/tests/representation/json/test_deck_part_header.py +++ /dev/null @@ -1,47 +0,0 @@ -import pytest - -from brain_brew.representation.json.deck_part_header import DeckPartHeader -from brain_brew.representation.json.json_file import JsonFile -from tests.test_files import TestFiles -from tests.representation.configuration.test_global_config import global_config - - -class TestConstructor: - @pytest.mark.parametrize("header_name", [ - TestFiles.Headers.FIRST, - TestFiles.Headers.FIRST_COMPLETE, - ]) - def test_runs(self, header_name, global_config): - file = DeckPartHeader(header_name) - - assert isinstance(file, DeckPartHeader) - assert file.file_location == TestFiles.Headers.LOC + TestFiles.Headers.FIRST_COMPLETE - assert len(file.get_data().keys()) == 10 - - def test_config_location_override(self, global_config): - loc = "place_for_stuff/" - filename = "what-a-great-file.json" - - global_config.deck_parts.headers = loc - - file = DeckPartHeader(filename, read_now=False, data_override={ - "__type__": "Deck", - "crowdanki_uuid": "72ac74b8-0077-11ea-959e-d8cb8ac9abf0", - "deck_config_uuid": "3cc64d85-e410-11e9-960e-d8cb8ac9abf0", - "name": "LL::1. Vocab" - }) - - assert file.file_location == loc + filename - - -@pytest.fixture() -def dp_headers_test1(global_config): - return DeckPartHeader.create(TestFiles.Headers.FIRST_COMPLETE) - - -@pytest.fixture() -def temp_dp_headers_file(tmpdir) -> DeckPartHeader: - file = tmpdir.mkdir("headers").join("file.json") - file.write("{}") - - return DeckPartHeader(file.strpath, read_now=False) diff --git a/tests/representation/json/test_deck_part_note_model.py b/tests/representation/json/test_deck_part_note_model.py deleted file mode 100644 index f7f745b..0000000 --- a/tests/representation/json/test_deck_part_note_model.py +++ /dev/null @@ -1,89 +0,0 @@ -from unittest.mock import Mock - -import pytest - -from brain_brew.representation.yaml.note_model_repr import DeckPartNoteModel, CANoteModelKeys -from tests.test_files import TestFiles - - -def mock_dp_nm(name, read_now): - mock = Mock() - mock.name = name - return mock - - -class TestConstructor: - @pytest.mark.parametrize("note_model_name", [ - TestFiles.CrowdAnkiNoteModels.TEST_COMPLETE, - TestFiles.CrowdAnkiNoteModels.TEST, - ]) - def test_run(self, global_config, note_model_name): - file = DeckPartNoteModel(note_model_name) - - assert isinstance(file, DeckPartNoteModel) - assert file.file_location == TestFiles.CrowdAnkiNoteModels.LOC + TestFiles.CrowdAnkiNoteModels.TEST_COMPLETE - assert len(file.get_data().keys()) == 13 - - assert file.name == "Test Model" - assert file.id == "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa" - assert file.fields == ["Word", "OtherWord"] - - def test_config_location_override(self, global_config): - loc = "place_for_stuff/" - filename = "what-a-great-file.json" - - global_config.deck_parts.note_models = loc - - file = DeckPartNoteModel(filename, read_now=False, data_override={ - CANoteModelKeys.NAME.value: "name", - CANoteModelKeys.ID.value: "id", - CANoteModelKeys.FIELDS.value: [] - }) - - assert file.file_location == loc + filename - - -@pytest.fixture() -def dp_note_model_test1(global_config) -> DeckPartNoteModel: - return DeckPartNoteModel.create_or_get(TestFiles.CrowdAnkiNoteModels.TEST_COMPLETE) - - -def test_read_fields(dp_note_model_test1): - expected = ["Word", "OtherWord"] - assert dp_note_model_test1.fields == expected - - -@pytest.mark.parametrize("fields", ([ - ["word", "unknown field"], - ["Word", "Unknown Field"] -])) -def test_check_field_overlap(dp_note_model_test1, fields): - expected_extra = ["unknown field"] - expected_missing = ["otherword"] - - missing, extra = dp_note_model_test1.check_field_overlap(fields) - - assert missing == expected_missing - assert extra == expected_extra - - -class TestZipFieldToData: - def test_runs_normally(self, dp_note_model_test1): - data_to_zip = ["Example word", "Another one"] - expected = {"word": "Example word", "otherword": "Another one"} - - zipped = dp_note_model_test1.zip_field_to_data(data_to_zip) - - assert zipped == expected - - def test_raises_exception_when_list_length_differs(self, dp_note_model_test1): - with pytest.raises(Exception): - dp_note_model_test1.zip_field_to_data(["Example"]) - - -@pytest.fixture() -def temp_dp_note_model_file(tmpdir) -> DeckPartNoteModel: - file = tmpdir.mkdir("note_models").join("file.json") - file.write("{}") - - return DeckPartNoteModel(file.strpath, read_now=False) diff --git a/tests/representation/json/test_deck_part_notes.py b/tests/representation/json/test_deck_part_notes.py deleted file mode 100644 index 0597292..0000000 --- a/tests/representation/json/test_deck_part_notes.py +++ /dev/null @@ -1,89 +0,0 @@ -import pytest - -from brain_brew.representation.json.json_file import JsonFile -from brain_brew.representation.json.deck_part_notes import DeckPartNotes, DeckPartNoteKeys -from tests.test_files import TestFiles -from tests.representation.configuration.test_global_config import global_config - - -class TestConstructor: - @pytest.mark.parametrize("file_to_read", [ - TestFiles.NoteFiles.TEST1_WITH_SHARED_TAGS_EMPTY_AND_GROUPING, - TestFiles.NoteFiles.TEST1_WITH_SHARED_TAGS, - TestFiles.NoteFiles.TEST1_WITH_GROUPING, - TestFiles.NoteFiles.TEST1_NO_GROUPING_OR_SHARED_TAGS, - ]) - def test_runs_and_unstructures_data(self, file_to_read, global_config, dp_notes_test1): - expected_result = dp_notes_test1.get_data() - notes = DeckPartNotes(file_to_read) - - assert isinstance(notes, DeckPartNotes) - assert notes.get_data() == expected_result - - -@pytest.fixture() -def dp_notes_test1(global_config) -> DeckPartNotes: - return DeckPartNotes.create(TestFiles.NoteFiles.TEST1_WITH_SHARED_TAGS_EMPTY_AND_GROUPING) - - -@pytest.fixture() -def dp_notes_test2(global_config) -> DeckPartNotes: - return DeckPartNotes.create(TestFiles.NoteFiles.TEST2_WITH_SHARED_TAGS_AND_GROUPING) - - -@pytest.fixture() -def temp_dp_notes_file(tmpdir) -> DeckPartNotes: - file = tmpdir.mkdir("notes").join("file.json") - file.write("{}") - - return DeckPartNotes(file.strpath, read_now=False) - - -class TestSortData: - @pytest.mark.parametrize("keys, reverse, result_column, expected_results", [ - (["guid"], False, "guid", [(0, "AAAA"), (1, "BBBB"), (2, "CCCC"), (14, "OOOO")]), - (["guid"], True, "guid", [(14, "AAAA"), (13, "BBBB"), (12, "CCCC"), (0, "OOOO")]), - # (["word"], False, "word", [(0, "banana"), (1, "bird"), (2, "cat"), (14, "you")]), - # (["word"], True, "word", [(14, "banana"), (13, "bird"), (12, "cat"), (0, "you")]), - # (["tags"], False, "tags", [(0, "besttag"), (1, "funny"), (2, "tag2, tag3"), (13, ""), (14, "")]), - ]) - def test_sort(self, dp_notes_test1: DeckPartNotes, keys, reverse, result_column, expected_results): - dp_notes_test1.sort_dict( - keys, reverse - ) - - sorted_data = dp_notes_test1.get_data()[DeckPartNoteKeys.NOTES.value] - - for result in expected_results: - assert sorted_data[result[0]][result_column] == result[1] - - def test_insensitive(self): - pass - - -# def test_set_data_from_override(): -# assert False -# -# -# def test_write_file(): -# assert False -# -# -# def test_read_file(): -# assert False -# -# -# def test_interpret_data(): -# assert False -# -# -# def test_read_note_config(): -# assert False -# -# -# def test_implement_note_structure(): -# assert False -# -# -# def test_remove_notes_structure(): -# assert False diff --git a/tests/representation/json/test_json_file.py b/tests/representation/json/test_json_file.py deleted file mode 100644 index f25cb27..0000000 --- a/tests/representation/json/test_json_file.py +++ /dev/null @@ -1,39 +0,0 @@ -import pytest - -from brain_brew.representation.json.json_file import JsonFile -from tests.test_files import TestFiles - - -def test_constructor(): - file_location = TestFiles.CrowdAnkiExport.TEST1_JSON - file = JsonFile(file_location) - - assert isinstance(file, JsonFile) - assert file.file_location == file_location - assert len(file.get_data().keys()) == 13 - - -def test_to_filename_json(): - expected = "read-this-file.json" - - assert expected == JsonFile.to_filename_json("read this file") - assert expected == JsonFile.to_filename_json("read-this-file") - assert expected == JsonFile.to_filename_json("read-this-file.json") - assert expected == JsonFile.to_filename_json("read this file") - - -def test_configure_file_location(): - expected = "folder/read-this-file.json" - - assert expected == JsonFile.get_json_file_location("folder/", "read this file") - assert expected == JsonFile.get_json_file_location("folder/", "read-this-file.json") - assert expected == JsonFile.get_json_file_location("", "folder/read this file") - assert expected == JsonFile.get_json_file_location("", "folder/read-this-file.json") - - -@pytest.fixture() -def temp_json_file(tmpdir) -> JsonFile: - file = tmpdir.mkdir("json").join("file.json") - file.write("{}") - - return JsonFile(file.strpath) diff --git a/tests/representation/yaml/test_note_model_repr.py b/tests/representation/yaml/test_note_model_repr.py index adb4c68..4263388 100644 --- a/tests/representation/yaml/test_note_model_repr.py +++ b/tests/representation/yaml/test_note_model_repr.py @@ -4,10 +4,12 @@ from brain_brew.representation.json.json_file import JsonFile from brain_brew.representation.yaml.my_yaml import YamlRepr from tests.test_files import TestFiles -from tests.test_helpers import debug_write_deck_part_to_file # CrowdAnki Files -------------------------------------------------------------------------- +from tests.test_helpers import debug_write_deck_part_to_file + + @pytest.fixture def ca_nm_data_word(): return JsonFile.read_file(TestFiles.CrowdAnkiNoteModels.LL_WORD_COMPLETE) @@ -118,5 +120,7 @@ def test_only_required_uses_defaults(self, ca_nm_word_no_defaults, ca_nm_data_wo encoded = model.encode() + # debug_write_deck_part_to_file(model, TestFiles.NoteModels.LL_WORD_NO_DEFAULTS) + assert encoded != ca_nm_data_word_no_defaults assert encoded == nm_data_word_no_defaults diff --git a/tests/representation/yaml/test_note_repr.py b/tests/representation/yaml/test_note_repr.py index e5d08a5..4cc7111 100644 --- a/tests/representation/yaml/test_note_repr.py +++ b/tests/representation/yaml/test_note_repr.py @@ -111,12 +111,12 @@ def _assert_dump_to_yaml(tmpdir, ystring, note_name): def test_all1(self, tmpdir): ystring = dedent(f'''\ {FIELDS}: - - first + - first {GUID}: '12345' {NOTE_MODEL}: model_name {TAGS}: - - noun - - other + - noun + - other ''') self._assert_dump_to_yaml(tmpdir, ystring, "test1") @@ -124,12 +124,12 @@ def test_all1(self, tmpdir): def test_all2(self, tmpdir): ystring = dedent(f'''\ {FIELDS}: - - english - - german + - english + - german {GUID}: sdfhfghsvsdv {NOTE_MODEL}: LL Test {TAGS}: - - marked + - marked ''') self._assert_dump_to_yaml(tmpdir, ystring, "test2") @@ -137,11 +137,11 @@ def test_all2(self, tmpdir): def test_no_note_model(self, tmpdir): ystring = dedent(f'''\ {FIELDS}: - - first + - first {GUID}: '12345' {TAGS}: - - noun - - other + - noun + - other ''') self._assert_dump_to_yaml(tmpdir, ystring, "no_note_model") @@ -150,7 +150,7 @@ def test_no_tags(self, tmpdir): for num, note in enumerate(["no_tags1", "no_tags2"]): ystring = dedent(f'''\ {FIELDS}: - - first + - first {GUID}: '12345' {NOTE_MODEL}: model_name ''') @@ -160,13 +160,13 @@ def test_no_tags(self, tmpdir): def test_with_flags(self, tmpdir): ystring = dedent(f'''\ {FIELDS}: - - first + - first {GUID}: '12345' {FLAGS}: 1 {NOTE_MODEL}: model_name {TAGS}: - - noun - - other + - noun + - other ''') self._assert_dump_to_yaml(tmpdir, ystring, "test1_with_flags") @@ -174,12 +174,12 @@ def test_with_flags(self, tmpdir): def test_with_default_flags(self, tmpdir): ystring = dedent(f'''\ {FIELDS}: - - first + - first {GUID}: '12345' {NOTE_MODEL}: model_name {TAGS}: - - noun - - other + - noun + - other ''') self._assert_dump_to_yaml(tmpdir, ystring, "test1_with_default_flags") @@ -197,20 +197,20 @@ def _assert_dump_to_yaml(tmpdir, ystring, note_grouping_name): def test_nothing_grouped(self, tmpdir): ystring = dedent(f'''\ {NOTES}: - - {FIELDS}: - - first - {GUID}: '12345' - {NOTE_MODEL}: model_name - {TAGS}: - - noun - - other - - {FIELDS}: - - english - - german - {GUID}: sdfhfghsvsdv - {NOTE_MODEL}: LL Test - {TAGS}: - - marked + - {FIELDS}: + - first + {GUID}: '12345' + {NOTE_MODEL}: model_name + {TAGS}: + - noun + - other + - {FIELDS}: + - english + - german + {GUID}: sdfhfghsvsdv + {NOTE_MODEL}: LL Test + {TAGS}: + - marked ''') self._assert_dump_to_yaml(tmpdir, ystring, "nothing_grouped") @@ -219,18 +219,18 @@ def test_note_model_grouped(self, tmpdir): ystring = dedent(f'''\ {NOTE_MODEL}: model_name {NOTES}: - - {FIELDS}: - - first - {GUID}: '12345' - {TAGS}: - - noun - - other - - {FIELDS}: - - second - {GUID}: '67890' - {TAGS}: - - noun - - other + - {FIELDS}: + - first + {GUID}: '12345' + {TAGS}: + - noun + - other + - {FIELDS}: + - second + {GUID}: '67890' + {TAGS}: + - noun + - other ''') self._assert_dump_to_yaml(tmpdir, ystring, "note_model_grouped") @@ -238,17 +238,17 @@ def test_note_model_grouped(self, tmpdir): def test_note_tags_grouped(self, tmpdir): ystring = dedent(f'''\ {TAGS}: - - noun - - other + - noun + - other {NOTES}: - - {FIELDS}: - - first - {GUID}: '12345' - {NOTE_MODEL}: model_name - - {FIELDS}: - - first - {GUID}: '12345' - {NOTE_MODEL}: model_name + - {FIELDS}: + - first + {GUID}: '12345' + {NOTE_MODEL}: model_name + - {FIELDS}: + - first + {GUID}: '12345' + {NOTE_MODEL}: model_name ''') self._assert_dump_to_yaml(tmpdir, ystring, "tags_grouped") @@ -257,15 +257,15 @@ def test_note_model_and_tags_grouped(self, tmpdir): ystring = dedent(f'''\ {NOTE_MODEL}: model_name {TAGS}: - - noun - - other + - noun + - other {NOTES}: - - {FIELDS}: - - first - {GUID}: '12345' - - {FIELDS}: - - first - {GUID}: '12345' + - {FIELDS}: + - first + {GUID}: '12345' + - {FIELDS}: + - first + {GUID}: '12345' ''') self._assert_dump_to_yaml(tmpdir, ystring, "model_and_tags_grouped") @@ -283,32 +283,32 @@ def _assert_dump_to_yaml(tmpdir, ystring, groups: list): def test_two_groupings(self, tmpdir): ystring = dedent(f'''\ {NOTE_GROUPINGS}: - - {NOTE_MODEL}: model_name + - {NOTE_MODEL}: model_name + {TAGS}: + - noun + - other + {NOTES}: + - {FIELDS}: + - first + {GUID}: '12345' + - {FIELDS}: + - first + {GUID}: '12345' + - {NOTES}: + - {FIELDS}: + - first + {GUID}: '12345' + {NOTE_MODEL}: model_name + {TAGS}: + - noun + - other + - {FIELDS}: + - english + - german + {GUID}: sdfhfghsvsdv + {NOTE_MODEL}: LL Test {TAGS}: - - noun - - other - {NOTES}: - - {FIELDS}: - - first - {GUID}: '12345' - - {FIELDS}: - - first - {GUID}: '12345' - - {NOTES}: - - {FIELDS}: - - first - {GUID}: '12345' - {NOTE_MODEL}: model_name - {TAGS}: - - noun - - other - - {FIELDS}: - - english - - german - {GUID}: sdfhfghsvsdv - {NOTE_MODEL}: LL Test - {TAGS}: - - marked + - marked ''') self._assert_dump_to_yaml(tmpdir, ystring, ["model_and_tags_grouped", "nothing_grouped"]) @@ -354,43 +354,43 @@ def test_two_groups_three_models(self): models = dpn.get_all_known_note_model_names() assert models == {'LL Test', 'model_name', 'different_model'} - class TestGetAllNotes: - class TestNoteGrouping: - def test_nothing_grouped(self): - group = NoteGrouping.from_dict(working_note_groupings["nothing_grouped"]) - notes = group.get_all_notes_copy() - assert len(notes) == 2 - - def test_model_grouped(self): - group = NoteGrouping.from_dict(working_note_groupings["note_model_grouped"]) - assert group.note_model == "model_name" - assert all([note.note_model is None for note in group.notes]) - - notes = group.get_all_notes_copy() - assert {note.note_model for note in notes} == {"model_name"} - - def test_tags_grouped(self): - group = NoteGrouping.from_dict(working_note_groupings["tags_grouped"]) - assert group.tags == ["noun", "other"] - assert all([note.tags is None or note.tags == [] for note in group.notes]) - - notes = group.get_all_notes_copy() - assert all([note.tags == ["noun", "other"] for note in notes]) - - def test_tags_grouped_as_addition(self): - group = NoteGrouping.from_dict(working_note_groupings["tags_grouped_as_addition"]) - assert group.tags == ["test", "recent"] - assert all([note.tags is not None for note in group.notes]) - - notes = group.get_all_notes_copy() - assert notes[0].tags == ['noun', 'other', "test", "recent"] - assert notes[1].tags == ['marked', "test", "recent"] - - def test_no_tags(self): - group = NoteGrouping.from_dict(working_note_groupings["tags_grouped"]) - group.tags = None - assert all([note.tags is None or note.tags == [] for note in group.notes]) - - notes = group.get_all_notes_copy() - assert all([note.tags == [] for note in notes]) - + # class TestGetAllNotes: + # class TestNoteGrouping: + # def test_nothing_grouped(self): + # group = NoteGrouping.from_dict(working_note_groupings["nothing_grouped"]) + # notes = group.get_all_notes_copy([], False) + # assert len(notes) == 2 + # + # def test_model_grouped(self): + # group = NoteGrouping.from_dict(working_note_groupings["note_model_grouped"]) + # assert group.note_model == "model_name" + # assert all([note.note_model is None for note in group.notes]) + # + # notes = group.get_all_notes_copy() + # assert {note.note_model for note in notes} == {"model_name"} + # + # def test_tags_grouped(self): + # group = NoteGrouping.from_dict(working_note_groupings["tags_grouped"]) + # assert group.tags == ["noun", "other"] + # assert all([note.tags is None or note.tags == [] for note in group.notes]) + # + # notes = group.get_all_notes_copy() + # assert all([note.tags == ["noun", "other"] for note in notes]) + # + # def test_tags_grouped_as_addition(self): + # group = NoteGrouping.from_dict(working_note_groupings["tags_grouped_as_addition"]) + # assert group.tags == ["test", "recent"] + # assert all([note.tags is not None for note in group.notes]) + # + # notes = group.get_all_notes_copy() + # assert notes[0].tags == ['noun', 'other', "test", "recent"] + # assert notes[1].tags == ['marked', "test", "recent"] + # + # def test_no_tags(self): + # group = NoteGrouping.from_dict(working_note_groupings["tags_grouped"]) + # group.tags = None + # assert all([note.tags is None or note.tags == [] for note in group.notes]) + # + # notes = group.get_all_notes_copy() + # assert all([note.tags == [] for note in notes]) + # diff --git a/tests/test_argument_reader.py b/tests/test_argument_reader.py index 0c8d5e8..86e6d63 100644 --- a/tests/test_argument_reader.py +++ b/tests/test_argument_reader.py @@ -32,16 +32,16 @@ def raise_exit(message): with patch.object(BBArgumentReader, "error", side_effect=raise_exit): parsed_args = arg_reader_test1.get_parsed(arguments) - @pytest.mark.parametrize("arguments, builder_file, config_file, run_reversed", [ + @pytest.mark.parametrize("arguments, builder_file, config_file, verify_only", [ (["test_builder.yaml"], "test_builder.yaml", None, False), - (["test_builder.yaml", "--reversed"], "test_builder.yaml", None, True), - (["test_builder.yaml", "-r"], "test_builder.yaml", None, True), + (["test_builder.yaml", "--verify"], "test_builder.yaml", None, True), + (["test_builder.yaml", "-v"], "test_builder.yaml", None, True), (["test_builder.yaml", "--config", "other_config.yaml"], "test_builder.yaml", "other_config.yaml", False), - (["test_builder.yaml", "--config", "other_config.yaml", "-r"], "test_builder.yaml", "other_config.yaml", True), + (["test_builder.yaml", "--config", "other_config.yaml", "-v"], "test_builder.yaml", "other_config.yaml", True), ]) - def test_correct_arguments(self, arg_reader_test1, arguments, builder_file, config_file, run_reversed): + def test_correct_arguments(self, arg_reader_test1, arguments, builder_file, config_file, verify_only): parsed_args = arg_reader_test1.parse_args(arguments) assert parsed_args.builder_file == builder_file assert parsed_args.config_file == config_file - assert parsed_args.run_reversed == run_reversed + assert parsed_args.verify_only == verify_only diff --git a/tests/test_builder.py b/tests/test_builder.py index 2c49dd0..9b7dce8 100644 --- a/tests/test_builder.py +++ b/tests/test_builder.py @@ -1,25 +1,12 @@ -from unittest.mock import patch, Mock - -from brain_brew.representation.build_config.top_level_task_builder import TopLevelTaskBuilder -from brain_brew.representation.deck_part_transformers.transform_csv_collection import TrNotesToCsvCollection -from brain_brew.representation.yaml.my_yaml import YamlRepr -from brain_brew.representation.yaml.note_model_repr import NoteModel -from brain_brew.representation.yaml.note_repr import Notes -from tests.representation.json.test_deck_part_note_model import mock_dp_nm -from tests.test_file_manager import get_new_file_manager -from tests.test_files import TestFiles - - -class TestConstructor: - def test_runs(self, global_config): - fm = get_new_file_manager() - - with patch.object(TrNotesToCsvCollection, "__init__", return_value=None) as mock_csv_tr, \ - patch.object(Notes, "from_deck_part_pool", return_value=Mock()), \ - patch.object(NoteModel, "from_deck_part_pool", side_effect=mock_dp_nm): - - data = YamlRepr.read_to_dict(TestFiles.BuildConfig.ONE_OF_EACH_TYPE) - builder = TopLevelTaskBuilder.from_dict(data, global_config, fm) - - assert len(builder.tasks) == 1 - assert mock_csv_tr.call_count == 1 +# class TestConstructor: +# def test_runs(self): +# with patch.object(CsvsGenerate, "__init__", return_value=None) as mock_csv_tr, \ +# patch.object(DeckPartHolder, "from_deck_part_pool", return_value=Mock()), \ +# patch.object(CsvFile, "create_or_get", return_value=Mock()): +# +# data = YamlRepr.read_to_dict(TestFiles.BuildConfig.ONE_OF_EACH_TYPE) +# builder = TopLevelTaskBuilder.from_list(data) +# builder.execute() +# +# assert len(builder.tasks) == 1 +# assert mock_csv_tr.call_count == 1 diff --git a/tests/test_file_manager.py b/tests/test_file_manager.py index 68b570a..8dbda96 100644 --- a/tests/test_file_manager.py +++ b/tests/test_file_manager.py @@ -1,7 +1,6 @@ import pytest from brain_brew.file_manager import FileManager -from tests.representation.configuration.test_global_config import global_config def get_new_file_manager(): @@ -9,27 +8,27 @@ def get_new_file_manager(): return FileManager() -class TestSingletonConstructor: - def test_runs(self, global_config): - fm = get_new_file_manager() - assert isinstance(fm, FileManager) - - def test_returns_existing_singleton(self): - fm = get_new_file_manager() - fm.known_files_dict = {'test': None} - fm2 = FileManager.get_instance() - - assert fm2.known_files_dict == {'test': None} - assert fm2 == fm - - def test_raises_error(self): - with pytest.raises(Exception): - FileManager() - FileManager() - - -class TestFindMediaFiles: - def test_finds(self): - fm = get_new_file_manager() - - assert len(fm.known_media_files_dict) == 2 +# class TestSingletonConstructor: +# def test_runs(self, global_config): +# fm = get_new_file_manager() +# assert isinstance(fm, FileManager) +# +# def test_returns_existing_singleton(self): +# fm = get_new_file_manager() +# fm.known_files_dict = {'test': None} +# fm2 = FileManager.get_instance() +# +# assert fm2.known_files_dict == {'test': None} +# assert fm2 == fm +# +# def test_raises_error(self): +# with pytest.raises(Exception): +# FileManager() +# FileManager() +# +# +# class TestFindMediaFiles: +# def test_finds(self): +# fm = get_new_file_manager() +# +# assert len(fm.known_media_files_dict) == 2 diff --git a/tests/test_files/build_files/builder1.yaml b/tests/test_files/build_files/builder1.yaml index 92706da..c45cda5 100644 --- a/tests/test_files/build_files/builder1.yaml +++ b/tests/test_files/build_files/builder1.yaml @@ -1,42 +1,42 @@ -tasks: - - generate_csv_collection: - notes: test_from_CA - note_model_mappings: - - note_models: - - LL Word - - LL Verb - - LL Noun - columns_to_fields: - guid: guid - tags: tags +- generate_csv_collection: + notes: test_from_CA - english: Word - danish: X Word - danish audio: X Pronunciation (Recording and/or IPA) - esperanto: Y Word - esperanto audio: Y Pronunciation (Recording and/or IPA) + note_model_mappings: + - note_models: + - LL Word + - LL Verb + - LL Noun + columns_to_fields: + guid: guid + tags: tags - present: Form Present - past: Form Past - present perfect: Form Perfect Present + english: Word + danish: X Word + danish audio: X Pronunciation (Recording and/or IPA) + esperanto: Y Word + esperanto audio: Y Pronunciation (Recording and/or IPA) - plural: Plural - indefinite plural: Indefinite Plural - definite plural: Definite Plural - personal_fields: - - picture - - extra - - morphman_focusmorph + present: Form Present + past: Form Past + present perfect: Form Perfect Present - file_mappings: - - file: source/vocab/main.csv - note_model: LL Word - sort_by_columns: [english] - reverse_sort: false + plural: Plural + indefinite plural: Indefinite Plural + definite plural: Definite Plural + personal_fields: + - picture + - extra + - morphman_focusmorph - derivatives: - - file: source/vocab/derivatives/danish/danish_verbs.csv - note_model: LL Verb - - file: source/vocab/derivatives/danish/danish_nouns.csv - note_model: LL Noun \ No newline at end of file + file_mappings: + - file: source/vocab/main.csv + note_model: LL Word + sort_by_columns: [english] + reverse_sort: false + + derivatives: + - file: source/vocab/derivatives/danish/danish_verbs.csv + note_model: LL Verb + - file: source/vocab/derivatives/danish/danish_nouns.csv + note_model: LL Noun \ No newline at end of file diff --git a/tests/test_files/build_files/csv_collection_config_with_only1.yaml b/tests/test_files/build_files/csv_collection_config_with_only1.yaml deleted file mode 100644 index 0f4a818..0000000 --- a/tests/test_files/build_files/csv_collection_config_with_only1.yaml +++ /dev/null @@ -1,13 +0,0 @@ -- csv: manager/tests/test_files/csv/test1.csv - note_model: LL Noun - sort_by_columns: [english] - reverse_sort: no - - columns: - guid: guid - tags: tags - english: word - danish: otherword - - personal_fields: - - Extra \ No newline at end of file diff --git a/tests/test_files/deck_parts/csvtonotes1_withgrouping.json b/tests/test_files/deck_parts/csvtonotes1_withgrouping.json deleted file mode 100644 index b011390..0000000 --- a/tests/test_files/deck_parts/csvtonotes1_withgrouping.json +++ /dev/null @@ -1,137 +0,0 @@ -{ - "_flags": { - "group_by_note_model": true, - "extract_shared_tags": false - }, - "Test Model": { - "notes": [ - { - "fields": [ - "you", - "du" - ], - "guid": "AAAA", - "tags": [ - "funny" - ] - }, - { - "fields": [ - "healthy", - "rask" - ], - "guid": "BBBB", - "tags": [ - "tag2", - "tag3" - ] - }, - { - "fields": [ - "tired", - "træt" - ], - "guid": "CCCC", - "tags": [ - "besttag" - ] - }, - { - "fields": [ - "banana", - "en banan" - ], - "guid": "DDDD", - "tags": [] - }, - { - "fields": [ - "cat", - "en kat" - ], - "guid": "EEEE", - "tags": [] - }, - { - "fields": [ - "dog", - "en hund" - ], - "guid": "FFFF", - "tags": [] - }, - { - "fields": [ - "fish", - "en fisk" - ], - "guid": "GGGG", - "tags": [] - }, - { - "fields": [ - "bird", - "en fugl" - ], - "guid": "HHHH", - "tags": [] - }, - { - "fields": [ - "cow", - "en ko" - ], - "guid": "IIII", - "tags": [] - }, - { - "fields": [ - "pig", - "et svin" - ], - "guid": "JJJJ", - "tags": [] - }, - { - "fields": [ - "mouse", - "en mus" - ], - "guid": "KKKK", - "tags": [] - }, - { - "fields": [ - "horse", - "en hest" - ], - "guid": "LLLL", - "tags": [] - }, - { - "fields": [ - "to learn", - "at lære" - ], - "guid": "MMMM", - "tags": [] - }, - { - "fields": [ - "to eat", - "at spise" - ], - "guid": "NNNN", - "tags": [] - }, - { - "fields": [ - "to drink", - "at drikke" - ], - "guid": "OOOO", - "tags": [] - } - ] - } -} \ No newline at end of file diff --git a/tests/test_files/deck_parts/csvtonotes1_withnogroupingorsharedtags.json b/tests/test_files/deck_parts/csvtonotes1_withnogroupingorsharedtags.json deleted file mode 100644 index 7f67ef3..0000000 --- a/tests/test_files/deck_parts/csvtonotes1_withnogroupingorsharedtags.json +++ /dev/null @@ -1,150 +0,0 @@ -{ - "_flags": { - "group_by_note_model": false, - "extract_shared_tags": false - }, - "notes": [ - { - "fields": [ - "you", - "du" - ], - "guid": "AAAA", - "note_model": "Test Model", - "tags": [ - "funny" - ] - }, - { - "fields": [ - "healthy", - "rask" - ], - "guid": "BBBB", - "note_model": "Test Model", - "tags": [ - "tag2", - "tag3" - ] - }, - { - "fields": [ - "tired", - "træt" - ], - "guid": "CCCC", - "note_model": "Test Model", - "tags": [ - "besttag" - ] - }, - { - "fields": [ - "banana", - "en banan" - ], - "guid": "DDDD", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "cat", - "en kat" - ], - "guid": "EEEE", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "dog", - "en hund" - ], - "guid": "FFFF", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "fish", - "en fisk" - ], - "guid": "GGGG", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "bird", - "en fugl" - ], - "guid": "HHHH", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "cow", - "en ko" - ], - "guid": "IIII", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "pig", - "et svin" - ], - "guid": "JJJJ", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "mouse", - "en mus" - ], - "guid": "KKKK", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "horse", - "en hest" - ], - "guid": "LLLL", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "to learn", - "at lære" - ], - "guid": "MMMM", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "to eat", - "at spise" - ], - "guid": "NNNN", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "to drink", - "at drikke" - ], - "guid": "OOOO", - "note_model": "Test Model", - "tags": [] - } - ] -} \ No newline at end of file diff --git a/tests/test_files/deck_parts/csvtonotes1_withsharedtags.json b/tests/test_files/deck_parts/csvtonotes1_withsharedtags.json deleted file mode 100644 index 137fb99..0000000 --- a/tests/test_files/deck_parts/csvtonotes1_withsharedtags.json +++ /dev/null @@ -1,151 +0,0 @@ -{ - "_flags": { - "group_by_note_model": false, - "extract_shared_tags": true - }, - "_shared_tags": [], - "notes": [ - { - "fields": [ - "you", - "du" - ], - "guid": "AAAA", - "note_model": "Test Model", - "tags": [ - "funny" - ] - }, - { - "fields": [ - "healthy", - "rask" - ], - "guid": "BBBB", - "note_model": "Test Model", - "tags": [ - "tag2", - "tag3" - ] - }, - { - "fields": [ - "tired", - "træt" - ], - "guid": "CCCC", - "note_model": "Test Model", - "tags": [ - "besttag" - ] - }, - { - "fields": [ - "banana", - "en banan" - ], - "guid": "DDDD", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "cat", - "en kat" - ], - "guid": "EEEE", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "dog", - "en hund" - ], - "guid": "FFFF", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "fish", - "en fisk" - ], - "guid": "GGGG", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "bird", - "en fugl" - ], - "guid": "HHHH", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "cow", - "en ko" - ], - "guid": "IIII", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "pig", - "et svin" - ], - "guid": "JJJJ", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "mouse", - "en mus" - ], - "guid": "KKKK", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "horse", - "en hest" - ], - "guid": "LLLL", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "to learn", - "at lære" - ], - "guid": "MMMM", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "to eat", - "at spise" - ], - "guid": "NNNN", - "note_model": "Test Model", - "tags": [] - }, - { - "fields": [ - "to drink", - "at drikke" - ], - "guid": "OOOO", - "note_model": "Test Model", - "tags": [] - } - ] -} \ No newline at end of file diff --git a/tests/test_files/deck_parts/csvtonotes1_withsharedtagsandgrouping_butnothingtogroup.json b/tests/test_files/deck_parts/csvtonotes1_withsharedtagsandgrouping_butnothingtogroup.json deleted file mode 100644 index 3e18c8a..0000000 --- a/tests/test_files/deck_parts/csvtonotes1_withsharedtagsandgrouping_butnothingtogroup.json +++ /dev/null @@ -1,138 +0,0 @@ -{ - "_flags": { - "group_by_note_model": true, - "extract_shared_tags": true - }, - "Test Model": { - "_shared_tags": [], - "notes": [ - { - "fields": [ - "you", - "du" - ], - "guid": "AAAA", - "tags": [ - "funny" - ] - }, - { - "fields": [ - "healthy", - "rask" - ], - "guid": "BBBB", - "tags": [ - "tag2", - "tag3" - ] - }, - { - "fields": [ - "tired", - "træt" - ], - "guid": "CCCC", - "tags": [ - "besttag" - ] - }, - { - "fields": [ - "banana", - "en banan" - ], - "guid": "DDDD", - "tags": [] - }, - { - "fields": [ - "cat", - "en kat" - ], - "guid": "EEEE", - "tags": [] - }, - { - "fields": [ - "dog", - "en hund" - ], - "guid": "FFFF", - "tags": [] - }, - { - "fields": [ - "fish", - "en fisk" - ], - "guid": "GGGG", - "tags": [] - }, - { - "fields": [ - "bird", - "en fugl" - ], - "guid": "HHHH", - "tags": [] - }, - { - "fields": [ - "cow", - "en ko" - ], - "guid": "IIII", - "tags": [] - }, - { - "fields": [ - "pig", - "et svin" - ], - "guid": "JJJJ", - "tags": [] - }, - { - "fields": [ - "mouse", - "en mus" - ], - "guid": "KKKK", - "tags": [] - }, - { - "fields": [ - "horse", - "en hest" - ], - "guid": "LLLL", - "tags": [] - }, - { - "fields": [ - "to learn", - "at lære" - ], - "guid": "MMMM", - "tags": [] - }, - { - "fields": [ - "to eat", - "at spise" - ], - "guid": "NNNN", - "tags": [] - }, - { - "fields": [ - "to drink", - "at drikke" - ], - "guid": "OOOO", - "tags": [] - } - ] - } -} \ No newline at end of file diff --git a/tests/test_files/deck_parts/csvtonotes2_withsharedtagsandgrouping.json b/tests/test_files/deck_parts/csvtonotes2_withsharedtagsandgrouping.json deleted file mode 100644 index bb7c43d..0000000 --- a/tests/test_files/deck_parts/csvtonotes2_withsharedtagsandgrouping.json +++ /dev/null @@ -1,101 +0,0 @@ -{ - "_flags": { - "group_by_note_model": true, - "extract_shared_tags": true - }, - "Test Model": { - "_shared_tags": [ - "LL::Noun" - ], - "notes": [ - { - "fields": [ - "banana", - "en banan" - ], - "guid": "DDDD", - "tags": [] - }, - { - "fields": [ - "cat", - "en kat" - ], - "guid": "EEEE", - "tags": [ - "Animal" - ] - }, - { - "fields": [ - "dog", - "en hund" - ], - "guid": "FFFF", - "tags": [ - "Animal" - ] - }, - { - "fields": [ - "fish", - "en fisk" - ], - "guid": "GGGG", - "tags": [ - "Animal" - ] - }, - { - "fields": [ - "bird", - "en fugl" - ], - "guid": "HHHH", - "tags": [ - "Animal" - ] - }, - { - "fields": [ - "cow", - "en ko" - ], - "guid": "IIII", - "tags": [ - "Animal" - ] - }, - { - "fields": [ - "pig", - "et svin" - ], - "guid": "JJJJ", - "tags": [ - "Animal" - ] - }, - { - "fields": [ - "mouse", - "en mus" - ], - "guid": "KKKK", - "tags": [ - "Animal" - ] - }, - { - "fields": [ - "horse", - "en hest" - ], - "guid": "LLLL", - "tags": [ - "Animal" - ] - } - ] - } -} \ No newline at end of file diff --git a/tests/test_files/deck_parts/headers/default-header.json b/tests/test_files/deck_parts/headers/default-header.json deleted file mode 100644 index a5685fe..0000000 --- a/tests/test_files/deck_parts/headers/default-header.json +++ /dev/null @@ -1,62 +0,0 @@ -{ - "__type__": "Deck", - "children": [], - "crowdanki_uuid": "16bef726-1426-11ea-a85d-d8cb8ac9abf0", - "deck_config_uuid": "3cc64d85-e410-11e9-960e-d8cb8ac9abf0", - "deck_configurations": [ - { - "__type__": "DeckConfig", - "autoplay": true, - "crowdanki_uuid": "3cc64d85-e410-11e9-960e-d8cb8ac9abf0", - "currentValue": 120, - "dyn": false, - "lapse": { - "delays": [ - 10, - 1440 - ], - "leechAction": 1, - "leechFails": 8, - "minInt": 1, - "mult": 0.25 - }, - "maxLife": 120, - "maxTaken": 60, - "name": "LL Default", - "new": { - "bury": false, - "delays": [ - 1, - 15 - ], - "initialFactor": 2500, - "ints": [ - 5, - 10, - 7 - ], - "order": 0, - "perDay": 3, - "separate": true - }, - "recover": 5, - "replayq": true, - "rev": { - "bury": true, - "ease4": 1.5, - "fuzz": 0.05, - "hardFactor": 1.2, - "ivlFct": 1.5, - "maxIvl": 36500, - "minSpace": 1, - "perDay": 70 - }, - "timer": 0 - } - ], - "desc": "", - "dyn": 0, - "extendNew": 10, - "extendRev": 50, - "name": "LL::1. Vocab" -} \ No newline at end of file diff --git a/tests/test_helpers.py b/tests/test_helpers.py index 719d009..5ccbcbb 100644 --- a/tests/test_helpers.py +++ b/tests/test_helpers.py @@ -3,4 +3,5 @@ def debug_write_deck_part_to_file(deck_part, filepath: str): dp = DeckPartHolder("Blah", filepath, deck_part) + dp.save_to_file = filepath dp.write_to_file() diff --git a/tests/test_utils.py b/tests/test_utils.py index 030654f..e52abe1 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,6 +1,6 @@ import pytest -from brain_brew.utils import find_media_in_field, str_to_lowercase_no_separators +from brain_brew.utils import find_media_in_field, str_to_lowercase_no_separators, split_tags, join_tags class TestFindMedia: @@ -34,3 +34,33 @@ class TestHelperFunctions: ]) def test_remove_spacers_from_str(self, str_to_tidy): assert str_to_lowercase_no_separators(str_to_tidy) == "generatecsvblahblah" + + +class TestSplitTags: + @pytest.mark.parametrize("str_to_split, expected_result", [ + ("tags1, tags2", ["tags1", "tags2"]), + ("tags1 tags2", ["tags1", "tags2"]), + ("tags1; tags2", ["tags1", "tags2"]), + ("tags1 tags2", ["tags1", "tags2"]), + ("tags1, tags2, tags3, tags4, tags5, tags6, tags7, tags8, tags9", + ["tags1", "tags2", "tags3", "tags4", "tags5", "tags6", "tags7", "tags8", "tags9"]), + ("tags1, tags2; tags3 tags4 tags5, tags6; tags7 tags8, tags9", + ["tags1", "tags2", "tags3", "tags4", "tags5", "tags6", "tags7", "tags8", "tags9"]), + ("tags1,tags2", ["tags1", "tags2"]), + ("tags1;tags2", ["tags1", "tags2"]), + ("tags1, tags2", ["tags1", "tags2"]), + ("tags1; tags2", ["tags1", "tags2"]), + ]) + def test_runs(self, str_to_split, expected_result): + assert split_tags(str_to_split) == expected_result + + +# class TestJoinTags: +# @pytest.mark.parametrize("join_with, expected_result", [ +# (", ", "test, test1, test2") +# ]) +# def test_joins(self, global_config, join_with, expected_result): +# list_to_join = ["test", "test1", "test2"] +# global_config.flags.join_values_with = join_with +# +# assert join_tags(list_to_join) == expected_result From 6e88f403f53520f795275bd5b0e0303445507c29 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sun, 6 Sep 2020 15:52:43 +0200 Subject: [PATCH 37/39] Warnings fixed --- brain_brew/utils.py | 12 +- .../yaml/note_models/LL-Word-No-Defaults.yaml | 258 +++++++++--------- .../note_models/LL-Word-Only-Required.yaml | 142 +++++----- 3 files changed, 207 insertions(+), 205 deletions(-) diff --git a/brain_brew/utils.py b/brain_brew/utils.py index 0a27075..b6a3676 100644 --- a/brain_brew/utils.py +++ b/brain_brew/utils.py @@ -26,8 +26,8 @@ def single_item_to_list(item): def all_combos_prepend_append(original_list: list, prepend_with: str, append_with: str): return list({append_or_not for normal in original_list - for prepend_or_not in (normal, prepend_with + normal) - for append_or_not in (prepend_or_not, prepend_or_not + append_with)}) + for prepend_or_not in (normal, prepend_with + normal) + for append_or_not in (prepend_or_not, prepend_or_not + append_with)}) def str_to_lowercase_no_separators(str_to_tidy: str): @@ -59,7 +59,7 @@ def find_all_files_in_directory(directory, recursive=False): def split_tags(tags_value: str) -> list: - split = [entry.strip() for entry in re.split(';\s*|,\s*|\s+', tags_value)] + split = [entry.strip() for entry in re.split(r';\s*|,\s*|\s+', tags_value)] while "" in split: split.remove("") return split @@ -96,9 +96,11 @@ def sort_dict(data, sort_by_keys, reverse_sort, case_insensitive_sort=None): if sort_by_keys: if case_insensitive_sort: - def sort_method(i): return tuple((i[column] == "", i[column].lower()) for column in sort_by_keys) + def sort_method(i): + return tuple((i[column] == "", i[column].lower()) for column in sort_by_keys) else: - def sort_method(i): return tuple((i[column] == "", i[column]) for column in sort_by_keys) + def sort_method(i): + return tuple((i[column] == "", i[column]) for column in sort_by_keys) return sorted(data, key=sort_method, reverse=reverse_sort) elif reverse_sort: diff --git a/tests/test_files/deck_parts/yaml/note_models/LL-Word-No-Defaults.yaml b/tests/test_files/deck_parts/yaml/note_models/LL-Word-No-Defaults.yaml index 9b909c6..01853ac 100644 --- a/tests/test_files/deck_parts/yaml/note_models/LL-Word-No-Defaults.yaml +++ b/tests/test_files/deck_parts/yaml/note_models/LL-Word-No-Defaults.yaml @@ -1,131 +1,131 @@ name: LL Word id: 057a8d66-bc4e-11e9-9822-d8cb8ac9abf0 css: ".card {\n font-family: arial;\n font-size: 20px;\n text-align: center;\n color:\ -\ black;\n background-color: white;\n}\n\n.card1,.card3, .card5 { background-color:\ -\ #B60F2D; }\n.card2,.card4, .card6 { background-color: #2E9017; }\n.card7 { background:\ -\ linear-gradient(90deg, #B60F2D 49.9%, #2E9017 50.1%); }\n\n.word {\n font-size:1.5em;\n\ -}\n\n.pronunciation{\n color:blue;\n}\n\n.extrainfo{\n color:lightgrey;\n}" + \ black;\n background-color: white;\n}\n\n.card1,.card3, .card5 { background-color:\ + \ #B60F2D; }\n.card2,.card4, .card6 { background-color: #2E9017; }\n.card7 { background:\ + \ linear-gradient(90deg, #B60F2D 49.9%, #2E9017 50.1%); }\n\n.word {\n font-size:1.5em;\n\ + }\n\n.pronunciation{\n color:blue;\n}\n\n.extrainfo{\n color:lightgrey;\n}" sort_field_num: 1 is_cloze: true latex_pre: "\\documentclass[12pt]{article}\n\\special{papersize=3in,5in}\n\\usepackage{amssymb,amsmath}\n\ -\\pagestyle{empty}\n\\setlength{\\parindent}{0in}\n\\begin{document}\nTEST" + \\pagestyle{empty}\n\\setlength{\\parindent}{0in}\n\\begin{document}\nTEST" latex_post: \end{document}TEST fields: - - name: Word - font: Liberation SansTEST - media: - - TEST - is_right_to_left: true - size: 10 - is_sticky: true - - name: X Word - font: Arial - media: - - TEST - is_right_to_left: true - size: 10 - is_sticky: true - - name: Y Word - font: Arial - media: - - TEST - is_right_to_left: true - size: 10 - is_sticky: true - - name: Picture - font: Arial - media: - - TEST - is_right_to_left: true - size: 10 - is_sticky: true - - name: Extra - font: Arial - media: - - TEST - is_right_to_left: true - size: 10 - is_sticky: true - - name: X Pronunciation (Recording and/or IPA) - font: Arial - media: - - TEST - is_right_to_left: true - size: 10 - is_sticky: true - - name: Y Pronunciation (Recording and/or IPA) - font: Arial - media: - - TEST - is_right_to_left: true - size: 10 - is_sticky: true +- name: Word + font: Liberation SansTEST + media: + - TEST + is_right_to_left: true + font_size: 10 + is_sticky: true +- name: X Word + font: Arial + media: + - TEST + is_right_to_left: true + font_size: 10 + is_sticky: true +- name: Y Word + font: Arial + media: + - TEST + is_right_to_left: true + font_size: 10 + is_sticky: true +- name: Picture + font: Arial + media: + - TEST + is_right_to_left: true + font_size: 10 + is_sticky: true +- name: Extra + font: Arial + media: + - TEST + is_right_to_left: true + font_size: 10 + is_sticky: true +- name: X Pronunciation (Recording and/or IPA) + font: Arial + media: + - TEST + is_right_to_left: true + font_size: 10 + is_sticky: true +- name: Y Pronunciation (Recording and/or IPA) + font: Arial + media: + - TEST + is_right_to_left: true + font_size: 10 + is_sticky: true templates: - - name: X Comprehension - question_format: "{{#X Word}}\n\t{{text:X Word}}\n\ - {{/X Word}}" - answer_format: "{{#X Word}}\n\t{{X Word}}\n{{/X Word}}\n\ +- name: X Comprehension + question_format: "{{#X Word}}\n\t{{text:X Word}}\n{{/X\ + \ Word}}" + answer_format: "{{#X Word}}\n\t{{X Word}}\n{{/X Word}}\n\ \n
\n\n{{Picture}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\ \t
{{X Pronunciation (Recording and/or IPA)}}\n\ {{/X Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}" - browser_question_format: TEST - browser_answer_format: TEST - deck_override_id: 1 - - name: Y Comprehension - question_format: "{{#Y Word}}\n\t{{text:Y Word}}\n\ - {{/Y Word}}" - answer_format: "{{#Y Word}}\n\t{{Y Word}}\n{{/Y Word}}\n\ + browser_question_format: TEST + browser_answer_format: TEST + deck_override_id: 1 +- name: Y Comprehension + question_format: "{{#Y Word}}\n\t{{text:Y Word}}\n{{/Y\ + \ Word}}" + answer_format: "{{#Y Word}}\n\t{{Y Word}}\n{{/Y Word}}\n\ \n
\n\n{{Picture}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\ \t
{{Y Pronunciation (Recording and/or IPA)}}\n\ {{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}" - browser_question_format: TEST - browser_answer_format: TEST - deck_override_id: 1 - - name: X Production - question_format: "{{#X Word}}{{#Picture}}\n\t{{Picture}}\n{{/Picture}}{{/X Word}}" - answer_format: "{{FrontSide}}\n\n
\n\n{{X Word}}\n\ + browser_question_format: TEST + browser_answer_format: TEST + deck_override_id: 1 +- name: X Production + question_format: "{{#X Word}}{{#Picture}}\n\t{{Picture}}\n{{/Picture}}{{/X Word}}" + answer_format: "{{FrontSide}}\n\n
\n\n{{X Word}}\n\ \n{{#X Pronunciation (Recording and/or IPA)}}\n\t
{{X Pronunciation (Recording and/or IPA)}}\n{{/X Pronunciation (Recording\ \ and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n\ {{/Extra}}" - browser_question_format: TEST - browser_answer_format: TEST - deck_override_id: 1 - - name: Y Production - question_format: "{{#Y Word}}{{#Picture}}\n\t{{Picture}}\n{{/Picture}}{{/Y Word}}" - answer_format: "{{FrontSide}}\n\n
\n\n{{Y Word}}\n\ + browser_question_format: TEST + browser_answer_format: TEST + deck_override_id: 1 +- name: Y Production + question_format: "{{#Y Word}}{{#Picture}}\n\t{{Picture}}\n{{/Picture}}{{/Y Word}}" + answer_format: "{{FrontSide}}\n\n
\n\n{{Y Word}}\n\ \n{{#Y Pronunciation (Recording and/or IPA)}}\n\t
{{Y Pronunciation (Recording and/or IPA)}}\n{{/Y Pronunciation (Recording\ \ and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n\ {{/Extra}}" - browser_question_format: TEST - browser_answer_format: TEST - deck_override_id: 1 - - name: X Spelling - question_format: "{{#X Word}}\n\t
Spell this word:
\n\n\t
Spell this word:
\n\n\t
{{type:X Word}}
\n\n\t
{{Picture}}\n{{/X Word}}" - answer_format: "{{FrontSide}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\ - \t
{{X Pronunciation (Recording and/or IPA)}}\n\ + answer_format: "{{FrontSide}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\t\ +
{{X Pronunciation (Recording and/or IPA)}}\n\ {{/X Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}" - browser_question_format: TEST - browser_answer_format: TEST - deck_override_id: 1 - - name: Y Spelling - question_format: "{{#Y Word}}\n\t
Spell this word:
\n\n\t
Spell this word:
\n\n\t
{{type:Y Word}}
\n\n\t
{{Picture}}\n{{/Y Word}}" - answer_format: "{{FrontSide}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\ - \t
{{Y Pronunciation (Recording and/or IPA)}}\n\ + answer_format: "{{FrontSide}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\t\ +
{{Y Pronunciation (Recording and/or IPA)}}\n\ {{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}" - browser_question_format: TEST - browser_answer_format: TEST - deck_override_id: 2 - - name: X and Y Production - question_format: "{{#X Word}}\n{{#Y Word}}\n\t{{Picture}}\n{{/Y Word}}\n{{/X Word}}" - answer_format: "{{FrontSide}}\n\n
\n\n
{{text:X\ + browser_question_format: TEST + browser_answer_format: TEST + deck_override_id: 2 +- name: X and Y Production + question_format: "{{#X Word}}\n{{#Y Word}}\n\t{{Picture}}\n{{/Y Word}}\n{{/X Word}}" + answer_format: "{{FrontSide}}\n\n
\n\n
{{text:X\ \ Word}}
\n
{{text:Y Word}}
\n\n{{#X Pronunciation\ \ (Recording and/or IPA)}}\n\t
{{X Pronunciation\ \ (Recording and/or IPA)}}\n{{/X Pronunciation (Recording and/or IPA)}}\n\ @@ -133,39 +133,39 @@ templates: >{{Y Pronunciation (Recording and/or IPA)}}\n{{/Y Pronunciation (Recording\ \ and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n\ {{/Extra}}" - browser_question_format: TEST - browser_answer_format: TEST - deck_override_id: 1 + browser_question_format: TEST + browser_answer_format: TEST + deck_override_id: 1 tags: - - TEST +- TEST version: - - TEST +- TEST __type__: NoteModelTEST required_fields_per_template: - - - 0 - - all - - - 1 - - - 1 - - all - - - 2 - - - 2 - - all - - - 1 - - 3 - - - 3 - - all - - - 2 - - 3 - - - 4 - - all - - - 1 - - 3 - - - 5 - - all - - - 2 - - 3 - - - 6 - - all - - - 1 - - 2 - - 3 +- - 0 + - all + - - 1 +- - 1 + - all + - - 2 +- - 2 + - all + - - 1 + - 3 +- - 3 + - all + - - 2 + - 3 +- - 4 + - all + - - 1 + - 3 +- - 5 + - all + - - 2 + - 3 +- - 6 + - all + - - 1 + - 2 + - 3 diff --git a/tests/test_files/deck_parts/yaml/note_models/LL-Word-Only-Required.yaml b/tests/test_files/deck_parts/yaml/note_models/LL-Word-Only-Required.yaml index 055097a..5107b89 100644 --- a/tests/test_files/deck_parts/yaml/note_models/LL-Word-Only-Required.yaml +++ b/tests/test_files/deck_parts/yaml/note_models/LL-Word-Only-Required.yaml @@ -1,74 +1,74 @@ name: LL Word id: 057a8d66-bc4e-11e9-9822-d8cb8ac9abf0 css: ".card {\n font-family: arial;\n font-size: 20px;\n text-align: center;\n color:\ -\ black;\n background-color: white;\n}\n\n.card1,.card3, .card5 { background-color:\ -\ #B60F2D; }\n.card2,.card4, .card6 { background-color: #2E9017; }\n.card7 { background:\ -\ linear-gradient(90deg, #B60F2D 49.9%, #2E9017 50.1%); }\n\n.word {\n font-size:1.5em;\n\ -}\n\n.pronunciation{\n color:blue;\n}\n\n.extrainfo{\n color:lightgrey;\n}" + \ black;\n background-color: white;\n}\n\n.card1,.card3, .card5 { background-color:\ + \ #B60F2D; }\n.card2,.card4, .card6 { background-color: #2E9017; }\n.card7 { background:\ + \ linear-gradient(90deg, #B60F2D 49.9%, #2E9017 50.1%); }\n\n.word {\n font-size:1.5em;\n\ + }\n\n.pronunciation{\n color:blue;\n}\n\n.extrainfo{\n color:lightgrey;\n}" fields: - - name: Word - size: 12 - - name: X Word - font: Arial - - name: Y Word - font: Arial - - name: Picture - font: Arial - size: 6 - - name: Extra - font: Arial - - name: X Pronunciation (Recording and/or IPA) - font: Arial - - name: Y Pronunciation (Recording and/or IPA) - font: Arial +- name: Word + font_size: 12 +- name: X Word + font: Arial +- name: Y Word + font: Arial +- name: Picture + font: Arial + font_size: 6 +- name: Extra + font: Arial +- name: X Pronunciation (Recording and/or IPA) + font: Arial +- name: Y Pronunciation (Recording and/or IPA) + font: Arial templates: - - name: X Comprehension - question_format: "{{#X Word}}\n\t{{text:X Word}}\n\ - {{/X Word}}" - answer_format: "{{#X Word}}\n\t{{X Word}}\n{{/X Word}}\n\ +- name: X Comprehension + question_format: "{{#X Word}}\n\t{{text:X Word}}\n{{/X\ + \ Word}}" + answer_format: "{{#X Word}}\n\t{{X Word}}\n{{/X Word}}\n\ \n
\n\n{{Picture}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\ \t
{{X Pronunciation (Recording and/or IPA)}}\n\ {{/X Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}" - - name: Y Comprehension - question_format: "{{#Y Word}}\n\t{{text:Y Word}}\n\ - {{/Y Word}}" - answer_format: "{{#Y Word}}\n\t{{Y Word}}\n{{/Y Word}}\n\ +- name: Y Comprehension + question_format: "{{#Y Word}}\n\t{{text:Y Word}}\n{{/Y\ + \ Word}}" + answer_format: "{{#Y Word}}\n\t{{Y Word}}\n{{/Y Word}}\n\ \n
\n\n{{Picture}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\ \t
{{Y Pronunciation (Recording and/or IPA)}}\n\ {{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}" - - name: X Production - question_format: "{{#X Word}}{{#Picture}}\n\t{{Picture}}\n{{/Picture}}{{/X Word}}" - answer_format: "{{FrontSide}}\n\n
\n\n{{X Word}}\n\ +- name: X Production + question_format: "{{#X Word}}{{#Picture}}\n\t{{Picture}}\n{{/Picture}}{{/X Word}}" + answer_format: "{{FrontSide}}\n\n
\n\n{{X Word}}\n\ \n{{#X Pronunciation (Recording and/or IPA)}}\n\t
{{X Pronunciation (Recording and/or IPA)}}\n{{/X Pronunciation (Recording\ \ and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n\ {{/Extra}}" - - name: Y Production - question_format: "{{#Y Word}}{{#Picture}}\n\t{{Picture}}\n{{/Picture}}{{/Y Word}}" - answer_format: "{{FrontSide}}\n\n
\n\n{{Y Word}}\n\ +- name: Y Production + question_format: "{{#Y Word}}{{#Picture}}\n\t{{Picture}}\n{{/Picture}}{{/Y Word}}" + answer_format: "{{FrontSide}}\n\n
\n\n{{Y Word}}\n\ \n{{#Y Pronunciation (Recording and/or IPA)}}\n\t
{{Y Pronunciation (Recording and/or IPA)}}\n{{/Y Pronunciation (Recording\ \ and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n\ {{/Extra}}" - - name: X Spelling - question_format: "{{#X Word}}\n\t
Spell this word:
\n\n\t
Spell this word:
\n\n\t
{{type:X Word}}
\n\n\t
{{Picture}}\n{{/X Word}}" - answer_format: "{{FrontSide}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\ - \t
{{X Pronunciation (Recording and/or IPA)}}\n\ + answer_format: "{{FrontSide}}\n\n{{#X Pronunciation (Recording and/or IPA)}}\n\t\ +
{{X Pronunciation (Recording and/or IPA)}}\n\ {{/X Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}" - - name: Y Spelling - question_format: "{{#Y Word}}\n\t
Spell this word:
\n\n\t
Spell this word:
\n\n\t
{{type:Y Word}}
\n\n\t
{{Picture}}\n{{/Y Word}}" - answer_format: "{{FrontSide}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\ - \t
{{Y Pronunciation (Recording and/or IPA)}}\n\ + answer_format: "{{FrontSide}}\n\n{{#Y Pronunciation (Recording and/or IPA)}}\n\t\ +
{{Y Pronunciation (Recording and/or IPA)}}\n\ {{/Y Pronunciation (Recording and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n{{/Extra}}" - - name: X and Y Production - question_format: "{{#X Word}}\n{{#Y Word}}\n\t{{Picture}}\n{{/Y Word}}\n{{/X Word}}" - answer_format: "{{FrontSide}}\n\n
\n\n
{{text:X\ +- name: X and Y Production + question_format: "{{#X Word}}\n{{#Y Word}}\n\t{{Picture}}\n{{/Y Word}}\n{{/X Word}}" + answer_format: "{{FrontSide}}\n\n
\n\n
{{text:X\ \ Word}}
\n
{{text:Y Word}}
\n\n{{#X Pronunciation\ \ (Recording and/or IPA)}}\n\t
{{X Pronunciation\ \ (Recording and/or IPA)}}\n{{/X Pronunciation (Recording and/or IPA)}}\n\ @@ -77,30 +77,30 @@ templates: \ and/or IPA)}}\n\n
\n{{#Extra}}\n\t
{{Extra}}\n\ {{/Extra}}" required_fields_per_template: - - - 0 - - all - - - 1 - - - 1 - - all - - - 2 - - - 2 - - all - - - 1 - - 3 - - - 3 - - all - - - 2 - - 3 - - - 4 - - all - - - 1 - - 3 - - - 5 - - all - - - 2 - - 3 - - - 6 - - all - - - 1 - - 2 - - 3 +- - 0 + - all + - - 1 +- - 1 + - all + - - 2 +- - 2 + - all + - - 1 + - 3 +- - 3 + - all + - - 2 + - 3 +- - 4 + - all + - - 1 + - 3 +- - 5 + - all + - - 2 + - 3 +- - 6 + - all + - - 1 + - 2 + - 3 From a334fc64634511ef0e113817df437b133b5f9be5 Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sun, 6 Sep 2020 16:29:11 +0200 Subject: [PATCH 38/39] Release V0.2.0 --- brain_brew/representation/yaml/note_repr.py | 1 - setup.py | 11 +++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/brain_brew/representation/yaml/note_repr.py b/brain_brew/representation/yaml/note_repr.py index f9c2caa..15da9cc 100644 --- a/brain_brew/representation/yaml/note_repr.py +++ b/brain_brew/representation/yaml/note_repr.py @@ -77,7 +77,6 @@ def encode(self) -> dict: return data_dict # TODO: Extract Shared Tags and Note Models - # TODO: Sort notes def verify_groupings(self): errors = [] diff --git a/setup.py b/setup.py index 5af8d8a..2835b3a 100644 --- a/setup.py +++ b/setup.py @@ -5,9 +5,9 @@ setuptools.setup( name="Brain-Brew", - version="0.1.5", + version="0.2.0", author="Jordan Munch O'Hare", - author_email="ohare93@gmail.com", + author_email="brainbrew@jordan.munchohare.com", description="Automated Anki flashcard creation and extraction to/from Csv ", long_description=long_description, long_description_content_type="text/markdown", @@ -17,6 +17,7 @@ 'console_scripts': [ 'brain_brew = brain_brew.main:main', 'brain-brew = brain_brew.main:main', + 'brainbrew = brain_brew.main:main', ] }, classifiers=[ @@ -24,8 +25,10 @@ "License :: Public Domain", "Operating System :: OS Independent", ], - python_requires='>=3.6', + python_requires='>=3.7', install_requires=[ - 'PyYAML' + 'PyYAML', + 'ruamel-yaml', + 'yamale' ] ) From 5919cf90f30f63f0d3ad5a6c3d0d410cea58f44f Mon Sep 17 00:00:00 2001 From: Jordan Munch O'Hare Date: Sun, 6 Sep 2020 16:56:20 +0200 Subject: [PATCH 39/39] Fix: Data file included in dist --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2835b3a..0789065 100644 --- a/setup.py +++ b/setup.py @@ -5,7 +5,7 @@ setuptools.setup( name="Brain-Brew", - version="0.2.0", + version="0.2.1", author="Jordan Munch O'Hare", author_email="brainbrew@jordan.munchohare.com", description="Automated Anki flashcard creation and extraction to/from Csv ", @@ -13,6 +13,7 @@ long_description_content_type="text/markdown", url="https://github.com/ohare93/brain-brew", packages=setuptools.find_packages(), + include_package_data=True, entry_points={ 'console_scripts': [ 'brain_brew = brain_brew.main:main',