Various fixes:

Fixed install requirement of Ruamel
DeckParts not read until Execute
Fixed Task name regex
Missing optionals for fields
Create missing files/dirs in more places

brain_brew/build_tasks/crowd_anki/crowd_anki_generate.py

@@ -20,7 +20,7 @@
 
 @dataclass
 class CrowdAnkiGenerate(TopLevelBuildTask):
-    task_regex = r'.*crowd[\s_-]+?anki.*'
+    task_regex = r'.*crowd[\s_-]*?anki.*'
 
     @dataclass
     class Representation(RepresentationBase):

brain_brew/build_tasks/crowd_anki/crowd_anki_to_deck_parts.py

@@ -15,7 +15,7 @@
 
 @dataclass
 class CrowdAnkiToDeckParts(DeckPartBuildTask):
-    task_regex = r'.*crowd[\s_-]+?anki.*'
+    task_regex = r'.*crowd[\s_-]*?anki.*'
 
     @dataclass
     class Representation(RepresentationBase):
@@ -50,7 +50,7 @@ def execute(self):
 
         note_models: List[NoteModel] = self.note_model_transform.execute(ca_wrapper)
 
-        nm_id_to_name: dict = {model.crowdanki_id: model.name for model in note_models}
+        nm_id_to_name: dict = {model.id: model.name for model in note_models}
         notes = self.notes_transform.execute(ca_wrapper, nm_id_to_name)
 
         headers = self.headers_transform.execute(ca_wrapper)

brain_brew/build_tasks/crowd_anki/headers_to_crowd_anki.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Optional, Union
 
 from brain_brew.build_tasks.crowd_anki.headers_from_crowdanki import headers_default_values
@@ -24,12 +24,15 @@ def from_repr(cls, data: Union[Representation, dict, str]):
             rep = cls.Representation(deck_part=data)  # Support single string being passed in
 
         return cls(
-            headers=DeckPartHolder.from_deck_part_pool(rep.deck_part).deck_part,
+            headers_to_read=rep.deck_part
         )
 
-    headers: Headers
+    headers_to_read: str
+    headers: Headers = field(init=False)
 
     def execute(self) -> dict:
+        self.headers = DeckPartHolder.from_deck_part_pool(self.headers_to_read).deck_part
+
         headers = self.headers_to_crowd_anki(self.headers.data_without_name)
 
         return headers

brain_brew/build_tasks/crowd_anki/note_models_from_crowd_anki.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Optional, Union, List
 import logging
 
@@ -14,7 +14,7 @@ class NoteModelsFromCrowdAnki:
     class NoteModelListItem(BaseDeckPartsFrom):
         @dataclass
         class Representation(BaseDeckPartsFrom.Representation):
-            model_name: Optional[str]
+            model_name: Optional[str] = field(default_factory=lambda: None)
             # TODO: fields: Optional[List[str]]
             # TODO: templates: Optional[List[str]]
 

brain_brew/build_tasks/crowd_anki/note_models_to_crowd_anki.py

@@ -1,4 +1,4 @@
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Optional, Union, List
 import logging
 
@@ -30,13 +30,15 @@ def from_repr(cls, data: Union[Representation, dict, str]):
                 rep = cls.Representation(deck_part=data)  # Support string
 
             return cls(
-                deck_part=DeckPartHolder.from_deck_part_pool(rep.deck_part).deck_part
+                deck_part_to_read=rep.deck_part
             )
 
         def get_note_model(self) -> NoteModel:
+            self.deck_part = DeckPartHolder.from_deck_part_pool(self.deck_part_to_read).deck_part
             return self.deck_part  # Todo: add filters in here
 
-        deck_part: NoteModel
+        deck_part: NoteModel = field(init=False)
+        deck_part_to_read: str
 
     @dataclass
     class Representation(RepresentationBase):

brain_brew/build_tasks/crowd_anki/notes_to_crowd_anki.py

@@ -22,18 +22,22 @@ class Representation(RepresentationBase):
     def from_repr(cls, data: Union[Representation, dict]):
         rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data)
         return cls(
-            notes=DeckPartHolder.from_deck_part_pool(rep.deck_part).deck_part,
+            notes_to_read=rep.deck_part,
             sort_order=SharedBaseNotes._get_sort_order(rep.sort_order),
             reverse_sort=SharedBaseNotes._get_reverse_sort(rep.reverse_sort),
             additional_items_to_add=rep.additional_items_to_add or {}
         )
 
-    notes: Notes
+    notes: Notes = field(init=False)
+
+    notes_to_read: str
     additional_items_to_add: dict
     sort_order: Optional[List[str]] = field(default_factory=lambda: None)
     reverse_sort: Optional[bool] = field(default_factory=lambda: None)
 
     def execute(self, nm_name_to_id: dict) -> List[dict]:
+        self.notes = DeckPartHolder.from_deck_part_pool(self.notes_to_read).deck_part
+
         notes = self.notes.get_sorted_notes_copy(sort_by_keys=self.sort_order, reverse_sort=self.reverse_sort)
 
         note_dicts = [self.note_to_ca_note(note, nm_name_to_id, self.additional_items_to_add) for note in notes]

brain_brew/build_tasks/csvs/csvs_generate.py

@@ -13,6 +13,7 @@
 class CsvsGenerate(SharedBaseCsvs, TopLevelBuildTask):
     task_regex = r'.*csv.*'
 
+    notes_to_read: str
     notes: DeckPartHolder[Notes] = field(default=None)
 
     @dataclass(init=False)
@@ -27,12 +28,14 @@ def __init__(self, notes, file_mappings, note_model_mappings):
     def from_repr(cls, data: Union[Representation, dict]):
         rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data)
         return cls(
-            notes=DeckPartHolder.from_deck_part_pool(rep.notes),
+            notes_to_read=rep.notes,
             file_mappings=rep.get_file_mappings(),
-            note_model_mappings=rep.get_note_model_mappings()
+            note_model_mappings_to_read=rep.note_model_mappings
         )
 
     def execute(self):
+        self.setup_note_model_mappings()
+        self.notes = DeckPartHolder.from_deck_part_pool(self.notes_to_read)
         self.verify_contents()
 
         notes: List[Note] = self.notes.deck_part.get_sorted_notes_copy(

brain_brew/build_tasks/csvs/notes_from_csvs.py

@@ -24,10 +24,11 @@ def from_repr(cls, data: Union[Representation, dict]):
             name=rep.name,
             save_to_file=rep.save_to_file,
             file_mappings=rep.get_file_mappings(),
-            note_model_mappings=rep.get_note_model_mappings()
+            note_model_mappings_to_read=rep.note_model_mappings
         )
 
     def execute(self):
+        self.setup_note_model_mappings()
         self.verify_contents()
 
         csv_data_by_guid: Dict[str, dict] = {}

brain_brew/build_tasks/csvs/shared_base_csvs.py

@@ -1,3 +1,4 @@
+import logging
 from dataclasses import dataclass, field
 from typing import List, Dict
 
@@ -20,15 +21,16 @@ def __init__(self, file_mappings, note_model_mappings):
         def get_file_mappings(self) -> List[FileMapping]:
             return list(map(FileMapping.from_repr, self.file_mappings))
 
-        def get_note_model_mappings(self):
-            def map_nmm(nmm_to_map: str):
-                nmm = NoteModelMapping.from_repr(nmm_to_map)
-                return nmm.get_note_model_mapping_dict()
+    file_mappings: List[FileMapping]
+    note_model_mappings_to_read: List[NoteModelMapping.Representation]
+    note_model_mappings: Dict[str, NoteModelMapping] = field(init=False)
 
-            return dict(*map(map_nmm, self.note_model_mappings))
+    def setup_note_model_mappings(self):
+        def map_nmm(nmm_to_map: str):
+            nmm = NoteModelMapping.from_repr(nmm_to_map)
+            return nmm.get_note_model_mapping_dict()
 
-    file_mappings: List[FileMapping]
-    note_model_mappings: Dict[str, NoteModelMapping]
+        self.note_model_mappings = dict(*map(map_nmm, self.note_model_mappings_to_read))
 
     def verify_contents(self):
         errors = []
@@ -64,7 +66,7 @@ def verify_contents(self):
                     missing_columns = [col for col in holder.deck_part.field_names_lowercase if
                                        col not in nm_map.csv_headers_map_to_note_fields(available_columns)]
                     if missing_columns:
-                        errors.append(KeyError(f"Csvs are missing columns from {holder.name}", missing_columns))
+                        logging.warning(f"Csvs are missing columns from {holder.name} {missing_columns}")
 
         if errors:
             raise Exception(errors)

brain_brew/build_tasks/deck_parts/from_deck_part.py

@@ -12,7 +12,7 @@
 
 @dataclass
 class FromDeckParts(DeckPartBuildTask):
-    task_regex = r'.*deck[\s_-]+?part.*'
+    task_regex = r'.*deck[\s_-]*?part.*'
 
     @dataclass
     class DeckPartToRead(RepresentationBase):

brain_brew/representation/build_config/generate_deck_parts.py

@@ -13,7 +13,7 @@
 
 @dataclass
 class GenerateDeckParts(TaskBuilder, TopLevelBuildTask):
-    task_regex = r'.*deck[\s_-]+?part.*'
+    task_regex = r'.*deck[\s_-]*?part.*'
 
     @classmethod
     def known_task_dict(cls) -> Dict[str, Type[BuildTask]]:

brain_brew/representation/configuration/csv_file_mapping.py

@@ -46,7 +46,7 @@ def from_repr(cls, data: Union[Representation, dict]):
         rep: cls.Representation = data if isinstance(data, cls.Representation) else cls.Representation.from_dict(data)
         return cls(
             csv_file=CsvFile.create_or_get(rep.file),
-            note_model=rep.note_model.strip() or None,
+            note_model=rep.note_model.strip() if rep.note_model else None,
             sort_by_columns=single_item_to_list(rep.sort_by_columns),
             reverse_sort=rep.reverse_sort or False,
             derivatives=list(map(cls.from_repr, rep.derivatives)) if rep.derivatives is not None else []
@@ -146,7 +146,7 @@ def set_relevant_data(self, data_set: Dict[str, dict]):
             if guid in self.compiled_data.keys():
                 changed_row = False
                 for key in data_set[guid]:
-                    if self.compiled_data[guid][key] != data_set[guid][key]:
+                    if key in self.compiled_data[guid] and self.compiled_data[guid][key] != data_set[guid][key]:
                         self.compiled_data[guid][key] = data_set[guid][key]
                         changed_row = True
                 if changed_row:

brain_brew/representation/configuration/note_model_mapping.py

@@ -156,4 +156,6 @@ def get_relevant_data(self, row):
         return relevant_data
 
     def field_values_in_note_model_order(self, note_model_name, fields_from_csv):
-        return [fields_from_csv[f] for f in self.note_models[note_model_name].deck_part.field_names_lowercase]
+        return [fields_from_csv[f] if f in fields_from_csv else ""
+                for f in self.note_models[note_model_name].deck_part.field_names_lowercase
+               ]

brain_brew/representation/generic/media_file.py

@@ -2,6 +2,8 @@
 import shutil
 from enum import Enum
 
+from brain_brew.utils import create_path_if_not_exists
+
 
 class MediaFile:
     class ManagementType(Enum):
@@ -30,7 +32,7 @@ def set_override(self, source_loc):
     def copy_source_to_target(self):
         if self.should_write():
             # TODO: If ManagementType.OVERRIDDEN check if override necessary
-            os.makedirs(os.path.dirname(self.file_location), exist_ok=True)
+            create_path_if_not_exists(self.file_location)
             shutil.copy2(self.source_loc, self.file_location)
 
     def should_write(self):

brain_brew/representation/json/crowd_anki_export.py

@@ -6,7 +6,7 @@
 from brain_brew.representation.generic.media_file import MediaFile
 from brain_brew.representation.json.json_file import JsonFile
 from brain_brew.representation.json.wrappers_for_crowd_anki import CrowdAnkiJsonWrapper
-from brain_brew.utils import filename_from_full_path, find_all_files_in_directory
+from brain_brew.utils import filename_from_full_path, find_all_files_in_directory, create_path_if_not_exists
 
 
 class CrowdAnkiExport(SourceFile):
@@ -23,8 +23,7 @@ def __init__(self, folder_location):
         if self.folder_location[-1] != "/":
             self.folder_location = self.folder_location + "/"
 
-        if not self.is_dir(self.folder_location):
-            raise FileNotFoundError(f"Missing CrowdAnkiExport '{self.folder_location}'")
+        create_path_if_not_exists(self.folder_location)
 
         self.json_file_location = self.find_json_file_in_folder()
         self.find_all_media()
@@ -39,8 +38,9 @@ def find_json_file_in_folder(self):
         if len(files) == 1:
             return files[0]
         elif not files:
-            logging.error(f"No json file found in folder '{self.folder_location}'")
-            raise FileNotFoundError(self.folder_location)
+            file_loc = self.folder_location + self.folder_location.split("/")[-2] + ".json"
+            logging.warning(f"Creating missing json file '{file_loc}'")
+            return file_loc
         else:
             logging.error(f"Multiple json files found in '{self.folder_location}': {files}")
             raise FileExistsError()
@@ -51,6 +51,7 @@ def find_all_media(self):
         self.contains_media = self.is_dir(self.media_loc)
 
         if not self.contains_media:
+            create_path_if_not_exists(self.media_loc)
             return
 
         media_files = find_all_files_in_directory(self.media_loc)
@@ -63,8 +64,8 @@ def find_all_media(self):
 
     def write_to_files(self, json_data):  # import_config_data
         JsonFile.write_file(self.json_file_location, json_data)
-        # for filename, media_file in self.known_media.items():
-        #     media_file.copy_source_to_target()
+        for filename, media_file in self.known_media.items():
+            media_file.copy_source_to_target()
 
     def read_json_file(self) -> CrowdAnkiJsonWrapper:
         return CrowdAnkiJsonWrapper(JsonFile.read_file(self.json_file_location))

brain_brew/representation/yaml/my_yaml.py

@@ -3,6 +3,8 @@
 import os
 from ruamel.yaml import YAML
 
+from brain_brew.utils import create_path_if_not_exists
+
 yaml_load = YAML(typ='safe')
 
 
@@ -30,9 +32,7 @@ def encode(self) -> dict:
     def dump_to_yaml(self, filepath):
         filepath = YamlRepr.append_yaml_if_needed(filepath)
 
-        if not Path(filepath).is_file():
-            logging.warning(f"Creating missing filepath '{filepath}'")
-            os.makedirs(os.path.dirname(filepath), exist_ok=True)
+        create_path_if_not_exists(filepath)
 
         with open(filepath, 'w') as fp:
             yaml_dump.dump(self.encode(), fp)

brain_brew/schemas/builder.yaml

@@ -1,16 +1,16 @@
 list(
-    map(include('generate_deck_parts'), key=regex('.*deck[\s_-]+?part.*', ignore_case=True)),
+    map(include('generate_deck_parts'), key=regex('.*deck[\s_-]*?part.*', ignore_case=True)),
     map(include('generate_csv'), key=regex('.*csv.*', ignore_case=True)),
-    map(include('generate_crowd_anki'), key=regex('.*crowd[\s_-]+?anki.*', ignore_case=True))
+    map(include('generate_crowd_anki'), key=regex('.*crowd[\s_-]*?anki.*', ignore_case=True))
 )
 
 ---
 
 generate_deck_parts:
   list(
-    map(include('from_deck_parts'), key=regex('.*deck[\s_-]+?part.*', ignore_case=True)),
+    map(include('from_deck_parts'), key=regex('.*deck[\s_-]*?part.*', ignore_case=True)),
     map(include('from_csv'), key=regex('.*csv.*', ignore_case=True)),
-    map(include('from_crowd_anki'), key=regex('.*crowd[\s_-]+?anki.*', ignore_case=True))
+    map(include('from_crowd_anki'), key=regex('.*crowd[\s_-]*?anki.*', ignore_case=True))
   )
 
 

brain_brew/utils.py

@@ -1,12 +1,11 @@
+import logging
 import os
-import pathlib
+from pathlib import Path
 import string
 import random
 import re
 from typing import List
 
-from brain_brew.representation.configuration.global_config import GlobalConfig
-
 
 def blank_str_if_none(s):
     return '' if s is None else s
@@ -58,6 +57,13 @@ def find_all_files_in_directory(directory, recursive=False):
     return found_files
 
 
+def create_path_if_not_exists(path):
+    dir_name = os.path.dirname(path)
+    if not Path().is_dir():
+        logging.warning(f"Creating missing filepath '{dir_name}'")
+        os.makedirs(os.path.dirname(dir_name), exist_ok=True)
+
+
 def split_tags(tags_value: str) -> list:
     split = [entry.strip() for entry in re.split(r';\s*|,\s*|\s+', tags_value)]
     while "" in split:
@@ -66,6 +72,7 @@ def split_tags(tags_value: str) -> list:
 
 
 def join_tags(tags_list: list) -> str:
+    from brain_brew.representation.configuration.global_config import GlobalConfig
     return GlobalConfig.get_instance().join_values_with.join(tags_list)
 
 
@@ -91,6 +98,7 @@ def base91(num: int) -> str:
 
 
 def sort_dict(data, sort_by_keys, reverse_sort, case_insensitive_sort=None):
+    from brain_brew.representation.configuration.global_config import GlobalConfig
     if case_insensitive_sort is None:
         case_insensitive_sort = GlobalConfig.get_instance().sort_case_insensitive