Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Get away from master-slave jargon #4

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Get away from master-slave jargon #4

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
88 changes: 44 additions & 44 deletions modules/bibencode/lib/bibencode_batch_engine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,7 +204,7 @@ def _task_write_message(message):

def clean_job_for_quality(batch_job_dict, fallback=True):
"""
Removes jobs from the batch description that are not suitable for the master
Removes jobs from the batch description that are not suitable for the main
video's quality. It applies only for encoding jobs!
@param batch_job_dict: the dict containing the batch description
@type batch_job_dict: dict
Expand Down Expand Up @@ -344,7 +344,7 @@ def san_bitrate(bitrate):
else:
raise Exception("Could not parse bitrate")

if not getval(batch_job, 'update_from_master'):
if not getval(batch_job, 'update_from_main'):
if not getval(batch_job, 'input'):
raise Exception("No input file in batch description")

Expand All @@ -354,11 +354,11 @@ def san_bitrate(bitrate):
if not getval(batch_job, 'jobs'):
raise Exception("No job list in batch description")

if getval(batch_job, 'update_from_master'):
if (not getval(batch_job, 'bibdoc_master_comment') and
not getval(batch_job, 'bibdoc_master_description') and
not getval(batch_job, 'bibdoc_master_subformat')):
raise Exception("If update_from_master ist set, a comment or"
if getval(batch_job, 'update_from_main'):
if (not getval(batch_job, 'bibdoc_main_comment') and
not getval(batch_job, 'bibdoc_main_description') and
not getval(batch_job, 'bibdoc_main_subformat')):
raise Exception("If update_from_main ist set, a comment or"
" description or subformat for matching must be given")

if getval(batch_job, 'marc_snippet'):
Expand Down Expand Up @@ -399,7 +399,7 @@ def upload_marcxml_file(marcxml):
# GENERAL #
#---------#

_task_write_message("----------- Handling Master -----------")
_task_write_message("----------- Handling Main -----------")

## Check the validity of the batch file here
batch_job = json_decode_file(batch_job_file)
Expand All @@ -417,23 +417,23 @@ def upload_marcxml_file(marcxml):
# UPDATE FROM MASTER #
#--------------------#

## We want to add new stuff to the video's record, using the master as input
if getval(batch_job, 'update_from_master'):
found_master = False
## We want to add new stuff to the video's record, using the main as input
if getval(batch_job, 'update_from_main'):
found_main = False
bibdocs = recdoc.list_bibdocs()
for bibdoc in bibdocs:
bibdocfiles = bibdoc.list_all_files()
for bibdocfile in bibdocfiles:
comment = bibdocfile.get_comment()
description = bibdocfile.get_description()
subformat = bibdocfile.get_subformat()
m_comment = getval(batch_job, 'bibdoc_master_comment', comment)
m_description = getval(batch_job, 'bibdoc_master_description', description)
m_subformat = getval(batch_job, 'bibdoc_master_subformat', subformat)
m_comment = getval(batch_job, 'bibdoc_main_comment', comment)
m_description = getval(batch_job, 'bibdoc_main_description', description)
m_subformat = getval(batch_job, 'bibdoc_main_subformat', subformat)
if (comment == m_comment and
description == m_description and
subformat == m_subformat):
found_master = True
found_main = True
batch_job['input'] = bibdocfile.get_full_path()
## Get the aspect of the from the record
try:
Expand All @@ -442,12 +442,12 @@ def upload_marcxml_file(marcxml):
except IndexError:
pass
break
if found_master:
if found_main:
break
if not found_master:
_task_write_message("Video master for record %d not found"
if not found_main:
_task_write_message("Video main for record %d not found"
% batch_job['recid'])
task_update_progress("Video master for record %d not found"
task_update_progress("Video main for record %d not found"
% batch_job['recid'])
## Maybe send an email?
return 1
Expand All @@ -461,10 +461,10 @@ def upload_marcxml_file(marcxml):

## Generate the docname from the input filename's name or given name
bibdoc_video_docname, bibdoc_video_extension = decompose_file(batch_job['input'])[1:]
if not bibdoc_video_extension or getval(batch_job, 'bibdoc_master_extension'):
bibdoc_video_extension = getval(batch_job, 'bibdoc_master_extension')
if getval(batch_job, 'bibdoc_master_docname'):
bibdoc_video_docname = getval(batch_job, 'bibdoc_master_docname')
if not bibdoc_video_extension or getval(batch_job, 'bibdoc_main_extension'):
bibdoc_video_extension = getval(batch_job, 'bibdoc_main_extension')
if getval(batch_job, 'bibdoc_main_docname'):
bibdoc_video_docname = getval(batch_job, 'bibdoc_main_docname')

write_message("Creating BibDoc for %s" % bibdoc_video_docname)
## If the bibdoc exists, receive it
Expand All @@ -480,28 +480,28 @@ def upload_marcxml_file(marcxml):
#--------#
# MASTER #
#--------#
if not getval(batch_job, 'update_from_master'):
if getval(batch_job, 'add_master'):
## Generate the right name for the master
## The master should be hidden first an then renamed
if not getval(batch_job, 'update_from_main'):
if getval(batch_job, 'add_main'):
## Generate the right name for the main
## The main should be hidden first an then renamed
## when it is really available
## !!! FIX !!!
_task_write_message("Adding %s master to the BibDoc"
_task_write_message("Adding %s main to the BibDoc"
% bibdoc_video_docname)
master_format = compose_format(
main_format = compose_format(
bibdoc_video_extension,
getval(batch_job, 'bibdoc_master_subformat', 'master')
getval(batch_job, 'bibdoc_main_subformat', 'main')
)
## If a file of the same format is there, something is wrong, remove it!
## it might be caused by a previous corrupted submission etc.
if bibdoc_video.format_already_exists_p(master_format):
bibdoc_video.delete_file(master_format, 1)
if bibdoc_video.format_already_exists_p(main_format):
bibdoc_video.delete_file(main_format, 1)
bibdoc_video.add_file_new_format(
batch_job['input'],
version=1,
description=getval(batch_job, 'bibdoc_master_description'),
comment=getval(batch_job, 'bibdoc_master_comment'),
docformat=master_format
description=getval(batch_job, 'bibdoc_main_description'),
comment=getval(batch_job, 'bibdoc_main_comment'),
docformat=main_format
)

#-----------#
Expand All @@ -516,9 +516,9 @@ def upload_marcxml_file(marcxml):
_task_write_message("----------- Job %s of %s -----------"
% (_BATCH_STEP, _BATCH_STEPS))

## Try to substitute docname with master docname
## Try to substitute docname with main docname
if getval(job, 'bibdoc_docname'):
job['bibdoc_docname'] = Template(job['bibdoc_docname']).safe_substitute({'bibdoc_master_docname': bibdoc_video_docname})
job['bibdoc_docname'] = Template(job['bibdoc_docname']).safe_substitute({'bibdoc_main_docname': bibdoc_video_docname})

#-------------#
# TRANSCODING #
Expand All @@ -541,15 +541,15 @@ def upload_marcxml_file(marcxml):
raise Exception("No container/extension defined")
## Get the docname and subformat
bibdoc_video_subformat = getval(job, 'bibdoc_subformat')
bibdoc_slave_video_docname = getval(job, 'bibdoc_docname', bibdoc_video_docname)
bibdoc_subordinate_video_docname = getval(job, 'bibdoc_docname', bibdoc_video_docname)
## The subformat is incompatible with ffmpegs name convention
## We do the encoding without and rename it afterwards
bibdoc_video_fullpath = compose_file(
bibdoc_video_directory,
bibdoc_slave_video_docname,
bibdoc_subordinate_video_docname,
bibdoc_video_extension
)
_task_write_message("Transcoding %s to %s;%s" % (bibdoc_slave_video_docname,
_task_write_message("Transcoding %s to %s;%s" % (bibdoc_subordinate_video_docname,
bibdoc_video_extension,
bibdoc_video_subformat))
## We encode now directly into the bibdocs directory
Expand Down Expand Up @@ -582,7 +582,7 @@ def upload_marcxml_file(marcxml):
bibdoc_video_extension,
bibdoc_video_subformat,
1,
bibdoc_slave_video_docname)
bibdoc_subordinate_video_docname)
)
#bibdoc_video._build_file_list()
bibdoc_video.touch()
Expand Down Expand Up @@ -653,7 +653,7 @@ def upload_marcxml_file(marcxml):
fname = os.path.join(tmpdir, filename)

bibdoc_frame_format = compose_format(bibdoc_frame_extension, bibdoc_frame_subformat)
## Same as with the master, if the format allready exists,
## Same as with the main, if the format allready exists,
## override it, because something went wrong before
if bibdoc_frame.format_already_exists_p(bibdoc_frame_format):
bibdoc_frame.delete_file(bibdoc_frame_format, 1)
Expand Down Expand Up @@ -698,8 +698,8 @@ def upload_marcxml_file(marcxml):
# ADD MASTER METADATA #
#---------------------#

if getval(batch_job, 'add_master_metadata'):
_task_write_message("Adding master metadata")
if getval(batch_job, 'add_main_metadata'):
_task_write_message("Adding main metadata")
pbcore = pbcore_metadata(input_file = getval(batch_job, 'input'),
pbcoreIdentifier = batch_job['recid'],
aspect_override = getval(batch_job, 'aspect'))
Expand Down
2 changes: 1 addition & 1 deletion modules/bibencode/lib/bibencode_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@
CFG_BIBENCODE_FFMPEG_RE_VIDEOINFO_ASTREAM = re.compile("^\s*Stream #(\d+.\d+)\(?(\w+)?\)?: Audio: ([a-zA-Z0-9\(\) ]*), (\d+) Hz, ([a-zA-Z0-9 ]+), (\w+), (\d+) kb\/s$")

## FFMPEG command for setting metadata
## This will create a copy of the master and write the metadata there
## This will create a copy of the main and write the metadata there
CFG_BIBENCODE_FFMPEG_METADATA_SET_COMMAND = "ffmpeg -y -i %s -acodec copy -vcodec copy %s"

## FFMPEG metadata argument template
Expand Down
16 changes: 8 additions & 8 deletions modules/bibfield/lib/bibfield.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,20 +61,20 @@ def plugin_builder(plugin_name, plugin_code):



def create_record(blob, master_format='marc', verbose=0, **additional_info):
def create_record(blob, main_format='marc', verbose=0, **additional_info):
"""
Creates a record object from the blob description using the apropiate reader
for it.

@return Record object
"""

reader = CFG_BIBFIELD_READERS['bibfield_%sreader.py' % (master_format,)](blob, **additional_info)
reader = CFG_BIBFIELD_READERS['bibfield_%sreader.py' % (main_format,)](blob, **additional_info)

return Record(reader.translate())


def create_records(blob, master_format='marc', verbose=0, **additional_info):
def create_records(blob, main_format='marc', verbose=0, **additional_info):
"""
Creates a list of records from the blod descriptions using the split_records
function to divide then.
Expand All @@ -83,9 +83,9 @@ def create_records(blob, master_format='marc', verbose=0, **additional_info):

@return List of record objects initiated by the functions create_record()
"""
record_blods = CFG_BIBFIELD_READERS['bibfield_%sreader.py' % (master_format,)].split_blob(blob, additional_info.get('schema', None))
record_blods = CFG_BIBFIELD_READERS['bibfield_%sreader.py' % (main_format,)].split_blob(blob, additional_info.get('schema', None))

return [create_record(record_blob, master_format, verbose=verbose, **additional_info) for record_blob in record_blods]
return [create_record(record_blob, main_format, verbose=verbose, **additional_info) for record_blob in record_blods]


def get_record(recid, reset_cache=False, fields=()):
Expand Down Expand Up @@ -113,18 +113,18 @@ def get_record(recid, reset_cache=False, fields=()):
#Then retrieve information and blob
if not record or reset_cache:
try:
master_format = run_sql("SELECT master_format FROM bibrec WHERE id=%s", (recid,))[0][0]
main_format = run_sql("SELECT main_format FROM bibrec WHERE id=%s", (recid,))[0][0]
except:
return None
schema = 'xml'
master_format = 'marc'
main_format = 'marc'
try:
from invenio.search_engine import print_record
blob = print_record(recid, format='xm')
except:
return None

reader = CFG_BIBFIELD_READERS['bibfield_%sreader.py' % (master_format,)](blob, schema=schema)
reader = CFG_BIBFIELD_READERS['bibfield_%sreader.py' % (main_format,)](blob, schema=schema)
record = Record(reader.translate())
#Update bibfmt for future uses
run_sql("REPLACE INTO bibfmt(id_bibrec, format, last_updated, value) VALUES (%s, 'recjson', NOW(), %s)",
Expand Down
Loading