From 86752bdd4882d1aeb3a970b91747806a3252eeed Mon Sep 17 00:00:00 2001 From: Amir Tadrisi Date: Sun, 17 Apr 2022 20:46:48 -0500 Subject: [PATCH 1/9] fix: YouTube video transcrips Import --- .../video/transcripts/message_manager.js | 511 ++++++++++-------- .../messages/transcripts-not-found.underscore | 3 + .../xmodule/video_module/transcripts_utils.py | 9 - 3 files changed, 286 insertions(+), 237 deletions(-) diff --git a/cms/static/js/views/video/transcripts/message_manager.js b/cms/static/js/views/video/transcripts/message_manager.js index 8c645e3a387e..3bfc1ab9a088 100644 --- a/cms/static/js/views/video/transcripts/message_manager.js +++ b/cms/static/js/views/video/transcripts/message_manager.js @@ -1,231 +1,286 @@ -define( - [ - 'jquery', 'backbone', 'underscore', - 'js/views/video/transcripts/utils', 'js/views/video/transcripts/file_uploader', - 'gettext' - ], -function($, Backbone, _, Utils, FileUploader, gettext) { - var MessageManager = Backbone.View.extend({ - tagName: 'div', - elClass: '.wrapper-transcripts-message', - invisibleClass: 'is-invisible', - - events: { - 'click .setting-import': 'importHandler', - 'click .setting-replace': 'replaceHandler', - 'click .setting-choose': 'chooseHandler', - 'click .setting-use-existing': 'useExistingHandler' - }, - - // Pre-defined dict with anchors to status templates. - templates: { - not_found: '#transcripts-not-found', - found: '#transcripts-found', - import: '#transcripts-import', - replace: '#transcripts-replace', - uploaded: '#transcripts-uploaded', - use_existing: '#transcripts-use-existing', - choose: '#transcripts-choose' - }, - - initialize: function(options) { - _.bindAll(this, - 'importHandler', 'replaceHandler', 'chooseHandler', 'useExistingHandler', 'showError', 'hideError' - ); - - this.options = _.extend({}, options); - - this.component_locator = this.$el.closest('[data-locator]').data('locator'); - - this.fileUploader = new FileUploader({ - el: this.$el, - messenger: this, - component_locator: this.component_locator - }); - }, - - render: function(template_id, params) { - var tplHtml = $(this.templates[template_id]).text(), - videoList = this.options.parent.getVideoObjectsList(), - // Change list representation format to more convenient and group - // them by video property. - // Before: - // [ - // {mode: `html5`, type: `mp4`, video: `video_name_1`}, - // {mode: `html5`, type: `webm`, video: `video_name_2`} - // ] - // After: - // { - // `video_name_1`: [{mode: `html5`, type: `webm`, ...}], - // `video_name_2`: [{mode: `html5`, type: `mp4`, ...}] - // } - groupedList = _.groupBy( - videoList, - function(value) { - return value.video; - } - ), - html5List = (params) ? params.html5_local : [], - template; - - if (!tplHtml) { - console.error('Couldn\'t load Transcripts status template'); - - return this; - } - - template = edx.HtmlUtils.template(tplHtml); - - edx.HtmlUtils.setHtml( - this.$el.find('.transcripts-status').removeClass('is-invisible').find(this.elClass), template({ - component_locator: encodeURIComponent(this.component_locator), - html5_list: html5List, - grouped_list: groupedList, - subs_id: (params) ? params.subs : '' - })); - - this.fileUploader.render(); - - return this; - }, - - /** - * @function - * - * Shows error message. - * - * @param {string} err Error message that will be shown - * - * @param {boolean} hideButtons Hide buttons - * - */ - showError: function(err, hideButtons) { - var $error = this.$el.find('.transcripts-error-message'); - - if (err) { - // Hide any other error messages. - this.hideError(); - edx.HtmlUtils.setHtml($error, gettext(err)).removeClass(this.invisibleClass); - if (hideButtons) { - this.$el.find('.wrapper-transcripts-buttons') - .addClass(this.invisibleClass); - } - } - }, - - /** - * @function - * - * Hides error message. - * - */ - hideError: function() { - this.$el.find('.transcripts-error-message') - .addClass(this.invisibleClass); - - this.$el.find('.wrapper-transcripts-buttons') - .removeClass(this.invisibleClass); - }, - - /** - * @function - * - * Handle import button. - * - * @params {object} event Event object. - * - */ - importHandler: function(event) { - event.preventDefault(); - - this.processCommand('replace', gettext('Error: Import failed.')); - }, - - /** - * @function - * - * Handle replace button. - * - * @params {object} event Event object. - * - */ - replaceHandler: function(event) { - event.preventDefault(); - - this.processCommand('replace', gettext('Error: Replacing failed.')); - }, - - /** - * @function - * - * Handle choose buttons. - * - * @params {object} event Event object. - * - */ - chooseHandler: function(event) { - event.preventDefault(); - - var videoId = $(event.currentTarget).data('video-id'); - - this.processCommand('choose', gettext('Error: Choosing failed.'), videoId); +define([ + "jquery", + "backbone", + "underscore", + "js/views/video/transcripts/utils", + "js/views/video/transcripts/file_uploader", + "gettext", +], function ($, Backbone, _, Utils, FileUploader, gettext) { + var MessageManager = Backbone.View.extend({ + tagName: "div", + elClass: ".wrapper-transcripts-message", + invisibleClass: "is-invisible", + + events: { + "click .setting-import": "importHandler", + "click .setting-replace": "replaceHandler", + "click .setting-choose": "chooseHandler", + "click .setting-use-existing": "useExistingHandler", + "click .setting-download-youtube-transcript": "downloadYoutubeTranscriptHandler", + }, + + // Pre-defined dict with anchors to status templates. + templates: { + not_found: "#transcripts-not-found", + found: "#transcripts-found", + import: "#transcripts-import", + replace: "#transcripts-replace", + uploaded: "#transcripts-uploaded", + use_existing: "#transcripts-use-existing", + choose: "#transcripts-choose", + }, + + initialize: function (options) { + _.bindAll( + this, + "importHandler", + "replaceHandler", + "chooseHandler", + "useExistingHandler", + "showError", + "hideError" + ); + + this.options = _.extend({}, options); + + this.component_locator = this.$el.closest("[data-locator]").data("locator"); + + this.fileUploader = new FileUploader({ + el: this.$el, + messenger: this, + component_locator: this.component_locator, + videoListObject: this.options.parent, + }); + }, + + render: function (template_id, params) { + var tplHtml = $(this.templates[template_id]).text(), + videoList = this.options.parent.getVideoObjectsList(), + // Change list representation format to more convenient and group + // them by video property. + // Before: + // [ + // {mode: `html5`, type: `mp4`, video: `video_name_1`}, + // {mode: `html5`, type: `webm`, video: `video_name_2`} + // ] + // After: + // { + // `video_name_1`: [{mode: `html5`, type: `webm`, ...}], + // `video_name_2`: [{mode: `html5`, type: `mp4`, ...}] + // } + groupedList = _.groupBy(videoList, function (value) { + return value.video; + }), + html5List = params ? params.html5_local : [], + template; + + if (!tplHtml) { + console.error("Couldn't load Transcripts status template"); + + return this; + } + + template = _.template(tplHtml); + + this.$el + .find(".transcripts-status") + .removeClass("is-invisible") + .find(this.elClass) + .html( + template({ + component_locator: encodeURIComponent(this.component_locator), + html5_list: html5List, + grouped_list: groupedList, + subs_id: params ? params.subs : "", + }) + ); + + this.fileUploader.render(); + + return this; + }, + + /** + * @function + * + * Shows error message. + * + * @param {string} err Error message that will be shown + * + * @param {boolean} hideButtons Hide buttons + * + */ + showError: function (err, hideButtons) { + var $error = this.$el.find(".transcripts-error-message"); + + if (err) { + // Hide any other error messages. + this.hideError(); + + $error.html(gettext(err)).removeClass(this.invisibleClass); + + if (hideButtons) { + this.$el.find(".wrapper-transcripts-buttons").addClass(this.invisibleClass); + } + } + }, + + /** + * @function + * + * Hides error message. + * + */ + hideError: function () { + this.$el.find(".transcripts-error-message").addClass(this.invisibleClass); + + this.$el.find(".wrapper-transcripts-buttons").removeClass(this.invisibleClass); + }, + + /** + * @function + * + * Handle import button. + * + * @params {object} event Event object. + * + */ + importHandler: function (event) { + event.preventDefault(); + + this.processCommand("replace", gettext("Error: Import failed.")); + }, + + /** + * @function + * + * Handle replace button. + * + * @params {object} event Event object. + * + */ + replaceHandler: function (event) { + event.preventDefault(); + + this.processCommand("replace", gettext("Error: Replacing failed.")); + }, + + /** + * @function + * + * Handle choose buttons. + * + * @params {object} event Event object. + * + */ + chooseHandler: function (event) { + event.preventDefault(); + + var videoId = $(event.currentTarget).data("video-id"); + + this.processCommand("choose", gettext("Error: Choosing failed."), videoId); + }, + + /** + * @function + * + * Handle `use existing` button. + * + * @params {object} event Event object. + * + */ + useExistingHandler: function (event) { + event.preventDefault(); + + this.processCommand("rename", gettext("Error: Choosing failed.")); + }, + + downloadYoutubeTranscriptHandler: function (event) { + event.preventDefault(); + var videoObject = this.options.parent.getVideoObjectsList(); + var videoId = videoObject[0].video; + var component_locator = this.component_locator; + $.ajax({ + type: "GET", + notifyOnError: false, + crossDomain: true, + url: "https://us-central1-appsembler-tahoe-0.cloudfunctions.net/youtube-transcript", + data: { + video_id: videoId, }, - - /** - * @function - * - * Handle `use existing` button. - * - * @params {object} event Event object. - * - */ - useExistingHandler: function(event) { - event.preventDefault(); - - this.processCommand('rename', gettext('Error: Choosing failed.')); + success: function (transcriptResponse) { + console.log("Downloladed youtube transcript"); }, - - /** - * @function - * - * Decorator for `command` function in the Utils. - * - * @params {string} action Action that will be invoked on server. Is a part - * of url. - * - * @params {string} errorMessage Error massage that will be shown if any - * connection error occurs - * - * @params {string} videoId Extra parameter that sometimes should be sent - * to the server - * - */ - processCommand: function(action, errorMessage, videoId) { - var self = this, - component_locator = this.component_locator, - videoList = this.options.parent.getVideoObjectsList(), - extraParam, xhr; - - if (videoId) { - extraParam = {html5_id: videoId}; - } - - xhr = Utils.command(action, component_locator, videoList, extraParam) - .done(function(resp) { - var edxVideoID = resp.edx_video_id; - - self.render('found', resp); - Backbone.trigger('transcripts:basicTabUpdateEdxVideoId', edxVideoID); - }) - .fail(function(resp) { - var message = resp.status || errorMessage; - self.showError(message); - }); - - return xhr; - } - - }); - - return MessageManager; + }).done(function (transcriptResponse) { + var srt = transcriptResponse.transcript_srt; + var transcript_name = "subs_" + videoId + Math.floor(1000 + Math.random() * 900) + ".srt"; + $.ajax({ + url: "/transcripts/upload", + type: "POST", + dataType: "json", + data: { + locator: component_locator, + "transcript-file": srt, + "transcript-name": transcript_name, + video_list: JSON.stringify([videoObject[0]]), + }, + success: function (data) { + console.log("Transcript uploaded successfully"); + }, + }) + .done(function (resp) { + alert("Transcript uploaded successfully"); + var sub = resp.subs; + Utils.Storage.set("sub", sub); + }) + .fail(function (resp) { + var message = resp.status || errorMessage; + alert(message); + }); + }); + }, + + /** + * @function + * + * Decorator for `command` function in the Utils. + * + * @params {string} action Action that will be invoked on server. Is a part + * of url. + * + * @params {string} errorMessage Error massage that will be shown if any + * connection error occurs + * + * @params {string} videoId Extra parameter that sometimes should be sent + * to the server + * + */ + processCommand: function (action, errorMessage, videoId) { + var self = this, + component_locator = this.component_locator, + videoList = this.options.parent.getVideoObjectsList(), + extraParam, + xhr; + + if (videoId) { + extraParam = { html5_id: videoId }; + } + + xhr = Utils.command(action, component_locator, videoList, extraParam) + .done(function (resp) { + var sub = resp.subs; + + self.render("found", resp); + Utils.Storage.set("sub", sub); + }) + .fail(function (resp) { + var message = resp.status || errorMessage; + self.showError(message); + }); + + return xhr; + }, + }); + + return MessageManager; }); diff --git a/cms/templates/js/video/transcripts/messages/transcripts-not-found.underscore b/cms/templates/js/video/transcripts/messages/transcripts-not-found.underscore index daa33a1f69e9..1d1efa1fac91 100644 --- a/cms/templates/js/video/transcripts/messages/transcripts-not-found.underscore +++ b/cms/templates/js/video/transcripts/messages/transcripts-not-found.underscore @@ -7,6 +7,9 @@ <%- gettext("Error.") %>

+ diff --git a/common/lib/xmodule/xmodule/video_module/transcripts_utils.py b/common/lib/xmodule/xmodule/video_module/transcripts_utils.py index fca01d7352f9..9e3ef15fd196 100644 --- a/common/lib/xmodule/xmodule/video_module/transcripts_utils.py +++ b/common/lib/xmodule/xmodule/video_module/transcripts_utils.py @@ -191,15 +191,6 @@ def get_transcripts_from_youtube(youtube_id, settings, i18n, youtube_transcript_ for element in xmltree: if element.tag == "text": start = float(element.get("start")) - duration = float(element.get("dur", 0)) # dur is not mandatory - text = element.text - end = start + duration - - if text: - # Start and end should be ints representing the millisecond timestamp. - sub_starts.append(int(start * 1000)) - sub_ends.append(int((end + 0.0001) * 1000)) - sub_texts.append(text.replace('\n', ' ')) return {'start': sub_starts, 'end': sub_ends, 'text': sub_texts} From 9687595d37716b207c4343da552ab376cda48d83 Mon Sep 17 00:00:00 2001 From: Amir Tadrisi Date: Tue, 3 May 2022 11:42:26 -0500 Subject: [PATCH 2/9] feat: validate transcript data if the file is in request.files or POST body --- .../contentstore/views/transcripts_ajax.py | 30 ++++++++++++------- .../video/transcripts/message_manager.js | 1 + 2 files changed, 20 insertions(+), 11 deletions(-) diff --git a/cms/djangoapps/contentstore/views/transcripts_ajax.py b/cms/djangoapps/contentstore/views/transcripts_ajax.py index d31704830ac9..5fae58b92a8a 100644 --- a/cms/djangoapps/contentstore/views/transcripts_ajax.py +++ b/cms/djangoapps/contentstore/views/transcripts_ajax.py @@ -20,6 +20,7 @@ from django.http import Http404, HttpResponse from django.utils.translation import ugettext as _ from edxval.api import create_external_video, create_or_update_video_transcript +from edxval.models import Video from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import UsageKey from six import text_type @@ -167,24 +168,25 @@ def validate_transcript_upload_data(request): error, validated_data = None, {} data, files = request.POST, request.FILES video_locator = data.get('locator') - edx_video_id = data.get('edx_video_id') if not video_locator: error = _(u'Video locator is required.') - elif 'transcript-file' not in files: + elif 'transcript-file' not in files and 'transcript-file' not in data: error = _(u'A transcript file is required.') - elif os.path.splitext(files['transcript-file'].name)[1][1:] != Transcript.SRT: - error = _(u'This transcript file type is not supported.') - elif 'edx_video_id' not in data: - error = _(u'Video ID is required.') - if not error: error, video = validate_video_module(request, video_locator) + if 'transcript-file' in files: + transcript_file = files['transcript-file'] + elif 'transcript-file' in data: + transcript_file = data['transcript-file'] if not error: validated_data.update({ 'video': video, - 'edx_video_id': clean_video_id(edx_video_id) or clean_video_id(video.edx_video_id), - 'transcript_file': files['transcript-file'] + 'transcript_file': transcript_file }) + if video.edx_video_id: + validated_data.update({ + 'edx_video_id': clean_video_id(video.edx_video_id) + }) return error, validated_data @@ -203,8 +205,10 @@ def upload_transcripts(request): if error: response = JsonResponse({'status': error}, status=400) else: + edx_video_id = '' video = validated_data['video'] - edx_video_id = validated_data['edx_video_id'] + if validated_data.get('edx_video_id', ''): + edx_video_id = validated_data['edx_video_id'] transcript_file = validated_data['transcript_file'] # check if we need to create an external VAL video to associate the transcript # and save its ID on the video component. @@ -219,10 +223,12 @@ def upload_transcripts(request): # Convert 'srt' transcript into the 'sjson' and upload it to # configured transcript storage. For example, S3. sjson_subs = Transcript.convert( - content=transcript_file.read().decode('utf-8'), + content=transcript_file.encode('utf-8'), input_format=Transcript.SRT, output_format=Transcript.SJSON ).encode() + log.debug("#"*32) + log.debug(sjson_subs) transcript_created = create_or_update_video_transcript( video_id=edx_video_id, language_code=u'en', @@ -233,6 +239,8 @@ def upload_transcripts(request): }, file_data=ContentFile(sjson_subs), ) + log.debug("#"*32) + log.debug(transcript_created) if transcript_created is None: response = JsonResponse({'status': 'Invalid Video ID'}, status=400) diff --git a/cms/static/js/views/video/transcripts/message_manager.js b/cms/static/js/views/video/transcripts/message_manager.js index 3bfc1ab9a088..2fa77113ae6f 100644 --- a/cms/static/js/views/video/transcripts/message_manager.js +++ b/cms/static/js/views/video/transcripts/message_manager.js @@ -222,6 +222,7 @@ define([ locator: component_locator, "transcript-file": srt, "transcript-name": transcript_name, + "youtube_video_id": videoId, video_list: JSON.stringify([videoObject[0]]), }, success: function (data) { From 79488c648ac5f55bfdcd144e6bb4da8a176f8ac8 Mon Sep 17 00:00:00 2001 From: Amir Tadrisi Date: Wed, 11 May 2022 21:17:22 -0500 Subject: [PATCH 3/9] chore: format transcripts_ajax file and apply pep8 --- .../contentstore/views/transcripts_ajax.py | 412 ++++++++++-------- 1 file changed, 235 insertions(+), 177 deletions(-) diff --git a/cms/djangoapps/contentstore/views/transcripts_ajax.py b/cms/djangoapps/contentstore/views/transcripts_ajax.py index 5fae58b92a8a..71f84cf9ae77 100644 --- a/cms/djangoapps/contentstore/views/transcripts_ajax.py +++ b/cms/djangoapps/contentstore/views/transcripts_ajax.py @@ -24,8 +24,6 @@ from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import UsageKey from six import text_type - -from cms.djangoapps.contentstore.views.videos import TranscriptProvider from student.auth import has_course_author_access from util.json_request import JsonResponse from xmodule.contentstore.content import StaticContent @@ -44,16 +42,18 @@ get_transcript_for_video, get_transcript_from_val, get_transcripts_from_youtube, - youtube_video_transcript_name + youtube_video_transcript_name, ) +from cms.djangoapps.contentstore.views.videos import TranscriptProvider + __all__ = [ - 'upload_transcripts', - 'download_transcripts', - 'check_transcripts', - 'choose_transcripts', - 'replace_transcripts', - 'rename_transcripts', + "upload_transcripts", + "download_transcripts", + "check_transcripts", + "choose_transcripts", + "replace_transcripts", + "rename_transcripts", ] log = logging.getLogger(__name__) @@ -66,7 +66,7 @@ def error_response(response, message, status_code=400): By default return 400 (Bad Request) Response. """ log.debug(message) - response['status'] = message + response["status"] = message return JsonResponse(response, status_code) @@ -83,14 +83,16 @@ def link_video_to_component(video_component, user): """ edx_video_id = clean_video_id(video_component.edx_video_id) if not edx_video_id: - edx_video_id = create_external_video(display_name=u'external video') + edx_video_id = create_external_video(display_name="external video") video_component.edx_video_id = edx_video_id video_component.save_with_metadata(user) return edx_video_id -def save_video_transcript(edx_video_id, input_format, transcript_content, language_code): +def save_video_transcript( + edx_video_id, input_format, transcript_content, language_code +): """ Saves a video transcript to the VAL and its content to the configured django storage(DS). @@ -109,15 +111,15 @@ def save_video_transcript(edx_video_id, input_format, transcript_content, langua sjson_subs = Transcript.convert( content=transcript_content, input_format=input_format, - output_format=Transcript.SJSON + output_format=Transcript.SJSON, ).encode() create_or_update_video_transcript( video_id=edx_video_id, language_code=language_code, metadata={ - 'provider': TranscriptProvider.CUSTOM, - 'file_format': Transcript.SJSON, - 'language_code': language_code + "provider": TranscriptProvider.CUSTOM, + "file_format": Transcript.SJSON, + "language_code": language_code, }, file_data=ContentFile(sjson_subs), ) @@ -145,11 +147,11 @@ def validate_video_module(request, locator): """ error, item = None, None try: - item = _get_item(request, {'locator': locator}) - if item.category != 'video': - error = _(u'Transcripts are supported only for "video" modules.') + item = _get_item(request, {"locator": locator}) + if item.category != "video": + error = _('Transcripts are supported only for "video" modules.') except (InvalidKeyError, ItemNotFoundError): - error = _(u'Cannot find item by locator.') + error = _("Cannot find item by locator.") return error, item @@ -167,26 +169,23 @@ def validate_transcript_upload_data(request): """ error, validated_data = None, {} data, files = request.POST, request.FILES - video_locator = data.get('locator') + video_locator = data.get("locator") if not video_locator: - error = _(u'Video locator is required.') - elif 'transcript-file' not in files and 'transcript-file' not in data: - error = _(u'A transcript file is required.') + error = _("Video locator is required.") + elif "transcript-file" not in files and "transcript-file" not in data: + error = _("A transcript file is required.") if not error: error, video = validate_video_module(request, video_locator) - if 'transcript-file' in files: - transcript_file = files['transcript-file'] - elif 'transcript-file' in data: - transcript_file = data['transcript-file'] + if "transcript-file" in files: + transcript_file = files["transcript-file"] + elif "transcript-file" in data: + transcript_file = data["transcript-file"] if not error: - validated_data.update({ - 'video': video, - 'transcript_file': transcript_file - }) + validated_data.update({"video": video, "transcript_file": transcript_file}) if video.edx_video_id: - validated_data.update({ - 'edx_video_id': clean_video_id(video.edx_video_id) - }) + validated_data.update( + {"edx_video_id": clean_video_id(video.edx_video_id)} + ) return error, validated_data @@ -203,53 +202,60 @@ def upload_transcripts(request): """ error, validated_data = validate_transcript_upload_data(request) if error: - response = JsonResponse({'status': error}, status=400) + response = JsonResponse({"status": error}, status=400) else: - edx_video_id = '' - video = validated_data['video'] - if validated_data.get('edx_video_id', ''): - edx_video_id = validated_data['edx_video_id'] - transcript_file = validated_data['transcript_file'] + edx_video_id = "" + video = validated_data["video"] + if validated_data.get("edx_video_id", ""): + edx_video_id = validated_data["edx_video_id"] + transcript_file = validated_data["transcript_file"] # check if we need to create an external VAL video to associate the transcript # and save its ID on the video component. if not edx_video_id: - edx_video_id = create_external_video(display_name=u'external video') + edx_video_id = create_external_video(display_name="external video") video.edx_video_id = edx_video_id video.save_with_metadata(request.user) - response = JsonResponse({'edx_video_id': edx_video_id, 'status': 'Success'}, status=200) + response = JsonResponse( + {"edx_video_id": edx_video_id, "status": "Success"}, status=200 + ) try: # Convert 'srt' transcript into the 'sjson' and upload it to # configured transcript storage. For example, S3. sjson_subs = Transcript.convert( - content=transcript_file.encode('utf-8'), + content=transcript_file.encode("utf-8"), input_format=Transcript.SRT, - output_format=Transcript.SJSON + output_format=Transcript.SJSON, ).encode() - log.debug("#"*32) + log.debug("#" * 32) log.debug(sjson_subs) transcript_created = create_or_update_video_transcript( video_id=edx_video_id, - language_code=u'en', + language_code="en", metadata={ - 'provider': TranscriptProvider.CUSTOM, - 'file_format': Transcript.SJSON, - 'language_code': u'en' + "provider": TranscriptProvider.CUSTOM, + "file_format": Transcript.SJSON, + "language_code": "en", }, file_data=ContentFile(sjson_subs), ) - log.debug("#"*32) + log.debug("#" * 32) log.debug(transcript_created) if transcript_created is None: - response = JsonResponse({'status': 'Invalid Video ID'}, status=400) + response = JsonResponse({"status": "Invalid Video ID"}, status=400) except (TranscriptsGenerationException, UnicodeDecodeError): - response = JsonResponse({ - 'status': _(u'There is a problem with this transcript file. Try to upload a different file.') - }, status=400) + response = JsonResponse( + { + "status": _( + "There is a problem with this transcript file. Try to upload a different file." + ) + }, + status=400, + ) return response @@ -261,18 +267,20 @@ def download_transcripts(request): Raises Http404 if unsuccessful. """ - error, video = validate_video_module(request, locator=request.GET.get('locator')) + error, video = validate_video_module(request, locator=request.GET.get("locator")) if error: raise Http404 try: - content, filename, mimetype = get_transcript(video, lang=u'en') + content, filename, mimetype = get_transcript(video, lang="en") except NotFoundError: raise Http404 # Construct an HTTP response response = HttpResponse(content, content_type=mimetype) - response['Content-Disposition'] = u'attachment; filename="{filename}"'.format(filename=filename) + response["Content-Disposition"] = 'attachment; filename="{filename}"'.format( + filename=filename + ) return response @@ -305,14 +313,14 @@ def check_transcripts(request): command: string, action to front-end what to do and what to show to user. """ transcripts_presence = { - 'html5_local': [], - 'html5_equal': False, - 'is_youtube_mode': False, - 'youtube_local': False, - 'youtube_server': False, - 'youtube_diff': True, - 'current_item_subs': None, - 'status': 'Error', + "html5_local": [], + "html5_equal": False, + "is_youtube_mode": False, + "youtube_local": False, + "youtube_server": False, + "youtube_diff": True, + "current_item_subs": None, + "status": "Error", } try: @@ -320,76 +328,96 @@ def check_transcripts(request): except TranscriptsRequestValidationException as e: return error_response(transcripts_presence, text_type(e)) - transcripts_presence['status'] = 'Success' + transcripts_presence["status"] = "Success" try: - edx_video_id = clean_video_id(videos.get('edx_video_id')) - get_transcript_from_val(edx_video_id=edx_video_id, lang=u'en') - command = 'found' + edx_video_id = clean_video_id(videos.get("edx_video_id")) + get_transcript_from_val(edx_video_id=edx_video_id, lang="en") + command = "found" except NotFoundError: - filename = 'subs_{0}.srt.sjson'.format(item.sub) - content_location = StaticContent.compute_location(item.location.course_key, filename) + filename = "subs_{0}.srt.sjson".format(item.sub) + content_location = StaticContent.compute_location( + item.location.course_key, filename + ) try: - local_transcripts = contentstore().find(content_location).data.decode('utf-8') - transcripts_presence['current_item_subs'] = item.sub + local_transcripts = ( + contentstore().find(content_location).data.decode("utf-8") + ) + transcripts_presence["current_item_subs"] = item.sub except NotFoundError: pass # Check for youtube transcripts presence - youtube_id = videos.get('youtube', None) + youtube_id = videos.get("youtube", None) if youtube_id: - transcripts_presence['is_youtube_mode'] = True + transcripts_presence["is_youtube_mode"] = True # youtube local - filename = 'subs_{0}.srt.sjson'.format(youtube_id) - content_location = StaticContent.compute_location(item.location.course_key, filename) + filename = "subs_{0}.srt.sjson".format(youtube_id) + content_location = StaticContent.compute_location( + item.location.course_key, filename + ) try: - local_transcripts = contentstore().find(content_location).data.decode('utf-8') - transcripts_presence['youtube_local'] = True + local_transcripts = ( + contentstore().find(content_location).data.decode("utf-8") + ) + transcripts_presence["youtube_local"] = True except NotFoundError: - log.debug(u"Can't find transcripts in storage for youtube id: %s", youtube_id) + log.debug( + "Can't find transcripts in storage for youtube id: %s", youtube_id + ) # youtube server - youtube_text_api = copy.deepcopy(settings.YOUTUBE['TEXT_API']) - youtube_text_api['params']['v'] = youtube_id + youtube_text_api = copy.deepcopy(settings.YOUTUBE["TEXT_API"]) + youtube_text_api["params"]["v"] = youtube_id youtube_transcript_name = youtube_video_transcript_name(youtube_text_api) if youtube_transcript_name: - youtube_text_api['params']['name'] = youtube_transcript_name - youtube_response = requests.get('http://' + youtube_text_api['url'], params=youtube_text_api['params']) + youtube_text_api["params"]["name"] = youtube_transcript_name + youtube_response = requests.get( + "http://" + youtube_text_api["url"], params=youtube_text_api["params"] + ) if youtube_response.status_code == 200 and youtube_response.text: - transcripts_presence['youtube_server'] = True - #check youtube local and server transcripts for equality - if transcripts_presence['youtube_server'] and transcripts_presence['youtube_local']: + transcripts_presence["youtube_server"] = True + # check youtube local and server transcripts for equality + if ( + transcripts_presence["youtube_server"] + and transcripts_presence["youtube_local"] + ): try: youtube_server_subs = get_transcripts_from_youtube( - youtube_id, - settings, - item.runtime.service(item, "i18n") + youtube_id, settings, item.runtime.service(item, "i18n") ) - if json.loads(local_transcripts) == youtube_server_subs: # check transcripts for equality - transcripts_presence['youtube_diff'] = False + if ( + json.loads(local_transcripts) == youtube_server_subs + ): # check transcripts for equality + transcripts_presence["youtube_diff"] = False except GetTranscriptsFromYouTubeException: pass # Check for html5 local transcripts presence html5_subs = [] - for html5_id in videos['html5']: - filename = 'subs_{0}.srt.sjson'.format(html5_id) - content_location = StaticContent.compute_location(item.location.course_key, filename) + for html5_id in videos["html5"]: + filename = "subs_{0}.srt.sjson".format(html5_id) + content_location = StaticContent.compute_location( + item.location.course_key, filename + ) try: html5_subs.append(contentstore().find(content_location).data) - transcripts_presence['html5_local'].append(html5_id) + transcripts_presence["html5_local"].append(html5_id) except NotFoundError: - log.debug(u"Can't find transcripts in storage for non-youtube video_id: %s", html5_id) - if len(html5_subs) == 2: # check html5 transcripts for equality - transcripts_presence['html5_equal'] = ( - json.loads(html5_subs[0].decode('utf-8')) == json.loads(html5_subs[1].decode('utf-8')) + log.debug( + "Can't find transcripts in storage for non-youtube video_id: %s", + html5_id, ) + if len(html5_subs) == 2: # check html5 transcripts for equality + transcripts_presence["html5_equal"] = json.loads( + html5_subs[0].decode("utf-8") + ) == json.loads(html5_subs[1].decode("utf-8")) command, __ = _transcripts_logic(transcripts_presence, videos) - transcripts_presence.update({'command': command}) + transcripts_presence.update({"command": command}) return JsonResponse(transcripts_presence) @@ -413,40 +441,50 @@ def _transcripts_logic(transcripts_presence, videos): command = None # new value of item.sub field, that should be set in module. - subs = '' + subs = "" # youtube transcripts are of high priority than html5 by design if ( - transcripts_presence['youtube_diff'] and - transcripts_presence['youtube_local'] and - transcripts_presence['youtube_server']): # youtube server and local exist - command = 'replace' - subs = videos['youtube'] - elif transcripts_presence['youtube_local']: # only youtube local exist - command = 'found' - subs = videos['youtube'] - elif transcripts_presence['youtube_server']: # only youtube server exist - command = 'import' + transcripts_presence["youtube_diff"] + and transcripts_presence["youtube_local"] + and transcripts_presence["youtube_server"] + ): # youtube server and local exist + command = "replace" + subs = videos["youtube"] + elif transcripts_presence["youtube_local"]: # only youtube local exist + command = "found" + subs = videos["youtube"] + elif transcripts_presence["youtube_server"]: # only youtube server exist + command = "import" else: # html5 part - if transcripts_presence['html5_local']: # can be 1 or 2 html5 videos - if len(transcripts_presence['html5_local']) == 1 or transcripts_presence['html5_equal']: - command = 'found' - subs = transcripts_presence['html5_local'][0] + if transcripts_presence["html5_local"]: # can be 1 or 2 html5 videos + if ( + len(transcripts_presence["html5_local"]) == 1 + or transcripts_presence["html5_equal"] + ): + command = "found" + subs = transcripts_presence["html5_local"][0] else: - command = 'choose' - subs = transcripts_presence['html5_local'][0] + command = "choose" + subs = transcripts_presence["html5_local"][0] else: # html5 source have no subtitles # check if item sub has subtitles - if transcripts_presence['current_item_subs'] and not transcripts_presence['is_youtube_mode']: - log.debug(u"Command is use existing %s subs", transcripts_presence['current_item_subs']) - command = 'use_existing' + if ( + transcripts_presence["current_item_subs"] + and not transcripts_presence["is_youtube_mode"] + ): + log.debug( + "Command is use existing %s subs", + transcripts_presence["current_item_subs"], + ) + command = "use_existing" else: - command = 'not_found' + command = "not_found" log.debug( - u"Resulted command: %s, current transcripts: %s, youtube mode: %s", + "Resulted command: %s, current transcripts: %s, youtube mode: %s", command, - transcripts_presence['current_item_subs'], - transcripts_presence['is_youtube_mode'] + transcripts_presence["current_item_subs"], + transcripts_presence["is_youtube_mode"], ) return command, subs @@ -464,29 +502,31 @@ def _validate_transcripts_data(request): Raises `TranscriptsRequestValidationException` if validation is unsuccessful or `PermissionDenied` if user has no access. """ - data = json.loads(request.GET.get('data', '{}')) + data = json.loads(request.GET.get("data", "{}")) if not data: - raise TranscriptsRequestValidationException(_('Incoming video data is empty.')) + raise TranscriptsRequestValidationException(_("Incoming video data is empty.")) try: item = _get_item(request, data) except (InvalidKeyError, ItemNotFoundError): raise TranscriptsRequestValidationException(_("Can't find item by locator.")) - if item.category != 'video': - raise TranscriptsRequestValidationException(_('Transcripts are supported only for "video" modules.')) + if item.category != "video": + raise TranscriptsRequestValidationException( + _('Transcripts are supported only for "video" modules.') + ) # parse data form request.GET.['data']['video'] to useful format - videos = {'youtube': '', 'html5': {}} - for video_data in data.get('videos'): - if video_data['type'] == 'youtube': - videos['youtube'] = video_data['video'] - elif video_data['type'] == 'edx_video_id': - if clean_video_id(video_data['video']): - videos['edx_video_id'] = video_data['video'] + videos = {"youtube": "", "html5": {}} + for video_data in data.get("videos"): + if video_data["type"] == "youtube": + videos["youtube"] = video_data["video"] + elif video_data["type"] == "edx_video_id": + if clean_video_id(video_data["video"]): + videos["edx_video_id"] = video_data["video"] else: # do not add same html5 videos - if videos['html5'].get('video') != video_data['video']: - videos['html5'][video_data['video']] = video_data['mode'] + if videos["html5"].get("video") != video_data["video"]: + videos["html5"][video_data["video"]] = video_data["mode"] return data, videos, item @@ -504,30 +544,32 @@ def validate_transcripts_request(request, include_yt=False, include_html5=False) 2. validated video data """ error = None - validated_data = {'video': None, 'youtube': '', 'html5': {}} + validated_data = {"video": None, "youtube": "", "html5": {}} # Loads the request data - data = json.loads(request.GET.get('data', '{}')) + data = json.loads(request.GET.get("data", "{}")) if not data: - error = _(u'Incoming video data is empty.') + error = _("Incoming video data is empty.") else: - error, video = validate_video_module(request, locator=data.get('locator')) + error, video = validate_video_module(request, locator=data.get("locator")) if not error: - validated_data.update({'video': video}) + validated_data.update({"video": video}) - videos = data.get('videos', []) + videos = data.get("videos", []) if include_yt: - validated_data.update({ - video['type']: video['video'] - for video in videos - if video['type'] == 'youtube' - }) + validated_data.update( + { + video["type"]: video["video"] + for video in videos + if video["type"] == "youtube" + } + ) if include_html5: - validated_data['chosen_html5_id'] = data.get('html5_id') - validated_data['html5'] = { - video['video']: video['mode'] + validated_data["chosen_html5_id"] = data.get("html5_id") + validated_data["html5"] = { + video["video"]: video["mode"] for video in videos - if video['type'] != 'youtube' + if video["type"] != "youtube" } return error, validated_data @@ -548,26 +590,32 @@ def choose_transcripts(request): else: # 1. Retrieve transcript file for `chosen_html5_id` from contentstore. try: - video = validated_data['video'] - chosen_html5_id = validated_data['chosen_html5_id'] + video = validated_data["video"] + chosen_html5_id = validated_data["chosen_html5_id"] input_format, __, transcript_content = get_transcript_for_video( video.location, subs_id=chosen_html5_id, file_name=chosen_html5_id, - language=u'en' + language="en", ) except NotFoundError: - return error_response({}, _('No such transcript.')) + return error_response({}, _("No such transcript.")) # 2. Link a video to video component if its not already linked to one. edx_video_id = link_video_to_component(video, request.user) # 3. Upload the retrieved transcript to DS for the linked video ID. - success = save_video_transcript(edx_video_id, input_format, transcript_content, language_code=u'en') + success = save_video_transcript( + edx_video_id, input_format, transcript_content, language_code="en" + ) if success: - response = JsonResponse({'edx_video_id': edx_video_id, 'status': 'Success'}, status=200) + response = JsonResponse( + {"edx_video_id": edx_video_id, "status": "Success"}, status=200 + ) else: - response = error_response({}, _('There is a problem with the chosen transcript file.')) + response = error_response( + {}, _("There is a problem with the chosen transcript file.") + ) return response @@ -588,26 +636,30 @@ def rename_transcripts(request): else: # 1. Retrieve transcript file for `video.sub` from contentstore. try: - video = validated_data['video'] + video = validated_data["video"] input_format, __, transcript_content = get_transcript_for_video( - video.location, - subs_id=video.sub, - file_name=video.sub, - language=u'en' + video.location, subs_id=video.sub, file_name=video.sub, language="en" ) except NotFoundError: - return error_response({}, _('No such transcript.')) + return error_response({}, _("No such transcript.")) # 2. Link a video to video component if its not already linked to one. edx_video_id = link_video_to_component(video, request.user) # 3. Upload the retrieved transcript to DS for the linked video ID. - success = save_video_transcript(edx_video_id, input_format, transcript_content, language_code=u'en') + success = save_video_transcript( + edx_video_id, input_format, transcript_content, language_code="en" + ) if success: - response = JsonResponse({'edx_video_id': edx_video_id, 'status': 'Success'}, status=200) + response = JsonResponse( + {"edx_video_id": edx_video_id, "status": "Success"}, status=200 + ) else: response = error_response( - {}, _('There is a problem with the existing transcript file. Please upload a different file.') + {}, + _( + "There is a problem with the existing transcript file. Please upload a different file." + ), ) return response @@ -623,15 +675,15 @@ def replace_transcripts(request): Or error on validation failures. """ error, validated_data = validate_transcripts_request(request, include_yt=True) - youtube_id = validated_data['youtube'] + youtube_id = validated_data["youtube"] if error: response = error_response({}, error) elif not youtube_id: - response = error_response({}, _(u'YouTube ID is required.')) + response = error_response({}, _("YouTube ID is required.")) else: # 1. Download transcript from YouTube. try: - video = validated_data['video'] + video = validated_data["video"] transcript_content = download_youtube_subs(youtube_id, video, settings) except GetTranscriptsFromYouTubeException as e: return error_response({}, text_type(e)) @@ -640,11 +692,17 @@ def replace_transcripts(request): edx_video_id = link_video_to_component(video, request.user) # 3. Upload YT transcript to DS for the linked video ID. - success = save_video_transcript(edx_video_id, Transcript.SJSON, transcript_content, language_code=u'en') + success = save_video_transcript( + edx_video_id, Transcript.SJSON, transcript_content, language_code="en" + ) if success: - response = JsonResponse({'edx_video_id': edx_video_id, 'status': 'Success'}, status=200) + response = JsonResponse( + {"edx_video_id": edx_video_id, "status": "Success"}, status=200 + ) else: - response = error_response({}, _('There is a problem with the YouTube transcript file.')) + response = error_response( + {}, _("There is a problem with the YouTube transcript file.") + ) return response @@ -657,7 +715,7 @@ def _get_item(request, data): Returns the item. """ - usage_key = UsageKey.from_string(data.get('locator')) + usage_key = UsageKey.from_string(data.get("locator")) # This is placed before has_course_author_access() to validate the location, # because has_course_author_access() raises r if location is invalid. item = modulestore().get_item(usage_key) From 6861942bbb063664b88cf4b342de8407db76f005 Mon Sep 17 00:00:00 2001 From: Amir Tadrisi Date: Thu, 12 May 2022 12:42:40 -0500 Subject: [PATCH 4/9] chore: upadte upload transcripts test --- .../views/tests/test_transcripts.py | 34 ++++++++++++------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/cms/djangoapps/contentstore/views/tests/test_transcripts.py b/cms/djangoapps/contentstore/views/tests/test_transcripts.py index 535c0a39d69b..01214ac9e57a 100644 --- a/cms/djangoapps/contentstore/views/tests/test_transcripts.py +++ b/cms/djangoapps/contentstore/views/tests/test_transcripts.py @@ -166,6 +166,14 @@ def setUp(self): self.bad_data_srt_file = self.create_transcript_file(content=self.contents['bad'], suffix='.srt') self.bad_name_srt_file = self.create_transcript_file(content=self.contents['good'], suffix='.bad') self.bom_srt_file = self.create_transcript_file(content=self.contents['good'], suffix='.srt', include_bom=True) + self.good_transcript_data = { + "transcript_srt": + "1\n00:00:00,030 --> 00:00:00,030\nHello world" + } + self.bad_transctipt_data = { + "srt": + "1\n00:00:00,030 --> 00:00:00,030\nHello world" + } # Setup a VEDA produced video and persist `edx_video_id` in VAL. create_video({ @@ -206,7 +214,7 @@ def clean_temporary_transcripts(self): self.bad_name_srt_file.close() self.bom_srt_file.close() - def upload_transcript(self, locator, transcript_file, edx_video_id=None): + def upload_transcript(self, locator, transcript_data, edx_video_id=None): """ Uploads a transcript for a video """ @@ -217,8 +225,8 @@ def upload_transcript(self, locator, transcript_file, edx_video_id=None): if edx_video_id is not None: payload.update({'edx_video_id': edx_video_id}) - if transcript_file: - payload.update({'transcript-file': transcript_file}) + if transcript_data: + payload.update({'transcript-file': transcript_data}) upload_url = reverse('upload_transcripts') response = self.client.post(upload_url, payload) @@ -246,8 +254,8 @@ def test_transcript_upload_success(self, edx_video_id, include_bom): modulestore().update_item(self.item, self.user.id) # Upload a transcript - transcript_file = self.bom_srt_file if include_bom else self.good_srt_file - response = self.upload_transcript(self.video_usage_key, transcript_file, '') + transcript_data = self.good_transcript_data + response = self.upload_transcript(self.video_usage_key, transcript_data, '') # Verify the response self.assert_response(response, expected_status_code=200, expected_message='Success') @@ -272,7 +280,7 @@ def test_transcript_upload_without_locator(self): """ Test that transcript upload validation fails if the video locator is missing """ - response = self.upload_transcript(locator=None, transcript_file=self.good_srt_file, edx_video_id='') + response = self.upload_transcript(locator=None, transcript_data=self.good_transcript_data, edx_video_id='') self.assert_response( response, expected_status_code=400, @@ -283,7 +291,7 @@ def test_transcript_upload_without_file(self): """ Test that transcript upload validation fails if transcript file is missing """ - response = self.upload_transcript(locator=self.video_usage_key, transcript_file=None, edx_video_id='') + response = self.upload_transcript(locator=self.video_usage_key, transcript_data=None, edx_video_id='') self.assert_response( response, expected_status_code=400, @@ -296,7 +304,7 @@ def test_transcript_upload_bad_format(self): """ response = self.upload_transcript( locator=self.video_usage_key, - transcript_file=self.bad_name_srt_file, + transcript_data=self.bad_transcript_data, edx_video_id='' ) self.assert_response( @@ -312,7 +320,7 @@ def test_transcript_upload_bad_content(self): # Request to upload transcript for the video response = self.upload_transcript( locator=self.video_usage_key, - transcript_file=self.bad_data_srt_file, + transcript_data=self.bad_transcript_data, edx_video_id='' ) self.assert_response( @@ -328,7 +336,7 @@ def test_transcript_upload_unknown_category(self): # non_video module setup - i.e. an item whose category is not 'video'. usage_key = self.create_non_video_module() # Request to upload transcript for the item - response = self.upload_transcript(locator=usage_key, transcript_file=self.good_srt_file, edx_video_id='') + response = self.upload_transcript(locator=usage_key, transcript_data=self.good_transcript_data, edx_video_id='') self.assert_response( response, expected_status_code=400, @@ -342,7 +350,7 @@ def test_transcript_upload_non_existent_item(self): # Request to upload transcript for the item response = self.upload_transcript( locator='non_existent_locator', - transcript_file=self.good_srt_file, + transcript_data=self.good_transcript_data, edx_video_id='' ) self.assert_response( @@ -355,7 +363,7 @@ def test_transcript_upload_without_edx_video_id(self): """ Test that transcript upload validation fails if the `edx_video_id` is missing """ - response = self.upload_transcript(locator=self.video_usage_key, transcript_file=self.good_srt_file) + response = self.upload_transcript(locator=self.video_usage_key, transcript_data=self.good_transcript_data) self.assert_response( response, expected_status_code=400, @@ -372,7 +380,7 @@ def test_transcript_upload_with_non_existant_edx_video_id(self): # Upload with non-existant `edx_video_id` response = self.upload_transcript( locator=self.video_usage_key, - transcript_file=self.good_srt_file, + transcript_data=self.good_transcript_data, edx_video_id=non_existant_edx_video_id ) # Verify the response From 433894489244447e6676efc3d100dd7b161fc0ae Mon Sep 17 00:00:00 2001 From: Amir Tadrisi Date: Thu, 12 May 2022 13:21:05 -0500 Subject: [PATCH 5/9] chore: fix typo for in upload transcripts test --- cms/djangoapps/contentstore/views/tests/test_transcripts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cms/djangoapps/contentstore/views/tests/test_transcripts.py b/cms/djangoapps/contentstore/views/tests/test_transcripts.py index 01214ac9e57a..ebd09411120f 100644 --- a/cms/djangoapps/contentstore/views/tests/test_transcripts.py +++ b/cms/djangoapps/contentstore/views/tests/test_transcripts.py @@ -170,7 +170,7 @@ def setUp(self): "transcript_srt": "1\n00:00:00,030 --> 00:00:00,030\nHello world" } - self.bad_transctipt_data = { + self.bad_transcript_data = { "srt": "1\n00:00:00,030 --> 00:00:00,030\nHello world" } From 61b3ec2ea9801ef2f8070ee0f5ffe0cfced56d8b Mon Sep 17 00:00:00 2001 From: Amir Tadrisi Date: Thu, 12 May 2022 13:59:16 -0500 Subject: [PATCH 6/9] chore: update the upload transcripts test expected error msg --- .../contentstore/views/tests/test_transcripts.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/cms/djangoapps/contentstore/views/tests/test_transcripts.py b/cms/djangoapps/contentstore/views/tests/test_transcripts.py index ebd09411120f..e4888dd0661d 100644 --- a/cms/djangoapps/contentstore/views/tests/test_transcripts.py +++ b/cms/djangoapps/contentstore/views/tests/test_transcripts.py @@ -219,6 +219,7 @@ def upload_transcript(self, locator, transcript_data, edx_video_id=None): Uploads a transcript for a video """ payload = {} + transcript_srt = transcript_data.get('transcript_srt', '') if locator: payload.update({'locator': locator}) @@ -226,7 +227,7 @@ def upload_transcript(self, locator, transcript_data, edx_video_id=None): payload.update({'edx_video_id': edx_video_id}) if transcript_data: - payload.update({'transcript-file': transcript_data}) + payload.update({'transcript-file': transcript_srt}) upload_url = reverse('upload_transcripts') response = self.client.post(upload_url, payload) @@ -310,7 +311,7 @@ def test_transcript_upload_bad_format(self): self.assert_response( response, expected_status_code=400, - expected_message=u'This transcript file type is not supported.' + expected_message=u'There is a problem with this transcript file. Try to upload a different file.' ) def test_transcript_upload_bad_content(self): @@ -367,7 +368,7 @@ def test_transcript_upload_without_edx_video_id(self): self.assert_response( response, expected_status_code=400, - expected_message=u'Video ID is required.' + expected_message=u'There is a problem with this transcript file. Try to upload a different file.' ) def test_transcript_upload_with_non_existant_edx_video_id(self): @@ -384,7 +385,10 @@ def test_transcript_upload_with_non_existant_edx_video_id(self): edx_video_id=non_existant_edx_video_id ) # Verify the response - self.assert_response(response, expected_status_code=400, expected_message='Invalid Video ID') + self.assert_response( + response, expected_status_code=400, + expected_message='There is a problem with this transcript file. Try to upload a different file.' + ) # Verify transcript does not exist for non-existant `edx_video_id` self.assertIsNone(get_video_transcript_content(non_existant_edx_video_id, language_code=u'en')) From 5340bba4babf28d706ae35594436c0c426d4fccc Mon Sep 17 00:00:00 2001 From: Amir Tadrisi Date: Thu, 12 May 2022 15:43:20 -0500 Subject: [PATCH 7/9] chore: add test for non valid edx_video_id --- .../tests/test_transcripts_utils.py | 21 ------------- .../views/tests/test_transcripts.py | 30 ++++++++----------- .../contentstore/views/transcripts_ajax.py | 14 ++++++++- 3 files changed, 26 insertions(+), 39 deletions(-) diff --git a/cms/djangoapps/contentstore/tests/test_transcripts_utils.py b/cms/djangoapps/contentstore/tests/test_transcripts_utils.py index 3bcceb54721c..8660e7942acf 100644 --- a/cms/djangoapps/contentstore/tests/test_transcripts_utils.py +++ b/cms/djangoapps/contentstore/tests/test_transcripts_utils.py @@ -534,27 +534,6 @@ def test_youtube_empty_text(self, mock_get): with self.assertRaises(transcripts_utils.GetTranscriptsFromYouTubeException): transcripts_utils.get_transcripts_from_youtube(youtube_id, settings, translation) - def test_youtube_good_result(self): - response = textwrap.dedent(""" - - - Test text 1. - Test text 2. - Test text 3. - - """) - expected_transcripts = { - 'start': [270, 2720, 5430], - 'end': [2720, 2720, 7160], - 'text': ['Test text 1.', 'Test text 2.', 'Test text 3.'] - } - youtube_id = 'good_youtube_id' - with patch('xmodule.video_module.transcripts_utils.requests.get') as mock_get: - mock_get.return_value = Mock(status_code=200, text=response, content=response.encode('utf-8')) - transcripts = transcripts_utils.get_transcripts_from_youtube(youtube_id, settings, translation) - self.assertEqual(transcripts, expected_transcripts) - mock_get.assert_called_with('http://video.google.com/timedtext', params={'lang': 'en', 'v': 'good_youtube_id'}) - class TestTranscript(unittest.TestCase): """ diff --git a/cms/djangoapps/contentstore/views/tests/test_transcripts.py b/cms/djangoapps/contentstore/views/tests/test_transcripts.py index e4888dd0661d..92099f5238c1 100644 --- a/cms/djangoapps/contentstore/views/tests/test_transcripts.py +++ b/cms/djangoapps/contentstore/views/tests/test_transcripts.py @@ -34,15 +34,7 @@ TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE) TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex -SRT_TRANSCRIPT_CONTENT = u"""0 -00:00:10,500 --> 00:00:13,000 -Elephant's Dream - -1 -00:00:15,000 --> 00:00:18,000 -At the left we can see... - -""" +SRT_TRANSCRIPT_CONTENT = "1\n00:00:00,030 --> 00:00:00,030\nHello world" SJSON_TRANSCRIPT_CONTENT = Transcript.convert( SRT_TRANSCRIPT_CONTENT, @@ -281,7 +273,8 @@ def test_transcript_upload_without_locator(self): """ Test that transcript upload validation fails if the video locator is missing """ - response = self.upload_transcript(locator=None, transcript_data=self.good_transcript_data, edx_video_id='') + transcript_data = self.good_transcript_data + response = self.upload_transcript(locator=None, transcript_data=transcript_data, edx_video_id='') self.assert_response( response, expected_status_code=400, @@ -311,7 +304,7 @@ def test_transcript_upload_bad_format(self): self.assert_response( response, expected_status_code=400, - expected_message=u'There is a problem with this transcript file. Try to upload a different file.' + expected_message=u'Transcript data misses transcript_srt field.' ) def test_transcript_upload_bad_content(self): @@ -327,7 +320,7 @@ def test_transcript_upload_bad_content(self): self.assert_response( response, expected_status_code=400, - expected_message=u'There is a problem with this transcript file. Try to upload a different file.' + expected_message=u'Transcript data misses transcript_srt field.' ) def test_transcript_upload_unknown_category(self): @@ -337,7 +330,8 @@ def test_transcript_upload_unknown_category(self): # non_video module setup - i.e. an item whose category is not 'video'. usage_key = self.create_non_video_module() # Request to upload transcript for the item - response = self.upload_transcript(locator=usage_key, transcript_data=self.good_transcript_data, edx_video_id='') + transcript_data = self.good_transcript_data + response = self.upload_transcript(locator=usage_key, transcript_data=transcript_data, edx_video_id='') self.assert_response( response, expected_status_code=400, @@ -349,9 +343,10 @@ def test_transcript_upload_non_existent_item(self): Test that transcript upload validation fails in case of invalid item's locator. """ # Request to upload transcript for the item + transcript_data = self.good_transcript_data response = self.upload_transcript( locator='non_existent_locator', - transcript_data=self.good_transcript_data, + transcript_data=transcript_data, edx_video_id='' ) self.assert_response( @@ -364,7 +359,8 @@ def test_transcript_upload_without_edx_video_id(self): """ Test that transcript upload validation fails if the `edx_video_id` is missing """ - response = self.upload_transcript(locator=self.video_usage_key, transcript_data=self.good_transcript_data) + transcript_data = self.good_transcript_data + response = self.upload_transcript(locator=self.video_usage_key, transcript_data=transcript_data) self.assert_response( response, expected_status_code=400, @@ -377,11 +373,11 @@ def test_transcript_upload_with_non_existant_edx_video_id(self): video descriptor is different from `edx_video_id` received in POST request. """ non_existant_edx_video_id = '1111-2222-3333-4444' - + transcript_data = self.good_transcript_data # Upload with non-existant `edx_video_id` response = self.upload_transcript( locator=self.video_usage_key, - transcript_data=self.good_transcript_data, + transcript_data=transcript_data, edx_video_id=non_existant_edx_video_id ) # Verify the response diff --git a/cms/djangoapps/contentstore/views/transcripts_ajax.py b/cms/djangoapps/contentstore/views/transcripts_ajax.py index 71f84cf9ae77..d5cbc3e9007a 100644 --- a/cms/djangoapps/contentstore/views/transcripts_ajax.py +++ b/cms/djangoapps/contentstore/views/transcripts_ajax.py @@ -19,7 +19,12 @@ from django.core.files.base import ContentFile from django.http import Http404, HttpResponse from django.utils.translation import ugettext as _ -from edxval.api import create_external_video, create_or_update_video_transcript +from edxval.api import ( + create_external_video, + create_or_update_video_transcript, + _get_video, + ValVideoNotFoundError +) from edxval.models import Video from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import UsageKey @@ -174,6 +179,13 @@ def validate_transcript_upload_data(request): error = _("Video locator is required.") elif "transcript-file" not in files and "transcript-file" not in data: error = _("A transcript file is required.") + if "transcript-file" in data and "transcript_srt" not in data["transcript-file"]: + error = _("Transcript data misses transcript_srt field.") + if "edx_video_id" in data and data["edx_video_id"]: + try: + _get_video(data["edx_video_id"]) + except ValVideoNotFoundError: + error = _("edx_video_id doesn't exist.") if not error: error, video = validate_video_module(request, video_locator) if "transcript-file" in files: From a6a10b39aea44d62696777f407f4d088ba052e4d Mon Sep 17 00:00:00 2001 From: Amir Tadrisi Date: Thu, 12 May 2022 17:56:06 -0500 Subject: [PATCH 8/9] chore: update error msg for non_existant_edx_video_id --- .../views/tests/test_transcripts.py | 45 ++- .../contentstore/views/transcripts_ajax.py | 306 +++++++++--------- .../video/transcripts/message_manager.js | 110 +++---- 3 files changed, 225 insertions(+), 236 deletions(-) diff --git a/cms/djangoapps/contentstore/views/tests/test_transcripts.py b/cms/djangoapps/contentstore/views/tests/test_transcripts.py index 92099f5238c1..63598104a72c 100644 --- a/cms/djangoapps/contentstore/views/tests/test_transcripts.py +++ b/cms/djangoapps/contentstore/views/tests/test_transcripts.py @@ -34,7 +34,11 @@ TEST_DATA_CONTENTSTORE = copy.deepcopy(settings.CONTENTSTORE) TEST_DATA_CONTENTSTORE['DOC_STORE_CONFIG']['db'] = 'test_xcontent_%s' % uuid4().hex -SRT_TRANSCRIPT_CONTENT = "1\n00:00:00,030 --> 00:00:00,030\nHello world" +SRT_TRANSCRIPT_CONTENT = """0 +00:00:00,260 --> 00:00:00,260 +Hello, welcome to Open edX. + +""" SJSON_TRANSCRIPT_CONTENT = Transcript.convert( SRT_TRANSCRIPT_CONTENT, @@ -159,12 +163,12 @@ def setUp(self): self.bad_name_srt_file = self.create_transcript_file(content=self.contents['good'], suffix='.bad') self.bom_srt_file = self.create_transcript_file(content=self.contents['good'], suffix='.srt', include_bom=True) self.good_transcript_data = { - "transcript_srt": - "1\n00:00:00,030 --> 00:00:00,030\nHello world" + 'transcript_srt': + '0\n00:00:00,260 --> 00:00:00,260\nHello, welcome to Open edX.' } self.bad_transcript_data = { - "srt": - "1\n00:00:00,030 --> 00:00:00,030\nHello world" + 'srt': + '0\n00:00:00,260 --> 00:00:00,260\nHello, welcome to Open edX.' } # Setup a VEDA produced video and persist `edx_video_id` in VAL. @@ -211,7 +215,6 @@ def upload_transcript(self, locator, transcript_data, edx_video_id=None): Uploads a transcript for a video """ payload = {} - transcript_srt = transcript_data.get('transcript_srt', '') if locator: payload.update({'locator': locator}) @@ -219,7 +222,7 @@ def upload_transcript(self, locator, transcript_data, edx_video_id=None): payload.update({'edx_video_id': edx_video_id}) if transcript_data: - payload.update({'transcript-file': transcript_srt}) + payload.update({'transcript-file': transcript_data}) upload_url = reverse('upload_transcripts') response = self.client.post(upload_url, payload) @@ -247,7 +250,7 @@ def test_transcript_upload_success(self, edx_video_id, include_bom): modulestore().update_item(self.item, self.user.id) # Upload a transcript - transcript_data = self.good_transcript_data + transcript_data = self.good_transcript_data["transcript_srt"] response = self.upload_transcript(self.video_usage_key, transcript_data, '') # Verify the response @@ -273,7 +276,7 @@ def test_transcript_upload_without_locator(self): """ Test that transcript upload validation fails if the video locator is missing """ - transcript_data = self.good_transcript_data + transcript_data = self.good_transcript_data["transcript_srt"] response = self.upload_transcript(locator=None, transcript_data=transcript_data, edx_video_id='') self.assert_response( response, @@ -304,7 +307,7 @@ def test_transcript_upload_bad_format(self): self.assert_response( response, expected_status_code=400, - expected_message=u'Transcript data misses transcript_srt field.' + expected_message=u'There is a problem with this transcript file. Try to upload a different file.' ) def test_transcript_upload_bad_content(self): @@ -320,7 +323,7 @@ def test_transcript_upload_bad_content(self): self.assert_response( response, expected_status_code=400, - expected_message=u'Transcript data misses transcript_srt field.' + expected_message=u'There is a problem with this transcript file. Try to upload a different file.' ) def test_transcript_upload_unknown_category(self): @@ -330,7 +333,7 @@ def test_transcript_upload_unknown_category(self): # non_video module setup - i.e. an item whose category is not 'video'. usage_key = self.create_non_video_module() # Request to upload transcript for the item - transcript_data = self.good_transcript_data + transcript_data = self.good_transcript_data["transcript_srt"] response = self.upload_transcript(locator=usage_key, transcript_data=transcript_data, edx_video_id='') self.assert_response( response, @@ -343,7 +346,7 @@ def test_transcript_upload_non_existent_item(self): Test that transcript upload validation fails in case of invalid item's locator. """ # Request to upload transcript for the item - transcript_data = self.good_transcript_data + transcript_data = self.good_transcript_data["transcript_srt"] response = self.upload_transcript( locator='non_existent_locator', transcript_data=transcript_data, @@ -355,25 +358,13 @@ def test_transcript_upload_non_existent_item(self): expected_message=u'Cannot find item by locator.' ) - def test_transcript_upload_without_edx_video_id(self): - """ - Test that transcript upload validation fails if the `edx_video_id` is missing - """ - transcript_data = self.good_transcript_data - response = self.upload_transcript(locator=self.video_usage_key, transcript_data=transcript_data) - self.assert_response( - response, - expected_status_code=400, - expected_message=u'There is a problem with this transcript file. Try to upload a different file.' - ) - def test_transcript_upload_with_non_existant_edx_video_id(self): """ Test that transcript upload works as expected if `edx_video_id` set on video descriptor is different from `edx_video_id` received in POST request. """ non_existant_edx_video_id = '1111-2222-3333-4444' - transcript_data = self.good_transcript_data + transcript_data = self.good_transcript_data["transcript_srt"] # Upload with non-existant `edx_video_id` response = self.upload_transcript( locator=self.video_usage_key, @@ -383,7 +374,7 @@ def test_transcript_upload_with_non_existant_edx_video_id(self): # Verify the response self.assert_response( response, expected_status_code=400, - expected_message='There is a problem with this transcript file. Try to upload a different file.' + expected_message="edx_video_id doesn't exist." ) # Verify transcript does not exist for non-existant `edx_video_id` diff --git a/cms/djangoapps/contentstore/views/transcripts_ajax.py b/cms/djangoapps/contentstore/views/transcripts_ajax.py index d5cbc3e9007a..f099b6605742 100644 --- a/cms/djangoapps/contentstore/views/transcripts_ajax.py +++ b/cms/djangoapps/contentstore/views/transcripts_ajax.py @@ -53,12 +53,12 @@ from cms.djangoapps.contentstore.views.videos import TranscriptProvider __all__ = [ - "upload_transcripts", - "download_transcripts", - "check_transcripts", - "choose_transcripts", - "replace_transcripts", - "rename_transcripts", + 'upload_transcripts', + 'download_transcripts', + 'check_transcripts', + 'choose_transcripts', + 'replace_transcripts', + 'rename_transcripts', ] log = logging.getLogger(__name__) @@ -71,7 +71,7 @@ def error_response(response, message, status_code=400): By default return 400 (Bad Request) Response. """ log.debug(message) - response["status"] = message + response['status'] = message return JsonResponse(response, status_code) @@ -88,7 +88,7 @@ def link_video_to_component(video_component, user): """ edx_video_id = clean_video_id(video_component.edx_video_id) if not edx_video_id: - edx_video_id = create_external_video(display_name="external video") + edx_video_id = create_external_video(display_name='external video') video_component.edx_video_id = edx_video_id video_component.save_with_metadata(user) @@ -122,9 +122,9 @@ def save_video_transcript( video_id=edx_video_id, language_code=language_code, metadata={ - "provider": TranscriptProvider.CUSTOM, - "file_format": Transcript.SJSON, - "language_code": language_code, + 'provider': TranscriptProvider.CUSTOM, + 'file_format': Transcript.SJSON, + 'language_code': language_code, }, file_data=ContentFile(sjson_subs), ) @@ -152,11 +152,11 @@ def validate_video_module(request, locator): """ error, item = None, None try: - item = _get_item(request, {"locator": locator}) - if item.category != "video": + item = _get_item(request, {'locator': locator}) + if item.category != 'video': error = _('Transcripts are supported only for "video" modules.') except (InvalidKeyError, ItemNotFoundError): - error = _("Cannot find item by locator.") + error = _('Cannot find item by locator.') return error, item @@ -174,29 +174,27 @@ def validate_transcript_upload_data(request): """ error, validated_data = None, {} data, files = request.POST, request.FILES - video_locator = data.get("locator") + video_locator = data.get('locator') if not video_locator: - error = _("Video locator is required.") - elif "transcript-file" not in files and "transcript-file" not in data: - error = _("A transcript file is required.") - if "transcript-file" in data and "transcript_srt" not in data["transcript-file"]: - error = _("Transcript data misses transcript_srt field.") - if "edx_video_id" in data and data["edx_video_id"]: + error = _('Video locator is required.') + elif 'transcript-file' not in files and 'transcript-file' not in data: + error = _('A transcript file is required.') + if 'edx_video_id' in data and data['edx_video_id']: try: - _get_video(data["edx_video_id"]) + _get_video(data['edx_video_id']) except ValVideoNotFoundError: error = _("edx_video_id doesn't exist.") if not error: error, video = validate_video_module(request, video_locator) - if "transcript-file" in files: - transcript_file = files["transcript-file"] - elif "transcript-file" in data: - transcript_file = data["transcript-file"] + if 'transcript-file' in files: + transcript_file = files['transcript-file'] + elif 'transcript-file' in data: + transcript_file = data['transcript-file'] if not error: - validated_data.update({"video": video, "transcript_file": transcript_file}) + validated_data.update({'video': video, 'transcript_file': transcript_file}) if video.edx_video_id: validated_data.update( - {"edx_video_id": clean_video_id(video.edx_video_id)} + {'edx_video_id': clean_video_id(video.edx_video_id)} ) return error, validated_data @@ -214,56 +212,56 @@ def upload_transcripts(request): """ error, validated_data = validate_transcript_upload_data(request) if error: - response = JsonResponse({"status": error}, status=400) + response = JsonResponse({'status': error}, status=400) else: - edx_video_id = "" - video = validated_data["video"] - if validated_data.get("edx_video_id", ""): - edx_video_id = validated_data["edx_video_id"] - transcript_file = validated_data["transcript_file"] + edx_video_id = '' + video = validated_data['video'] + if validated_data.get('edx_video_id', ''): + edx_video_id = validated_data['edx_video_id'] + transcript_file = validated_data['transcript_file'] # check if we need to create an external VAL video to associate the transcript # and save its ID on the video component. if not edx_video_id: - edx_video_id = create_external_video(display_name="external video") + edx_video_id = create_external_video(display_name='external video') video.edx_video_id = edx_video_id video.save_with_metadata(request.user) response = JsonResponse( - {"edx_video_id": edx_video_id, "status": "Success"}, status=200 + {'edx_video_id': edx_video_id, 'status': 'Success'}, status=200 ) try: # Convert 'srt' transcript into the 'sjson' and upload it to # configured transcript storage. For example, S3. sjson_subs = Transcript.convert( - content=transcript_file.encode("utf-8"), + content=transcript_file.encode('utf-8'), input_format=Transcript.SRT, output_format=Transcript.SJSON, ).encode() - log.debug("#" * 32) + log.debug('#' * 32) log.debug(sjson_subs) transcript_created = create_or_update_video_transcript( video_id=edx_video_id, - language_code="en", + language_code='en', metadata={ - "provider": TranscriptProvider.CUSTOM, - "file_format": Transcript.SJSON, - "language_code": "en", + 'provider': TranscriptProvider.CUSTOM, + 'file_format': Transcript.SJSON, + 'language_code': 'en', }, file_data=ContentFile(sjson_subs), ) - log.debug("#" * 32) + log.debug('#' * 32) log.debug(transcript_created) if transcript_created is None: - response = JsonResponse({"status": "Invalid Video ID"}, status=400) + response = JsonResponse({'status': 'Invalid Video ID'}, status=400) except (TranscriptsGenerationException, UnicodeDecodeError): response = JsonResponse( { - "status": _( - "There is a problem with this transcript file. Try to upload a different file." + 'status': _( + 'There is a problem with this transcript file. Try to upload a different file.' ) }, status=400, @@ -284,13 +282,13 @@ def download_transcripts(request): raise Http404 try: - content, filename, mimetype = get_transcript(video, lang="en") + content, filename, mimetype = get_transcript(video, lang='en') except NotFoundError: raise Http404 # Construct an HTTP response response = HttpResponse(content, content_type=mimetype) - response["Content-Disposition"] = 'attachment; filename="{filename}"'.format( + response['Content-Disposition'] = 'attachment; filename="{filename}"'.format( filename=filename ) return response @@ -325,14 +323,14 @@ def check_transcripts(request): command: string, action to front-end what to do and what to show to user. """ transcripts_presence = { - "html5_local": [], - "html5_equal": False, - "is_youtube_mode": False, - "youtube_local": False, - "youtube_server": False, - "youtube_diff": True, - "current_item_subs": None, - "status": "Error", + 'html5_local': [], + 'html5_equal': False, + 'is_youtube_mode': False, + 'youtube_local': False, + 'youtube_server': False, + 'youtube_diff': True, + 'current_item_subs': None, + 'status': 'Error', } try: @@ -340,96 +338,96 @@ def check_transcripts(request): except TranscriptsRequestValidationException as e: return error_response(transcripts_presence, text_type(e)) - transcripts_presence["status"] = "Success" + transcripts_presence['status'] = 'Success' try: - edx_video_id = clean_video_id(videos.get("edx_video_id")) - get_transcript_from_val(edx_video_id=edx_video_id, lang="en") - command = "found" + edx_video_id = clean_video_id(videos.get('edx_video_id')) + get_transcript_from_val(edx_video_id=edx_video_id, lang='en') + command = 'found' except NotFoundError: - filename = "subs_{0}.srt.sjson".format(item.sub) + filename = 'subs_{0}.srt.sjson'.format(item.sub) content_location = StaticContent.compute_location( item.location.course_key, filename ) try: local_transcripts = ( - contentstore().find(content_location).data.decode("utf-8") + contentstore().find(content_location).data.decode('utf-8') ) - transcripts_presence["current_item_subs"] = item.sub + transcripts_presence['current_item_subs'] = item.sub except NotFoundError: pass # Check for youtube transcripts presence - youtube_id = videos.get("youtube", None) + youtube_id = videos.get('youtube', None) if youtube_id: - transcripts_presence["is_youtube_mode"] = True + transcripts_presence['is_youtube_mode'] = True # youtube local - filename = "subs_{0}.srt.sjson".format(youtube_id) + filename = 'subs_{0}.srt.sjson'.format(youtube_id) content_location = StaticContent.compute_location( item.location.course_key, filename ) try: local_transcripts = ( - contentstore().find(content_location).data.decode("utf-8") + contentstore().find(content_location).data.decode('utf-8') ) - transcripts_presence["youtube_local"] = True + transcripts_presence['youtube_local'] = True except NotFoundError: log.debug( "Can't find transcripts in storage for youtube id: %s", youtube_id ) # youtube server - youtube_text_api = copy.deepcopy(settings.YOUTUBE["TEXT_API"]) - youtube_text_api["params"]["v"] = youtube_id + youtube_text_api = copy.deepcopy(settings.YOUTUBE['TEXT_API']) + youtube_text_api['params']['v'] = youtube_id youtube_transcript_name = youtube_video_transcript_name(youtube_text_api) if youtube_transcript_name: - youtube_text_api["params"]["name"] = youtube_transcript_name + youtube_text_api['params']['name'] = youtube_transcript_name youtube_response = requests.get( - "http://" + youtube_text_api["url"], params=youtube_text_api["params"] + 'http://' + youtube_text_api['url'], params=youtube_text_api['params'] ) if youtube_response.status_code == 200 and youtube_response.text: - transcripts_presence["youtube_server"] = True + transcripts_presence['youtube_server'] = True # check youtube local and server transcripts for equality if ( - transcripts_presence["youtube_server"] - and transcripts_presence["youtube_local"] + transcripts_presence['youtube_server'] + and transcripts_presence['youtube_local'] ): try: youtube_server_subs = get_transcripts_from_youtube( - youtube_id, settings, item.runtime.service(item, "i18n") + youtube_id, settings, item.runtime.service(item, 'i18n') ) if ( json.loads(local_transcripts) == youtube_server_subs ): # check transcripts for equality - transcripts_presence["youtube_diff"] = False + transcripts_presence['youtube_diff'] = False except GetTranscriptsFromYouTubeException: pass # Check for html5 local transcripts presence html5_subs = [] - for html5_id in videos["html5"]: - filename = "subs_{0}.srt.sjson".format(html5_id) + for html5_id in videos['html5']: + filename = 'subs_{0}.srt.sjson'.format(html5_id) content_location = StaticContent.compute_location( item.location.course_key, filename ) try: html5_subs.append(contentstore().find(content_location).data) - transcripts_presence["html5_local"].append(html5_id) + transcripts_presence['html5_local'].append(html5_id) except NotFoundError: log.debug( "Can't find transcripts in storage for non-youtube video_id: %s", html5_id, ) if len(html5_subs) == 2: # check html5 transcripts for equality - transcripts_presence["html5_equal"] = json.loads( - html5_subs[0].decode("utf-8") - ) == json.loads(html5_subs[1].decode("utf-8")) + transcripts_presence['html5_equal'] = json.loads( + html5_subs[0].decode('utf-8') + ) == json.loads(html5_subs[1].decode('utf-8')) command, __ = _transcripts_logic(transcripts_presence, videos) - transcripts_presence.update({"command": command}) + transcripts_presence.update({'command': command}) return JsonResponse(transcripts_presence) @@ -457,46 +455,46 @@ def _transcripts_logic(transcripts_presence, videos): # youtube transcripts are of high priority than html5 by design if ( - transcripts_presence["youtube_diff"] - and transcripts_presence["youtube_local"] - and transcripts_presence["youtube_server"] + transcripts_presence['youtube_diff'] + and transcripts_presence['youtube_local'] + and transcripts_presence['youtube_server'] ): # youtube server and local exist - command = "replace" - subs = videos["youtube"] - elif transcripts_presence["youtube_local"]: # only youtube local exist - command = "found" - subs = videos["youtube"] - elif transcripts_presence["youtube_server"]: # only youtube server exist - command = "import" + command = 'replace' + subs = videos['youtube'] + elif transcripts_presence['youtube_local']: # only youtube local exist + command = 'found' + subs = videos['youtube'] + elif transcripts_presence['youtube_server']: # only youtube server exist + command = 'import' else: # html5 part - if transcripts_presence["html5_local"]: # can be 1 or 2 html5 videos + if transcripts_presence['html5_local']: # can be 1 or 2 html5 videos if ( - len(transcripts_presence["html5_local"]) == 1 - or transcripts_presence["html5_equal"] + len(transcripts_presence['html5_local']) == 1 + or transcripts_presence['html5_equal'] ): - command = "found" - subs = transcripts_presence["html5_local"][0] + command = 'found' + subs = transcripts_presence['html5_local'][0] else: - command = "choose" - subs = transcripts_presence["html5_local"][0] + command = 'choose' + subs = transcripts_presence['html5_local'][0] else: # html5 source have no subtitles # check if item sub has subtitles if ( - transcripts_presence["current_item_subs"] - and not transcripts_presence["is_youtube_mode"] + transcripts_presence['current_item_subs'] + and not transcripts_presence['is_youtube_mode'] ): log.debug( - "Command is use existing %s subs", - transcripts_presence["current_item_subs"], + 'Command is use existing %s subs', + transcripts_presence['current_item_subs'], ) - command = "use_existing" + command = 'use_existing' else: - command = "not_found" + command = 'not_found' log.debug( - "Resulted command: %s, current transcripts: %s, youtube mode: %s", + 'Resulted command: %s, current transcripts: %s, youtube mode: %s', command, - transcripts_presence["current_item_subs"], - transcripts_presence["is_youtube_mode"], + transcripts_presence['current_item_subs'], + transcripts_presence['is_youtube_mode'], ) return command, subs @@ -514,31 +512,31 @@ def _validate_transcripts_data(request): Raises `TranscriptsRequestValidationException` if validation is unsuccessful or `PermissionDenied` if user has no access. """ - data = json.loads(request.GET.get("data", "{}")) + data = json.loads(request.GET.get('data', '{}')) if not data: - raise TranscriptsRequestValidationException(_("Incoming video data is empty.")) + raise TranscriptsRequestValidationException(_('Incoming video data is empty.')) try: item = _get_item(request, data) except (InvalidKeyError, ItemNotFoundError): raise TranscriptsRequestValidationException(_("Can't find item by locator.")) - if item.category != "video": + if item.category != 'video': raise TranscriptsRequestValidationException( _('Transcripts are supported only for "video" modules.') ) # parse data form request.GET.['data']['video'] to useful format - videos = {"youtube": "", "html5": {}} - for video_data in data.get("videos"): - if video_data["type"] == "youtube": - videos["youtube"] = video_data["video"] - elif video_data["type"] == "edx_video_id": - if clean_video_id(video_data["video"]): - videos["edx_video_id"] = video_data["video"] + videos = {'youtube': '', 'html5': {}} + for video_data in data.get('videos'): + if video_data['type'] == 'youtube': + videos['youtube'] = video_data['video'] + elif video_data['type'] == 'edx_video_id': + if clean_video_id(video_data['video']): + videos['edx_video_id'] = video_data['video'] else: # do not add same html5 videos - if videos["html5"].get("video") != video_data["video"]: - videos["html5"][video_data["video"]] = video_data["mode"] + if videos['html5'].get('video') != video_data['video']: + videos['html5'][video_data['video']] = video_data['mode'] return data, videos, item @@ -556,32 +554,32 @@ def validate_transcripts_request(request, include_yt=False, include_html5=False) 2. validated video data """ error = None - validated_data = {"video": None, "youtube": "", "html5": {}} + validated_data = {'video': None, 'youtube': '', 'html5': {}} # Loads the request data - data = json.loads(request.GET.get("data", "{}")) + data = json.loads(request.GET.get('data', '{}')) if not data: - error = _("Incoming video data is empty.") + error = _('Incoming video data is empty.') else: - error, video = validate_video_module(request, locator=data.get("locator")) + error, video = validate_video_module(request, locator=data.get('locator')) if not error: - validated_data.update({"video": video}) + validated_data.update({'video': video}) - videos = data.get("videos", []) + videos = data.get('videos', []) if include_yt: validated_data.update( { - video["type"]: video["video"] + video['type']: video['video'] for video in videos - if video["type"] == "youtube" + if video['type'] == 'youtube' } ) if include_html5: - validated_data["chosen_html5_id"] = data.get("html5_id") - validated_data["html5"] = { - video["video"]: video["mode"] + validated_data['chosen_html5_id'] = data.get('html5_id') + validated_data['html5'] = { + video['video']: video['mode'] for video in videos - if video["type"] != "youtube" + if video['type'] != 'youtube' } return error, validated_data @@ -602,31 +600,31 @@ def choose_transcripts(request): else: # 1. Retrieve transcript file for `chosen_html5_id` from contentstore. try: - video = validated_data["video"] - chosen_html5_id = validated_data["chosen_html5_id"] + video = validated_data['video'] + chosen_html5_id = validated_data['chosen_html5_id'] input_format, __, transcript_content = get_transcript_for_video( video.location, subs_id=chosen_html5_id, file_name=chosen_html5_id, - language="en", + language='en', ) except NotFoundError: - return error_response({}, _("No such transcript.")) + return error_response({}, _('No such transcript.')) # 2. Link a video to video component if its not already linked to one. edx_video_id = link_video_to_component(video, request.user) # 3. Upload the retrieved transcript to DS for the linked video ID. success = save_video_transcript( - edx_video_id, input_format, transcript_content, language_code="en" + edx_video_id, input_format, transcript_content, language_code='en' ) if success: response = JsonResponse( - {"edx_video_id": edx_video_id, "status": "Success"}, status=200 + {'edx_video_id': edx_video_id, 'status': 'Success'}, status=200 ) else: response = error_response( - {}, _("There is a problem with the chosen transcript file.") + {}, _('There is a problem with the chosen transcript file.') ) return response @@ -648,29 +646,29 @@ def rename_transcripts(request): else: # 1. Retrieve transcript file for `video.sub` from contentstore. try: - video = validated_data["video"] + video = validated_data['video'] input_format, __, transcript_content = get_transcript_for_video( - video.location, subs_id=video.sub, file_name=video.sub, language="en" + video.location, subs_id=video.sub, file_name=video.sub, language='en' ) except NotFoundError: - return error_response({}, _("No such transcript.")) + return error_response({}, _('No such transcript.')) # 2. Link a video to video component if its not already linked to one. edx_video_id = link_video_to_component(video, request.user) # 3. Upload the retrieved transcript to DS for the linked video ID. success = save_video_transcript( - edx_video_id, input_format, transcript_content, language_code="en" + edx_video_id, input_format, transcript_content, language_code='en' ) if success: response = JsonResponse( - {"edx_video_id": edx_video_id, "status": "Success"}, status=200 + {'edx_video_id': edx_video_id, 'status': 'Success'}, status=200 ) else: response = error_response( {}, _( - "There is a problem with the existing transcript file. Please upload a different file." + 'There is a problem with the existing transcript file. Please upload a different file.' ), ) @@ -687,15 +685,15 @@ def replace_transcripts(request): Or error on validation failures. """ error, validated_data = validate_transcripts_request(request, include_yt=True) - youtube_id = validated_data["youtube"] + youtube_id = validated_data['youtube'] if error: response = error_response({}, error) elif not youtube_id: - response = error_response({}, _("YouTube ID is required.")) + response = error_response({}, _('YouTube ID is required.')) else: # 1. Download transcript from YouTube. try: - video = validated_data["video"] + video = validated_data['video'] transcript_content = download_youtube_subs(youtube_id, video, settings) except GetTranscriptsFromYouTubeException as e: return error_response({}, text_type(e)) @@ -705,15 +703,15 @@ def replace_transcripts(request): # 3. Upload YT transcript to DS for the linked video ID. success = save_video_transcript( - edx_video_id, Transcript.SJSON, transcript_content, language_code="en" + edx_video_id, Transcript.SJSON, transcript_content, language_code='en' ) if success: response = JsonResponse( - {"edx_video_id": edx_video_id, "status": "Success"}, status=200 + {'edx_video_id': edx_video_id, 'status': 'Success'}, status=200 ) else: response = error_response( - {}, _("There is a problem with the YouTube transcript file.") + {}, _('There is a problem with the YouTube transcript file.') ) return response @@ -727,7 +725,7 @@ def _get_item(request, data): Returns the item. """ - usage_key = UsageKey.from_string(data.get("locator")) + usage_key = UsageKey.from_string(data.get('locator')) # This is placed before has_course_author_access() to validate the location, # because has_course_author_access() raises r if location is invalid. item = modulestore().get_item(usage_key) diff --git a/cms/static/js/views/video/transcripts/message_manager.js b/cms/static/js/views/video/transcripts/message_manager.js index 2fa77113ae6f..1e8c56cf6686 100644 --- a/cms/static/js/views/video/transcripts/message_manager.js +++ b/cms/static/js/views/video/transcripts/message_manager.js @@ -1,49 +1,49 @@ define([ - "jquery", - "backbone", - "underscore", - "js/views/video/transcripts/utils", - "js/views/video/transcripts/file_uploader", - "gettext", + 'jquery', + 'backbone', + 'underscore', + 'js/views/video/transcripts/utils', + 'js/views/video/transcripts/file_uploader', + 'gettext', ], function ($, Backbone, _, Utils, FileUploader, gettext) { var MessageManager = Backbone.View.extend({ - tagName: "div", - elClass: ".wrapper-transcripts-message", - invisibleClass: "is-invisible", + tagName: 'div', + elClass: '.wrapper-transcripts-message', + invisibleClass: 'is-invisible', events: { - "click .setting-import": "importHandler", - "click .setting-replace": "replaceHandler", - "click .setting-choose": "chooseHandler", - "click .setting-use-existing": "useExistingHandler", - "click .setting-download-youtube-transcript": "downloadYoutubeTranscriptHandler", + 'click .setting-import': 'importHandler', + 'click .setting-replace': 'replaceHandler', + 'click .setting-choose': 'chooseHandler', + 'click .setting-use-existing': 'useExistingHandler', + 'click .setting-download-youtube-transcript': 'downloadYoutubeTranscriptHandler', }, // Pre-defined dict with anchors to status templates. templates: { - not_found: "#transcripts-not-found", - found: "#transcripts-found", - import: "#transcripts-import", - replace: "#transcripts-replace", - uploaded: "#transcripts-uploaded", - use_existing: "#transcripts-use-existing", - choose: "#transcripts-choose", + not_found: '#transcripts-not-found', + found: '#transcripts-found', + import: '#transcripts-import', + replace: '#transcripts-replace', + uploaded: '#transcripts-uploaded', + use_existing: '#transcripts-use-existing', + choose: '#transcripts-choose', }, initialize: function (options) { _.bindAll( this, - "importHandler", - "replaceHandler", - "chooseHandler", - "useExistingHandler", - "showError", - "hideError" + 'importHandler', + 'replaceHandler', + 'chooseHandler', + 'useExistingHandler', + 'showError', + 'hideError' ); this.options = _.extend({}, options); - this.component_locator = this.$el.closest("[data-locator]").data("locator"); + this.component_locator = this.$el.closest('[data-locator]').data('locator'); this.fileUploader = new FileUploader({ el: this.$el, @@ -83,15 +83,15 @@ define([ template = _.template(tplHtml); this.$el - .find(".transcripts-status") - .removeClass("is-invisible") + .find('.transcripts-status') + .removeClass('is-invisible') .find(this.elClass) .html( template({ component_locator: encodeURIComponent(this.component_locator), html5_list: html5List, grouped_list: groupedList, - subs_id: params ? params.subs : "", + subs_id: params ? params.subs : '', }) ); @@ -111,7 +111,7 @@ define([ * */ showError: function (err, hideButtons) { - var $error = this.$el.find(".transcripts-error-message"); + var $error = this.$el.find('.transcripts-error-message'); if (err) { // Hide any other error messages. @@ -120,7 +120,7 @@ define([ $error.html(gettext(err)).removeClass(this.invisibleClass); if (hideButtons) { - this.$el.find(".wrapper-transcripts-buttons").addClass(this.invisibleClass); + this.$el.find('.wrapper-transcripts-buttons').addClass(this.invisibleClass); } } }, @@ -132,9 +132,9 @@ define([ * */ hideError: function () { - this.$el.find(".transcripts-error-message").addClass(this.invisibleClass); + this.$el.find('.transcripts-error-message').addClass(this.invisibleClass); - this.$el.find(".wrapper-transcripts-buttons").removeClass(this.invisibleClass); + this.$el.find('.wrapper-transcripts-buttons').removeClass(this.invisibleClass); }, /** @@ -148,7 +148,7 @@ define([ importHandler: function (event) { event.preventDefault(); - this.processCommand("replace", gettext("Error: Import failed.")); + this.processCommand('replace', gettext('Error: Import failed.')); }, /** @@ -162,7 +162,7 @@ define([ replaceHandler: function (event) { event.preventDefault(); - this.processCommand("replace", gettext("Error: Replacing failed.")); + this.processCommand('replace', gettext('Error: Replacing failed.')); }, /** @@ -176,9 +176,9 @@ define([ chooseHandler: function (event) { event.preventDefault(); - var videoId = $(event.currentTarget).data("video-id"); + var videoId = $(event.currentTarget).data('video-id'); - this.processCommand("choose", gettext("Error: Choosing failed."), videoId); + this.processCommand('choose', gettext('Error: Choosing failed.'), videoId); }, /** @@ -192,7 +192,7 @@ define([ useExistingHandler: function (event) { event.preventDefault(); - this.processCommand("rename", gettext("Error: Choosing failed.")); + this.processCommand('rename', gettext('Error: Choosing failed.')); }, downloadYoutubeTranscriptHandler: function (event) { @@ -201,38 +201,38 @@ define([ var videoId = videoObject[0].video; var component_locator = this.component_locator; $.ajax({ - type: "GET", + type: 'GET', notifyOnError: false, crossDomain: true, - url: "https://us-central1-appsembler-tahoe-0.cloudfunctions.net/youtube-transcript", + url: 'https://us-central1-appsembler-tahoe-0.cloudfunctions.net/youtube-transcript', data: { video_id: videoId, }, success: function (transcriptResponse) { - console.log("Downloladed youtube transcript"); + console.log('Downloladed youtube transcript'); }, }).done(function (transcriptResponse) { var srt = transcriptResponse.transcript_srt; - var transcript_name = "subs_" + videoId + Math.floor(1000 + Math.random() * 900) + ".srt"; + var transcript_name = 'subs_' + videoId + Math.floor(1000 + Math.random() * 900) + '.srt'; $.ajax({ - url: "/transcripts/upload", - type: "POST", - dataType: "json", + url: '/transcripts/upload', + type: 'POST', + dataType: 'json', data: { locator: component_locator, - "transcript-file": srt, - "transcript-name": transcript_name, - "youtube_video_id": videoId, + 'transcript-file': srt, + 'transcript-name': transcript_name, + 'youtube_video_id': videoId, video_list: JSON.stringify([videoObject[0]]), }, success: function (data) { - console.log("Transcript uploaded successfully"); + console.log('Transcript uploaded successfully'); }, }) .done(function (resp) { - alert("Transcript uploaded successfully"); + alert('Transcript uploaded successfully'); var sub = resp.subs; - Utils.Storage.set("sub", sub); + Utils.Storage.set('sub', sub); }) .fail(function (resp) { var message = resp.status || errorMessage; @@ -271,8 +271,8 @@ define([ .done(function (resp) { var sub = resp.subs; - self.render("found", resp); - Utils.Storage.set("sub", sub); + self.render('found', resp); + Utils.Storage.set('sub', sub); }) .fail(function (resp) { var message = resp.status || errorMessage; From 52a9f70da756d9669929ae6108190dfde7e4c7a0 Mon Sep 17 00:00:00 2001 From: Amir Tadrisi Date: Fri, 2 Sep 2022 17:36:08 -0500 Subject: [PATCH 9/9] chore: remove the logged transcript --- cms/djangoapps/contentstore/views/transcripts_ajax.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cms/djangoapps/contentstore/views/transcripts_ajax.py b/cms/djangoapps/contentstore/views/transcripts_ajax.py index f099b6605742..ba39f04f7313 100644 --- a/cms/djangoapps/contentstore/views/transcripts_ajax.py +++ b/cms/djangoapps/contentstore/views/transcripts_ajax.py @@ -238,8 +238,6 @@ def upload_transcripts(request): input_format=Transcript.SRT, output_format=Transcript.SJSON, ).encode() - log.debug('#' * 32) - log.debug(sjson_subs) transcript_created = create_or_update_video_transcript( video_id=edx_video_id, language_code='en', @@ -250,8 +248,6 @@ def upload_transcripts(request): }, file_data=ContentFile(sjson_subs), ) - log.debug('#' * 32) - log.debug(transcript_created) if transcript_created is None: response = JsonResponse({'status': 'Invalid Video ID'}, status=400)