From dd4213c0d28cf38fff900d8ab51918abe47bac0b Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Thu, 1 Jun 2023 15:38:24 -0700 Subject: [PATCH 01/11] refactor: improve typing of StaticFile named tuple --- .../core/lib/xblock_serializer/block_serializer.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/openedx/core/lib/xblock_serializer/block_serializer.py b/openedx/core/lib/xblock_serializer/block_serializer.py index 92bb7475c3d6..a09355d87233 100644 --- a/openedx/core/lib/xblock_serializer/block_serializer.py +++ b/openedx/core/lib/xblock_serializer/block_serializer.py @@ -1,9 +1,10 @@ """ Code for serializing a modulestore XBlock to OLX. """ +from __future__ import annotations import logging import os -from collections import namedtuple +from typing import NamedTuple from lxml import etree @@ -11,14 +12,19 @@ log = logging.getLogger(__name__) -# A static file required by an XBlock -StaticFile = namedtuple('StaticFile', ['name', 'url', 'data']) + +class StaticFile(NamedTuple): + """ A static file required by an XBlock """ + name: str + url: str | None + data: bytes | None class XBlockSerializer: """ A class that can serialize an XBlock to OLX. """ + static_files: list[StaticFile] def __init__(self, block): """ From 75058559bdee3045c5ec508635feac6e3fdbbf14 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Thu, 1 Jun 2023 18:20:01 -0700 Subject: [PATCH 02/11] feat: copy static asset files into the clipboard --- cms/envs/common.py | 8 +++ .../core/djangoapps/content_staging/admin.py | 9 ++- .../core/djangoapps/content_staging/data.py | 13 ++++- .../djangoapps/content_staging/filters.py | 58 +++++++++++++++++++ .../migrations/0002_stagedcontentfile.py | 24 ++++++++ .../core/djangoapps/content_staging/models.py | 15 +++++ .../core/djangoapps/content_staging/views.py | 55 +++++++++++++++++- 7 files changed, 178 insertions(+), 4 deletions(-) create mode 100644 openedx/core/djangoapps/content_staging/filters.py create mode 100644 openedx/core/djangoapps/content_staging/migrations/0002_stagedcontentfile.py diff --git a/cms/envs/common.py b/cms/envs/common.py index 41ef3f60c2ef..459c24de329d 100644 --- a/cms/envs/common.py +++ b/cms/envs/common.py @@ -2728,3 +2728,11 @@ DISCUSSIONS_INCONTEXT_FEEDBACK_URL = '' DISCUSSIONS_INCONTEXT_LEARNMORE_URL = '' + +OPEN_EDX_FILTERS_CONFIG = { + "org.openedx.content_authoring.staged_content.static_filter_source.v1": { + "pipeline": [ + "openedx.core.djangoapps.content_staging.filters.IgnoreLargeFiles", + ] + } +} diff --git a/openedx/core/djangoapps/content_staging/admin.py b/openedx/core/djangoapps/content_staging/admin.py index 6cc567d87d7c..9b0fcef8258f 100644 --- a/openedx/core/djangoapps/content_staging/admin.py +++ b/openedx/core/djangoapps/content_staging/admin.py @@ -4,7 +4,13 @@ from django.contrib import admin from django.urls import reverse from django.utils.html import format_html -from .models import StagedContent, UserClipboard +from .models import StagedContent, StagedContentFile, UserClipboard + + +class StagedContentFileInline(admin.TabularInline): + """ Inline admin UI for StagedContentFile """ + model = StagedContentFile + readonly_fields = ('filename', 'md5_hash', 'data_file') @admin.register(StagedContent) @@ -14,6 +20,7 @@ class StagedContentAdmin(admin.ModelAdmin): list_filter = ('purpose', 'status', 'block_type') search_fields = ('user__username', 'display_name', 'suggested_url_name') readonly_fields = ('id', 'user', 'created', 'purpose', 'status', 'block_type', 'olx') + inlines = (StagedContentFileInline, ) @admin.register(UserClipboard) diff --git a/openedx/core/djangoapps/content_staging/data.py b/openedx/core/djangoapps/content_staging/data.py index f9138915833a..08f9cc38c758 100644 --- a/openedx/core/djangoapps/content_staging/data.py +++ b/openedx/core/djangoapps/content_staging/data.py @@ -1,12 +1,13 @@ """ Public python data types for content staging """ +from __future__ import annotations from attrs import field, frozen, validators from datetime import datetime from django.db.models import TextChoices from django.utils.translation import gettext_lazy as _ -from opaque_keys.edx.keys import UsageKey +from opaque_keys.edx.keys import UsageKey, AssetKey class StagedContentStatus(TextChoices): @@ -43,6 +44,16 @@ class StagedContentData: display_name: str = field(validator=validators.instance_of(str)) +@frozen +class StagedContentFileData: + """ Read-only data model for a single file used by some staged content """ + filename: str = field(validator=validators.instance_of(str)) + # Everything below is optional: + data: bytes | None = field(validator=validators.optional(validators.instance_of(bytes))) + source_asset_key: AssetKey | None = field(validator=validators.optional(validators.instance_of(AssetKey))) + md5_hash: str | None = field(validator=validators.optional(validators.instance_of(str))) + + @frozen class UserClipboardData: """ Read-only data model for User Clipboard data (copied OLX) """ diff --git a/openedx/core/djangoapps/content_staging/filters.py b/openedx/core/djangoapps/content_staging/filters.py new file mode 100644 index 000000000000..3759313b5a98 --- /dev/null +++ b/openedx/core/djangoapps/content_staging/filters.py @@ -0,0 +1,58 @@ +""" +Filters that affect the behavior of staged content (and the clipboard) +""" +# pylint: disable=unused-argument +from __future__ import annotations + +from attrs import asdict + +from openedx_filters import PipelineStep +from openedx_filters.tooling import OpenEdxPublicFilter +from .data import StagedContentFileData +from .models import StagedContent + + +class StagingStaticAssetFilter(OpenEdxPublicFilter): + """ + A filter used to determine which static assets associate with an XBlock(s) + should be staged in the StagedContent app (e.g. the clipboard). + + This API is considered BETA. Once it is stable, this definition should be moved into openedx_filters. + """ + + filter_type = "org.openedx.content_authoring.staged_content.static_filter_source.v1" + + @classmethod + def run_filter(cls, staged_content: StagedContent, file_datas: list[StagedContentFileData]): + """ + Run this filter, which requires the following arguments: + staged_content (StagedContent): details of the content being staged, as saved to the DB. + file_datas (list[StagedContentFileData]): details of the files being staged + """ + data = super().run_pipeline(staged_content=staged_content, file_datas=file_datas) + return data.get("file_datas") + + +class IgnoreLargeFiles(PipelineStep): + """ + Don't copy files over 10MB into the clipboard + """ + + # pylint: disable=arguments-differ + def run_filter(self, staged_content: StagedContent, file_datas: list[StagedContentFileData]): + """ + Filter the list of file_datas to remove any large files + """ + limit = 10 * 1024 * 1024 + + def remove_large_data(fd: StagedContentFileData): + """ Remove 'data' from the immutable StagedContentFileData object, if it's above the size limit """ + if fd.data and len(fd.data) > limit: + # these data objects are immutable so make a copy with data=None: + return StagedContentFileData(**{**asdict(fd), "data": None}) + return fd + + return {"file_datas": [remove_large_data(fd) for fd in file_datas]} + +#class DetectPythonLibZip(PipelineStep): +# python_lib_filename = getattr(settings, 'PYTHON_LIB_FILENAME', DEFAULT_PYTHON_LIB_FILENAME) diff --git a/openedx/core/djangoapps/content_staging/migrations/0002_stagedcontentfile.py b/openedx/core/djangoapps/content_staging/migrations/0002_stagedcontentfile.py new file mode 100644 index 000000000000..38b6279358e8 --- /dev/null +++ b/openedx/core/djangoapps/content_staging/migrations/0002_stagedcontentfile.py @@ -0,0 +1,24 @@ +# Generated by Django 3.2.19 on 2023-06-02 00:47 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ('content_staging', '0001_initial'), + ] + + operations = [ + migrations.CreateModel( + name='StagedContentFile', + fields=[ + ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('filename', models.CharField(max_length=255)), + ('data_file', models.FileField(blank=True, upload_to='staged-content-temp/')), + ('md5_hash', models.CharField(blank=True, max_length=32)), + ('for_content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='files', to='content_staging.stagedcontent')), + ], + ), + ] diff --git a/openedx/core/djangoapps/content_staging/models.py b/openedx/core/djangoapps/content_staging/models.py index 9f89177f278c..713b39fa6727 100644 --- a/openedx/core/djangoapps/content_staging/models.py +++ b/openedx/core/djangoapps/content_staging/models.py @@ -67,6 +67,21 @@ def __str__(self): return f'Staged {self.block_type} block "{self.display_name}" ({self.status})' +class StagedContentFile(models.Model): + """ + A data file ("Static Asset") associated with some StagedContent. + + These usually come from a course's Files & Uploads page, but can also come + from per-xblock file storage (e.g. video transcripts or images used in + v2 content libraries). + """ + for_content = models.ForeignKey(StagedContent, on_delete=models.CASCADE, related_name="files") + filename = models.CharField(max_length=255, blank=False) + # Everything below is optional: + data_file = models.FileField(upload_to="staged-content-temp/", blank=True) + md5_hash = models.CharField(max_length=32, blank=True) + + class UserClipboard(models.Model): """ Each user has a clipboard that can hold one item at a time, where an item diff --git a/openedx/core/djangoapps/content_staging/views.py b/openedx/core/djangoapps/content_staging/views.py index ecebffda98c7..d1041f06686e 100644 --- a/openedx/core/djangoapps/content_staging/views.py +++ b/openedx/core/djangoapps/content_staging/views.py @@ -1,8 +1,11 @@ """ REST API views for content staging """ +from __future__ import annotations +import hashlib import logging +from django.core.files.base import ContentFile from django.db import transaction from django.http import HttpResponse from django.shortcuts import get_object_or_404 @@ -19,11 +22,14 @@ from openedx.core.lib.api.view_utils import view_auth_classes from openedx.core.lib.xblock_serializer.api import serialize_xblock_to_olx from xmodule import block_metadata_utils +from xmodule.contentstore.content import StaticContent +from xmodule.contentstore.django import contentstore from xmodule.modulestore.django import modulestore from xmodule.modulestore.exceptions import ItemNotFoundError -from .data import CLIPBOARD_PURPOSE, StagedContentStatus -from .models import StagedContent, UserClipboard +from .data import CLIPBOARD_PURPOSE, StagedContentFileData, StagedContentStatus +from .filters import StagingStaticAssetFilter +from .models import StagedContent, StagedContentFile, UserClipboard from .serializers import UserClipboardSerializer, PostToClipboardSerializer from .tasks import delete_expired_clipboards @@ -149,6 +155,51 @@ def post(self, request): serializer = UserClipboardSerializer(clipboard, context={"request": request}) # Log an event so we can analyze how this feature is used: log.info(f"Copied {usage_key.block_type} component \"{usage_key}\" to their clipboard.") + + # Try to copy the static files. If this fails, we still consider the overall copy attempt to have succeeded, + # because intra-course pasting will still work fine, and in any case users can manually resolve the file issue. + try: + files_to_save: list[StagedContentFileData] = [] + for f in block_data.static_files: + source_asset_key = ( + StaticContent.get_asset_key_from_path(course_key, f.url) + if (f.url and f.url.startswith('/')) else None + ) + # Compute the MD5 hash and get the content: + content: bytes | None = f.data + if content: + md5_hash = hashlib.md5(f.data).hexdigest() + elif source_asset_key: + sc = contentstore().find(source_asset_key) + md5_hash = sc.content_digest + content = sc.data + else: + md5_hash = "" # Unknown + + # Load the data: + + entry = StagedContentFileData( + filename=f.name, + data=content, + source_asset_key=source_asset_key, + md5_hash=md5_hash, + ) + files_to_save.append(entry) + + # run filters on files_to_save. + # e.g. remove large files, add python_lib.zip which may not otherwise be detected + files_to_save = StagingStaticAssetFilter.run_filter(staged_content=staged_content, file_datas=files_to_save) + + for f in files_to_save: + StagedContentFile.objects.create( + for_content=staged_content, + filename=f.filename, + data_file=ContentFile(content=f.data, name=f.filename) if f.data else None, + md5_hash=f.md5_hash or "", + ) + except Exception as err: # pylint: disable=broad-except + log.exception(f"Unable to copy static files to clipboard for component {usage_key}") + # Enqueue a (potentially slow) task to delete the old staged content try: delete_expired_clipboards.delay(expired_ids) From 5f75a8345750bd364799929572c3e7d5576b22be Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Sat, 3 Jun 2023 16:24:06 -0700 Subject: [PATCH 03/11] feat: save the source key in StagedContentFile --- openedx/core/djangoapps/content_staging/admin.py | 2 +- openedx/core/djangoapps/content_staging/api.py | 2 +- openedx/core/djangoapps/content_staging/data.py | 7 ++++++- .../migrations/0002_stagedcontentfile.py | 1 + openedx/core/djangoapps/content_staging/models.py | 4 ++++ openedx/core/djangoapps/content_staging/views.py | 11 +++++++---- 6 files changed, 20 insertions(+), 7 deletions(-) diff --git a/openedx/core/djangoapps/content_staging/admin.py b/openedx/core/djangoapps/content_staging/admin.py index 9b0fcef8258f..a40a13913be4 100644 --- a/openedx/core/djangoapps/content_staging/admin.py +++ b/openedx/core/djangoapps/content_staging/admin.py @@ -10,7 +10,7 @@ class StagedContentFileInline(admin.TabularInline): """ Inline admin UI for StagedContentFile """ model = StagedContentFile - readonly_fields = ('filename', 'md5_hash', 'data_file') + readonly_fields = ('filename', 'md5_hash', 'source_key_str', 'data_file') @admin.register(StagedContent) diff --git a/openedx/core/djangoapps/content_staging/api.py b/openedx/core/djangoapps/content_staging/api.py index 8e4345895644..88ccda22e74a 100644 --- a/openedx/core/djangoapps/content_staging/api.py +++ b/openedx/core/djangoapps/content_staging/api.py @@ -5,7 +5,7 @@ from django.http import HttpRequest -from .data import StagedContentData, StagedContentStatus, UserClipboardData +from .data import StagedContentData, StagedContentFileData, StagedContentStatus, UserClipboardData from .models import UserClipboard as _UserClipboard, StagedContent as _StagedContent from .serializers import UserClipboardSerializer as _UserClipboardSerializer diff --git a/openedx/core/djangoapps/content_staging/data.py b/openedx/core/djangoapps/content_staging/data.py index 08f9cc38c758..3e9168870cbf 100644 --- a/openedx/core/djangoapps/content_staging/data.py +++ b/openedx/core/djangoapps/content_staging/data.py @@ -50,7 +50,12 @@ class StagedContentFileData: filename: str = field(validator=validators.instance_of(str)) # Everything below is optional: data: bytes | None = field(validator=validators.optional(validators.instance_of(bytes))) - source_asset_key: AssetKey | None = field(validator=validators.optional(validators.instance_of(AssetKey))) + # If this asset came from Files & Uploads in a course, this is an AssetKey + # as a string. If this asset came from an XBlock's filesystem, this is the + # UsageKey of the XBlock. + source_key: AssetKey | UsageKey | None = field( + validator=validators.optional(validators.instance_of((AssetKey, UsageKey))) + ) md5_hash: str | None = field(validator=validators.optional(validators.instance_of(str))) diff --git a/openedx/core/djangoapps/content_staging/migrations/0002_stagedcontentfile.py b/openedx/core/djangoapps/content_staging/migrations/0002_stagedcontentfile.py index 38b6279358e8..5c3501ac0d9d 100644 --- a/openedx/core/djangoapps/content_staging/migrations/0002_stagedcontentfile.py +++ b/openedx/core/djangoapps/content_staging/migrations/0002_stagedcontentfile.py @@ -17,6 +17,7 @@ class Migration(migrations.Migration): ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), ('filename', models.CharField(max_length=255)), ('data_file', models.FileField(blank=True, upload_to='staged-content-temp/')), + ('source_key_str', models.CharField(blank=True, max_length=255)), ('md5_hash', models.CharField(blank=True, max_length=32)), ('for_content', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='files', to='content_staging.stagedcontent')), ], diff --git a/openedx/core/djangoapps/content_staging/models.py b/openedx/core/djangoapps/content_staging/models.py index 713b39fa6727..d1a790e3b2f7 100644 --- a/openedx/core/djangoapps/content_staging/models.py +++ b/openedx/core/djangoapps/content_staging/models.py @@ -79,6 +79,10 @@ class StagedContentFile(models.Model): filename = models.CharField(max_length=255, blank=False) # Everything below is optional: data_file = models.FileField(upload_to="staged-content-temp/", blank=True) + # If this asset came from Files & Uploads in a course, this is an AssetKey + # as a string. If this asset came from an XBlock's filesystem, this is the + # UsageKey of the XBlock. + source_key_str = models.CharField(max_length=255, blank=True) md5_hash = models.CharField(max_length=32, blank=True) diff --git a/openedx/core/djangoapps/content_staging/views.py b/openedx/core/djangoapps/content_staging/views.py index d1041f06686e..07a6dff0f924 100644 --- a/openedx/core/djangoapps/content_staging/views.py +++ b/openedx/core/djangoapps/content_staging/views.py @@ -161,7 +161,7 @@ def post(self, request): try: files_to_save: list[StagedContentFileData] = [] for f in block_data.static_files: - source_asset_key = ( + source_key = ( StaticContent.get_asset_key_from_path(course_key, f.url) if (f.url and f.url.startswith('/')) else None ) @@ -169,8 +169,10 @@ def post(self, request): content: bytes | None = f.data if content: md5_hash = hashlib.md5(f.data).hexdigest() - elif source_asset_key: - sc = contentstore().find(source_asset_key) + # This asset came from the XBlock's filesystem, e.g. a video block's transcript file + source_key = usage_key + elif source_key: + sc = contentstore().find(source_key) md5_hash = sc.content_digest content = sc.data else: @@ -181,7 +183,7 @@ def post(self, request): entry = StagedContentFileData( filename=f.name, data=content, - source_asset_key=source_asset_key, + source_key=source_key, md5_hash=md5_hash, ) files_to_save.append(entry) @@ -195,6 +197,7 @@ def post(self, request): for_content=staged_content, filename=f.filename, data_file=ContentFile(content=f.data, name=f.filename) if f.data else None, + source_key_str=str(source_key) if source_key else "", md5_hash=f.md5_hash or "", ) except Exception as err: # pylint: disable=broad-except From ee6e698b1173c2c68661d2594da09e0eee2179cd Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Sat, 3 Jun 2023 16:57:15 -0700 Subject: [PATCH 04/11] feat: paste static assets --- cms/djangoapps/contentstore/helpers.py | 119 ++++++++++++++++-- .../xblock_services/xblock_service.py | 14 +-- .../core/djangoapps/content_staging/api.py | 44 +++++++ 3 files changed, 160 insertions(+), 17 deletions(-) diff --git a/cms/djangoapps/contentstore/helpers.py b/cms/djangoapps/contentstore/helpers.py index cbaf4b8f80cf..9805ed2538cc 100644 --- a/cms/djangoapps/contentstore/helpers.py +++ b/cms/djangoapps/contentstore/helpers.py @@ -1,31 +1,34 @@ """ Helper methods for Studio views. """ - +from __future__ import annotations +import logging import urllib from lxml import etree +from mimetypes import guess_type +from attrs import frozen, Factory from django.utils.translation import gettext as _ -from opaque_keys.edx.keys import UsageKey +from opaque_keys.edx.keys import AssetKey, CourseKey, UsageKey from opaque_keys.edx.locator import DefinitionLocator, LocalId from xblock.core import XBlock from xblock.fields import ScopeIds from xblock.runtime import IdGenerator +from xmodule.contentstore.content import StaticContent +from xmodule.contentstore.django import contentstore +from xmodule.exceptions import NotFoundError from xmodule.modulestore.django import modulestore # from cms.djangoapps.contentstore.views.preview import _load_preview_block from cms.djangoapps.models.settings.course_grading import CourseGradingModel from common.djangoapps.student import auth from common.djangoapps.student.roles import CourseCreatorRole, OrgContentCreatorRole - -try: - # Technically this is a django app plugin, so we should not error if it's not installed: - import openedx.core.djangoapps.content_staging.api as content_staging_api -except ImportError: - content_staging_api = None +import openedx.core.djangoapps.content_staging.api as content_staging_api from .utils import reverse_course_url, reverse_library_url, reverse_usage_url +log = logging.getLogger(__name__) + # Note: Grader types are used throughout the platform but most usages are simply in-line # strings. In addition, new grader types can be defined on the fly anytime one is needed # (because they're just strings). This dict is an attempt to constrain the sprawl in Studio. @@ -210,6 +213,14 @@ def create_definition(self, block_type, slug=None) -> DefinitionLocator: return DefinitionLocator(block_type, LocalId(block_type)) +@frozen +class StaticFileNotices: + """ Information about what static files were updated (or not) when pasting content into another course """ + new_files: list[str] = Factory(list) + conflicting_files: list[str] = Factory(list) + error_files: list[str] = Factory(list) + + def import_staged_content_from_user_clipboard(parent_key: UsageKey, request): """ Import a block (and any children it has) from "staged" OLX. @@ -226,9 +237,10 @@ def import_staged_content_from_user_clipboard(parent_key: UsageKey, request): user_clipboard = content_staging_api.get_user_clipboard(request.user.id) if not user_clipboard: # Clipboard is empty or expired/error/loading - return None + return None, StaticFileNotices() block_type = user_clipboard.content.block_type olx_str = content_staging_api.get_staged_content_olx(user_clipboard.content.id) + static_files = content_staging_api.get_staged_content_static_files(user_clipboard.content.id) node = etree.fromstring(olx_str) store = modulestore() with store.bulk_operations(parent_key.course_key): @@ -247,6 +259,11 @@ def import_staged_content_from_user_clipboard(parent_key: UsageKey, request): # 'keys' and uses 'id_generator', but the default XBlock parse_xml ignores 'id_generator' and uses 'keys'. # For children of this block, obviously only id_generator is used. xblock_class = runtime.load_block_type(block_type) + # Note: if we find a case where any XBlock needs access to the block-specific static files that were saved to + # export_fs during copying, we could make them available here via runtime.resources_fs before calling parse_xml. + # However, currently the only known case for that is video block's transcript files, and those will + # automatically be "carried over" to the new XBlock even in a different course because the video ID is the same, + # and VAL will thus make the transcript available. temp_xblock = xblock_class.parse_xml(node, runtime, keys, id_generator) if xblock_class.has_children and temp_xblock.children: raise NotImplementedError("We don't yet support pasting XBlocks with children") @@ -257,7 +274,89 @@ def import_staged_content_from_user_clipboard(parent_key: UsageKey, request): new_xblock = store.update_item(temp_xblock, request.user.id, allow_not_found=True) parent_xblock.children.append(new_xblock.location) store.update_item(parent_xblock, request.user.id) - return new_xblock + + # Now handle static files that need to go into Files & Uploads: + notices = _import_files_into_course( + course_key=parent_key.context_key, + staged_content_id=user_clipboard.content.id, + static_files=static_files, + ) + + return new_xblock, notices + + +def _import_files_into_course( + course_key: CourseKey, + staged_content_id: int, + static_files: list[content_staging_api.StagedContentFileData], +) -> StaticFileNotices: + """ + For the given staged static asset files (which are in "Staged Content" such as the user's clipbaord, but which + need to end up in the course's Files & Uploads page), import them into the destination course, unless they already + exist. + """ + # List of files that were newly added to the destination course + new_files = [] + # List of files that conflicted with identically named files already in the destination course + conflicting_files = [] + # List of files that had an error (shouldn't happen unless we have some kind of bug) + error_files = [] + for file_data_obj in static_files: + if not isinstance(file_data_obj.source_key, AssetKey): + # This static asset was managed by the XBlock and instead of being added to "Files & Uploads", it is stored + # using some other system. We could make it available via runtime.resources_fs during XML parsing, but it's + # not needed here. + continue + # At this point, we know this is a "Files & Uploads" asset that we may need to copy into the course: + try: + result = _import_file_into_course(course_key, staged_content_id, file_data_obj) + if result is True: + new_files.append(file_data_obj.filename) + elif result is None: + pass # This file already exists; no action needed. + else: + conflicting_files.append(file_data_obj.filename) + except Exception: # lint-amnesty, pylint: disable=broad-except + error_files.append(file_data_obj.filename) + log.exception(f"Failed to import Files & Uploads file {file_data_obj.filename}") + return StaticFileNotices( + new_files=new_files, + conflicting_files=conflicting_files, + error_files=error_files, + ) + + +def _import_file_into_course( + course_key: CourseKey, + staged_content_id: int, + file_data_obj: content_staging_api.StagedContentFileData, +) -> bool | None: + """ + Import a single staged static asset file into the course, unless it already exists. + Returns True if it was imported, False if there's a conflict, or None if + the file already existed (no action needed). + """ + filename = file_data_obj.filename + new_key = course_key.make_asset_key("asset", filename) + try: + current_file = contentstore().find(new_key) + except NotFoundError: + current_file = None + if not current_file: + # This static asset should be imported into the new course: + content_type = guess_type(filename)[0] + data = content_staging_api.get_staged_content_static_file_data(staged_content_id, filename) + if data is None: + raise NotFoundError(file_data_obj.source_key) + content = StaticContent(new_key, name=filename, content_type=content_type, data=data) + contentstore().save(content) + return True + elif current_file.content_digest == file_data_obj.md5_hash: + # The file already exists and matches exactly, so no action is needed + return None + else: + # There is a conflict with some other file that has the same name. + return False def is_item_in_course_tree(item): diff --git a/cms/djangoapps/contentstore/xblock_services/xblock_service.py b/cms/djangoapps/contentstore/xblock_services/xblock_service.py index 696760a2db9d..f1fb8eb048c8 100644 --- a/cms/djangoapps/contentstore/xblock_services/xblock_service.py +++ b/cms/djangoapps/contentstore/xblock_services/xblock_service.py @@ -13,6 +13,7 @@ from datetime import datetime from uuid import uuid4 +from attrs import asdict from django.conf import settings from django.contrib.auth.decorators import login_required from django.contrib.auth.models import (User) # lint-amnesty, pylint: disable=imported-auth-user @@ -562,7 +563,7 @@ def _create_block(request): if request.json.get("staged_content") == "clipboard": # Paste from the user's clipboard (content_staging app clipboard, not browser clipboard) into 'usage_key': try: - created_xblock = import_staged_content_from_user_clipboard( + created_xblock, notices = import_staged_content_from_user_clipboard( parent_key=usage_key, request=request ) except Exception: # pylint: disable=broad-except @@ -576,12 +577,11 @@ def _create_block(request): return JsonResponse( {"error": _("Your clipboard is empty or invalid.")}, status=400 ) - return JsonResponse( - { - "locator": str(created_xblock.location), - "courseKey": str(created_xblock.location.course_key), - } - ) + return JsonResponse({ + "locator": str(created_xblock.location), + "courseKey": str(created_xblock.location.course_key), + "static_file_notices": asdict(notices), + }) category = request.json["category"] if isinstance(usage_key, LibraryUsageLocator): diff --git a/openedx/core/djangoapps/content_staging/api.py b/openedx/core/djangoapps/content_staging/api.py index 88ccda22e74a..87616b706dd6 100644 --- a/openedx/core/djangoapps/content_staging/api.py +++ b/openedx/core/djangoapps/content_staging/api.py @@ -4,6 +4,8 @@ from __future__ import annotations from django.http import HttpRequest +from opaque_keys import InvalidKeyError +from opaque_keys.edx.keys import AssetKey, UsageKey from .data import StagedContentData, StagedContentFileData, StagedContentStatus, UserClipboardData from .models import UserClipboard as _UserClipboard, StagedContent as _StagedContent @@ -75,3 +77,45 @@ def get_staged_content_olx(staged_content_id: int) -> str | None: return sc.olx except _StagedContent.DoesNotExist: return None + + +def get_staged_content_static_files(staged_content_id: int) -> list[StagedContentFileData]: + """ + Get the filenames and metadata for any static files used by the given staged content. + + Does not check permissions! + """ + sc = _StagedContent.objects.get(pk=staged_content_id) + + def str_to_key(source_key_str: str): + if not str: + return None + try: + return AssetKey.from_string(source_key_str) + except InvalidKeyError: + return UsageKey.from_string(source_key_str) + + return [ + StagedContentFileData( + filename=f.filename, + # For performance, we don't load data unless it's needed, so there's + # a separate API call for that. + data=None, + source_key=str_to_key(f.source_key_str), + md5_hash=f.md5_hash, + ) + for f in sc.files.all() + ] + + +def get_staged_content_static_file_data(staged_content_id: int, filename: str) -> bytes | None: + """ + Get the data for the static asset associated with the given staged content. + + Does not check permissions! + """ + sc = _StagedContent.objects.get(pk=staged_content_id) + file_data_obj = sc.files.filter(filename=filename).first() + if file_data_obj: + return file_data_obj.data_file.open().read() + return None From ed915658fce379dd39070a8382d97e18f4850889 Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Wed, 7 Jun 2023 19:33:03 -0700 Subject: [PATCH 05/11] feat: show notification in studio about pasted assets --- cms/static/js/views/pages/container.js | 56 +++++++++++++++++++++++++- 1 file changed, 54 insertions(+), 2 deletions(-) diff --git a/cms/static/js/views/pages/container.js b/cms/static/js/views/pages/container.js index 0586cffc3ec0..b1ec303bd566 100644 --- a/cms/static/js/views/pages/container.js +++ b/cms/static/js/views/pages/container.js @@ -6,10 +6,12 @@ define(['jquery', 'underscore', 'backbone', 'gettext', 'js/views/pages/base_page 'common/js/components/utils/view_utils', 'js/views/container', 'js/views/xblock', 'js/views/components/add_xblock', 'js/views/modals/edit_xblock', 'js/views/modals/move_xblock_modal', 'js/models/xblock_info', 'js/views/xblock_string_field_editor', 'js/views/xblock_access_editor', - 'js/views/pages/container_subviews', 'js/views/unit_outline', 'js/views/utils/xblock_utils'], + 'js/views/pages/container_subviews', 'js/views/unit_outline', 'js/views/utils/xblock_utils', + 'common/js/components/views/feedback_notification', 'common/js/components/views/feedback_prompt', +], function($, _, Backbone, gettext, BasePage, ViewUtils, ContainerView, XBlockView, AddXBlockComponent, EditXBlockModal, MoveXBlockModal, XBlockInfo, XBlockStringFieldEditor, XBlockAccessEditor, - ContainerSubviews, UnitOutlineView, XBlockUtils) { + ContainerSubviews, UnitOutlineView, XBlockUtils, NotificationView, PromptView) { 'use strict'; var XBlockContainerPage = BasePage.extend({ @@ -255,10 +257,60 @@ function($, _, Backbone, gettext, BasePage, ViewUtils, ContainerView, XBlockView staged_content: "clipboard", }).then((data) => { this.onNewXBlock(placeholderElement, scrollOffset, false, data); + return data; }).fail(() => { // Remove the placeholder if the paste failed placeholderElement.remove(); }); + }).done((data) => { + const { + conflicting_files: conflictingFiles, + error_files: errorFiles, + new_files: newFiles, + } = data.static_file_notices; + + const notices = []; + if (errorFiles.length) { + notices.push((next) => new PromptView.Error({ + title: gettext("Some errors occurred"), + message: ( + gettext("The following required files could not be added to the course:") + + " " + errorFiles.join(", ") + ), + actions: {primary: {text: gettext("OK"), click: (x) => { x.hide(); next(); }}}, + })); + } + if (conflictingFiles.length) { + notices.push((next) => new PromptView.Warning({ + title: gettext("You may need to update a file(s) manually"), + message: ( + gettext( + "The following files already exist in this course but don't match the " + + "version used by the component you pasted:" + ) + " " + conflictingFiles.join(", ") + ), + actions: {primary: {text: gettext("OK"), click: (x) => { x.hide(); next(); }}}, + })); + } + if (newFiles.length) { + notices.push(() => new NotificationView.Confirmation({ + title: gettext("New files were added to this course's Files & Uploads"), + message: ( + gettext("The following required files were imported to this course:") + + " " + newFiles.join(", ") + ), + closeIcon: true, + })); + } + if (notices.length) { + // Show the notices, one at a time: + const showNext = () => { + const view = notices.shift()(showNext); + view.show(); + } + // Delay to avoid conflict with the "Pasting..." notification. + setTimeout(showNext, 1250); + } }); }, From 5c2ad99842d1b41c52b0cc964e70d1e9250b7ded Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Thu, 15 Jun 2023 15:43:39 -0700 Subject: [PATCH 06/11] fix: HTML XBlocks would lose the editor="raw" setting when copy-pasted. --- .../lib/xblock_serializer/block_serializer.py | 4 +++ .../core/lib/xblock_serializer/test_api.py | 28 ++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/openedx/core/lib/xblock_serializer/block_serializer.py b/openedx/core/lib/xblock_serializer/block_serializer.py index a09355d87233..87110471aa4d 100644 --- a/openedx/core/lib/xblock_serializer/block_serializer.py +++ b/openedx/core/lib/xblock_serializer/block_serializer.py @@ -106,6 +106,10 @@ def _serialize_html_block(self, block) -> etree.Element: olx_node.attrib["url_name"] = block.scope_ids.usage_id.block_id if block.display_name: olx_node.attrib["display_name"] = block.display_name + if block.fields["editor"].is_set_on(block): + olx_node.attrib["editor"] = block.editor + if block.use_latex_compiler: + olx_node.attrib["use_latex_compiler"] = "true" olx_node.text = etree.CDATA("\n" + block.data + "\n") return olx_node diff --git a/openedx/core/lib/xblock_serializer/test_api.py b/openedx/core/lib/xblock_serializer/test_api.py index f9258f18551a..3d9d67333cd0 100644 --- a/openedx/core/lib/xblock_serializer/test_api.py +++ b/openedx/core/lib/xblock_serializer/test_api.py @@ -6,7 +6,7 @@ from openedx.core.djangolib.testing.utils import skip_unless_cms from xmodule.modulestore.django import modulestore from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase -from xmodule.modulestore.tests.factories import ToyCourseFactory +from xmodule.modulestore.tests.factories import BlockFactory, CourseFactory, ToyCourseFactory from . import api @@ -134,6 +134,32 @@ def test_html_with_static_asset_blockstore(self): # This is the only other difference - an extra field with the blockstore-specific definition ID: self.assertEqual(serialized_blockstore.def_id, "html/just_img") + def test_html_with_fields(self): + """ Test an HTML Block with non-default fields like editor='raw' """ + course = CourseFactory.create(display_name='test course', run="Testing_course") + html_block = BlockFactory.create( + parent_location=course.location, + category="html", + display_name="Non-default HTML Block", + editor="raw", + use_latex_compiler=True, + data="🍔", + ) + serialized = api.serialize_xblock_to_olx(html_block) + self.assertXmlEqual( + serialized.olx_str, + """ + + """ + ) + def test_export_sequential(self): """ Export a sequential from the toy course, including all of its children. From 63d152ddadba40b292237b7fa2205a486e29befb Mon Sep 17 00:00:00 2001 From: Braden MacDonald Date: Thu, 15 Jun 2023 17:23:10 -0700 Subject: [PATCH 07/11] feat: copy python_lib.zip to the clipboard when it seems to be in use --- .../djangoapps/content_staging/filters.py | 3 - .../lib/xblock_serializer/block_serializer.py | 14 ++--- openedx/core/lib/xblock_serializer/data.py | 12 ++++ .../core/lib/xblock_serializer/test_api.py | 57 ++++++++++++++++++- openedx/core/lib/xblock_serializer/utils.py | 39 ++++++++++++- 5 files changed, 111 insertions(+), 14 deletions(-) create mode 100644 openedx/core/lib/xblock_serializer/data.py diff --git a/openedx/core/djangoapps/content_staging/filters.py b/openedx/core/djangoapps/content_staging/filters.py index 3759313b5a98..a6e0741c3129 100644 --- a/openedx/core/djangoapps/content_staging/filters.py +++ b/openedx/core/djangoapps/content_staging/filters.py @@ -53,6 +53,3 @@ def remove_large_data(fd: StagedContentFileData): return fd return {"file_datas": [remove_large_data(fd) for fd in file_datas]} - -#class DetectPythonLibZip(PipelineStep): -# python_lib_filename = getattr(settings, 'PYTHON_LIB_FILENAME', DEFAULT_PYTHON_LIB_FILENAME) diff --git a/openedx/core/lib/xblock_serializer/block_serializer.py b/openedx/core/lib/xblock_serializer/block_serializer.py index 87110471aa4d..9b6e970b114f 100644 --- a/openedx/core/lib/xblock_serializer/block_serializer.py +++ b/openedx/core/lib/xblock_serializer/block_serializer.py @@ -4,22 +4,15 @@ from __future__ import annotations import logging import os -from typing import NamedTuple from lxml import etree +from .data import StaticFile from . import utils log = logging.getLogger(__name__) -class StaticFile(NamedTuple): - """ A static file required by an XBlock """ - name: str - url: str | None - data: bytes | None - - class XBlockSerializer: """ A class that can serialize an XBlock to OLX. @@ -45,6 +38,11 @@ def __init__(self, block): if path not in [sf.name for sf in self.static_files]: self.static_files.append(StaticFile(name=path, url=asset['url'], data=None)) + if block.scope_ids.usage_id.block_type == 'problem': + py_lib_zip_file = utils.get_python_lib_zip_if_using(self.olx_str, course_key) + if py_lib_zip_file: + self.static_files.append(py_lib_zip_file) + def _serialize_block(self, block) -> etree.Element: """ Serialize an XBlock to OLX/XML. """ if block.scope_ids.usage_id.block_type == 'html': diff --git a/openedx/core/lib/xblock_serializer/data.py b/openedx/core/lib/xblock_serializer/data.py new file mode 100644 index 000000000000..141fff7dfd57 --- /dev/null +++ b/openedx/core/lib/xblock_serializer/data.py @@ -0,0 +1,12 @@ +""" +Simple data structures used for XBlock serialization +""" +from __future__ import annotations +from typing import NamedTuple + + +class StaticFile(NamedTuple): + """ A static file required by an XBlock """ + name: str + url: str | None + data: bytes | None diff --git a/openedx/core/lib/xblock_serializer/test_api.py b/openedx/core/lib/xblock_serializer/test_api.py index 3d9d67333cd0..2867b4ae082f 100644 --- a/openedx/core/lib/xblock_serializer/test_api.py +++ b/openedx/core/lib/xblock_serializer/test_api.py @@ -4,9 +4,10 @@ from xml.etree import ElementTree from openedx.core.djangolib.testing.utils import skip_unless_cms -from xmodule.modulestore.django import modulestore -from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase +from xmodule.modulestore.django import contentstore, modulestore +from xmodule.modulestore.tests.django_utils import SharedModuleStoreTestCase, upload_file_to_course from xmodule.modulestore.tests.factories import BlockFactory, CourseFactory, ToyCourseFactory +from xmodule.util.sandboxing import DEFAULT_PYTHON_LIB_FILENAME from . import api @@ -191,3 +192,55 @@ def test_export_sequential_blockstore(self): """) + + def test_capa_python_lib(self): + """ Test capa problem blocks with and without python_lib.zip """ + course = CourseFactory.create(display_name='Python Testing course', run="PY") + upload_file_to_course( + course_key=course.id, + contentstore=contentstore(), + source_file='./common/test/data/uploads/python_lib.zip', + target_filename=DEFAULT_PYTHON_LIB_FILENAME, + ) + + regular_problem = BlockFactory.create( + parent_location=course.location, + category="problem", + display_name="Problem No Python", + max_attempts=3, + data="", + ) + + python_problem = BlockFactory.create( + parent_location=course.location, + category="problem", + display_name="Python Problem", + data='This uses python: ...', + ) + + # The regular problem doesn't use python so shouldn't contain python_lib.zip: + + serialized = api.serialize_xblock_to_olx(regular_problem) + assert not serialized.static_files + self.assertXmlEqual( + serialized.olx_str, + """ + + + + """ + ) + + # The python problem should contain python_lib.zip: + + serialized = api.serialize_xblock_to_olx(python_problem) + assert len(serialized.static_files) == 1 + assert serialized.static_files[0].name == "python_lib.zip" + self.assertXmlEqual( + serialized.olx_str, + """ + + This uses python: ... + + """ + ) diff --git a/openedx/core/lib/xblock_serializer/utils.py b/openedx/core/lib/xblock_serializer/utils.py index fd2253ccad44..6541aaa7f791 100644 --- a/openedx/core/lib/xblock_serializer/utils.py +++ b/openedx/core/lib/xblock_serializer/utils.py @@ -1,21 +1,26 @@ """ Helper functions for XBlock serialization """ +from __future__ import annotations import logging import re from contextlib import contextmanager +from django.conf import settings from fs.memoryfs import MemoryFS from fs.wrapfs import WrapFS from opaque_keys import InvalidKeyError from opaque_keys.edx.keys import AssetKey, CourseKey + +from common.djangoapps.static_replace import replace_static_urls from xmodule.assetstore.assetmgr import AssetManager from xmodule.contentstore.content import StaticContent from xmodule.exceptions import NotFoundError from xmodule.modulestore.exceptions import ItemNotFoundError +from xmodule.util.sandboxing import DEFAULT_PYTHON_LIB_FILENAME from xmodule.xml_block import XmlMixin -from common.djangoapps.static_replace import replace_static_urls +from .data import StaticFile log = logging.getLogger(__name__) @@ -89,6 +94,38 @@ def collect_assets_from_text(text, course_id, include_content=False): yield info +def get_python_lib_zip_if_using(olx: str, course_id: CourseKey) -> StaticFile | None: + """ + When python_lib is in use, capa problems that contain python code should be assumed to depend on it. + + Note: for any given problem that uses python, there is no way to tell if it + actually uses any imports from python_lib.zip because the imports could be + named anything. So we just have to assume that any python problems may be + using python_lib.zip + """ + MATCH_STRINGS = ( + '''', False), + ('''\n''', True), + ('''\n''', True), + ('''\n''', True), + ('''\n''', True), + ) + def test_has_python_script(self, olx: str, has_script: bool): + """ + Test the _has_python_script() helper + """ + assert utils._has_python_script(olx) == has_script # pylint: disable=protected-access diff --git a/openedx/core/lib/xblock_serializer/utils.py b/openedx/core/lib/xblock_serializer/utils.py index 6541aaa7f791..389571e4cb3b 100644 --- a/openedx/core/lib/xblock_serializer/utils.py +++ b/openedx/core/lib/xblock_serializer/utils.py @@ -103,29 +103,33 @@ def get_python_lib_zip_if_using(olx: str, course_id: CourseKey) -> StaticFile | named anything. So we just have to assume that any python problems may be using python_lib.zip """ - MATCH_STRINGS = ( - '