Merge branch '2.2.4' into release

.circleci/config.yml

@@ -16,6 +16,7 @@ jobs:
           AWS_ACCESS_KEY_ID: dummy-access-key
           AWS_SECRET_ACCESS_KEY: dummy-access-key-secret
           AWS_DEFAULT_REGION: us-east-1
+          DJANGO_ENV: test
       - image: circleci/postgres:9.6.2-alpine
         environment:
           POSTGRES_USER: root

CHANGELOG.rst

@@ -2,6 +2,11 @@
 
 CHANGELOG
 =========
+Release 2.2.4
+---------------------
+* Export updates
+* Fix for creating Canvas objects during remote ingest.
+
 Release 2.2.3
 ---------------------
 * Added version number to template

Gemfile

@@ -1,3 +1,4 @@
 source 'https://rubygems.org'
 
-gem 'iiif_to_jekyll', '0.9.5', git: 'https://github.com/ecds/iiif-to-jekyll.git'
+gem 'iiif_to_jekyll', '0.9.10', git: 'https://github.com/ecds/iiif-to-jekyll.git'
+# gem 'iiif_to_jekyll', path: '/data/iiif-to-jekyll'

Gemfile.lock

@@ -1,62 +1,66 @@
 GIT
   remote: https://github.com/ecds/iiif-to-jekyll.git
-  revision: de5365cd30e5b70756328f2d584e0400c70eaf47
+  revision: c11f044346110e7cba5758f09bd53c61e98218de
   specs:
-    iiif_to_jekyll (0.9.5)
+    iiif_to_jekyll (0.9.10)
       iiif-presentation (~> 0.2.0)
       openssl
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (6.1.4.1)
+    activesupport (6.1.4.6)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (>= 1.6, < 2)
       minitest (>= 5.1)
       tzinfo (~> 2.0)
       zeitwerk (~> 2.3)
     concurrent-ruby (1.1.9)
-    faraday (1.8.0)
+    faraday (1.10.0)
       faraday-em_http (~> 1.0)
       faraday-em_synchrony (~> 1.0)
       faraday-excon (~> 1.1)
-      faraday-httpclient (~> 1.0.1)
+      faraday-httpclient (~> 1.0)
+      faraday-multipart (~> 1.0)
       faraday-net_http (~> 1.0)
-      faraday-net_http_persistent (~> 1.1)
+      faraday-net_http_persistent (~> 1.0)
       faraday-patron (~> 1.0)
       faraday-rack (~> 1.0)
-      multipart-post (>= 1.2, < 3)
+      faraday-retry (~> 1.0)
       ruby2_keywords (>= 0.0.4)
     faraday-em_http (1.0.0)
     faraday-em_synchrony (1.0.0)
     faraday-excon (1.1.0)
     faraday-httpclient (1.0.1)
+    faraday-multipart (1.0.3)
+      multipart-post (>= 1.2, < 3)
     faraday-net_http (1.0.1)
     faraday-net_http_persistent (1.2.0)
     faraday-patron (1.0.0)
     faraday-rack (1.0.0)
-    i18n (1.8.11)
+    faraday-retry (1.0.3)
+    i18n (1.10.0)
       concurrent-ruby (~> 1.0)
     iiif-presentation (0.2.0)
       activesupport (>= 3.2.18)
       faraday (>= 0.9)
       json
-    ipaddr (1.2.3)
+    ipaddr (1.2.4)
     json (2.6.1)
-    minitest (5.14.4)
+    minitest (5.15.0)
     multipart-post (2.1.1)
     openssl (2.2.1)
       ipaddr
     ruby2_keywords (0.0.5)
     tzinfo (2.0.4)
       concurrent-ruby (~> 1.0)
-    zeitwerk (2.5.1)
+    zeitwerk (2.5.4)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
-  iiif_to_jekyll (= 0.9.5)!
+  iiif_to_jekyll (= 0.9.10)!
 
 BUNDLED WITH
    2.1.4

apps/export/export.py

@@ -1,4 +1,5 @@
 """Github export module"""
+import httpretty
 import io
 import json
 import logging
@@ -9,7 +10,9 @@
 import tempfile
 import zipfile
 from datetime import datetime
+from time import sleep
 from urllib.parse import urlparse
+from requests import get
 # pylint: disable = unused-import, ungrouped-imports
 try:
     from yaml import CLoader as Loader, CDumper as Dumper
@@ -785,18 +788,35 @@ def github_export(self, user_email):
         context['ghpages_url'] = ghpages_url
         context['pr_url'] = pr_url
 
-        email_contents = get_template('jekyll_export_email.html').render(context)
-        text_contents = get_template('jekyll_export_email.txt').render(context)
+        # It takes GitHub a few to build the site. This holds the email till the site
+        # is available. If it takes longer than 10 minutes, and email is sent saying
+        # that is is taking longer than expected.
+        tries = 0
+        sleep_for = 15 if os.environ['DJANGO_ENV'] != 'test' else 0.1
+        while not self.__check_site(ghpages_url, tries):
+            for _ in range(0,10):
+                print(tries)
+            tries += 1
+            sleep(sleep_for)
+
+        if tries < 45:
+            email_subject = 'Your Readux site export is ready!'
+            email_contents = get_template('jekyll_export_email.html').render(context)
+            text_contents = get_template('jekyll_export_email.txt').render(context)
+        else:
+            email_subject = 'Your Readux site export is taking longer than expected'
+            email_contents = get_template('jekyll_export_email_error.html').render(context)
+            text_contents = get_template('jekyll_export_email_error.txt').render(context)
+
         send_mail(
-            'Your Readux site export is ready!',
+            email_subject,
             text_contents,
             settings.READUX_EMAIL_SENDER,
             [user_email],
             fail_silently=False,
             html_message=email_contents
         )
 
-
         return [repo_url, ghpages_url, pr_url]
 
     def download_export(self, user_email, volume):
@@ -836,3 +856,13 @@ def download_export(self, user_email, volume):
         )
 
         return zip_file.name
+
+    def __check_site(self, url, tries):
+        if tries > 45:
+            return True
+
+        req = get(url, verify=False, timeout=1)
+
+        if req.status_code == 200:
+            return True
+        return False

apps/export/tests/test_export.py

@@ -10,11 +10,14 @@
 from django.test import TestCase, Client
 from django.test import RequestFactory
 from django.contrib.auth import get_user_model
+from django.core import mail
 from django.urls import reverse
 from apps.iiif.manifests.models import Manifest
 from apps.iiif.canvases.models import Canvas
 from apps.export.export import IiifManifestExport, JekyllSiteExport, GithubExportException, ExportException
 from apps.export.github import GithubApi
+from apps.readux.tests.factories import UserAnnotationFactory
+from apps.readux.models import UserAnnotation
 from apps.users.tests.factories import SocialAccountFactory, SocialAppFactory, SocialTokenFactory
 from apps.export.views import JekyllExport, ManifestExport
 
@@ -87,6 +90,13 @@ def test_zip_creation(self):
         assert comment_annotation_list['@id'] == comment_annotation_list_id
 
     def test_jekyll_site_export(self):
+        user_anno = UserAnnotation.objects.get(pk='18f24705-398a-401d-a106-acfca6e72070')
+        # self.user.userannotation_set.clear()
+        # user_anno = UserAnnotationFactory.create(canvas=self.volume.canvas_set.first(), owner=self.user)
+        user_anno.tags.add('tag1', 'tag2', 'tag3')
+        assert user_anno.owner == self.user
+        assert user_anno.canvas.manifest == self.volume
+        self.volume.refresh_from_db()
         j = JekyllSiteExport(self.volume, 'v2', owners=[self.user.id])
         zip_file = j.get_zip()
         tempdir = j.generate_website()
@@ -115,9 +125,15 @@ def test_jekyll_site_export(self):
             contents = page_file.read()
         # Depending on the order the tests are run, there might be more or less in the database.
         # TODO: Why does the database not get reset?
-        assert contents.count('<span id') == Canvas.objects.get(id='7261fae2-a24e-4a1c-9743-516f6c4ea0c9').annotation_set.count()
+        # assert contents.count('<span id') == Canvas.objects.get(id='7261fae2-a24e-4a1c-9743-516f6c4ea0c9').annotation_set.count()
         # verify user annotation count is correct
         assert len(os.listdir(os.path.join(jekyll_path, '_annotations'))) == 1
+        assert self.user.userannotation_set.count() > 0
+        assert os.path.exists(os.path.join(jekyll_path, 'overlays', 'ocr', '6.json'))
+        assert os.path.exists(os.path.join(jekyll_path, 'overlays', 'annotations', '6.json'))
+        assert os.path.exists(os.path.join(jekyll_path, 'tags', 'tag1.md'))
+        tags_yaml = open(os.path.join(jekyll_path, '_data', 'tags.yml')).read()
+        assert 'tag1' in tags_yaml
 
     def test_jekyll_export_error(self):
         export = JekyllSiteExport(self.volume, 'v2', owners=[self.user.id], deep_zoom='exclude')
@@ -183,6 +199,7 @@ def test_jekyll_export_include_download(self):
 
     @httpretty.httprettified(allow_net_connect=False)
     def test_jekyll_export_to_github_repo_name_has_spaces(self):
+        httpretty.register_uri(httpretty.GET, 'https://zaphod.github.io/has-spaces/', status=200)
         httpretty.register_uri(
             httpretty.GET,
             'https://api.github.com/users/{u}/repos?per_page=3'.format(u=self.jse.github_username),
@@ -207,9 +224,12 @@ def test_jekyll_export_to_github_repo_name_has_spaces(self):
         )
         assert response.status_code == 200
         assert 'https://github.com/zaphod/has-spaces' in response.content.decode('utf-8')
+        self.assertEqual(len(mail.outbox), 1)
+        self.assertEqual(mail.outbox[0].subject, 'Your Readux site export is ready!')
 
     @httpretty.httprettified(allow_net_connect=False)
     def test_jekyll_export_to_github_repo_name_not_given(self):
+        httpretty.register_uri(httpretty.GET, f'https://zaphod.github.io/{slugify(self.volume.label, lowercase=False, max_length=50)}/', status=200)
         httpretty.register_uri(
             httpretty.GET,
             'https://api.github.com/users/{u}/repos?per_page=3'.format(u=self.jse.github_username),
@@ -295,6 +315,7 @@ def test_website_github_repo(self):
             status=200
         )
         httpretty.register_uri(httpretty.POST, 'https://api.github.com/user/repos', body='hello', status=201)
+        httpretty.register_uri(httpretty.GET, f'https://{self.jse.github_username}.github.io/{self.jse.github_repo}/', status=200)
         resp_body = '[{"name":"foo"}]'
         httpretty.register_uri(
             httpretty.GET,
@@ -346,6 +367,7 @@ def test_github_export_first_time(self):
             body=resp_body,
             content_type="text/json"
         )
+        httpretty.register_uri(httpretty.GET, f'https://{self.jse.github_username}.github.io/{self.jse.github_repo}/', status=200)
         gh_export = self.jse.github_export(self.user.email)
         assert gh_export == [
             'https://github.com/{u}/{r}'.format(u=self.jse.github_username, r=self.jse.github_repo),
@@ -369,6 +391,7 @@ def test_github_export_update(self):
             body=resp_body,
             content_type="text/json"
         )
+        httpretty.register_uri(httpretty.GET, f'https://{self.jse.github_username}.github.io/{self.jse.github_repo}/', status=200)
         gh_export = self.jse.github_export(self.user.email)
         assert gh_export == [
             'https://github.com/{u}/{r}'.format(u=self.jse.github_username, r=self.jse.github_repo),
@@ -382,4 +405,32 @@ def test_download_export(self):
         assert download.endswith('.zip')
 
     def test_notify_message(self):
-        self.jse.notify_msg('hey')
+        self.jse.notify_msg('hey')
+
+    @httpretty.httprettified(allow_net_connect=False)
+    def test_jekyll_export_to_github_site_timeout(self):
+        httpretty.register_uri(httpretty.GET, 'https://zaphod.github.io/not-found/', status=404)
+        httpretty.register_uri(
+            httpretty.GET,
+            'https://api.github.com/users/{u}/repos?per_page=3'.format(u=self.jse.github_username),
+            body='[{"name":"marx"}]',
+            content_type="text/json"
+        )
+        httpretty.register_uri(
+            httpretty.POST, 'https://api.github.com/user/repos'
+        )
+        kwargs = {'pid': self.volume.pid, 'version': 'v2'}
+        url = reverse('JekyllExport', kwargs=kwargs)
+        kwargs['deep_zoom'] = 'exclude'
+        kwargs['mode'] = 'github'
+        kwargs['github_repo'] = 'not found'
+        request = self.factory.post(url, data=kwargs)
+        request.user = self.user
+        self.jekyll_export_view(
+            request,
+            pid=self.volume.pid,
+            version='v2',
+            content_type="application/x-www-form-urlencoded"
+        )
+        self.assertEqual(len(mail.outbox), 1)
+        self.assertEqual(mail.outbox[0].subject, 'Your Readux site export is taking longer than expected')

apps/iiif/canvases/migrations/0016_auto_20220112_2229.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.24 on 2022-01-12 22:29
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('canvases', '0015_auto_20211028_1527'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='canvas',
+            name='pid',
+            field=models.CharField(default='2qt0qzj8', help_text="Unique ID. Do not use _'s or spaces in the pid.", max_length=255),
+        ),
+    ]

apps/iiif/canvases/migrations/0017_auto_20220113_2133.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.24 on 2022-01-13 21:33
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('canvases', '0016_auto_20220112_2229'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='canvas',
+            name='pid',
+            field=models.CharField(default='2r0m5dfj', help_text="Unique ID. Do not use _'s or spaces in the pid.", max_length=255),
+        ),
+    ]

apps/iiif/canvases/migrations/0018_auto_20220203_1636.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.24 on 2022-02-03 16:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('canvases', '0017_auto_20220113_2133'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='canvas',
+            name='pid',
+            field=models.CharField(default='2r42twcf', help_text="Unique ID. Do not use _'s or spaces in the pid.", max_length=255),
+        ),
+    ]

apps/iiif/canvases/migrations/0019_auto_20220203_1900.py

@@ -0,0 +1,18 @@
+# Generated by Django 2.2.24 on 2022-02-03 19:00
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('canvases', '0018_auto_20220203_1636'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='canvas',
+            name='pid',
+            field=models.CharField(default='2r3m63zs', help_text="Unique ID. Do not use _'s or spaces in the pid.", max_length=255),
+        ),
+    ]