From 3d213d24579c121999146ca2f49ddc5d32c3fd06 Mon Sep 17 00:00:00 2001 From: Oludare Oludare Date: Wed, 31 May 2017 16:19:28 -0400 Subject: [PATCH 01/16] Cherry pick --- api/serializers.py | 8 ++++++-- api/urls.py | 3 +++ api/views/__init__.py | 1 + api/views/workflow.py | 4 ++-- 4 files changed, 12 insertions(+), 4 deletions(-) diff --git a/api/serializers.py b/api/serializers.py index 1eaf5f3f0..4c4fff633 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -5,7 +5,7 @@ from rest_framework_json_api import serializers from share import models -from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult +from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult, HarvestLogs from api import fields @@ -132,12 +132,16 @@ class Meta: 'is_active', 'gravatar', 'locale', 'time_zone', 'is_trusted' ) - class SourceSerializer(ShareModelSerializer): class Meta: model = models.Source fields = ('name', 'home_page', 'long_title', 'icon') +class HarvestLogSerializer(ShareModelSerializer): + class Meta: + model = models.HarvestLog + fields = '__all__' + class SiteBannerSerializer(ShareModelSerializer): color = serializers.SerializerMethodField() diff --git a/api/urls.py b/api/urls.py index ec60b1804..28b364806 100644 --- a/api/urls.py +++ b/api/urls.py @@ -14,6 +14,8 @@ from api.serializers import BaseShareSerializer from api.views.share import ShareObjectViewSet + + app_name = 'api' router = DefaultRouter() @@ -86,6 +88,7 @@ def register_url(self, subclass, viewset): register_route(r'rawdata', views.RawDatumViewSet) register_route(r'user', views.ShareUserViewSet) register_route(r'sources', views.SourceViewSet) +register_route(r'harvest', views.HarvestLogViewSet) router.register(r'normalizeddata', views.NormalizedDataViewSet, base_name='normalizeddata') diff --git a/api/views/__init__.py b/api/views/__init__.py index 68bec4096..9b557d372 100644 --- a/api/views/__init__.py +++ b/api/views/__init__.py @@ -5,3 +5,4 @@ from .registration import * # noqa from .schema import * # noqa from .banner import * # noqa +from .harvestlogs import * # noqa diff --git a/api/views/workflow.py b/api/views/workflow.py index 30e1c906c..e79fb70a9 100644 --- a/api/views/workflow.py +++ b/api/views/workflow.py @@ -20,8 +20,8 @@ from api.pagination import CursorPagination from api.authentication import APIV1TokenBackPortAuthentication from api.permissions import ReadOnlyOrTokenHasScopeOrIsAuthenticated -from api.serializers import FullNormalizedDataSerializer, BasicNormalizedDataSerializer, \ - RawDatumSerializer, ShareUserSerializer, SourceSerializer +from api.serializers import FullNormalizedDataSerializer, BasicNormalizedDataSerializer,RawDatumSerializer, ShareUserSerializer, SourceSerializer + from share.models import RawDatum, NormalizedData, Source, SourceConfig, Transformer, ShareUser from share.tasks import disambiguate from share.harvest.serialization import DictSerializer From 7c3f6bc6e0fcebd6224c06d897c669ea0b538e0c Mon Sep 17 00:00:00 2001 From: Oludare Oludare Date: Wed, 31 May 2017 16:22:59 -0400 Subject: [PATCH 02/16] harvestlogs view file --- api/views/harvestlogs.py | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 api/views/harvestlogs.py diff --git a/api/views/harvestlogs.py b/api/views/harvestlogs.py new file mode 100644 index 000000000..79b68b6e5 --- /dev/null +++ b/api/views/harvestlogs.py @@ -0,0 +1,10 @@ +from rest_framework import viewsets + +from api.serializers import HarvestLogSerializer +from share.models import HarvestLog + +class HarvestLogViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = HarvestLogSerializer + + def get_queryset(self): + return HarvestLog.objects.all() From 1f356d9d2594c731e38b8f7022da5b34d94408ae Mon Sep 17 00:00:00 2001 From: Oludare Oludare Date: Tue, 6 Jun 2017 09:48:56 -0400 Subject: [PATCH 03/16] Creating new endpoint for Source Config --- api/serializers.py | 4 ++++ api/urls.py | 1 + api/views/__init__.py | 1 + api/views/sourceConfig.py | 10 ++++++++++ 4 files changed, 16 insertions(+) create mode 100644 api/views/sourceConfig.py diff --git a/api/serializers.py b/api/serializers.py index 4c4fff633..39fd5f940 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -142,6 +142,10 @@ class Meta: model = models.HarvestLog fields = '__all__' +class SourceConfigSerializer(ShareModelSerializer): + class Meta: + model = models.SourceConfig + fields = '__all__' class SiteBannerSerializer(ShareModelSerializer): color = serializers.SerializerMethodField() diff --git a/api/urls.py b/api/urls.py index 28b364806..d48896a64 100644 --- a/api/urls.py +++ b/api/urls.py @@ -89,6 +89,7 @@ def register_url(self, subclass, viewset): register_route(r'user', views.ShareUserViewSet) register_route(r'sources', views.SourceViewSet) register_route(r'harvest', views.HarvestLogViewSet) +register_route(r'sourceConfig', views.SourceConfigViewSet) router.register(r'normalizeddata', views.NormalizedDataViewSet, base_name='normalizeddata') diff --git a/api/views/__init__.py b/api/views/__init__.py index 9b557d372..ca4da1cb9 100644 --- a/api/views/__init__.py +++ b/api/views/__init__.py @@ -6,3 +6,4 @@ from .schema import * # noqa from .banner import * # noqa from .harvestlogs import * # noqa +from .sourceConfig import * # noqa diff --git a/api/views/sourceConfig.py b/api/views/sourceConfig.py new file mode 100644 index 000000000..7485a3021 --- /dev/null +++ b/api/views/sourceConfig.py @@ -0,0 +1,10 @@ +from rest_framework import viewsets + +from api.serializers import SourceConfigSerializer +from share.models import SourceConfig + +class SourceConfigViewSet(viewsets.ReadOnlyModelViewSet): + serializer_class = SourceConfigSerializer + + def get_queryset(self): + return SourceConfig.objects.all() From 346130f3081e5a02e52e40660be1bfa9cb9c95ca Mon Sep 17 00:00:00 2001 From: Oludare Oludare Date: Tue, 6 Jun 2017 11:18:42 -0400 Subject: [PATCH 04/16] fix atempt --- api/views/sourceConfig.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/views/sourceConfig.py b/api/views/sourceConfig.py index 7485a3021..ecc76f054 100644 --- a/api/views/sourceConfig.py +++ b/api/views/sourceConfig.py @@ -1,5 +1,5 @@ from rest_framework import viewsets - +# trying to fix shit from api.serializers import SourceConfigSerializer from share.models import SourceConfig From bf052545055e6b2ccfa509093fb908fd9c2eb808 Mon Sep 17 00:00:00 2001 From: Oludare Oludare Date: Wed, 31 May 2017 16:19:28 -0400 Subject: [PATCH 05/16] Cherry pick --- api/serializers.py | 8 ++++++-- api/views/__init__.py | 2 +- api/views/sourceConfig.py | 31 +++++++++++++++++++++++++++---- api/views/workflow.py | 2 ++ 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/api/serializers.py b/api/serializers.py index 39fd5f940..3921ffb0e 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -5,10 +5,13 @@ from rest_framework_json_api import serializers from share import models -from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult, HarvestLogs -from api import fields +from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult, HarvestLog + +from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult, logs + +from api import fields class BaseShareSerializer(serializers.ModelSerializer): @@ -147,6 +150,7 @@ class Meta: model = models.SourceConfig fields = '__all__' + class SiteBannerSerializer(ShareModelSerializer): color = serializers.SerializerMethodField() diff --git a/api/views/__init__.py b/api/views/__init__.py index ca4da1cb9..1d6041d2e 100644 --- a/api/views/__init__.py +++ b/api/views/__init__.py @@ -5,5 +5,5 @@ from .registration import * # noqa from .schema import * # noqa from .banner import * # noqa -from .harvestlogs import * # noqa +from .harvestlog import * # noqa from .sourceConfig import * # noqa diff --git a/api/views/sourceConfig.py b/api/views/sourceConfig.py index ecc76f054..5c5f67689 100644 --- a/api/views/sourceConfig.py +++ b/api/views/sourceConfig.py @@ -1,10 +1,33 @@ from rest_framework import viewsets -# trying to fix shit +from api.views import ShareObjectViewSet +from rest_framework import filters from api.serializers import SourceConfigSerializer from share.models import SourceConfig +from api.pagination import FuzzyPageNumberPagination +from django.db.models import Count +from django.db.models import Aggregate +from share.models import HarvestLog +from django.db.models import OuterRef, Subquery +from django.db.models import IntegerField -class SourceConfigViewSet(viewsets.ReadOnlyModelViewSet): +class SourceConfigViewSet(ShareObjectViewSet): serializer_class = SourceConfigSerializer + pagination_class = FuzzyPageNumberPagination + queryset = SourceConfig.objects.all() - def get_queryset(self): - return SourceConfig.objects.all() + # def get_queryset(self): + # recent_harvests = HarvestLog.objects.filter( + # source_config_id = OuterRef(OuterRef('id')), + # status__in = [HarvestLog.STATUS.failed, HarvestLog.STATUS.succeeded] + # ).order_by('-date_started')[:10] + # recent_fails = HarvestLog.objects.filter( + # status = HarvestLog.STATUS.failed, + # id__in = Subquery(recent_harvests.values('id')), + # # output_field = models.IntegerField()) + # ) + # queryset = SourceConfig.objects.all().annotate( + # fails=Count(Subquery(recent_fails.values('id')), + # # output_field = models.IntegerField())) + # ) + # queryset = queryset.order_by('fails') + # return queryset diff --git a/api/views/workflow.py b/api/views/workflow.py index e79fb70a9..3d3abe495 100644 --- a/api/views/workflow.py +++ b/api/views/workflow.py @@ -56,6 +56,7 @@ class SourceViewSet(viewsets.ReadOnlyModelViewSet): def get_queryset(self): return Source.objects.exclude(icon='').exclude(is_deleted=True) + def create(self, request, *args, **kwargs): try: long_title = request.data['long_title'] @@ -129,6 +130,7 @@ def create(self, request, *args, **kwargs): ) + class NormalizedDataViewSet(viewsets.ModelViewSet): """View showing all normalized data in the SHARE Dataset. From 6f8a5aa1f1af2f853348c57125c06762eb91780e Mon Sep 17 00:00:00 2001 From: Oludare Oludare Date: Fri, 9 Jun 2017 13:42:57 -0400 Subject: [PATCH 06/16] no message --- api/urls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/urls.py b/api/urls.py index d48896a64..47daf496a 100644 --- a/api/urls.py +++ b/api/urls.py @@ -88,8 +88,8 @@ def register_url(self, subclass, viewset): register_route(r'rawdata', views.RawDatumViewSet) register_route(r'user', views.ShareUserViewSet) register_route(r'sources', views.SourceViewSet) -register_route(r'harvest', views.HarvestLogViewSet) -register_route(r'sourceConfig', views.SourceConfigViewSet) +register_route(r'HarvestLog', views.HarvestLogViewSet) +register_route(r'SourceConfig', views.SourceConfigViewSet) router.register(r'normalizeddata', views.NormalizedDataViewSet, base_name='normalizeddata') From 4edb803926bd131edcc954d6d9adb61143e7cdf2 Mon Sep 17 00:00:00 2001 From: Emily Carden Date: Mon, 19 Jun 2017 14:43:37 -0400 Subject: [PATCH 07/16] Cherry pick --- api/serializers.py | 2 +- api/views/harvestlogs.py | 7 +++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/api/serializers.py b/api/serializers.py index 3921ffb0e..737622f63 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -6,7 +6,7 @@ from share import models -from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult, HarvestLog +from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult, HarvestLog, SourceConfig from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult, logs diff --git a/api/views/harvestlogs.py b/api/views/harvestlogs.py index 79b68b6e5..7401ba970 100644 --- a/api/views/harvestlogs.py +++ b/api/views/harvestlogs.py @@ -1,10 +1,9 @@ from rest_framework import viewsets +from api.views import ShareObjectViewSet from api.serializers import HarvestLogSerializer from share.models import HarvestLog -class HarvestLogViewSet(viewsets.ReadOnlyModelViewSet): +class HarvestLogViewSet(ShareObjectViewSet): serializer_class = HarvestLogSerializer - - def get_queryset(self): - return HarvestLog.objects.all() + queryset = HarvestLog.objects.all() From 64f959bd2b6be98827b40f5d98b976e0c6cb41c7 Mon Sep 17 00:00:00 2001 From: Emily Carden Date: Mon, 19 Jun 2017 14:43:37 -0400 Subject: [PATCH 08/16] Cherry pick --- api/serializers.py | 1 + api/views/sourceConfig.py | 17 ----------------- 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/api/serializers.py b/api/serializers.py index 737622f63..074db7dd3 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -6,6 +6,7 @@ from share import models + from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult, HarvestLog, SourceConfig diff --git a/api/views/sourceConfig.py b/api/views/sourceConfig.py index 5c5f67689..ee31d233f 100644 --- a/api/views/sourceConfig.py +++ b/api/views/sourceConfig.py @@ -14,20 +14,3 @@ class SourceConfigViewSet(ShareObjectViewSet): serializer_class = SourceConfigSerializer pagination_class = FuzzyPageNumberPagination queryset = SourceConfig.objects.all() - - # def get_queryset(self): - # recent_harvests = HarvestLog.objects.filter( - # source_config_id = OuterRef(OuterRef('id')), - # status__in = [HarvestLog.STATUS.failed, HarvestLog.STATUS.succeeded] - # ).order_by('-date_started')[:10] - # recent_fails = HarvestLog.objects.filter( - # status = HarvestLog.STATUS.failed, - # id__in = Subquery(recent_harvests.values('id')), - # # output_field = models.IntegerField()) - # ) - # queryset = SourceConfig.objects.all().annotate( - # fails=Count(Subquery(recent_fails.values('id')), - # # output_field = models.IntegerField())) - # ) - # queryset = queryset.order_by('fails') - # return queryset From 1dea1848e547655e5527b8e074fff3b6a842183f Mon Sep 17 00:00:00 2001 From: Emily Carden Date: Wed, 21 Jun 2017 14:55:22 -0400 Subject: [PATCH 09/16] new backend filter for source config id's --- api/views/harvestlogs.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/api/views/harvestlogs.py b/api/views/harvestlogs.py index 7401ba970..02fa70b62 100644 --- a/api/views/harvestlogs.py +++ b/api/views/harvestlogs.py @@ -1,9 +1,24 @@ from rest_framework import viewsets +from rest_framework import filters from api.views import ShareObjectViewSet +from share.util import IDObfuscator from api.serializers import HarvestLogSerializer from share.models import HarvestLog + +class SourceConfigFilterBackend(filters.BaseFilterBackend): + """ + Filter that only allows users to see their own objects. + """ + def filter_queryset(self, request, queryset, view): + if 'source_config_id' in request.GET: + decoded = IDObfuscator.decode_id(request.GET['source_config_id']) + return queryset.filter(source_config_id=decoded) + else: + return queryset + class HarvestLogViewSet(ShareObjectViewSet): serializer_class = HarvestLogSerializer queryset = HarvestLog.objects.all() + filter_backends = (SourceConfigFilterBackend, ) From a2da01122d3d1fb424e0ed4cee91ebac67149b54 Mon Sep 17 00:00:00 2001 From: Oludare Oludare Date: Wed, 21 Jun 2017 15:53:49 -0400 Subject: [PATCH 10/16] Cherry pick --- api/views/harvestlogs.py | 8 +++++++- api/views/share.py | 2 ++ api/views/sourceConfig.py | 2 ++ docs/elasticsearch.rst | 1 + 4 files changed, 12 insertions(+), 1 deletion(-) diff --git a/api/views/harvestlogs.py b/api/views/harvestlogs.py index 02fa70b62..9d44c7ee4 100644 --- a/api/views/harvestlogs.py +++ b/api/views/harvestlogs.py @@ -3,9 +3,14 @@ from api.views import ShareObjectViewSet from share.util import IDObfuscator +from rest_framework import viewsets, views +# from rest_framework import filters +from django_filters.rest_framework import DjangoFilterBackend from api.serializers import HarvestLogSerializer from share.models import HarvestLog +from api.views import ShareObjectViewSet +<<<<<<< HEAD class SourceConfigFilterBackend(filters.BaseFilterBackend): """ @@ -19,6 +24,7 @@ def filter_queryset(self, request, queryset, view): return queryset class HarvestLogViewSet(ShareObjectViewSet): + queryset= HarvestLog.objects.all() serializer_class = HarvestLogSerializer - queryset = HarvestLog.objects.all() filter_backends = (SourceConfigFilterBackend, ) + filter_fields = ['source_config__id', 'status',] diff --git a/api/views/share.py b/api/views/share.py index 798762c2a..3a0ed5248 100644 --- a/api/views/share.py +++ b/api/views/share.py @@ -28,6 +28,7 @@ class ShareObjectViewSet(viewsets.ReadOnlyModelViewSet): # Override get_queryset to handle items marked as deleted. def get_queryset(self, list=True): + # import ipdb; ipdb.set_trace() queryset = super().get_queryset() if list and hasattr(queryset.model, 'is_deleted'): return queryset.exclude(is_deleted=True) @@ -35,6 +36,7 @@ def get_queryset(self, list=True): # Override to convert encoded pk to an actual pk def get_object(self): + import ipdb; ipdb.set_trace() queryset = self.filter_queryset(self.get_queryset(False)) # Perform the lookup filtering. diff --git a/api/views/sourceConfig.py b/api/views/sourceConfig.py index ee31d233f..4d4258a4e 100644 --- a/api/views/sourceConfig.py +++ b/api/views/sourceConfig.py @@ -14,3 +14,5 @@ class SourceConfigViewSet(ShareObjectViewSet): serializer_class = SourceConfigSerializer pagination_class = FuzzyPageNumberPagination queryset = SourceConfig.objects.all() + + diff --git a/docs/elasticsearch.rst b/docs/elasticsearch.rst index 5c6f59b37..3d0645b25 100644 --- a/docs/elasticsearch.rst +++ b/docs/elasticsearch.rst @@ -26,6 +26,7 @@ Elasticsearch can be used to search the following fields in the normalized data: 'contributors' 'funders' 'publishers' + 'id' Accessing the Search API From fcdf94ea6277155342fa2d9a178e11819b85feca Mon Sep 17 00:00:00 2001 From: Emily Carden Date: Wed, 19 Jul 2017 15:41:26 -0400 Subject: [PATCH 11/16] query for statuses --- api/views/harvestlogs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/views/harvestlogs.py b/api/views/harvestlogs.py index 9d44c7ee4..9a9d23549 100644 --- a/api/views/harvestlogs.py +++ b/api/views/harvestlogs.py @@ -20,6 +20,8 @@ def filter_queryset(self, request, queryset, view): if 'source_config_id' in request.GET: decoded = IDObfuscator.decode_id(request.GET['source_config_id']) return queryset.filter(source_config_id=decoded) + if 'status' in request.GET: + return queryset.filter(status=request.GET['status']) else: return queryset From aabb219eeb7f4bf7a6b45bd377789f4651b1455c Mon Sep 17 00:00:00 2001 From: Emily Carden Date: Tue, 8 Aug 2017 11:45:24 -0400 Subject: [PATCH 12/16] no message --- api/views/harvestlogs.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/views/harvestlogs.py b/api/views/harvestlogs.py index 9a9d23549..2cafc6f25 100644 --- a/api/views/harvestlogs.py +++ b/api/views/harvestlogs.py @@ -10,7 +10,6 @@ from share.models import HarvestLog from api.views import ShareObjectViewSet -<<<<<<< HEAD class SourceConfigFilterBackend(filters.BaseFilterBackend): """ @@ -22,6 +21,7 @@ def filter_queryset(self, request, queryset, view): return queryset.filter(source_config_id=decoded) if 'status' in request.GET: return queryset.filter(status=request.GET['status']) + else: return queryset From f876dd588f338e0ec2d4645691e6e4ef3ba0219e Mon Sep 17 00:00:00 2001 From: Emily Carden Date: Tue, 8 Aug 2017 11:52:32 -0400 Subject: [PATCH 13/16] no message --- api/views/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/views/__init__.py b/api/views/__init__.py index 1d6041d2e..ca4da1cb9 100644 --- a/api/views/__init__.py +++ b/api/views/__init__.py @@ -5,5 +5,5 @@ from .registration import * # noqa from .schema import * # noqa from .banner import * # noqa -from .harvestlog import * # noqa +from .harvestlogs import * # noqa from .sourceConfig import * # noqa From b1200f16a99fe315225d911551b46460059d62ba Mon Sep 17 00:00:00 2001 From: Emily Carden Date: Tue, 8 Aug 2017 14:56:24 -0400 Subject: [PATCH 14/16] Cherry pick --- api/views/harvestlogs.py | 25 ++++++++----------------- 1 file changed, 8 insertions(+), 17 deletions(-) diff --git a/api/views/harvestlogs.py b/api/views/harvestlogs.py index 2cafc6f25..4216a5cdf 100644 --- a/api/views/harvestlogs.py +++ b/api/views/harvestlogs.py @@ -1,32 +1,23 @@ from rest_framework import viewsets from rest_framework import filters +from django_filters.filters import MultipleChoiceFilter from api.views import ShareObjectViewSet from share.util import IDObfuscator -from rest_framework import viewsets, views -# from rest_framework import filters -from django_filters.rest_framework import DjangoFilterBackend from api.serializers import HarvestLogSerializer from share.models import HarvestLog -from api.views import ShareObjectViewSet - -class SourceConfigFilterBackend(filters.BaseFilterBackend): - """ - Filter that only allows users to see their own objects. - """ - def filter_queryset(self, request, queryset, view): +class SourceConfigFilterBackend(MultipleChoiceFilter): + def filter_queryset(self, request, queryset, view, conjoined=True): if 'source_config_id' in request.GET: decoded = IDObfuscator.decode_id(request.GET['source_config_id']) - return queryset.filter(source_config_id=decoded) + queryset = queryset.filter(source_config_id=decoded) if 'status' in request.GET: - return queryset.filter(status=request.GET['status']) - - else: - return queryset + queryset = queryset.filter(status__in=request.GET.getlist('status')) + return queryset class HarvestLogViewSet(ShareObjectViewSet): - queryset= HarvestLog.objects.all() serializer_class = HarvestLogSerializer + queryset = HarvestLog.objects.all() filter_backends = (SourceConfigFilterBackend, ) - filter_fields = ['source_config__id', 'status',] + filter_fields = ('source_config_id', 'status',) From 3db1a1c99728def1016a485281f57cc96223f1a5 Mon Sep 17 00:00:00 2001 From: Emily Carden Date: Fri, 11 Aug 2017 12:17:58 -0400 Subject: [PATCH 15/16] Formatting from code review --- api/serializers.py | 12 +++++------- api/urls.py | 6 ++---- api/views/__init__.py | 4 ++-- api/views/{harvestlogs.py => harvest_logs.py} | 11 ++++++----- api/views/share.py | 8 +++----- api/views/{sourceConfig.py => source_config.py} | 15 ++++----------- api/views/workflow.py | 15 +++++++-------- 7 files changed, 29 insertions(+), 42 deletions(-) rename api/views/{harvestlogs.py => harvest_logs.py} (85%) rename api/views/{sourceConfig.py => source_config.py} (57%) diff --git a/api/serializers.py b/api/serializers.py index 074db7dd3..2851a6a8d 100644 --- a/api/serializers.py +++ b/api/serializers.py @@ -1,19 +1,14 @@ from collections import OrderedDict - from django.contrib.auth.models import AnonymousUser from rest_framework_json_api import serializers from share import models - - -from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult, HarvestLog, SourceConfig - - -from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult, logs +from share.models import ProviderRegistration, SiteBanner, CeleryTaskResult from api import fields + class BaseShareSerializer(serializers.ModelSerializer): def __init__(self, *args, **kwargs): @@ -136,16 +131,19 @@ class Meta: 'is_active', 'gravatar', 'locale', 'time_zone', 'is_trusted' ) + class SourceSerializer(ShareModelSerializer): class Meta: model = models.Source fields = ('name', 'home_page', 'long_title', 'icon') + class HarvestLogSerializer(ShareModelSerializer): class Meta: model = models.HarvestLog fields = '__all__' + class SourceConfigSerializer(ShareModelSerializer): class Meta: model = models.SourceConfig diff --git a/api/urls.py b/api/urls.py index 47daf496a..b98e5e2cc 100644 --- a/api/urls.py +++ b/api/urls.py @@ -14,8 +14,6 @@ from api.serializers import BaseShareSerializer from api.views.share import ShareObjectViewSet - - app_name = 'api' router = DefaultRouter() @@ -88,8 +86,8 @@ def register_url(self, subclass, viewset): register_route(r'rawdata', views.RawDatumViewSet) register_route(r'user', views.ShareUserViewSet) register_route(r'sources', views.SourceViewSet) -register_route(r'HarvestLog', views.HarvestLogViewSet) -register_route(r'SourceConfig', views.SourceConfigViewSet) +register_route(r'harvest_logs', views.HarvestLogViewSet) +register_route(r'source_config', views.SourceConfigViewSet) router.register(r'normalizeddata', views.NormalizedDataViewSet, base_name='normalizeddata') diff --git a/api/views/__init__.py b/api/views/__init__.py index ca4da1cb9..dc1df2245 100644 --- a/api/views/__init__.py +++ b/api/views/__init__.py @@ -5,5 +5,5 @@ from .registration import * # noqa from .schema import * # noqa from .banner import * # noqa -from .harvestlogs import * # noqa -from .sourceConfig import * # noqa +from .harvest_logs import * # noqa +from .source_config import * # noqa diff --git a/api/views/harvestlogs.py b/api/views/harvest_logs.py similarity index 85% rename from api/views/harvestlogs.py rename to api/views/harvest_logs.py index 4216a5cdf..370f98140 100644 --- a/api/views/harvestlogs.py +++ b/api/views/harvest_logs.py @@ -1,14 +1,14 @@ -from rest_framework import viewsets -from rest_framework import filters from django_filters.filters import MultipleChoiceFilter -from api.views import ShareObjectViewSet + +from share.models import HarvestLog from share.util import IDObfuscator from api.serializers import HarvestLogSerializer -from share.models import HarvestLog +from api.views import ShareObjectViewSet + class SourceConfigFilterBackend(MultipleChoiceFilter): - def filter_queryset(self, request, queryset, view, conjoined=True): + def filter_queryset(self, request, queryset, view): if 'source_config_id' in request.GET: decoded = IDObfuscator.decode_id(request.GET['source_config_id']) queryset = queryset.filter(source_config_id=decoded) @@ -16,6 +16,7 @@ def filter_queryset(self, request, queryset, view, conjoined=True): queryset = queryset.filter(status__in=request.GET.getlist('status')) return queryset + class HarvestLogViewSet(ShareObjectViewSet): serializer_class = HarvestLogSerializer queryset = HarvestLog.objects.all() diff --git a/api/views/share.py b/api/views/share.py index 3a0ed5248..a76566e26 100644 --- a/api/views/share.py +++ b/api/views/share.py @@ -9,13 +9,13 @@ from django.views.decorators.http import require_GET from django.views.generic.base import RedirectView +from share.util import IDObfuscator, InvalidID +from share.models import Source + from api.pagination import CursorPagination from api.permissions import IsDeletedPremissions from api import serializers as api_serializers -from share.util import IDObfuscator, InvalidID -from share.models import Source - class ShareObjectViewSet(viewsets.ReadOnlyModelViewSet): ordering = ('-id', ) @@ -28,7 +28,6 @@ class ShareObjectViewSet(viewsets.ReadOnlyModelViewSet): # Override get_queryset to handle items marked as deleted. def get_queryset(self, list=True): - # import ipdb; ipdb.set_trace() queryset = super().get_queryset() if list and hasattr(queryset.model, 'is_deleted'): return queryset.exclude(is_deleted=True) @@ -36,7 +35,6 @@ def get_queryset(self, list=True): # Override to convert encoded pk to an actual pk def get_object(self): - import ipdb; ipdb.set_trace() queryset = self.filter_queryset(self.get_queryset(False)) # Perform the lookup filtering. diff --git a/api/views/sourceConfig.py b/api/views/source_config.py similarity index 57% rename from api/views/sourceConfig.py rename to api/views/source_config.py index 4d4258a4e..e634ee94b 100644 --- a/api/views/sourceConfig.py +++ b/api/views/source_config.py @@ -1,18 +1,11 @@ -from rest_framework import viewsets -from api.views import ShareObjectViewSet -from rest_framework import filters -from api.serializers import SourceConfigSerializer from share.models import SourceConfig + from api.pagination import FuzzyPageNumberPagination -from django.db.models import Count -from django.db.models import Aggregate -from share.models import HarvestLog -from django.db.models import OuterRef, Subquery -from django.db.models import IntegerField +from api.serializers import SourceConfigSerializer +from api.views import ShareObjectViewSet + class SourceConfigViewSet(ShareObjectViewSet): serializer_class = SourceConfigSerializer pagination_class = FuzzyPageNumberPagination queryset = SourceConfig.objects.all() - - diff --git a/api/views/workflow.py b/api/views/workflow.py index 3d3abe495..b48df7216 100644 --- a/api/views/workflow.py +++ b/api/views/workflow.py @@ -16,18 +16,19 @@ from rest_framework.serializers import ValidationError from rest_framework.permissions import DjangoModelPermissionsOrAnonReadOnly -from api import schemas -from api.pagination import CursorPagination -from api.authentication import APIV1TokenBackPortAuthentication -from api.permissions import ReadOnlyOrTokenHasScopeOrIsAuthenticated -from api.serializers import FullNormalizedDataSerializer, BasicNormalizedDataSerializer,RawDatumSerializer, ShareUserSerializer, SourceSerializer - from share.models import RawDatum, NormalizedData, Source, SourceConfig, Transformer, ShareUser from share.tasks import disambiguate from share.harvest.serialization import DictSerializer from share.harvest.base import FetchResult from share.util import IDObfuscator +from api import schemas +from api.pagination import CursorPagination +from api.authentication import APIV1TokenBackPortAuthentication +from api.permissions import ReadOnlyOrTokenHasScopeOrIsAuthenticated +from api.serializers import FullNormalizedDataSerializer, BasicNormalizedDataSerializer, RawDatumSerializer, ShareUserSerializer, SourceSerializer + + logger = logging.getLogger(__name__) __all__ = ('NormalizedDataViewSet', 'RawDatumViewSet', 'ShareUserViewSet', 'SourceViewSet', 'V1DataView') @@ -56,7 +57,6 @@ class SourceViewSet(viewsets.ReadOnlyModelViewSet): def get_queryset(self): return Source.objects.exclude(icon='').exclude(is_deleted=True) - def create(self, request, *args, **kwargs): try: long_title = request.data['long_title'] @@ -130,7 +130,6 @@ def create(self, request, *args, **kwargs): ) - class NormalizedDataViewSet(viewsets.ModelViewSet): """View showing all normalized data in the SHARE Dataset. From abb21264eae04e1126e21dec64471f0ea43856a3 Mon Sep 17 00:00:00 2001 From: Emily Carden Date: Fri, 11 Aug 2017 13:23:52 -0400 Subject: [PATCH 16/16] Change url names --- api/urls.py | 2 +- api/views/harvest_logs.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/api/urls.py b/api/urls.py index b98e5e2cc..bfa7e8a61 100644 --- a/api/urls.py +++ b/api/urls.py @@ -87,7 +87,7 @@ def register_url(self, subclass, viewset): register_route(r'user', views.ShareUserViewSet) register_route(r'sources', views.SourceViewSet) register_route(r'harvest_logs', views.HarvestLogViewSet) -register_route(r'source_config', views.SourceConfigViewSet) +register_route(r'source_configs', views.SourceConfigViewSet) router.register(r'normalizeddata', views.NormalizedDataViewSet, base_name='normalizeddata') diff --git a/api/views/harvest_logs.py b/api/views/harvest_logs.py index 370f98140..46c0e6754 100644 --- a/api/views/harvest_logs.py +++ b/api/views/harvest_logs.py @@ -4,7 +4,7 @@ from share.util import IDObfuscator from api.serializers import HarvestLogSerializer -from api.views import ShareObjectViewSet +# from api.views import ShareObjectViewSet class SourceConfigFilterBackend(MultipleChoiceFilter):