From 4e41af0bd89c5cb1dd5425126a63e290b35b7545 Mon Sep 17 00:00:00 2001 From: Nikolai R Kristiansen Date: Wed, 25 Oct 2017 21:36:20 +0200 Subject: [PATCH 1/3] Bump to latest django, drf TODO: Fix regex filter (write a test) --- dpdapi/filters.py | 18 +++++++++--------- dpdapi/models.py | 5 ----- dpdapi/serializers.py | 3 +++ dpdapi/views.py | 17 ++++++++--------- dpdapisite/settings.py | 7 +------ dpdapisite/urls.py | 2 +- requirements.txt | 16 ++++++++-------- 7 files changed, 30 insertions(+), 38 deletions(-) diff --git a/dpdapi/filters.py b/dpdapi/filters.py index 13f6cc8..54b93ce 100644 --- a/dpdapi/filters.py +++ b/dpdapi/filters.py @@ -1,18 +1,18 @@ import django_filters from dpdapi.models import Alias -from rest_framework.filters import BaseFilterBackend + +from django_filters import filters + +filters.LOOKUP_TYPES = ['iregex', 'exact', 'iexact'] -class AliasRegexFilter(django_filters.FilterSet): +class AliasFilter(django_filters.FilterSet): + # TODO: These do not work # Ref: https://docs.djangoproject.com/en/1.7/ref/models/querysets/#iregex - source_regex = django_filters.CharFilter(name="source", lookup_type='iregex') - destination_regex = django_filters.CharFilter(name="destination", lookup_type='iregex') + source_regex = django_filters.CharFilter(name='source', lookup_expr=['iregex']) + destination_regex = django_filters.CharFilter(name='destination', lookup_expr=['iregex']) class Meta: model = Alias - - -class AliasRegexFilterBackend(BaseFilterBackend): - def filter_queryset(self, request, queryset, view): - return AliasRegexFilter(request.query_params, queryset=queryset).qs + fields = ['domain', 'domain__name', 'source_regex', 'destination_regex'] diff --git a/dpdapi/models.py b/dpdapi/models.py index e0e44ef..2447983 100644 --- a/dpdapi/models.py +++ b/dpdapi/models.py @@ -1,6 +1,4 @@ -from __future__ import unicode_literals from django.db import models -from django.utils.encoding import python_2_unicode_compatible class BaseModel(models.Model): @@ -11,7 +9,6 @@ class BaseModel(models.Model): abstract = True -@python_2_unicode_compatible class Alias(BaseModel): source = models.EmailField(max_length=254) destination = models.EmailField(max_length=254) @@ -32,7 +29,6 @@ class Alias(BaseModel): unique_together = ('source', 'destination', 'domain') -@python_2_unicode_compatible class Domain(BaseModel): name = models.CharField(max_length=254, unique=True) @@ -45,7 +41,6 @@ class Domain(BaseModel): super(Domain, self).save(*args, **kwargs) -@python_2_unicode_compatible class User(BaseModel): email = models.EmailField(max_length=254, unique=True) password = models.CharField(max_length=109) diff --git a/dpdapi/serializers.py b/dpdapi/serializers.py index f2c73b2..0ac067a 100644 --- a/dpdapi/serializers.py +++ b/dpdapi/serializers.py @@ -14,6 +14,7 @@ class AliasSerializer(serializers.ModelSerializer): class Meta: model = Alias validators = [] # Disable unique togheter check + fields = '__all__' class AliasDeleteSerializer(serializers.ModelSerializer): @@ -25,11 +26,13 @@ class AliasDeleteSerializer(serializers.ModelSerializer): class DomainSerializer(serializers.ModelSerializer): class Meta: model = Domain + fields = '__all__' class UserSerializer(serializers.ModelSerializer): class Meta: model = User + fields = '__all__' extra_kwargs = {'password': {'write_only': True}} def _hash_password(self, raw_password): diff --git a/dpdapi/views.py b/dpdapi/views.py index ec6f3ea..b8e370d 100644 --- a/dpdapi/views.py +++ b/dpdapi/views.py @@ -1,20 +1,19 @@ from __future__ import unicode_literals import logging import operator -from django.utils import six + +from functools import reduce from django.db.models import Q +from django_filters.rest_framework import DjangoFilterBackend from rest_framework.decorators import list_route from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response -from rest_framework import status, filters, viewsets +from rest_framework import status, viewsets -from dpdapi.filters import AliasRegexFilterBackend +from dpdapi.filters import AliasFilter from dpdapi.models import Alias, Domain, User from dpdapi.serializers import AliasSerializer, AliasDeleteSerializer, DomainSerializer, UserSerializer -if six.PY3: - from functools import reduce - logger = logging.getLogger(__name__) @@ -23,8 +22,8 @@ class AliasViewSet(viewsets.ModelViewSet): queryset = Alias.objects.all() serializer_class = AliasSerializer permission_classes = [IsAuthenticated] - filter_backends = (filters.DjangoFilterBackend, AliasRegexFilterBackend) - filter_fields = ('domain', 'domain__name',) + filter_backends = (DjangoFilterBackend,) + filter_class = AliasFilter @list_route(methods=['post']) def create_bulk(self, request): @@ -65,7 +64,7 @@ class DomainViewSet(viewsets.ModelViewSet): queryset = Domain.objects.all() serializer_class = DomainSerializer permission_classes = [IsAuthenticated] - filter_backends = (filters.DjangoFilterBackend,) + filter_backends = (DjangoFilterBackend,) filter_fields = ('name',) diff --git a/dpdapisite/settings.py b/dpdapisite/settings.py index 1866b80..4da1f5c 100644 --- a/dpdapisite/settings.py +++ b/dpdapisite/settings.py @@ -59,6 +59,7 @@ INSTALLED_APPS = ( INSTALLED_APPS += ( 'rest_framework', + 'django_filters', 'dpdapi', ) @@ -89,21 +90,15 @@ DATABASES = { # Internationalization # https://docs.djangoproject.com/en/1.7/topics/i18n/ - LANGUAGE_CODE = 'en-us' - TIME_ZONE = 'Europe/Oslo' - USE_I18N = True - USE_L10N = True - USE_TZ = True # Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/1.7/howto/static-files/ - STATIC_URL = '/static/' REST_FRAMEWORK = { diff --git a/dpdapisite/urls.py b/dpdapisite/urls.py index 3d35aee..f8e8e65 100644 --- a/dpdapisite/urls.py +++ b/dpdapisite/urls.py @@ -1,8 +1,8 @@ from django.conf.urls import include, url from django.contrib import admin from django.views.generic import RedirectView -from rest_framework import urls as rest_framework_urls +from rest_framework import urls as rest_framework_urls from dpdapi import urls as dpdapi_urls diff --git a/requirements.txt b/requirements.txt index 3fe2024..ca12fd5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ -Django>=1.9,<1.10 -django-extensions==1.6.1 -django-filter==0.12.0 -djangorestframework>=3.3,<3.4 -Fabric==1.10.2 -gunicorn==19.4.5 -mysqlclient==1.3.7 -passlib==1.6.5 +Django==1.11.6 +django-extensions==1.9.6 +django-filter==1.1.0 +djangorestframework==3.7.1 +Fabric==1.14.0 +gunicorn==19.7.1 +mysqlclient==1.3.12 +passlib==1.7.1 From 2a037fbe8ff6111ae73d9cfef5e8a2212a442bd5 Mon Sep 17 00:00:00 2001 From: Nikolai R Kristiansen Date: Wed, 1 Nov 2017 21:44:15 +0100 Subject: [PATCH 2/3] Try fixing regex lookup Adds tests --- dpdapi/filters.py | 11 ++++++++--- dpdapi/tests.py | 31 +++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 5 deletions(-) diff --git a/dpdapi/filters.py b/dpdapi/filters.py index 54b93ce..320b5bd 100644 --- a/dpdapi/filters.py +++ b/dpdapi/filters.py @@ -10,9 +10,14 @@ filters.LOOKUP_TYPES = ['iregex', 'exact', 'iexact'] class AliasFilter(django_filters.FilterSet): # TODO: These do not work # Ref: https://docs.djangoproject.com/en/1.7/ref/models/querysets/#iregex - source_regex = django_filters.CharFilter(name='source', lookup_expr=['iregex']) - destination_regex = django_filters.CharFilter(name='destination', lookup_expr=['iregex']) + source_regex = django_filters.CharFilter(name='source') + destination_regex = django_filters.CharFilter(name='destination') class Meta: model = Alias - fields = ['domain', 'domain__name', 'source_regex', 'destination_regex'] + fields = { + 'domain': ['exact'], + 'domain__name': ['iexact'], + 'source_regex': ['iregex', 'regex'], + 'destination_regex': ['iregex', 'regex'] + } diff --git a/dpdapi/tests.py b/dpdapi/tests.py index 7ce503c..b9e1804 100644 --- a/dpdapi/tests.py +++ b/dpdapi/tests.py @@ -1,3 +1,30 @@ -from django.test import TestCase +from django.contrib.auth.models import User +from django.urls import reverse +from rest_framework.test import APITestCase -# Create your tests here. +from dpdapi.models import Domain, Alias + + +class RegexTestCase(APITestCase): + aliases = ['ops-alert@example.com', 'yolo-announce@example.com'] + + def setUp(self): + user = User.objects.create(username='yolo') + d = Domain.objects.create(name='example.com') + for a in self.aliases: + Alias.objects.create(source=a, destination='@example.com', domain=d) + self.client.force_login(user) + + def test_get_all(self): + res = self.client.get(reverse('alias-list')) + self.assertEqual(len(res.data), len(self.aliases)) + + def test_domain_filter(self): + params = {'domain__name': 'example.com'} + res = self.client.get(reverse('alias-list'), data=params) + self.assertEqual(len(res.data), len(self.aliases)) + + def test_regex_filter(self): + params = {'source_regex': '^ops-|^ops@'} + res = self.client.get(reverse('alias-list'), data=params) + self.assertEqual(len(res.data), 1) From f0a78588110906abce06dd25e61a2d25105534b8 Mon Sep 17 00:00:00 2001 From: Nikolai R Kristiansen Date: Wed, 1 Nov 2017 23:29:57 +0100 Subject: [PATCH 3/3] Fix regex lookup Note: Renames filter fields: - source_regex to source__iregex - destination_regex to destination__iregex --- dpdapi/filters.py | 23 +++++++++-------------- dpdapi/tests.py | 10 ++++++++-- dpdapi/views.py | 10 ++++++---- dpdapisite/settings.py | 1 + 4 files changed, 24 insertions(+), 20 deletions(-) diff --git a/dpdapi/filters.py b/dpdapi/filters.py index 320b5bd..ec63073 100644 --- a/dpdapi/filters.py +++ b/dpdapi/filters.py @@ -1,23 +1,18 @@ import django_filters +from rest_framework.filters import BaseFilterBackend from dpdapi.models import Alias -from django_filters import filters - -filters.LOOKUP_TYPES = ['iregex', 'exact', 'iexact'] - - -class AliasFilter(django_filters.FilterSet): - # TODO: These do not work - # Ref: https://docs.djangoproject.com/en/1.7/ref/models/querysets/#iregex - source_regex = django_filters.CharFilter(name='source') - destination_regex = django_filters.CharFilter(name='destination') +class AliasRegexFilter(django_filters.FilterSet): class Meta: model = Alias fields = { - 'domain': ['exact'], - 'domain__name': ['iexact'], - 'source_regex': ['iregex', 'regex'], - 'destination_regex': ['iregex', 'regex'] + 'source': ['iregex', 'regex'], + 'destination': ['iregex', 'regex'] } + + +class AliasRegexFilterBackend(BaseFilterBackend): + def filter_queryset(self, request, queryset, view): + return AliasRegexFilter(request.query_params, queryset=queryset).qs diff --git a/dpdapi/tests.py b/dpdapi/tests.py index b9e1804..de8bc75 100644 --- a/dpdapi/tests.py +++ b/dpdapi/tests.py @@ -1,4 +1,5 @@ from django.contrib.auth.models import User +from django.test import override_settings from django.urls import reverse from rest_framework.test import APITestCase @@ -12,7 +13,7 @@ class RegexTestCase(APITestCase): user = User.objects.create(username='yolo') d = Domain.objects.create(name='example.com') for a in self.aliases: - Alias.objects.create(source=a, destination='@example.com', domain=d) + Alias.objects.create(source=a, destination='ninja@example.com', domain=d) self.client.force_login(user) def test_get_all(self): @@ -24,7 +25,12 @@ class RegexTestCase(APITestCase): res = self.client.get(reverse('alias-list'), data=params) self.assertEqual(len(res.data), len(self.aliases)) + def test_domain_filter_no_hits(self): + params = {'domain__name': 'zexample.com'} + res = self.client.get(reverse('alias-list'), data=params) + self.assertEqual(len(res.data), 0) + def test_regex_filter(self): - params = {'source_regex': '^ops-|^ops@'} + params = {'source__regex': '^ops-|^ops@'} res = self.client.get(reverse('alias-list'), data=params) self.assertEqual(len(res.data), 1) diff --git a/dpdapi/views.py b/dpdapi/views.py index b8e370d..385acab 100644 --- a/dpdapi/views.py +++ b/dpdapi/views.py @@ -6,24 +6,26 @@ from functools import reduce from django.db.models import Q from django_filters.rest_framework import DjangoFilterBackend from rest_framework.decorators import list_route +from rest_framework.mixins import ListModelMixin from rest_framework.permissions import IsAuthenticated from rest_framework.response import Response from rest_framework import status, viewsets +from rest_framework.viewsets import GenericViewSet -from dpdapi.filters import AliasFilter +from dpdapi.filters import AliasRegexFilterBackend from dpdapi.models import Alias, Domain, User from dpdapi.serializers import AliasSerializer, AliasDeleteSerializer, DomainSerializer, UserSerializer logger = logging.getLogger(__name__) -class AliasViewSet(viewsets.ModelViewSet): +class AliasViewSet(ListModelMixin, GenericViewSet): """ Aliases """ queryset = Alias.objects.all() serializer_class = AliasSerializer permission_classes = [IsAuthenticated] - filter_backends = (DjangoFilterBackend,) - filter_class = AliasFilter + filter_backends = (DjangoFilterBackend, AliasRegexFilterBackend) + filter_fields = ('domain', 'domain__name',) @list_route(methods=['post']) def create_bulk(self, request): diff --git a/dpdapisite/settings.py b/dpdapisite/settings.py index 4da1f5c..9ee7dc1 100644 --- a/dpdapisite/settings.py +++ b/dpdapisite/settings.py @@ -58,6 +58,7 @@ INSTALLED_APPS = ( ) INSTALLED_APPS += ( + 'django_extensions', 'rest_framework', 'django_filters', 'dpdapi',