From b0edaae32eb08b2b57fbaf90bcd32683b750e9e4 Mon Sep 17 00:00:00 2001 From: Jan Tuomi Date: Wed, 20 Sep 2017 23:17:55 +0300 Subject: [PATCH] Remove conflict resolver and add dynamic payments --- members/forms.py | 71 +++++++++- members/migrations/0014_auto_20170920_1457.py | 25 ++++ members/models.py | 3 +- members/templates/error.html | 2 +- members/templates/member_add_many.html | 13 +- .../templates/member_add_many_confirm.html | 26 ++++ members/templates/member_duplicates.html | 40 ------ members/urls.py | 12 +- members/views.py | 132 ++++++++---------- setup.sh | 2 +- 10 files changed, 197 insertions(+), 129 deletions(-) create mode 100644 members/migrations/0014_auto_20170920_1457.py create mode 100644 members/templates/member_add_many_confirm.html delete mode 100644 members/templates/member_duplicates.html diff --git a/members/forms.py b/members/forms.py index a6e47ce..221b8b7 100644 --- a/members/forms.py +++ b/members/forms.py @@ -3,12 +3,81 @@ from django.utils.translation import ugettext_lazy as _ from members.models import Member, Payment, Request +import csv +import datetime +import logging + + +class CSVValidationError(Exception): + def __init__(self, form_errors): + self.form_errors = form_errors + class MemberForm(forms.ModelForm): class Meta: model = Member - fields = ['first_name', 'last_name', 'email', 'AYY', 'jas', 'POR'] + fields = ['first_name', 'last_name', 'email', 'POR', 'AYY', 'jas'] + + class ImportResult: + def __init__(self, members, payments): + self.members = members + self.payments = payments + + def clean_email(self): + email = self.cleaned_data['email'] + + if Member.objects.filter(email=email).exists(): + raise forms.ValidationError('Member with email "{}" already exists.'.format(email), code='exists') + + return email + + def clean_jas(self): + return bool(int(self.data['jas'])) + + def clean_AYY(self): + return bool(int(self.data['AYY'])) + + @staticmethod + def csv_to_models(data, payment_source='AYY'): + clean_data = data.strip().split('\n') + clean_data = [row.rstrip(',') for row in clean_data] + csv_reader = csv.DictReader(clean_data, fieldnames=MemberForm.Meta.fields) + + members = [] + payments = [] + for line in csv_reader: + for key, value in line.items(): + line[key] = value.strip() + + email = line['email'] + member_exists = False + if Member.objects.filter(email=email).exists(): + member_exists = True + + if not member_exists: + form = MemberForm(line) + if not form.is_valid(): + raise CSVValidationError(form.errors) + + model = form.save(commit=False) + members.append(model) + + else: + member = Member.objects.get(email=email) + payment_data = { + 'source': payment_source, + 'member': member.id, + 'date': datetime.datetime.now(), + } + form = PaymentForm(payment_data) + if not form.is_valid(): + raise CSVValidationError(form.errors) + + model = form.save(commit=False) + payments.append(model) + + return MemberForm.ImportResult(members, payments) class PaymentForm(forms.ModelForm): diff --git a/members/migrations/0014_auto_20170920_1457.py b/members/migrations/0014_auto_20170920_1457.py new file mode 100644 index 0000000..1397954 --- /dev/null +++ b/members/migrations/0014_auto_20170920_1457.py @@ -0,0 +1,25 @@ +# -*- coding: utf-8 -*- +# Generated by Django 1.11 on 2017-09-20 11:57 +from __future__ import unicode_literals + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('members', '0013_auto_20170601_1822'), + ] + + operations = [ + migrations.AlterField( + model_name='member', + name='email', + field=models.EmailField(max_length=254, unique=True, verbose_name='Email'), + ), + migrations.AlterField( + model_name='request', + name='email', + field=models.EmailField(max_length=254, unique=True, verbose_name='Email'), + ), + ] diff --git a/members/models.py b/members/models.py index a8c46d0..8cbcae8 100644 --- a/members/models.py +++ b/members/models.py @@ -13,7 +13,7 @@ class BaseMember(models.Model): ''' first_name = models.CharField(_("First name"), max_length=127) last_name = models.CharField(_("Last name"), max_length=127) - email = models.EmailField(_("Email")) + email = models.EmailField(_("Email"), unique=True) POR = models.CharField(_("Place of residence"), max_length=255) # place of residence AYY = models.BooleanField(_("AYY"), default=False) jas = models.BooleanField(_("JAS"), default=False) @@ -26,7 +26,6 @@ class BaseMember(models.Model): @staticmethod def from_csv(data): - print("Imported CSV data: {}".format(data)) clean_data = data.strip().split('\n') csv_reader = csv.reader(clean_data) diff --git a/members/templates/error.html b/members/templates/error.html index b8c30ae..745eb56 100644 --- a/members/templates/error.html +++ b/members/templates/error.html @@ -9,7 +9,7 @@
- {{ error }} + {{ error|safe }}
diff --git a/members/templates/member_add_many.html b/members/templates/member_add_many.html index fc083eb..9d9b4e8 100644 --- a/members/templates/member_add_many.html +++ b/members/templates/member_add_many.html @@ -5,7 +5,7 @@ {% block content %}
-

Lisää useampi jäsen

+

{% trans "Add many members" %}

@@ -25,9 +25,18 @@
{% csrf_token %} -
+
+
+
+ + +
diff --git a/members/templates/member_add_many_confirm.html b/members/templates/member_add_many_confirm.html new file mode 100644 index 0000000..2cff70a --- /dev/null +++ b/members/templates/member_add_many_confirm.html @@ -0,0 +1,26 @@ +{% extends "members_base.html" %} + +{% load i18n %} + +{% block content %} +
+
+

{% trans "Confirm adding these entries?" %}

+
+ +
+ + {{ members|safe }} +
+
+ + {{ payments|safe }} +
+
+ {% csrf_token %} +
+ +
+ +
+{% endblock content %} diff --git a/members/templates/member_duplicates.html b/members/templates/member_duplicates.html deleted file mode 100644 index c6f2995..0000000 --- a/members/templates/member_duplicates.html +++ /dev/null @@ -1,40 +0,0 @@ -{% extends "members_base.html" %} - -{% load i18n %} -{% load bootstrap3 %} - -{% block content %} -
-
-

{% trans "Conflicting member entries" %}

-
- -
-

{% blocktrans %}Found conflicting member entries. Choose how to handle the problematic data.{% endblocktrans %}

- - {% for conflict in conflicts %} -
-
- - {{ conflict.first_member_form }} -
-
-
- - {{ conflict.second_member_form }} -
-
-
-
{% csrf_token %} -

{% blocktrans %}Which one has the correct information for this member?{% endblocktrans %}

- - - - -
-
-
- {% endfor %} -
-
-{% endblock content %} diff --git a/members/urls.py b/members/urls.py index 9738af6..397409a 100644 --- a/members/urls.py +++ b/members/urls.py @@ -16,8 +16,7 @@ from members.views import member_update from members.views import member_delete_confirm from members.views import member_delete from members.views import payment_list -from members.views import member_duplicates -from members.views import resolve_conflict +from members.views import add_many_confirm # rest api from members.views import MemberDetail @@ -89,6 +88,9 @@ urlpatterns = [ # delete confirmation view url(r'^delete_payment_confirm/(?P\d+)$', payment_delete_confirm), + # post endpoint for confirming multiple entries + url(r'^add_many_confirm$', add_many_confirm), + # settings page url(r'^settings$', settings_page), @@ -104,10 +106,4 @@ urlpatterns = [ # rest api url url(r'^api/members/(?P\d+)$', MemberDetail.as_view()), - # member duplicate resolution view - url(r'^duplicates$', member_duplicates), - - # post target for resolving a conflict - url(r'^resolve_conflict$', resolve_conflict) - ] diff --git a/members/views.py b/members/views.py index da03f3d..2792dd6 100644 --- a/members/views.py +++ b/members/views.py @@ -6,6 +6,7 @@ from django.http import HttpResponse, HttpResponseRedirect from django.core.mail import send_mail from django.conf import settings from django.utils.translation import ugettext as _ +from django.forms.models import model_to_dict # Email validation from django.db.models.signals import post_save @@ -25,13 +26,18 @@ import requests import logging import html import csv +import pickle from smtplib import SMTPAuthenticationError from members.models import Member, Request, Payment, MemberConflict -from members.forms import MemberForm, PaymentForm, ApplicationForm +from members.forms import MemberForm, PaymentForm, ApplicationForm, CSVValidationError from members.tables import MemberTable, PaymentTable, RequestTable +def error_view(request, message): + return render(request, 'error.html', {'error': str(message)}) + + def validate_recaptcha(response): ''' Recaptcha is used in member applications @@ -421,16 +427,61 @@ def settings_page(request, *args, **kwargs): def import_csv(request, *args, **kwargs): try: data = request.POST['textfield'] + payment_source = request.POST['payment_source'] except: return render(request, 'error.html', {'error': _('Missing "textfield" POST request field')}) - success = Member.from_csv(data) - if success: - logging.info('Imported CSV data:\n'.format(data)) - notification = "{}.".format(_("Successfully imported multiple members")) + try: + result = MemberForm.csv_to_models(data, payment_source=payment_source) + except CSVValidationError as ex: + logging.exception('Model validation error') + return error_view(request, ex.form_errors) + except Exception as ex: + logging.exception('Other error in CSV import') + return error_view(request, ex) + + member_table = MemberTable(result.members, + request=request, + exclude=['id', 'options'], + attrs={'class': 'table table-bordered table-hover'}) + + member_table_html = convert_table_to_html(member_table, request) + + payment_table = PaymentTable(result.payments, + request=request, + exclude=['id', 'options'], + attrs={'class': 'table table-bordered table-hover'}) + + payment_table_html = convert_table_to_html(payment_table, request) + + request.session['models'] = result + context = { + 'members': member_table_html, + 'payments': payment_table_html + } + return render(request, 'member_add_many_confirm.html', context) + + +@ensure_csrf_cookie +@require_http_methods(["POST"]) +@permission_required('members.change_member', login_url='/login') +def add_many_confirm(request, *args, **kwargs): + models = request.session['models'] + + try: + members, payments = models.members, models.payments + for member in members: + member.save() + + for payment in payments: + payment.save() + + msg = "Successfully imported {} members and {} payments." + notification = _(msg).format(len(members), len(payments)) return HttpResponseRedirect('/members/list?notification={}'.format(html.escape(notification))) - else: - return render(request, 'error.html', {'error': _('Failed to import members')}) + except Exception as ex: + logging.exception('Failed to save models after "add many."') + return error_view(request, _('Failed to import members')) @ensure_csrf_cookie @@ -452,54 +503,6 @@ def export_csv(request, *args, **kwargs): return response -@ensure_csrf_cookie -@require_http_methods(["GET"]) -@permission_required('members.change_member', login_url='/login') -def member_duplicates(request, *args, **kwargs): - conflicts = MemberConflict.objects.all() - context = { - 'conflicts': conflicts - } - - return render(request, 'member_duplicates.html', context) - - -@ensure_csrf_cookie -@require_http_methods(["POST"]) -@permission_required('members.change_member', login_url='/login') -def resolve_conflict(request, *args, **kwargs): - action = request.POST.get('action', None) - if action not in ['first', 'second', 'both']: - return render(request, 'error.html', {'error': '{}: {}'.format(('Incorrect action value'), action)}) - - id = request.POST.get('id', None) - if id is None: - return render(request, 'error.html', {'error': '{}: {}'.format(('Incorrect id value'), id)}) - - conflict = MemberConflict.objects.get(id=id) - first_member = conflict.first_member - second_member = conflict.second_member - - if action == 'first': - for payment in second_member.payments.all(): - payment.member = first_member - payment.save() - second_member.delete() - elif action == 'second': - for payment in first_member.payments.all(): - payment.member = second_member - payment.save() - first_member.delete() - - conflict.delete() - - if MemberConflict.objects.exists(): - return HttpResponseRedirect('/members/duplicates') - else: - notification = _('Successfully resolved all member conflicts.') - return HttpResponseRedirect('/members/list?notification={}'.format(html.escape(notification))) - - def send_mail_wrapper(subject, message, email_to): send_mail(subject, message, @@ -536,25 +539,6 @@ def email_on_accept(sender, instance, created, **kwargs): logging.error('Failed to send email to accepted member!') -def check_for_duplicates(instance): - name_candidates = Member.objects.filter(first_name=instance.first_name, - last_name=instance.last_name) - email_candidates = Member.objects.filter(email=instance.email) - - candidates = name_candidates | email_candidates - duplicates = candidates.exclude(id=instance.id) - - if len(duplicates) > 0: - conflict = MemberConflict(first_member=instance, - second_member=duplicates[0]) - conflict.save() - - -@receiver(post_save, sender=Member) -def duplicate_receiver(sender, instance, created, **kwargs): - check_for_duplicates(instance) - - # Can be used to retrieve single member information via REST API class MemberDetail(generics.RetrieveAPIView): queryset = Member.objects.all() diff --git a/setup.sh b/setup.sh index 8f71754..6ac4281 100755 --- a/setup.sh +++ b/setup.sh @@ -16,7 +16,7 @@ then USE_NPM="false" fi -$INTERACTIVE || echo "Running in non-interactive mode." && env +$INTERACTIVE || (echo "Running in non-interactive mode." && env) $INTERACTIVE && read -p "Are these programs installed? [y/n]" -n 1 -r || REPLY="y" echo ""