Merge branch 'full_archive' into 'dev'

Full archive

See merge request federez/re2o!414

api/views.py

@@ -455,7 +455,7 @@ class UserViewSet(viewsets.ReadOnlyModelViewSet):
 class HomeCreationViewSet(viewsets.ReadOnlyModelViewSet):
     """Exposes infos of `users.models.Users` objects to create homes.
     """
-    queryset = users.User.objects.exclude(Q(state=users.User.STATE_DISABLED) | Q(state=users.User.STATE_NOT_YET_ACTIVE))
+    queryset = users.User.objects.exclude(Q(state=users.User.STATE_DISABLED) | Q(state=users.User.STATE_NOT_YET_ACTIVE) | Q(state=users.User.STATE_FULL_ARCHIVE))
     serializer_class = serializers.BasicUserSerializer
 
 

logs/views.py

@@ -217,8 +217,8 @@ def stats_general(request):
         ).count()
         ip_dict[ip_range] = [ip_range, ip_range.vlan, all_ip.count(),
                              used_ip, active_ip, all_ip.count()-used_ip]
-    _all_adherent = all_adherent()
+    _all_adherent = all_adherent(including_asso=False)
-    _all_has_access = all_has_access()
+    _all_has_access = all_has_access(including_asso=False)
     _all_baned = all_baned()
     _all_whitelisted = all_whitelisted()
     _all_active_interfaces_count = all_active_interfaces_count()
@@ -257,6 +257,14 @@ def stats_general(request):
                      .count()),
                     Club.objects.filter(state=Club.STATE_ARCHIVE).count()
                 ],
+                'full_archive_users': [
+                    _("Full Archived users"),
+                    User.objects.filter(state=User.STATE_FULL_ARCHIVE).count(),
+                    (Adherent.objects
+                     .filter(state=Adherent.STATE_FULL_ARCHIVE)
+                     .count()),
+                    Club.objects.filter(state=Club.STATE_FULL_ARCHIVE).count()
+                ],
                 'not_active_users': [
                     _("Not yet active users"),
                     User.objects.filter(state=User.STATE_NOT_YET_ACTIVE).count(),

machines/models.py

@@ -41,6 +41,8 @@ from django.db.models.signals import post_save, post_delete
 from django.dispatch import receiver
 from django.forms import ValidationError
 from django.utils import timezone
+from django.db import transaction
+from reversion import revisions as reversion
 from django.utils.functional import cached_property
 from django.utils.translation import ugettext_lazy as _
 from macaddress.fields import MACAddressField, default_dialect
@@ -222,6 +224,16 @@ class Machine(RevMixin, FieldPermissionModelMixin, models.Model):
         """Return a name : user provided name or first interface name"""
         return self.name or self.short_name
 
+    @classmethod
+    def mass_delete(cls, machine_queryset):
+        """Mass delete for machine queryset"""
+        Domain.objects.filter(cname__interface_parent__machine__in=machine_queryset)._raw_delete(machine_queryset.db)
+        Domain.objects.filter(interface_parent__machine__in=machine_queryset)._raw_delete(machine_queryset.db)
+        Ipv6List.objects.filter(interface__machine__in=machine_queryset)._raw_delete(machine_queryset.db)
+        Interface.objects.filter(machine__in=machine_queryset).filter(port_lists__isnull=False).delete()
+        Interface.objects.filter(machine__in=machine_queryset)._raw_delete(machine_queryset.db)
+        machine_queryset._raw_delete(machine_queryset.db)
+
     @cached_property
     def all_complete_names(self):
         """Renvoie tous les tls complets de la machine"""
@@ -1134,6 +1146,21 @@ class Interface(RevMixin, AclMixin, FieldPermissionModelMixin, models.Model):
         """ Sans commentaire, désassigne une ipv4"""
         self.ipv4 = None
 
+    @classmethod
+    def mass_unassign_ipv4(cls, interface_list):
+        """Unassign ipv4 to multiple interfaces"""
+        with transaction.atomic(), reversion.create_revision():
+            interface_list.update(ipv4=None)
+            reversion.set_comment(_("IPv4 unassigning"))
+
+    @classmethod
+    def mass_assign_ipv4(cls, interface_list):
+        for interface in interface_list:
+            with transaction.atomic(), reversion.create_revision():
+                interface.assign_ipv4()
+                interface.save()
+                reversion.set_comment(_("IPv4 assigning"))
+
     def update_type(self):
         """ Lorsque le machinetype est changé de type d'ip, on réassigne"""
         self.clean()

preferences/models.py

@@ -256,7 +256,7 @@ class OptionalTopologie(AclMixin, PreferencesModel):
         """Return the ip of the interface that the switch have to contact to get it's config"""
         if self.switchs_ip_type:
             from machines.models import Role, Interface
-            return Interface.objects.filter(machine__interface__in=Role.interface_for_roletype("switch-conf-server")).filter(type__ip_type=self.switchs_ip_type).first()
+            return Interface.objects.filter(machine__interface__in=Role.interface_for_roletype("switch-conf-server")).filter(machine_type__ip_type=self.switchs_ip_type).first()
         else:
             return None
 
@@ -282,11 +282,11 @@ class OptionalTopologie(AclMixin, PreferencesModel):
         def return_ips_dict(interfaces):
             return {'ipv4' : [str(interface.ipv4) for interface in interfaces], 'ipv6' : Ipv6List.objects.filter(interface__in=interfaces).values_list('ipv6', flat=True)}
 
-        ntp_servers = Role.all_interfaces_for_roletype("ntp-server").filter(type__ip_type=self.switchs_ip_type)
+        ntp_servers = Role.all_interfaces_for_roletype("ntp-server").filter(machine_type__ip_type=self.switchs_ip_type)
-        log_servers = Role.all_interfaces_for_roletype("log-server").filter(type__ip_type=self.switchs_ip_type)
+        log_servers = Role.all_interfaces_for_roletype("log-server").filter(machine_type__ip_type=self.switchs_ip_type)
-        radius_servers = Role.all_interfaces_for_roletype("radius-server").filter(type__ip_type=self.switchs_ip_type)
+        radius_servers = Role.all_interfaces_for_roletype("radius-server").filter(machine_type__ip_type=self.switchs_ip_type)
         dhcp_servers = Role.all_interfaces_for_roletype("dhcp-server")
-        dns_recursive_servers = Role.all_interfaces_for_roletype("dns-recursive-server").filter(type__ip_type=self.switchs_ip_type)
+        dns_recursive_servers = Role.all_interfaces_for_roletype("dns-recursive-server").filter(machine_type__ip_type=self.switchs_ip_type)
         subnet = None
         subnet6 = None
         if self.switchs_ip_type:

re2o/utils.py

@@ -44,15 +44,15 @@ from machines.models import Interface, Machine
 from users.models import Adherent, User, Ban, Whitelist
 from preferences.models import AssoOption
 
-def all_adherent(search_time=None):
+def all_adherent(search_time=None, including_asso=True):
     """ Fonction renvoyant tous les users adherents. Optimisee pour n'est
     qu'une seule requete sql
     Inspecte les factures de l'user et ses cotisation, regarde si elles
     sont posterieur à now (end_time)"""
     if search_time is None:
         search_time = timezone.now()
-    return User.objects.filter(
+    filter_user = (
-        facture__in=Facture.objects.filter(
+        Q(facture__in=Facture.objects.filter(
             vente__in=Vente.objects.filter(
                 Q(type_cotisation='All') | Q(type_cotisation='Adhesion'),
                 cotisation__in=Cotisation.objects.filter(
@@ -62,7 +62,12 @@ def all_adherent(search_time=None):
                 ).filter(Q(date_start__lt=search_time) & Q(date_end__gt=search_time))
             )
         )
-    ).distinct()
+    ))
+    if including_asso:
+        asso_user = AssoOption.get_cached_value('utilisateur_asso')
+        if asso_user:
+            filter_user |= Q(id=asso_user.id)
+    return User.objects.filter(filter_user).distinct()
 
 
 def all_baned(search_time=None):
@@ -87,7 +92,7 @@ def all_whitelisted(search_time=None):
     ).distinct()
 
 
-def all_has_access(search_time=None):
+def all_has_access(search_time=None, including_asso=True):
     """ Return all connected users : active users and whitelisted +
     asso_user defined in AssoOption pannel
     ----
@@ -111,6 +116,7 @@ def all_has_access(search_time=None):
              )
          )))
     )
+    if including_asso:
         asso_user = AssoOption.get_cached_value('utilisateur_asso')
         if asso_user:
             filter_user |= Q(id=asso_user.id)

search/forms.py

@@ -34,6 +34,7 @@ CHOICES_USER = (
     ('1', _("Disabled")),
     ('2', _("Archived")),
     ('3', _("Not yet active")),
+    ('4', _("Full archived")),
 )
 
 CHOICES_AFF = (

topologie/models.py

@@ -361,12 +361,12 @@ class Switch(AclMixin, Machine):
     @cached_property
     def interfaces_subnet(self):
         """Return dict ip:subnet for all ip of the switch"""
-        return dict((str(interface.ipv4), interface.type.ip_type.ip_set_full_info) for interface in self.interface_set.all())
+        return dict((str(interface.ipv4), interface.machine_type.ip_type.ip_set_full_info) for interface in self.interface_set.all())
 
     @cached_property
     def interfaces6_subnet(self):
         """Return dict ip6:subnet for all ipv6 of the switch"""
-        return dict((str(interface.ipv6().first()), interface.type.ip_type.ip6_set_full_info) for interface in self.interface_set.all())
+        return dict((str(interface.ipv6().first()), interface.machine_type.ip_type.ip6_set_full_info) for interface in self.interface_set.all())
 
     @cached_property
     def list_modules(self):

users/forms.py

@@ -297,6 +297,11 @@ class MassArchiveForm(forms.Form):
     du formulaire la date de depart avant laquelle archiver les
     users"""
     date = forms.DateTimeField(help_text='%d/%m/%y')
+    full_archive = forms.BooleanField(
+        label=_("Make a full archive operation ? (WARNING : CRITICAL OPERATION IF TRUE)"),
+        initial=False,
+        required=False
+    )
 
     def clean(self):
         cleaned_data = super(MassArchiveForm, self).clean()

users/migrations/0081_auto_20190317_0302.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.10.7 on 2019-03-17 02:02
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('users', '0080_auto_20190108_1726'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='user',
+            name='state',
+            field=models.IntegerField(choices=[(0, 'Active'), (1, 'Disabled'), (2, 'Archived'), (3, 'Not yet active'), (4, 'Full Archived')], default=3),
+        ),
+    ]

users/models.py

@@ -188,11 +188,13 @@ class User(RevMixin, FieldPermissionModelMixin, AbstractBaseUser,
     STATE_DISABLED = 1
     STATE_ARCHIVE = 2
     STATE_NOT_YET_ACTIVE = 3
+    STATE_FULL_ARCHIVE = 4
     STATES = (
         (0, _("Active")),
         (1, _("Disabled")),
         (2, _("Archived")),
         (3, _("Not yet active")),
+        (4, _("Full Archived")),
     )
 
     surname = models.CharField(max_length=255)
@@ -335,11 +337,17 @@ class User(RevMixin, FieldPermissionModelMixin, AbstractBaseUser,
         return self.state == self.STATE_ACTIVE or self.state == self.STATE_NOT_YET_ACTIVE
 
     def set_active(self):
-        """Enable this user if he subscribed successfully one time before"""
+        """Enable this user if he subscribed successfully one time before
+        Reenable it if it was archived
+        Do nothing if disabed"""
         if self.state == self.STATE_NOT_YET_ACTIVE:
             if self.facture_set.filter(valid=True).filter(Q(vente__type_cotisation='All') | Q(vente__type_cotisation='Adhesion')).exists() or OptionalUser.get_cached_value('all_users_active'):
                 self.state = self.STATE_ACTIVE
                 self.save()
+        if self.state == self.STATE_ARCHIVE or self.state == self.STATE_FULL_ARCHIVE:
+            self.state = self.STATE_ACTIVE
+            self.unarchive()
+            self.save()
 
     @property
     def is_staff(self):
@@ -519,46 +527,103 @@ class User(RevMixin, FieldPermissionModelMixin, AbstractBaseUser,
         )['total'] or 0
         return somme_credit - somme_debit
 
-    def user_interfaces(self, active=True):
+    @classmethod
+    def users_interfaces(cls, users, active=True, all_interfaces=False):
         """ Renvoie toutes les interfaces dont les machines appartiennent à
         self. Par defaut ne prend que les interfaces actives"""
+        if all_interfaces:
             return Interface.objects.filter(
-            machine__in=Machine.objects.filter(user=self, active=active)
+                machine__in=Machine.objects.filter(user__in=users)
             ).select_related('domain__extension')
+        else:
+            return Interface.objects.filter(
+                machine__in=Machine.objects.filter(user__in=users, active=active)
+            ).select_related('domain__extension')
+
+    def user_interfaces(self, active=True, all_interfaces=False):
+        """ Renvoie toutes les interfaces dont les machines appartiennent à
+        self. Par defaut ne prend que les interfaces actives"""
+        return self.users_interfaces([self], active=active, all_interfaces=all_interfaces)
 
     def assign_ips(self):
         """ Assign une ipv4 aux machines d'un user """
         interfaces = self.user_interfaces()
-        for interface in interfaces:
-            if not interface.ipv4:
         with transaction.atomic(), reversion.create_revision():
-                    interface.assign_ipv4()
+            Interface.mass_assign_ipv4(interfaces)
             reversion.set_comment(_("IPv4 assigning"))
-                    interface.save()
 
     def unassign_ips(self):
         """ Désassigne les ipv4 aux machines de l'user"""
         interfaces = self.user_interfaces()
-        for interface in interfaces:
         with transaction.atomic(), reversion.create_revision():
-                interface.unassign_ipv4()
+            Interface.mass_unassign_ipv4(interfaces)
             reversion.set_comment(_("IPv4 unassigning"))
-                interface.save()
+
+    @classmethod
+    def mass_unassign_ips(cls, users_list):
+        interfaces = cls.users_interfaces(users_list)
+        with transaction.atomic(), reversion.create_revision():
+            Interface.mass_unassign_ipv4(interfaces)
+            reversion.set_comment(_("IPv4 assigning"))
+
+    @classmethod
+    def mass_disable_email(cls, queryset_users):
+        """Disable email account and redirection"""
+        queryset_users.update(local_email_enabled=False)
+        queryset_users.update(local_email_redirect=False)
+
+    @classmethod
+    def mass_delete_data(cls, queryset_users):
+        """This users will be completely archived, so only keep mandatory data"""
+        cls.mass_disable_email(queryset_users)
+        Machine.mass_delete(Machine.objects.filter(user__in=queryset_users))
+        cls.ldap_delete_users(queryset_users)
 
     def disable_email(self):
         """Disable email account and redirection"""
-        self.email = ""
+        self.local_email_enabled=False
-        self.local_email_enabled = False
+        self.local_email_redirect=False
-        self.local_email_redirect = False
+
+    def delete_data(self):
+        """This user will be completely archived, so only keep mandatory data"""
+        self.disable_email()
+        self.machine_set.all().delete()
+
+    @classmethod
+    def mass_archive(cls, users_list):
+        """Mass Archive several users, take a queryset
+        Copy Queryset to avoid eval problem with queryset update"""
+        #Force eval of queryset
+        bool(users_list)
+        users_list = users_list.all()
+        cls.mass_unassign_ips(users_list)
+        users_list.update(state=User.STATE_ARCHIVE)
+
+    @classmethod
+    def mass_full_archive(cls, users_list):
+        """Mass Archive several users, take a queryset
+        Copy Queryset to avoid eval problem with queryset update"""
+        #Force eval of queryset
+        bool(users_list)
+        users_list = users_list.all()
+        cls.mass_unassign_ips(users_list)
+        cls.mass_delete_data(users_list)
+        users_list.update(state=User.STATE_FULL_ARCHIVE)
 
     def archive(self):
         """ Filling the user; no more active"""
         self.unassign_ips()
-        self.disable_email()
+
+    def full_archive(self):
+        """Full Archive = Archive + Service access complete deletion"""
+        self.archive()
+        self.delete_data()
+        self.ldap_del()
 
     def unarchive(self):
         """Unfilling the user"""
         self.assign_ips()
+        self.ldap_sync()
 
     def state_sync(self):
         """Archive, or unarchive, if the user was not active/or archived before"""
@@ -566,6 +631,8 @@ class User(RevMixin, FieldPermissionModelMixin, AbstractBaseUser,
             self.unarchive()
         elif self.__original_state != self.STATE_ARCHIVE and self.state == self.STATE_ARCHIVE:
             self.archive()
+        elif self.__original_state != self.STATE_FULL_ARCHIVE and self.state == self.STATE_FULL_ARCHIVE:
+            self.full_archive()
 
     def ldap_sync(self, base=True, access_refresh=True, mac_refresh=True,
                   group_refresh=False):
@@ -578,15 +645,11 @@ class User(RevMixin, FieldPermissionModelMixin, AbstractBaseUser,
         mac_refresh : synchronise les machines de l'user
         group_refresh : synchronise les group de l'user
         Si l'instance n'existe pas, on crée le ldapuser correspondant"""
-        if sys.version_info[0] >= 3 and self.state != self.STATE_ARCHIVE and\
+        if sys.version_info[0] >= 3 and (self.state == self.STATE_ACTIVE or self.state == self.STATE_ARCHIVE or self.state == self.STATE_DISABLED):
-           self.state != self.STATE_DISABLED:
             self.refresh_from_db()
             try:
                 user_ldap = LdapUser.objects.get(uidNumber=self.uid_number)
             except LdapUser.DoesNotExist:
-                #  Freshly created users are NOT synced in ldap base
-                if self.state == self.STATE_NOT_YET_ACTIVE:
-                    return
                 user_ldap = LdapUser(uidNumber=self.uid_number)
                 base = True
                 access_refresh = True
@@ -639,6 +702,11 @@ class User(RevMixin, FieldPermissionModelMixin, AbstractBaseUser,
         except LdapUser.DoesNotExist:
             pass
 
+    @classmethod
+    def ldap_delete_users(cls, queryset_users):
+        """Delete multiple users in ldap"""
+        LdapUser.objects.filter(name__in=list(queryset_users.values_list('pseudo', flat=True)))
+
     def notif_inscription(self):
         """ Prend en argument un objet user, envoie un mail de bienvenue """
         template = loader.get_template('users/email_welcome')
@@ -1025,7 +1093,7 @@ class User(RevMixin, FieldPermissionModelMixin, AbstractBaseUser,
             .filter(local_part=self.pseudo.lower()).exclude(user_id=self.id)
             ):
             raise ValidationError(_("This username is already used."))
-        if not self.local_email_enabled and not self.email and not (self.state == self.STATE_ARCHIVE):
+        if not self.local_email_enabled and not self.email and not (self.state == self.STATE_FULL_ARCHIVE):
             raise ValidationError(_("There is neither a local email address nor an external"
                       " email address for this user.")
             )

users/templates/users/profil.html

@@ -253,6 +253,8 @@ with this program; if not, write to the Free Software Foundation, Inc.,
                             <dd><i class="text-danger">{% trans "Archived" %}</i></dd>
                         {% elif users.state == 3 %}
                             <dd><i class="text-danger">{% trans "Not yet member" %}</i></dd>
+                        {% elif users.state == 4 %}
+                            <dd><i class="text-danger">{% trans "Full Archived" %}</i></dd>
                         {% endif %}
                     </div>
 

users/views.py

@@ -776,26 +776,27 @@ def del_listright(request, instances):
 @can_change(User, 'state')
 def mass_archive(request):
     """ Permet l'archivage massif"""
-    to_archive_date = MassArchiveForm(request.POST or None)
+    pagination_number = GeneralOption.get_cached_value('pagination_number')
+    to_archive_form = MassArchiveForm(request.POST or None)
     to_archive_list = []
-    if to_archive_date.is_valid():
+    if to_archive_form.is_valid():
-        date = to_archive_date.cleaned_data['date']
+        date = to_archive_form.cleaned_data['date']
-        to_archive_list = [user for user in
+        full_archive = to_archive_form.cleaned_data['full_archive']
-                           User.objects.exclude(state=User.STATE_ARCHIVE)
+        to_archive_list = User.objects.exclude(id__in=all_has_access()).exclude(id__in=all_has_access(search_time=date)).exclude(state=User.STATE_NOT_YET_ACTIVE).exclude(state=User.STATE_FULL_ARCHIVE)
-                           if not user.end_access()
+        if not full_archive:
-                           or user.end_access() < date]
+            to_archive_list = to_archive_list.exclude(state=User.STATE_ARCHIVE)
         if "valider" in request.POST:
-            for user in to_archive_list:
+            if full_archive:
-                with transaction.atomic(), reversion.create_revision():
+                User.mass_full_archive(to_archive_list)
-                    user.archive()
+            else:
-                    user.save()
+                User.mass_archive(to_archive_list)
-                    reversion.set_comment(_("Archiving"))
+            messages.success(request, _("%s users were archived.") %
-            messages.success(request, _("%s users were archived.") % len(
+                to_archive_list.count()
-                to_archive_list
+            )
-            ))
             return redirect(reverse('users:index'))
+        to_archive_list = re2o_paginator(request, to_archive_list, pagination_number)
     return form(
-        {'userform': to_archive_date, 'to_archive_list': to_archive_list},
+        {'userform': to_archive_form, 'to_archive_list': to_archive_list},
         'users/mass_archive.html',
         request
     )