[feature] Added automatic provisioning of Subnets and IPs openwisp#400

- Added subnet division rules for defining charecterstic of
  provisioned subnets and IPs
- Added "VPN" rule type that provisions subents and IPs for a
  device when a new VpnClient is created
- Added support for adding custom subnet division rule types
- Added support for updating subnet divisioon rules
  Currently only "number_of__ips" and "label" fields can be updated
- Added an inline admin for subnet division rules in SubnetAdmin
- Added a filter on SubnetAdmin to allow filtering by related device name
- Added a filter on DeviceAdmin to allow filtering by related subnet
  It also allows filtering using parent of master subnet
- Added setting for hiding provisioned subnets
- Extends SubnetAdmin and IpAdmin from openwisp-ipam

NOTE: Adds openwisp-ipam as direct dependency

Closes openwisp#400

openwisp_controller/config/admin.py

@@ -723,6 +723,7 @@ class VpnAdmin(
         'organization',
         'uuid',
         'key',
+        'subnet',
         'ca',
         'cert',
         'backend',

openwisp_controller/config/base/config.py

@@ -482,6 +482,31 @@ def get_vpn_context(self):
                 )
         return c
 
+    def get_subnet_division_context(self):
+        # NOTE: Use regex to know which subnet division variables
+        # are used in this config and only provide those contexts
+
+        context = {}
+        qs = self.subnetdivisionindex_set.values(
+            'keyword', 'subnet__subnet', 'ip__ip_address'
+        )
+        for entry in qs:
+            if entry['ip__ip_address'] is None:
+                context[entry['keyword']] = str(entry['subnet__subnet'])
+            else:
+                context[entry['keyword']] = str(entry['ip__ip_address'])
+
+        prefixlen = (
+            self.subnetdivisionindex_set.select_related('rule')
+            .values('rule__label', 'rule__size')
+            .first()
+        )
+        if prefixlen:
+            context[f'{prefixlen["rule__label"]}_prefixlen'] = str(
+                prefixlen['rule__size']
+            )
+        return context
+
     def get_context(self, system=False):
         """
         additional context passed to netjsonconfig
@@ -500,6 +525,7 @@ def get_context(self, system=False):
             if self.context and not system:
                 extra.update(self.context)
         extra.update(self.get_vpn_context())
+        extra.update(self.get_subnet_division_context())
         if app_settings.HARDWARE_ID_ENABLED and self._has_device():
             extra.update({'hardware_id': str(self.device.hardware_id)})
         c.update(sorted(extra.items()))

openwisp_controller/config/base/vpn.py

@@ -45,6 +45,14 @@ class AbstractVpn(ShareableOrgMixinUniqueName, BaseConfig):
         help_text=_('Select VPN configuration backend'),
     )
     notes = models.TextField(blank=True)
+    subnet = models.ForeignKey(
+        get_model_name('openwisp_ipam', 'Subnet'),
+        verbose_name=_('Subnet'),
+        help_text=_('Subnet IP addresses used by VPN clients, if applicable'),
+        blank=True,
+        null=True,
+        on_delete=models.SET_NULL,
+    )
     # diffie hellman parameters are required
     # in some VPN solutions (eg: OpenVPN)
     dh = models.TextField(blank=True)
@@ -142,6 +150,13 @@ def get_context(self):
             c.update([('cert', self.cert.certificate), ('key', self.cert.private_key)])
         if self.dh:
             c.update([('dh', self.dh)])
+        if self.subnet:
+            c.update(
+                {
+                    'subnet': str(self.subnet.subnet),
+                    'subnet_prefixlen': str(self.subnet.subnet.prefixlen),
+                }
+            )
         c.update(sorted(super().get_context().items()))
         return c
 

openwisp_controller/config/migrations/0038_vpn_subnet.py

@@ -0,0 +1,28 @@
+# Generated by Django 3.1.7 on 2021-03-08 12:17
+
+import django.db.models.deletion
+from django.conf import settings
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.OPENWISP_IPAM_SUBNET_MODEL),
+        ('config', '0037_alter_taggedtemplate'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='vpn',
+            name='subnet',
+            field=models.ForeignKey(
+                blank=True,
+                help_text='Subnet IP addresses used by VPN clients, if applicable',
+                null=True,
+                on_delete=django.db.models.deletion.SET_NULL,
+                to=settings.OPENWISP_IPAM_SUBNET_MODEL,
+                verbose_name='Subnet',
+            ),
+        ),
+    ]

openwisp_controller/config/tests/test_api.py

@@ -274,7 +274,7 @@ def test_device_download_api(self):
         d1 = self._create_device()
         self._create_config(device=d1)
         path = reverse('config_api:download_device_config', args=[d1.pk])
-        with self.assertNumQueries(6):
+        with self.assertNumQueries(8):
             r = self.client.get(path)
         self.assertEqual(r.status_code, 200)
 

openwisp_controller/config/tests/test_controller.py

@@ -133,7 +133,7 @@ def test_get_cached_checksum(self):
         url = reverse('controller:device_checksum', args=[d.pk])
 
         with self.subTest('first request does not return value from cache'):
-            with self.assertNumQueries(3):
+            with self.assertNumQueries(5):
                 with patch.object(
                     controller_views_logger, 'debug'
                 ) as mocked_view_debug:
@@ -1107,12 +1107,12 @@ def test_ip_fields_not_duplicated(self):
         c2 = self._create_config(device=d2)
         org2 = self._create_org(name='org2', shared_secret='123456')
         c3 = self._create_config(organization=org2)
-        with self.assertNumQueries(6):
+        with self.assertNumQueries(8):
             self.client.get(
                 reverse('controller:device_checksum', args=[c3.device.pk]),
                 {'key': c3.device.key, 'management_ip': '192.168.1.99'},
             )
-        with self.assertNumQueries(6):
+        with self.assertNumQueries(8):
             self.client.get(
                 reverse('controller:device_checksum', args=[c1.device.pk]),
                 {'key': c1.device.key, 'management_ip': '192.168.1.99'},
@@ -1125,7 +1125,7 @@ def test_ip_fields_not_duplicated(self):
             )
         # triggers more queries because devices with conflicting addresses
         # need to be updated, luckily it does not happen often
-        with self.assertNumQueries(8):
+        with self.assertNumQueries(10):
             self.client.get(
                 reverse('controller:device_checksum', args=[c2.device.pk]),
                 {'key': c2.device.key, 'management_ip': '192.168.1.99'},
@@ -1154,14 +1154,14 @@ def test_organization_shares_management_ip_address_space(self):
         org1_config = self._create_config(organization=org1)
         org2 = self._create_org(name='org2', shared_secret='org2')
         org2_config = self._create_config(organization=org2)
-        with self.assertNumQueries(6):
+        with self.assertNumQueries(8):
             self.client.get(
                 reverse('controller:device_checksum', args=[org1_config.device_id]),
                 {'key': org1_config.device.key, 'management_ip': '192.168.1.99'},
             )
         # Device from another organization sends conflicting management IP
         # Extra queries due to conflict resolution
-        with self.assertNumQueries(8):
+        with self.assertNumQueries(10):
             self.client.get(
                 reverse('controller:device_checksum', args=[org2_config.device_id]),
                 {'key': org2_config.device.key, 'management_ip': '192.168.1.99'},

openwisp_controller/subnet_division/__init__.py

@@ -0,0 +1 @@
+default_app_config = 'openwisp_controller.subnet_division.apps.SubnetDivisionConfig'

openwisp_controller/subnet_division/admin.py

@@ -0,0 +1,121 @@
+from django.contrib import admin
+from django.urls import reverse
+from django.utils.html import mark_safe
+from django.utils.translation import gettext_lazy as _
+from openwisp_ipam.admin import IpAddressAdmin as BaseIpAddressAdmin
+from openwisp_ipam.admin import SubnetAdmin as BaseSubnetAdmin
+from swapper import load_model
+
+from openwisp_controller.config.admin import DeviceAdmin
+from openwisp_users.multitenancy import MultitenantAdminMixin, MultitenantOrgFilter
+from openwisp_utils.admin import TimeReadonlyAdminMixin
+
+from . import settings as app_settings
+from .filters import DeviceFilter, SubnetFilter, SubnetListFilter
+
+SubnetDivisionRule = load_model('subnet_division', 'SubnetDivisionRule')
+SubnetDivisionIndex = load_model('subnet_division', 'SubnetDivisionIndex')
+Subnet = load_model('openwisp_ipam', 'Subnet')
+IpAddress = load_model('openwisp_ipam', 'IpAddress')
+Device = load_model('config', 'Device')
+
+
+class SubnetDivisionRuleInlineAdmin(
+    MultitenantAdminMixin, TimeReadonlyAdminMixin, admin.StackedInline
+):
+    model = SubnetDivisionRule
+    extra = 0
+
+
+# Monkey patching DeviceAdmin to allow filtering using subnet
+DeviceAdmin.list_filter.append(SubnetFilter)
+
+# NOTE: Monkey patching SubnetAdmin didn't work for adding readonly_field
+# to change_view because of TimeReadonlyAdminMixin.
+
+admin.site.unregister(Subnet)
+admin.site.unregister(IpAddress)
+
+
+@admin.register(Subnet)
+class SubnetAdmin(BaseSubnetAdmin):
+    list_filter = BaseSubnetAdmin.list_filter + [DeviceFilter]
+    inlines = [SubnetDivisionRuleInlineAdmin] + BaseSubnetAdmin.inlines
+
+    def get_queryset(self, request):
+        qs = super().get_queryset(request)
+        subnet_division_index_qs = (
+            SubnetDivisionIndex.objects.filter(
+                subnet_id__in=qs.filter(master_subnet__isnull=False).values('id'),
+                ip__isnull=True,
+            )
+            .select_related('config__device')
+            .values_list('subnet_id', 'config__device__name')
+        )
+        self._lookup = {}
+        for subnet_id, device_name in subnet_division_index_qs:
+            self._lookup[subnet_id] = device_name
+
+        if app_settings.HIDE_GENERATED_SUBNETS:
+            qs = qs.exclude(
+                id__in=SubnetDivisionIndex.objects.filter(
+                    ip__isnull=True, subnet__isnull=False
+                ).values_list('subnet_id')
+            )
+
+        return qs
+
+    def get_readonly_fields(self, request, obj=None):
+        fields = super().get_readonly_fields(request, obj)
+        if obj is not None and 'related_device' not in fields:
+            fields = ('related_device',) + fields
+        return fields
+
+    def get_list_display(self, request):
+        fields = super().get_list_display(request)
+        return fields + ['related_device']
+
+    def related_device(self, obj):
+        app_label = Device._meta.app_label
+        url = reverse(f'admin:{app_label}_device_changelist')
+        if obj.master_subnet is None:
+            msg_string = _('See all devices')
+            return mark_safe(
+                f'<a href="{url}?subnet={str(obj.subnet)}">{msg_string}</a>'
+            )
+        else:
+            device = self._lookup[obj.id]
+            return mark_safe(f'<a href="{url}?subnet={str(obj.subnet)}">{device}</a>')
+
+    def has_change_permission(self, request, obj=None):
+        permission = super().has_change_permission(request, obj)
+        if not obj:
+            return permission
+        automated = SubnetDivisionIndex.objects.filter(subnet_id=obj.id).exists()
+        return permission and not automated
+
+
+@admin.register(IpAddress)
+class IpAddressAdmin(BaseIpAddressAdmin):
+    list_filter = [
+        ('subnet', SubnetListFilter),
+        ('subnet__organization', MultitenantOrgFilter),
+    ]
+
+    def get_queryset(self, request):
+        qs = super().get_queryset(request)
+
+        if app_settings.HIDE_GENERATED_SUBNETS:
+            qs = qs.exclude(
+                id__in=SubnetDivisionIndex.objects.filter(ip__isnull=False).values_list(
+                    'ip_id'
+                )
+            )
+        return qs
+
+    def has_change_permission(self, request, obj=None):
+        permission = super().has_change_permission(request, obj)
+        if not obj:
+            return permission
+        automated = SubnetDivisionIndex.objects.filter(ip_id=obj.id).exists()
+        return permission and not automated

openwisp_controller/subnet_division/apps.py

@@ -0,0 +1,28 @@
+from django.apps import AppConfig
+from django.utils.module_loading import import_string
+from django.utils.translation import gettext_lazy as _
+
+
+class SubnetDivisionConfig(AppConfig):
+    name = 'openwisp_controller.subnet_division'
+    verbose_name = _('Subnet Division')
+    default_auto_field = 'django.db.models.AutoField'
+
+    def ready(self):
+        super().ready()
+        from . import handlers  # noqa
+        from . import settings as app_settings
+
+        for rule_path, name in app_settings.SUBNET_DIVISION_TYPES:
+            rule_class = import_string(rule_path)
+            rule_class.validate_rule_type()
+            rule_class.provision_signal.connect(
+                receiver=rule_class.provision_receiver,
+                sender=rule_class.provision_sender,
+                dispatch_uid=rule_class.provision_dispatch_uid,
+            )
+            rule_class.destroyer_signal.connect(
+                receiver=rule_class.destroyer_receiver,
+                sender=rule_class.destroyer_sender,
+                dispatch_uid=rule_class.destroyer_dispatch_uid,
+            )