Add logic to reduce full resync of firewall zonea

Chore: Formatting

api/v1beta1/firewallgroup_types.go

@@ -36,7 +36,7 @@ type FirewallGroupSpec struct {
 	// ManualAddresses is a list of manual IPs or CIDRs (IPv4 or IPv6)
 	// +optional
 	ManualAddresses []string `json:"manualAddresses,omitempty"`
-	ManualPorts []string `json:"manualPorts,omitempty"`
+	ManualPorts     []string `json:"manualPorts,omitempty"`
 
 	// AutoIncludeSelector defines which services to extract addresses from
 	// +optional

api/v1beta1/firewallzone_types.go

@@ -28,11 +28,11 @@ type FirewallZoneSpec struct {
 	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
 
-	Name string `json:"name,omitempty"`
-	ID string `json:"_id,omitempty"`
-	DefaultZone bool `json:"default_zone,omitempty"`
-	ZoneKey string `json:"zone_key,omitempty"`
-	NetworkIDs []string `json:"network_ids,omitempty"`
+	Name        string   `json:"name,omitempty"`
+	ID          string   `json:"_id,omitempty"`
+	DefaultZone bool     `json:"default_zone,omitempty"`
+	ZoneKey     string   `json:"zone_key,omitempty"`
+	NetworkIDs  []string `json:"network_ids,omitempty"`
 }
 
 // FirewallZoneStatus defines the observed state of FirewallZone.

api/v1beta1/networkconfiguration_types.go

@@ -29,6 +29,7 @@ type NetworkconfigurationSpec struct {
 	// Important: Run "make" to regenerate code after modifying this file
 
 	// Foo is an example field of Networkconfiguration. Edit networkconfiguration_types.go to remove/update
+	ID                        string `json:"_id,omitempty"`
 	Enabled                   bool   `json:"enabled,omitempty"`
 	FirewallZone              string `json:"firewall_zone,omitempty"`
 	GatewayType               string `json:"gateway_type,omitempty"`
@@ -39,7 +40,6 @@ type NetworkconfigurationSpec struct {
 	Ipv6SettingPreference     string `json:"ipv6_setting_preference,omitempty"`
 	Ipv6Subnet                string `json:"ipv6_subnet,omitempty"`
 	Name                      string `json:"name"`
-	NetworkID                 string `json:"network_id,omitempty"`
 	Networkgroup              string `json:"networkgroup,omitempty"`
 	Purpose                   string `json:"purpose,omitempty"`
 	SettingPreference         string `json:"setting_preference,omitempty"`
@@ -51,16 +51,16 @@ type NetworkconfigurationSpec struct {
 type NetworkconfigurationStatus struct {
 	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
 	// Important: Run "make" to regenerate code after modifying this file
-	FirewallZoneID            string `json:"firewall_zone_id,omitempty"`
-	Ipv6SubnetStatus          string `json:"ipv6_subnet_status,omitempty"`
+	FirewallZoneID   string `json:"firewall_zone_id,omitempty"`
+	Ipv6SubnetStatus string `json:"ipv6_subnet_status,omitempty"`
 
 	// SyncedWithUnifi indicates whether the addresses are successfully pushed
-        // +optional
-	SyncedWithUnifi           bool `json:"syncedWithUnifi,omitempty"`
+	// +optional
+	SyncedWithUnifi bool `json:"syncedWithUnifi,omitempty"`
 
 	// LastSyncTime is the last time the object was synced
-        // +optional
-        LastSyncTime *metav1.Time `json:"lastSyncTime,omitempty"`
+	// +optional
+	LastSyncTime *metav1.Time `json:"lastSyncTime,omitempty"`
 }
 
 // +kubebuilder:object:root=true

api/v1beta1/zz_generated.deepcopy.go

@@ -92,6 +92,11 @@ func (in *FirewallGroupSpec) DeepCopyInto(out *FirewallGroupSpec) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
+	if in.ManualPorts != nil {
+		in, out := &in.ManualPorts, &out.ManualPorts
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
 	if in.AutoIncludeSelector != nil {
 		in, out := &in.AutoIncludeSelector, &out.AutoIncludeSelector
 		*out = new(v1.LabelSelector)

cmd/main.go

@@ -38,9 +38,9 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/webhook"
 
 	unifiv1beta1 "github.com/vegardengen/unifi-network-operator/api/v1beta1"
+	"github.com/vegardengen/unifi-network-operator/internal/config"
 	"github.com/vegardengen/unifi-network-operator/internal/controller"
 	"github.com/vegardengen/unifi-network-operator/internal/unifi"
-	"github.com/vegardengen/unifi-network-operator/internal/config"
 	// +kubebuilder:scaffold:imports
 )
 
@@ -204,7 +204,7 @@ func main() {
 		os.Exit(1)
 	}
 
-       configLoader := config.NewConfigLoader(mgr.GetClient())
+	configLoader := config.NewConfigLoader(mgr.GetClient())
 
 	// Unifi client
 	setupLog.Info("Setting up UniFi client")
@@ -216,27 +216,27 @@ func main() {
 	setupLog.Info("Finished Setting up UniFi client")
 
 	if err = (&controller.NetworkconfigurationReconciler{
-		Client:      mgr.GetClient(),
-		Scheme:      mgr.GetScheme(),
-		UnifiClient: unifiClient,
+		Client:       mgr.GetClient(),
+		Scheme:       mgr.GetScheme(),
+		UnifiClient:  unifiClient,
 		ConfigLoader: configLoader,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "Networkconfiguration")
 		os.Exit(1)
 	}
 	if err = (&controller.FirewallZoneReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
-		UnifiClient: unifiClient,
+		Client:       mgr.GetClient(),
+		Scheme:       mgr.GetScheme(),
+		UnifiClient:  unifiClient,
 		ConfigLoader: configLoader,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "FirewallZone")
 		os.Exit(1)
 	}
 	if err = (&controller.FirewallRuleReconciler{
-		Client: mgr.GetClient(),
-		Scheme: mgr.GetScheme(),
-		UnifiClient: unifiClient,
+		Client:       mgr.GetClient(),
+		Scheme:       mgr.GetScheme(),
+		UnifiClient:  unifiClient,
 		ConfigLoader: configLoader,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "FirewallRule")
@@ -245,9 +245,9 @@ func main() {
 	// +kubebuilder:scaffold:builder
 
 	if err = (&controller.FirewallGroupReconciler{
-		Client:      mgr.GetClient(),
-		Scheme:      mgr.GetScheme(),
-		UnifiClient: unifiClient,
+		Client:       mgr.GetClient(),
+		Scheme:       mgr.GetScheme(),
+		UnifiClient:  unifiClient,
 		ConfigLoader: configLoader,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "FirewallGroup")

config/crd/bases/unifi.engen.priv.no_networkconfigurations.yaml

@@ -40,9 +40,11 @@ spec:
           spec:
             description: NetworkconfigurationSpec defines the desired state of Networkconfiguration.
             properties:
-              enabled:
+              _id:
                 description: Foo is an example field of Networkconfiguration. Edit
                   networkconfiguration_types.go to remove/update
+                type: string
+              enabled:
                 type: boolean
               firewall_zone:
                 type: string
@@ -62,8 +64,6 @@ spec:
                 type: string
               name:
                 type: string
-              network_id:
-                type: string
               networkgroup:
                 type: string
               purpose:

internal/config/config.go

@@ -1,45 +1,44 @@
 package config
 
 import (
-    "context"
-    "sync"
+	"context"
+	"sync"
 
-    corev1 "k8s.io/api/core/v1"
-    "k8s.io/apimachinery/pkg/types"
-    "sigs.k8s.io/controller-runtime/pkg/client"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
 type ConfigLoaderType struct {
-    Client client.Client
+	Client client.Client
 
-    mu      sync.Mutex
-    loaded  bool
-    config  *corev1.ConfigMap
-    err     error
+	mu     sync.Mutex
+	loaded bool
+	config *corev1.ConfigMap
+	err    error
 }
 
 func NewConfigLoader(k8sClient client.Client) *ConfigLoaderType {
-    return &ConfigLoaderType{Client: k8sClient}
+	return &ConfigLoaderType{Client: k8sClient}
 }
 
 func (c *ConfigLoaderType) GetConfig(ctx context.Context, name string) (*corev1.ConfigMap, error) {
-    c.mu.Lock()
-    defer c.mu.Unlock()
+	c.mu.Lock()
+	defer c.mu.Unlock()
 
-    if c.loaded {
-        return c.config, c.err
-    }
+	if c.loaded {
+		return c.config, c.err
+	}
 
-    cm := &corev1.ConfigMap{}
-    err := c.Client.Get(ctx, types.NamespacedName{
-        Name:      name,
-        Namespace: "unifi-network-operator-system",
-    }, cm)
+	cm := &corev1.ConfigMap{}
+	err := c.Client.Get(ctx, types.NamespacedName{
+		Name:      name,
+		Namespace: "unifi-network-operator-system",
+	}, cm)
 
-    c.loaded = true
-    c.config = cm
-    c.err = err
+	c.loaded = true
+	c.config = cm
+	c.err = err
 
-    return cm, err
+	return cm, err
 }
-

internal/controller/firewallzone_controller.go

@@ -91,6 +91,61 @@ func (r *FirewallZoneReconciler) Reconcile(ctx context.Context, req ctrl.Request
 		return ctrl.Result{}, err
 	}
 
+	fullSyncZone := "gateway"
+	if cfg.Data["fullSyncZone"] != "" {
+		fullSyncZone = cfg.Data["fullSyncZone"]
+	}
+
+	fullSync := false
+	var zoneObj unifiv1beta1.FirewallZone
+	if err := r.Get(ctx, req.NamespacedName, &zoneObj); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	log.Info(fmt.Sprintf("fullSyncZone: %s Zone: %s", fullSyncZone, zoneObj.Name))
+	if zoneObj.Name == fullSyncZone {
+		fullSync = true
+		log.Info("Going into fullsync mode")
+	}
+
+	err = r.UnifiClient.Reauthenticate()
+	if err != nil {
+		return ctrl.Result{}, err
+	}
+
+	if !fullSync {
+		firewallzones, err := r.UnifiClient.Client.ListFirewallZones(context.Background(), r.UnifiClient.SiteID)
+		if err != nil {
+			log.Error(err, "Could not list firewall zones")
+			return ctrl.Result{}, err
+		}
+		found := false
+		for _, unifizone := range firewallzones {
+			if unifizone.Name == zoneObj.Spec.Name {
+				found = true
+				zoneSpec := unifiv1beta1.FirewallZoneSpec{
+					Name:        unifizone.Name,
+					ID:          unifizone.ID,
+					DefaultZone: unifizone.DefaultZone,
+					NetworkIDs:  unifizone.NetworkIDs,
+				}
+				zoneObj.Spec = zoneSpec
+				err := r.Update(ctx, &zoneObj)
+				if err != nil {
+					return ctrl.Result{}, err
+				}
+			}
+		}
+		if !found {
+			err := r.Delete(ctx, &zoneObj)
+			if err != nil {
+				return ctrl.Result{}, err
+			}
+		}
+		return ctrl.Result{}, nil
+	}
+	log.Info("Starting fullsync mode")
+
 	var fwzCRDs unifiv1beta1.FirewallZoneList
 	_ = r.List(ctx, &fwzCRDs, client.InNamespace(defaultNs))
 

internal/controller/networkconfiguration_controller.go

@@ -19,6 +19,7 @@ package controller
 import (
 	"context"
 	"fmt"
+	"time"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -62,9 +63,21 @@ func (r *NetworkconfigurationReconciler) Reconcile(ctx context.Context, req ctrl
 	defaultNs := cfg.Data["defaultNamespace"]
 	log.Info(defaultNs)
 
-	var networkCRDs unifiv1.NetworkconfigurationList
-	if err := r.List(ctx, &networkCRDs); err != nil {
-		return ctrl.Result{}, err
+	fullSyncNetwork := "core"
+	if cfg.Data["fullSyncNetwork"] != "" {
+		fullSyncNetwork = cfg.Data["fullSyncNetwork"]
+	}
+
+	fullSync := false
+	var networkObj unifiv1.Networkconfiguration
+	if err := r.Get(ctx, req.NamespacedName, &networkObj); err != nil {
+		return ctrl.Result{}, client.IgnoreNotFound(err)
+	}
+
+	log.Info(fmt.Sprintf("fullSyncNetwork: %s Network: %s", fullSyncNetwork, networkObj.Name))
+	if networkObj.Name == fullSyncNetwork {
+		fullSync = true
+		log.Info("Going into fullsync mode")
 	}
 
 	err = r.UnifiClient.Reauthenticate()
@@ -72,49 +85,132 @@ func (r *NetworkconfigurationReconciler) Reconcile(ctx context.Context, req ctrl
 		return ctrl.Result{}, err
 	}
 
-	k8sNetworks := make(map[string]*unifiv1.Networkconfiguration)
-	for i := range networkCRDs.Items {
-		log.Info(fmt.Sprintf("Inserting network %s\n", networkCRDs.Items[i].Spec.NetworkID))
-		k8sNetworks[networkCRDs.Items[i].Spec.NetworkID] = &networkCRDs.Items[i]
+	if !fullSync {
+		networks, err := r.UnifiClient.Client.ListNetwork(context.Background(), r.UnifiClient.SiteID)
+		if err != nil {
+			log.Error(err, "Could not list netwrorks")
+			return ctrl.Result{}, err
+		}
+		found := false
+		for _, unifinetwork := range networks {
+			if unifinetwork.Name == networkObj.Spec.Name {
+				found = true
+				networkSpec := unifiv1.NetworkconfigurationSpec{
+					Name:                      unifinetwork.Name,
+					ID:                        unifinetwork.ID,
+					IPSubnet:                  unifinetwork.IPSubnet,
+					Ipv6InterfaceType:         unifinetwork.IPV6InterfaceType,
+					Ipv6PdAutoPrefixidEnabled: unifinetwork.IPV6PDAutoPrefixidEnabled,
+					Ipv6RaEnabled:             unifinetwork.IPV6RaEnabled,
+					Ipv6SettingPreference:     unifinetwork.IPV6SettingPreference,
+					Ipv6Subnet:                unifinetwork.IPV6Subnet,
+					Purpose:                   unifinetwork.Purpose,
+					Networkgroup:              unifinetwork.NetworkGroup,
+					SettingPreference:         unifinetwork.SettingPreference,
+					Vlan:                      int64(unifinetwork.VLAN),
+					VlanEnabled:               unifinetwork.VLANEnabled,
+				}
+				networkObj.Spec = networkSpec
+				err := r.Update(ctx, &networkObj)
+				if err != nil {
+					return ctrl.Result{}, err
+				}
+			}
+		}
+		if !found {
+			err := r.Delete(ctx, &networkObj)
+			if err != nil {
+				return ctrl.Result{}, err
+			}
+		}
+		return ctrl.Result{}, nil
 	}
+	log.Info("Starting fullsync mode")
+	var networkCRDs unifiv1.NetworkconfigurationList
+	_ = r.List(ctx, &networkCRDs, client.InNamespace(defaultNs))
 
 	networks, err := r.UnifiClient.Client.ListNetwork(context.Background(), r.UnifiClient.SiteID)
 	if err != nil {
-		log.Error(err, "Failed to list Unifi Networks")
-		return ctrl.Result{}, err
+		log.Error(err, "Could not list netwrorks")
+		return ctrl.Result{RequeueAfter: 10 * time.Minute}, err
+	}
+	log.Info(fmt.Sprintf("Number of resources: %d Number of networks in Unifi: %d", len(networkCRDs.Items), len(networks)))
+
+	var firewallZoneCRDs unifiv1.FirewallZoneList
+	err = r.List(ctx, &firewallZoneCRDs, client.InNamespace(defaultNs))
+	if err != nil {
+		log.Error(err, "Could not list firewall zones")
+		return ctrl.Result{RequeueAfter: 10 * time.Minute}, err
 	}
 
-	seenNetworks := map[string]bool{}
+	networkNamesUnifi := make(map[string]struct{})
+	for _, unifinetwork := range networks {
+		networkNamesUnifi[unifinetwork.Name] = struct{}{}
+	}
 
-	for _, network := range networks {
-		networkID := network.ID
-		seenNetworks[networkID] = true
-		log.Info(fmt.Sprintf("Searching for  %s\n", networkID))
-
-		if existing, found := k8sNetworks[networkID]; found {
-			log.Info(fmt.Sprintf("Found network match: %s/%s", existing.Spec.NetworkID, networkID))
-		} else {
-			if network.Purpose == "corporate" {
-				log.Info(fmt.Sprintf("New network: %s with ID %s", network.Name, network.ID))
-				var networkObject unifiv1.Networkconfiguration
-				networkObject.Name = network.Name
-				networkObject.Spec.Name = network.Name
-				networkObject.Spec.NetworkID = network.ID
-				networkObject.Spec.IPSubnet = network.IPSubnet
-				networkObject.Spec.Ipv6InterfaceType = network.IPV6InterfaceType
-				networkObject.Spec.Ipv6PdAutoPrefixidEnabled = network.IPV6PDAutoPrefixidEnabled
-				networkObject.Spec.Ipv6RaEnabled = network.IPV6RaEnabled
-				networkObject.Spec.Ipv6SettingPreference = network.IPV6SettingPreference
-				networkObject.Spec.Ipv6Subnet = network.IPV6Subnet
-				networkObject.Spec.Purpose = network.Purpose
-				networkObject.Spec.Networkgroup = network.NetworkGroup
-				networkObject.Spec.SettingPreference = network.SettingPreference
-				networkObject.Spec.VlanEnabled = network.VLANEnabled
+	// Step 2: Collect zones in fwzCRDs that are NOT in firewall_zones
+	for _, network := range networkCRDs.Items {
+		if _, found := networkNamesUnifi[network.Spec.Name]; !found {
+			err := r.Delete(ctx, &network)
+			if err != nil {
+				return ctrl.Result{RequeueAfter: 10 * time.Minute}, err
+			}
+		}
+	}
+	networkNamesCRDs := make(map[string]struct{})
+	for _, networkCRD := range networkCRDs.Items {
+		networkNamesCRDs[networkCRD.Spec.Name] = struct{}{}
+	}
+	for _, unifinetwork := range networks {
+		if unifinetwork.Purpose == "corporate" {
+			networkSpec := unifiv1.NetworkconfigurationSpec{
+				Name:                      unifinetwork.Name,
+				ID:                        unifinetwork.ID,
+				IPSubnet:                  unifinetwork.IPSubnet,
+				Ipv6InterfaceType:         unifinetwork.IPV6InterfaceType,
+				Ipv6PdAutoPrefixidEnabled: unifinetwork.IPV6PDAutoPrefixidEnabled,
+				Ipv6RaEnabled:             unifinetwork.IPV6RaEnabled,
+				Ipv6SettingPreference:     unifinetwork.IPV6SettingPreference,
+				Ipv6Subnet:                unifinetwork.IPV6Subnet,
+				Purpose:                   unifinetwork.Purpose,
+				Networkgroup:              unifinetwork.NetworkGroup,
+				SettingPreference:         unifinetwork.SettingPreference,
+				Vlan:                      int64(unifinetwork.VLAN),
+				VlanEnabled:               unifinetwork.VLANEnabled,
+			}
+			if _, found := networkNamesCRDs[unifinetwork.Name]; !found {
+				firewallZoneNamesCRDs := make(map[string]struct{})
+				firewallZoneIdsCRDs := make(map[string]struct{})
+				for _, firewallZoneCRD := range firewallZoneCRDs.Items {
+					firewallZoneNamesCRDs[firewallZoneCRD.Spec.Name] = struct{}{}
+					firewallZoneIdsCRDs[firewallZoneCRD.Spec.ID] = struct{}{}
+				}
+				networkCRD := &unifiv1.Networkconfiguration{
+					ObjectMeta: ctrl.ObjectMeta{
+						Name:      toKubeName(unifinetwork.Name),
+						Namespace: defaultNs,
+					},
+					Spec: networkSpec,
+				}
+				err = r.Create(ctx, networkCRD)
+				if err != nil {
+					return ctrl.Result{RequeueAfter: 10 * time.Minute}, err
+				}
+			} else {
+				for _, networkCRD := range networkCRDs.Items {
+					if networkCRD.Spec.Name == unifinetwork.Name {
+						networkCRD.Spec = networkSpec
+					}
+					err := r.Update(ctx, &networkCRD)
+					if err != nil {
+						return ctrl.Result{RequeueAfter: 10 * time.Minute}, err
+					}
+				}
 			}
 		}
 	}
 
-	return ctrl.Result{}, nil
+	return ctrl.Result{RequeueAfter: 10 * time.Minute}, nil
 }
 
 // SetupWithManager sets up the controller with the Manager.

internal/unifi_network_operator_utils/unifi_network_operator_utils.go

@@ -0,0 +1,33 @@
+/* https://github.com/clbx/kube-port-forward-controller */
+
+package unifi_network_operator_utils
+
+import (
+	"regexp"
+	"strings"
+)
+
+func isIPv6(ip string) bool {
+	return strings.Contains(ip, ":")
+}
+
+func toKubeName(input string) string {
+	// Lowercase the input
+	name := strings.ToLower(input)
+
+	// Replace any non-alphanumeric characters with dashes
+	re := regexp.MustCompile(`[^a-z0-9\-\.]+`)
+	name = re.ReplaceAllString(name, "-")
+
+	// Trim leading and trailing non-alphanumerics
+	name = strings.Trim(name, "-.")
+
+	// Ensure it's not empty and doesn't exceed 253 characters
+	if len(name) == 0 {
+		name = "default"
+	} else if len(name) > 253 {
+		name = name[:253]
+	}
+
+	return name
+}