[EV-3924] Add network policy egress rules for threat feed pull domains

pkg/controller/intrusiondetection/intrusiondetection_controller.go

@@ -17,6 +17,10 @@ package intrusiondetection
 import (
 	"context"
 	"fmt"
+	"maps"
+	"net/url"
+	"slices"
+	"sort"
 
 	esv1 "github.com/elastic/cloud-on-k8s/v2/pkg/apis/elasticsearch/v1"
 
@@ -71,9 +75,10 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error {
 	licenseAPIReady := &utils.ReadyFlag{}
 	dpiAPIReady := &utils.ReadyFlag{}
 	tierWatchReady := &utils.ReadyFlag{}
+	threatFeedAPIReady := &utils.ReadyFlag{}
 
 	// Create the reconciler
-	reconciler := newReconciler(mgr, opts, licenseAPIReady, dpiAPIReady, tierWatchReady)
+	reconciler := newReconciler(mgr, opts, licenseAPIReady, dpiAPIReady, tierWatchReady, threatFeedAPIReady)
 
 	// Create a new controller
 	c, err := ctrlruntime.NewController("intrusiondetection-controller", mgr, controller.Options{Reconciler: reconcile.Reconciler(reconciler)})
@@ -100,6 +105,8 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error {
 			[]client.Object{&v3.DeepPacketInspection{TypeMeta: metav1.TypeMeta{Kind: v3.KindDeepPacketInspection}}})
 		policiesToWatch = append(policiesToWatch, types.NamespacedName{Name: dpi.DeepPacketInspectionPolicyName, Namespace: dpi.DeepPacketInspectionNamespace})
 	}
+	go utils.WaitToAddResourceWatch(c, opts.K8sClientset, log, threatFeedAPIReady,
+		[]client.Object{&v3.GlobalThreatFeed{TypeMeta: metav1.TypeMeta{Kind: v3.KindGlobalThreatFeed}}})
 	go utils.WaitToAddNetworkPolicyWatches(c, opts.K8sClientset, log, policiesToWatch)
 	go utils.WaitToAddLicenseKeyWatch(c, opts.K8sClientset, log, licenseAPIReady)
 	go utils.WaitToAddTierWatch(networkpolicy.CalicoTierName, c, opts.K8sClientset, log, tierWatchReady)
@@ -160,15 +167,16 @@ func Add(mgr manager.Manager, opts options.ControllerOptions) error {
 }
 
 // newReconciler returns a new reconcile.Reconciler
-func newReconciler(mgr manager.Manager, opts options.ControllerOptions, licenseAPIReady *utils.ReadyFlag, dpiAPIReady *utils.ReadyFlag, tierWatchReady *utils.ReadyFlag) reconcile.Reconciler {
+func newReconciler(mgr manager.Manager, opts options.ControllerOptions, licenseAPIReady *utils.ReadyFlag, dpiAPIReady *utils.ReadyFlag, tierWatchReady *utils.ReadyFlag, threatFeedAPIReady *utils.ReadyFlag) reconcile.Reconciler {
 	r := &ReconcileIntrusionDetection{
-		client:          mgr.GetClient(),
-		scheme:          mgr.GetScheme(),
-		status:          status.New(mgr.GetClient(), tigeraStatusName, opts.KubernetesVersion),
-		licenseAPIReady: licenseAPIReady,
-		dpiAPIReady:     dpiAPIReady,
-		tierWatchReady:  tierWatchReady,
-		opts:            opts,
+		client:             mgr.GetClient(),
+		scheme:             mgr.GetScheme(),
+		status:             status.New(mgr.GetClient(), tigeraStatusName, opts.KubernetesVersion),
+		licenseAPIReady:    licenseAPIReady,
+		dpiAPIReady:        dpiAPIReady,
+		tierWatchReady:     tierWatchReady,
+		threatFeedAPIReady: threatFeedAPIReady,
+		opts:               opts,
 	}
 	r.status.Run(opts.ShutdownContext)
 	return r
@@ -181,13 +189,14 @@ var _ reconcile.Reconciler = &ReconcileIntrusionDetection{}
 type ReconcileIntrusionDetection struct {
 	// This client, initialized using mgr.Client() above, is a split client
 	// that reads objects from the cache and writes to the apiserver
-	client          client.Client
-	scheme          *runtime.Scheme
-	status          status.StatusManager
-	licenseAPIReady *utils.ReadyFlag
-	dpiAPIReady     *utils.ReadyFlag
-	tierWatchReady  *utils.ReadyFlag
-	opts            options.ControllerOptions
+	client             client.Client
+	scheme             *runtime.Scheme
+	status             status.StatusManager
+	licenseAPIReady    *utils.ReadyFlag
+	dpiAPIReady        *utils.ReadyFlag
+	tierWatchReady     *utils.ReadyFlag
+	threatFeedAPIReady *utils.ReadyFlag
+	opts               options.ControllerOptions
 }
 
 func getIntrusionDetection(ctx context.Context, cli client.Client, mt bool, ns string) (*operatorv1.IntrusionDetection, error) {
@@ -439,6 +448,18 @@ func (r *ReconcileIntrusionDetection) Reconcile(ctx context.Context, request rec
 		return reconcile.Result{}, err
 	}
 
+	// Collect domains from GlobalThreatFeed HTTP pull URLs so that the network policy
+	// allows the intrusion-detection-controller to reach them.
+	var threatFeedsDomains []string
+	if r.threatFeedAPIReady.IsReady() {
+		globalThreatFeeds := &v3.GlobalThreatFeedList{}
+		if err := r.client.List(ctx, globalThreatFeeds); err != nil {
+			r.status.SetDegraded(operatorv1.ResourceReadError, "Failed to retrieve GlobalThreatFeed resources", err, reqLogger)
+			return reconcile.Result{}, err
+		}
+		threatFeedsDomains = threatFeedPullDomains(globalThreatFeeds.Items)
+	}
+
 	reqLogger.V(3).Info("rendering components")
 	// Render the desired objects from the CRD and create or update them.
 	hasNoLicense := !utils.IsFeatureActive(license, common.ThreatDefenseFeature)
@@ -459,6 +480,7 @@ func (r *ReconcileIntrusionDetection) Reconcile(ctx context.Context, request rec
 		Tenant:                       tenant,
 		ExternalElastic:              r.opts.ElasticExternal,
 		SyslogForwardingIsEnabled:    syslogForwardingIsEnabled(lc),
+		ThreatFeedsDomains:           threatFeedsDomains,
 	}
 	setUp := render.NewSetup(&render.SetUpConfiguration{
 		OpenShift:       r.opts.DetectedProvider.IsOpenShift(),
@@ -666,3 +688,28 @@ func (r *ReconcileIntrusionDetection) fillDefaults(ctx context.Context, ids *ope
 
 	return nil
 }
+
+// threatFeedPullDomains extracts unique hostnames from GlobalThreatFeed HTTP pull
+// URLs so that the network policy can allow egress to those domains.
+func threatFeedPullDomains(feeds []v3.GlobalThreatFeed) []string {
+	seen := map[string]struct{}{}
+	for i := range feeds {
+		feed := &feeds[i]
+		if feed.Spec.Pull == nil || feed.Spec.Pull.HTTP == nil {
+			continue
+		}
+		if feed.Spec.Mode != nil && *feed.Spec.Mode == v3.ThreatFeedModeDisabled {
+			continue
+		}
+		u, err := url.Parse(feed.Spec.Pull.HTTP.URL)
+		if err != nil || u.Hostname() == "" {
+			continue
+		}
+		seen[u.Hostname()] = struct{}{}
+	}
+	domains := slices.Collect(maps.Keys(seen))
+	// Sorts the domains to ensure no change detected in the network policy
+	// if the order of the domains changes.
+	sort.Strings(domains)
+	return domains
+}

pkg/controller/intrusiondetection/intrusiondetection_controller_test.go

@@ -103,12 +103,13 @@ var _ = Describe("IntrusionDetection controller tests", func() {
 		mockStatus.On("SetMetaData", mock.Anything).Return()
 
 		r = ReconcileIntrusionDetection{
-			client:          c,
-			scheme:          scheme,
-			status:          mockStatus,
-			licenseAPIReady: &utils.ReadyFlag{},
-			dpiAPIReady:     &utils.ReadyFlag{},
-			tierWatchReady:  &utils.ReadyFlag{},
+			client:             c,
+			scheme:             scheme,
+			status:             mockStatus,
+			licenseAPIReady:    &utils.ReadyFlag{},
+			dpiAPIReady:        &utils.ReadyFlag{},
+			tierWatchReady:     &utils.ReadyFlag{},
+			threatFeedAPIReady: &utils.ReadyFlag{},
 			opts: options.ControllerOptions{
 				DetectedProvider: operatorv1.ProviderNone,
 			},
@@ -323,12 +324,13 @@ var _ = Describe("IntrusionDetection controller tests", func() {
 			readyFlag = &utils.ReadyFlag{}
 			readyFlag.MarkAsReady()
 			r = ReconcileIntrusionDetection{
-				client:          c,
-				scheme:          scheme,
-				status:          mockStatus,
-				licenseAPIReady: readyFlag,
-				dpiAPIReady:     readyFlag,
-				tierWatchReady:  readyFlag,
+				client:             c,
+				scheme:             scheme,
+				status:             mockStatus,
+				licenseAPIReady:    readyFlag,
+				dpiAPIReady:        readyFlag,
+				tierWatchReady:     readyFlag,
+				threatFeedAPIReady: readyFlag,
 				opts: options.ControllerOptions{
 					DetectedProvider: operatorv1.ProviderNone,
 				},
@@ -805,4 +807,70 @@ var _ = Describe("IntrusionDetection controller tests", func() {
 			Expect(pullSecret.Kind).To(Equal("Tenant"))
 		})
 	})
+
+	Context("threatFeedPullDomains", func() {
+		enabled := v3.ThreatFeedModeEnabled
+		disabled := v3.ThreatFeedModeDisabled
+
+		It("should extract unique hostnames from enabled feeds with HTTP pull URLs", func() {
+			feeds := []v3.GlobalThreatFeed{
+				{
+					Spec: v3.GlobalThreatFeedSpec{
+						Pull: &v3.Pull{HTTP: &v3.HTTPPull{URL: "https://feeds.example.com/v1/ips"}},
+					},
+				},
+				{
+					Spec: v3.GlobalThreatFeedSpec{
+						Pull: &v3.Pull{HTTP: &v3.HTTPPull{URL: "https://intel.threatprovider.io/domains"}},
+					},
+				},
+				{
+					Spec: v3.GlobalThreatFeedSpec{
+						Mode: &enabled,
+						Pull: &v3.Pull{HTTP: &v3.HTTPPull{URL: "https://feeds.example.com/v2/domains"}},
+					},
+				},
+			}
+			domains := threatFeedPullDomains(feeds)
+			Expect(domains).To(Equal([]string{"feeds.example.com", "intel.threatprovider.io"}))
+		})
+
+		It("should skip disabled feeds", func() {
+			feeds := []v3.GlobalThreatFeed{
+				{
+					Spec: v3.GlobalThreatFeedSpec{
+						Mode: &disabled,
+						Pull: &v3.Pull{HTTP: &v3.HTTPPull{URL: "https://disabled.example.com/ips"}},
+					},
+				},
+				{
+					Spec: v3.GlobalThreatFeedSpec{
+						Pull: &v3.Pull{HTTP: &v3.HTTPPull{URL: "https://enabled.example.com/ips"}},
+					},
+				},
+			}
+			domains := threatFeedPullDomains(feeds)
+			Expect(domains).To(Equal([]string{"enabled.example.com"}))
+		})
+
+		It("should skip feeds without HTTP pull configured", func() {
+			feeds := []v3.GlobalThreatFeed{
+				{
+					Spec: v3.GlobalThreatFeedSpec{},
+				},
+				{
+					Spec: v3.GlobalThreatFeedSpec{
+						Pull: &v3.Pull{HTTP: &v3.HTTPPull{URL: "https://valid.example.com/ips"}},
+					},
+				},
+			}
+			domains := threatFeedPullDomains(feeds)
+			Expect(domains).To(Equal([]string{"valid.example.com"}))
+		})
+
+		It("should return empty for no feeds", func() {
+			domains := threatFeedPullDomains(nil)
+			Expect(domains).To(BeEmpty())
+		})
+	})
 })

pkg/render/intrusion_detection.go

@@ -112,10 +112,11 @@ type IntrusionDetectionConfiguration struct {
 	TrustedCertBundle            certificatemanagement.TrustedBundleRO
 	IntrusionDetectionCertSecret certificatemanagement.KeyPairInterface
 
-	Namespace       string
-	BindNamespaces  []string
-	Tenant          *operatorv1.Tenant
-	ExternalElastic bool
+	Namespace          string
+	BindNamespaces     []string
+	Tenant             *operatorv1.Tenant
+	ExternalElastic    bool
+	ThreatFeedsDomains []string
 }
 
 type intrusionDetectionComponent struct {
@@ -1028,6 +1029,17 @@ func (c *intrusionDetectionComponent) intrusionDetectionControllerCalicoSystemPo
 		},
 	}
 	egressRules = networkpolicy.AppendDNSEgressRules(egressRules, c.cfg.OpenShift)
+	// Allow egress to threat feed HTTP pull domains.
+	if len(c.cfg.ThreatFeedsDomains) > 0 {
+		egressRules = append(egressRules, v3.Rule{
+			Action:   v3.Allow,
+			Protocol: &networkpolicy.TCPProtocol,
+			Destination: v3.EntityRule{
+				Domains: c.cfg.ThreatFeedsDomains,
+				Ports:   networkpolicy.Ports(443),
+			},
+		})
+	}
 	if c.cfg.ManagedCluster {
 		egressRules = append(egressRules, v3.Rule{
 			Action:      v3.Allow,

pkg/render/intrusion_detection_test.go

@@ -528,6 +528,40 @@ var _ = Describe("Intrusion Detection rendering tests", func() {
 		)
 	})
 
+	It("should include threat feed domains in the network policy egress rules", func() {
+		cfg.ThreatFeedsDomains = []string{"feeds.example.com", "intel.threatprovider.io"}
+		component := render.IntrusionDetection(cfg)
+		resources, _ := component.Objects()
+
+		netPol := rtest.GetResource(resources, "calico-system.intrusion-detection-controller", render.IntrusionDetectionNamespace, "projectcalico.org", "v3", "NetworkPolicy").(*v3.NetworkPolicy)
+		Expect(netPol).NotTo(BeNil())
+
+		// Find the threat feed domain egress rule.
+		var found bool
+		for _, rule := range netPol.Spec.Egress {
+			if rule.Action == v3.Allow && len(rule.Destination.Domains) > 0 {
+				Expect(rule.Destination.Domains).To(Equal([]string{"feeds.example.com", "intel.threatprovider.io"}))
+				Expect(rule.Destination.Ports).To(Equal(networkpolicy.Ports(443)))
+				found = true
+				break
+			}
+		}
+		Expect(found).To(BeTrue(), "Expected to find a threat feed domains egress rule")
+	})
+
+	It("should not include threat feed domain rule when no domains are configured", func() {
+		cfg.ThreatFeedsDomains = nil
+		component := render.IntrusionDetection(cfg)
+		resources, _ := component.Objects()
+
+		netPol := rtest.GetResource(resources, "calico-system.intrusion-detection-controller", render.IntrusionDetectionNamespace, "projectcalico.org", "v3", "NetworkPolicy").(*v3.NetworkPolicy)
+		Expect(netPol).NotTo(BeNil())
+
+		for _, rule := range netPol.Spec.Egress {
+			Expect(rule.Destination.Domains).To(BeEmpty(), "Expected no domain-based egress rule when ThreatFeedsDomains is empty")
+		}
+	})
+
 	It("should render an init container for pods when certificate management is enabled", func() {
 		ca, _ := tls.MakeCA(rmeta.DefaultOperatorCASignerName())
 		cert, _, _ := ca.Config.GetPEMBytes() // create a valid pem block