diff --git a/pkg/provider/kubernetes/gateway/fixtures/grpcroute/with_backend_tls_policy.yml b/pkg/provider/kubernetes/gateway/fixtures/grpcroute/with_backend_tls_policy.yml new file mode 100644 index 000000000..2e583101a --- /dev/null +++ b/pkg/provider/kubernetes/gateway/fixtures/grpcroute/with_backend_tls_policy.yml @@ -0,0 +1,84 @@ +--- +kind: GatewayClass +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: my-gateway-class +spec: + controllerName: traefik.io/gateway-controller + +--- +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: my-gateway + namespace: default +spec: + gatewayClassName: my-gateway-class + listeners: # Use GatewayClass defaults for listener definition. + - name: web + protocol: HTTP + port: 80 + allowedRoutes: + kinds: + - kind: GRPCRoute + group: gateway.networking.k8s.io + namespaces: + from: Same + +--- +kind: GRPCRoute +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: grpc-app-1 + namespace: default +spec: + parentRefs: + - name: my-gateway + kind: Gateway + group: gateway.networking.k8s.io + hostnames: + - foo.com + rules: + - backendRefs: + - name: whoami + port: 80 + weight: 1 + +--- +kind: BackendTLSPolicy +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: backend-tls-policy + namespace: default +spec: + targetRefs: + - group: "" + kind: Service + name: whoami + validation: + hostname: whoami + caCertificateRefs: + - group: "" + kind: ConfigMap + name: ca-file + - group: core + kind: ConfigMap + name: ca-file-2 + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ca-file + namespace: default +data: + ca.crt: "CA1" + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ca-file-2 + namespace: default +data: + ca.crt: "CA2" diff --git a/pkg/provider/kubernetes/gateway/fixtures/grpcroute/with_backend_tls_policy_system.yml b/pkg/provider/kubernetes/gateway/fixtures/grpcroute/with_backend_tls_policy_system.yml new file mode 100644 index 000000000..0cae5d07a --- /dev/null +++ b/pkg/provider/kubernetes/gateway/fixtures/grpcroute/with_backend_tls_policy_system.yml @@ -0,0 +1,60 @@ +--- +kind: GatewayClass +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: my-gateway-class +spec: + controllerName: traefik.io/gateway-controller + +--- +kind: Gateway +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: my-gateway + namespace: default +spec: + gatewayClassName: my-gateway-class + listeners: # Use GatewayClass defaults for listener definition. + - name: http + protocol: HTTP + port: 80 + allowedRoutes: + kinds: + - kind: GRPCRoute + group: gateway.networking.k8s.io + namespaces: + from: Same + +--- +kind: GRPCRoute +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: grpc-app-1 + namespace: default +spec: + parentRefs: + - name: my-gateway + kind: Gateway + group: gateway.networking.k8s.io + hostnames: + - foo.com + rules: + - backendRefs: + - name: whoami + port: 80 + weight: 1 + +--- +kind: BackendTLSPolicy +apiVersion: gateway.networking.k8s.io/v1 +metadata: + name: backend-tls-policy + namespace: default +spec: + targetRefs: + - group: core + kind: Service + name: whoami + validation: + hostname: whoami + wellKnownCACertificates: System diff --git a/pkg/provider/kubernetes/gateway/grpcroute.go b/pkg/provider/kubernetes/gateway/grpcroute.go index 42826a8bb..efaa42807 100644 --- a/pkg/provider/kubernetes/gateway/grpcroute.go +++ b/pkg/provider/kubernetes/gateway/grpcroute.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "regexp" + "slices" "strconv" "strings" @@ -168,7 +169,7 @@ func (p *Provider) loadGRPCRoute(ctx context.Context, listener gatewayListener, default: var serviceCondition *metav1.Condition - router.Service, serviceCondition = p.loadGRPCService(conf, routerName, routeRule, route) + router.Service, serviceCondition = p.loadGRPCService(ctx, listener, conf, routerName, routeRule, route) if serviceCondition != nil { condition = *serviceCondition } @@ -181,7 +182,7 @@ func (p *Provider) loadGRPCRoute(ctx context.Context, listener gatewayListener, return conf, condition } -func (p *Provider) loadGRPCService(conf *dynamic.Configuration, routeKey string, routeRule gatev1.GRPCRouteRule, route *gatev1.GRPCRoute) (string, *metav1.Condition) { +func (p *Provider) loadGRPCService(ctx context.Context, listener gatewayListener, conf *dynamic.Configuration, routeKey string, routeRule gatev1.GRPCRouteRule, route *gatev1.GRPCRoute) (string, *metav1.Condition) { name := routeKey + "-wrr" if _, ok := conf.HTTP.Services[name]; ok { return name, nil @@ -190,7 +191,7 @@ func (p *Provider) loadGRPCService(conf *dynamic.Configuration, routeKey string, var wrr dynamic.WeightedRoundRobin var condition *metav1.Condition for _, backendRef := range routeRule.BackendRefs { - svcName, svc, errCondition := p.loadGRPCBackendRef(route, backendRef) + svcName, svc, errCondition := p.loadGRPCBackendRef(ctx, listener, conf, route, backendRef) weight := ptr.To(int(ptr.Deref(backendRef.Weight, 1))) if errCondition != nil { condition = errCondition @@ -219,7 +220,7 @@ func (p *Provider) loadGRPCService(conf *dynamic.Configuration, routeKey string, return name, condition } -func (p *Provider) loadGRPCBackendRef(route *gatev1.GRPCRoute, backendRef gatev1.GRPCBackendRef) (string, *dynamic.Service, *metav1.Condition) { +func (p *Provider) loadGRPCBackendRef(ctx context.Context, listener gatewayListener, conf *dynamic.Configuration, route *gatev1.GRPCRoute, backendRef gatev1.GRPCBackendRef) (string, *dynamic.Service, *metav1.Condition) { kind := ptr.Deref(backendRef.Kind, kindService) group := groupCore @@ -271,11 +272,16 @@ func (p *Provider) loadGRPCBackendRef(route *gatev1.GRPCRoute, backendRef gatev1 portStr := strconv.FormatInt(int64(port), 10) serviceName = provider.Normalize(serviceName + "-" + portStr + "-grpc") - lb, errCondition := p.loadGRPCServers(namespace, route, backendRef) + lb, st, errCondition := p.loadGRPCServers(ctx, namespace, route, backendRef, listener) if errCondition != nil { return serviceName, nil, errCondition } + if st != nil { + lb.ServersTransport = serviceName + conf.HTTP.ServersTransports[serviceName] = st + } + return serviceName, &dynamic.Service{LoadBalancer: lb}, nil } @@ -325,10 +331,10 @@ func (p *Provider) loadGRPCMiddlewares(conf *dynamic.Configuration, namespace, r return middlewareNames, nil } -func (p *Provider) loadGRPCServers(namespace string, route *gatev1.GRPCRoute, backendRef gatev1.GRPCBackendRef) (*dynamic.ServersLoadBalancer, *metav1.Condition) { +func (p *Provider) loadGRPCServers(ctx context.Context, namespace string, route *gatev1.GRPCRoute, backendRef gatev1.GRPCBackendRef, listener gatewayListener) (*dynamic.ServersLoadBalancer, *dynamic.ServersTransport, *metav1.Condition) { backendAddresses, svcPort, err := p.getBackendAddresses(namespace, backendRef.BackendRef) if err != nil { - return nil, &metav1.Condition{ + return nil, nil, &metav1.Condition{ Type: string(gatev1.RouteConditionResolvedRefs), Status: metav1.ConditionFalse, ObservedGeneration: route.Generation, @@ -338,26 +344,138 @@ func (p *Provider) loadGRPCServers(namespace string, route *gatev1.GRPCRoute, ba } } - if svcPort.Protocol != corev1.ProtocolTCP { - return nil, &metav1.Condition{ + backendTLSPolicies, err := p.client.ListBackendTLSPoliciesForService(namespace, string(backendRef.Name)) + if err != nil { + return nil, nil, &metav1.Condition{ Type: string(gatev1.RouteConditionResolvedRefs), Status: metav1.ConditionFalse, ObservedGeneration: route.Generation, LastTransitionTime: metav1.Now(), - Reason: string(gatev1.RouteReasonUnsupportedProtocol), - Message: fmt.Sprintf("Cannot load GRPCBackendRef %s/%s: only TCP protocol is supported", namespace, backendRef.Name), + Reason: string(gatev1.RouteReasonRefNotPermitted), + Message: fmt.Sprintf("Cannot list BackendTLSPolicies for Service %s/%s: %s", namespace, string(backendRef.Name), err), } } - protocol, err := getGRPCServiceProtocol(svcPort) - if err != nil { - return nil, &metav1.Condition{ - Type: string(gatev1.RouteConditionResolvedRefs), - Status: metav1.ConditionFalse, - ObservedGeneration: route.Generation, - LastTransitionTime: metav1.Now(), - Reason: string(gatev1.RouteReasonUnsupportedProtocol), - Message: fmt.Sprintf("Cannot load GRPCBackendRef %s/%s: only \"kubernetes.io/h2c\" and \"https\" appProtocol is supported", namespace, backendRef.Name), + // Sort BackendTLSPolicies by creation timestamp, then by name to match the BackendTLSPolicy requirements. + slices.SortStableFunc(backendTLSPolicies, func(a, b *gatev1.BackendTLSPolicy) int { + cmpTime := a.CreationTimestamp.Time.Compare(b.CreationTimestamp.Time) + if cmpTime == 0 { + return strings.Compare(a.Name, b.Name) + } + return cmpTime + }) + + var serversTransport *dynamic.ServersTransport + for _, policy := range backendTLSPolicies { + for _, targetRef := range policy.Spec.TargetRefs { + // Skip targetRefs that doesn't match the backendRef, + // since a BackendTLSPolicy can select multiple services. + if targetRef.Name != backendRef.Name { + continue + } + // Skip the targetRef if the sectionName doesn't match the backendRef port. + if targetRef.SectionName != nil && svcPort.Name != string(*targetRef.SectionName) { + continue + } + + policyAncestorStatus := gatev1.PolicyAncestorStatus{ + AncestorRef: gatev1.ParentReference{ + Group: ptr.To(gatev1.Group(groupGateway)), + Kind: ptr.To(gatev1.Kind(kindGateway)), + Namespace: ptr.To(gatev1.Namespace(namespace)), + Name: gatev1.ObjectName(listener.GWName), + SectionName: ptr.To(gatev1.SectionName(listener.Name)), + }, + ControllerName: controllerName, + } + + // Multiple BackendTLSPolicies can match the same service port, meaning that there is a conflict. + if serversTransport != nil { + policyAncestorStatus.Conditions = append(policyAncestorStatus.Conditions, + metav1.Condition{ + Type: string(gatev1.BackendTLSPolicyConditionResolvedRefs), + Status: metav1.ConditionFalse, + ObservedGeneration: policy.Generation, + LastTransitionTime: metav1.Now(), + Reason: string(gatev1.BackendTLSPolicyReasonResolvedRefs), + }, + metav1.Condition{ + Type: string(gatev1.PolicyConditionAccepted), + Status: metav1.ConditionFalse, + ObservedGeneration: policy.Generation, + LastTransitionTime: metav1.Now(), + Reason: string(gatev1.PolicyReasonConflicted), + }, + ) + + status := gatev1.PolicyStatus{ + Ancestors: []gatev1.PolicyAncestorStatus{policyAncestorStatus}, + } + if err := p.client.UpdateBackendTLSPolicyStatus(ctx, ktypes.NamespacedName{Namespace: policy.Namespace, Name: policy.Name}, status); err != nil { + log.Ctx(ctx).Warn().Err(err).Msg("Unable to update conflicting BackendTLSPolicy status") + } + + continue + } + + var resolvedRefCondition metav1.Condition + serversTransport, resolvedRefCondition = p.loadServersTransport(namespace, policy) + + policyAncestorStatus.Conditions = append(policyAncestorStatus.Conditions, resolvedRefCondition) + if resolvedRefCondition.Status == metav1.ConditionFalse { + policyAncestorStatus.Conditions = append(policyAncestorStatus.Conditions, metav1.Condition{ + Type: string(gatev1.PolicyConditionAccepted), + Status: metav1.ConditionFalse, + ObservedGeneration: policy.Generation, + LastTransitionTime: metav1.Now(), + Reason: string(gatev1.BackendTLSPolicyReasonNoValidCACertificate), + }) + } else { + policyAncestorStatus.Conditions = append(policyAncestorStatus.Conditions, metav1.Condition{ + Type: string(gatev1.PolicyConditionAccepted), + Status: metav1.ConditionTrue, + ObservedGeneration: policy.Generation, + LastTransitionTime: metav1.Now(), + Reason: string(gatev1.PolicyReasonAccepted), + }) + } + + status := gatev1.PolicyStatus{ + Ancestors: []gatev1.PolicyAncestorStatus{policyAncestorStatus}, + } + if err := p.client.UpdateBackendTLSPolicyStatus(ctx, ktypes.NamespacedName{Namespace: policy.Namespace, Name: policy.Name}, status); err != nil { + log.Ctx(ctx).Warn().Err(err).Msg("Unable to update BackendTLSPolicy status") + } + + // When something went wrong during the loading of a ServersTransport, + // we stop here and return a route condition error. + if resolvedRefCondition.Status == metav1.ConditionFalse { + return nil, nil, &metav1.Condition{ + Type: string(gatev1.RouteConditionResolvedRefs), + Status: metav1.ConditionFalse, + ObservedGeneration: route.Generation, + LastTransitionTime: metav1.Now(), + Reason: string(gatev1.RouteReasonRefNotPermitted), + Message: fmt.Sprintf("Cannot apply BackendTLSPolicy for Service %s/%s: %s", namespace, string(backendRef.Name), resolvedRefCondition.Message), + } + } + } + } + + // If a ServersTransport is set, it means a BackendTLSPolicy matched the service port, and we can safely assume the protocol is HTTPS. + // When no ServersTransport is set, we need to determine the protocol based on the service port. + protocol := "https" + if serversTransport == nil { + protocol, err = getGRPCServiceProtocol(svcPort) + if err != nil { + return nil, nil, &metav1.Condition{ + Type: string(gatev1.RouteConditionResolvedRefs), + Status: metav1.ConditionFalse, + ObservedGeneration: route.Generation, + LastTransitionTime: metav1.Now(), + Reason: string(gatev1.RouteReasonUnsupportedProtocol), + Message: fmt.Sprintf("Cannot load GRPCBackendRef %s/%s: %s", namespace, backendRef.Name, err), + } } } @@ -369,7 +487,8 @@ func (p *Provider) loadGRPCServers(namespace string, route *gatev1.GRPCRoute, ba URL: fmt.Sprintf("%s://%s", protocol, net.JoinHostPort(ba.IP, strconv.Itoa(int(ba.Port)))), }) } - return lb, nil + + return lb, serversTransport, nil } func buildGRPCMatchRule(hostnames []gatev1.Hostname, match gatev1.GRPCRouteMatch) (string, int) { diff --git a/pkg/provider/kubernetes/gateway/kubernetes_test.go b/pkg/provider/kubernetes/gateway/kubernetes_test.go index ccf623eb0..651628969 100644 --- a/pkg/provider/kubernetes/gateway/kubernetes_test.go +++ b/pkg/provider/kubernetes/gateway/kubernetes_test.go @@ -3508,6 +3508,186 @@ func TestLoadHTTPRoutes_filterExtensionRef(t *testing.T) { } } +func TestLoadGRPCRoutes(t *testing.T) { + testCases := []struct { + desc string + paths []string + expected *dynamic.Configuration + entryPoints map[string]Entrypoint + }{ + { + desc: "Simple GRPCRoute and BackendTLSPolicy with CA certificate", + paths: []string{"services.yml", "grpcroute/with_backend_tls_policy.yml"}, + entryPoints: map[string]Entrypoint{"web": { + Address: ":80", + }}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "grpcroute-default-grpc-app-1-gw-default-my-gateway-ep-web-0-6a1e0890d475642f7c64": { + EntryPoints: []string{"web"}, + Service: "grpcroute-default-grpc-app-1-gw-default-my-gateway-ep-web-0-6a1e0890d475642f7c64-wrr", + Rule: `Host("foo.com") && PathPrefix("/")`, + Priority: 22, + RuleSyntax: "default", + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "grpcroute-default-grpc-app-1-gw-default-my-gateway-ep-web-0-6a1e0890d475642f7c64-wrr": { + Weighted: &dynamic.WeightedRoundRobin{ + Services: []dynamic.WRRService{ + { + Name: "default-whoami-80-grpc", + Weight: ptr.To(1), + }, + }, + }, + }, + "default-whoami-80-grpc": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Strategy: dynamic.BalancerStrategyWRR, + Servers: []dynamic.Server{ + { + URL: "https://10.10.0.1:80", + }, + { + URL: "https://10.10.0.2:80", + }, + }, + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: ptypes.Duration(100 * time.Millisecond), + }, + ServersTransport: "default-whoami-80-grpc", + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{ + "default-whoami-80-grpc": { + ServerName: "whoami", + RootCAs: []types.FileOrContent{ + "CA1", + "CA2", + }, + }, + }, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + { + desc: "Simple GRPCRoute and BackendTLSPolicy with System CA", + paths: []string{"services.yml", "grpcroute/with_backend_tls_policy_system.yml"}, + entryPoints: map[string]Entrypoint{"web": { + Address: ":80", + }}, + expected: &dynamic.Configuration{ + UDP: &dynamic.UDPConfiguration{ + Routers: map[string]*dynamic.UDPRouter{}, + Services: map[string]*dynamic.UDPService{}, + }, + TCP: &dynamic.TCPConfiguration{ + Routers: map[string]*dynamic.TCPRouter{}, + Middlewares: map[string]*dynamic.TCPMiddleware{}, + Services: map[string]*dynamic.TCPService{}, + ServersTransports: map[string]*dynamic.TCPServersTransport{}, + }, + HTTP: &dynamic.HTTPConfiguration{ + Routers: map[string]*dynamic.Router{ + "grpcroute-default-grpc-app-1-gw-default-my-gateway-ep-web-0-6a1e0890d475642f7c64": { + EntryPoints: []string{"web"}, + Service: "grpcroute-default-grpc-app-1-gw-default-my-gateway-ep-web-0-6a1e0890d475642f7c64-wrr", + Rule: `Host("foo.com") && PathPrefix("/")`, + Priority: 22, + RuleSyntax: "default", + }, + }, + Middlewares: map[string]*dynamic.Middleware{}, + Services: map[string]*dynamic.Service{ + "grpcroute-default-grpc-app-1-gw-default-my-gateway-ep-web-0-6a1e0890d475642f7c64-wrr": { + Weighted: &dynamic.WeightedRoundRobin{ + Services: []dynamic.WRRService{ + { + Name: "default-whoami-80-grpc", + Weight: ptr.To(1), + }, + }, + }, + }, + "default-whoami-80-grpc": { + LoadBalancer: &dynamic.ServersLoadBalancer{ + Strategy: dynamic.BalancerStrategyWRR, + Servers: []dynamic.Server{ + { + URL: "https://10.10.0.1:80", + }, + { + URL: "https://10.10.0.2:80", + }, + }, + PassHostHeader: ptr.To(true), + ResponseForwarding: &dynamic.ResponseForwarding{ + FlushInterval: ptypes.Duration(100 * time.Millisecond), + }, + ServersTransport: "default-whoami-80-grpc", + }, + }, + }, + ServersTransports: map[string]*dynamic.ServersTransport{ + "default-whoami-80-grpc": { + ServerName: "whoami", + }, + }, + }, + TLS: &dynamic.TLSConfiguration{}, + }, + }, + } + + for _, test := range testCases { + t.Run(test.desc, func(t *testing.T) { + t.Parallel() + + if test.expected == nil { + return + } + + k8sObjects, gwObjects := readResources(t, test.paths) + + kubeClient := kubefake.NewClientset(k8sObjects...) + gwClient := newGatewaySimpleClientSet(t, gwObjects...) + + client := newClientImpl(kubeClient, gwClient) + + eventCh, err := client.WatchAll(nil, make(chan struct{})) + require.NoError(t, err) + + if len(k8sObjects) > 0 || len(gwObjects) > 0 { + <-eventCh + } + + p := Provider{ + EntryPoints: test.entryPoints, + client: client, + } + + conf := p.loadConfigurationFromGateways(t.Context()) + assert.Equal(t, test.expected, conf) + }) + } +} + func TestLoadGRPCRoutes_filterExtensionRef(t *testing.T) { testCases := []struct { desc string