From 685c17c52b6866df7606fc7491c0cce7cc6c6bbc Mon Sep 17 00:00:00 2001 From: Sheogorath <sheogorath@shivering-isles.com> Date: Tue, 22 Nov 2022 00:42:26 +0100 Subject: [PATCH] feat(shields): Initial shields deployment This patch provides an initial version of shields for the cluster, deploying shields in a fairly locked down setup. This includes blocking all ingress and egress traffic except of the ingress controller, monitoring and outgoing web traffic to the public internet. As part of this some new shared network policies are created, added and renamed. These aim to improve the namespace isolation and provisioning of controlled network access. --- apps/base/shields/kustomization.yaml | 10 ++ apps/base/shields/namespace.yaml | 31 ++++++ apps/base/shields/networkpolicy.yaml | 18 ++++ apps/base/shields/shields.yaml | 96 +++++++++++++++++++ apps/k8s01/shields/certificate.yaml | 64 +++++++++++++ apps/k8s01/shields/egress-policy.yaml | 18 ++++ apps/k8s01/shields/ingress.yaml | 75 +++++++++++++++ apps/k8s01/shields/kustomization.yaml | 15 +++ infrastructure/drivers/kustomization.yaml | 2 +- shared/networkpolicies/allow-to-kubedns.yaml | 21 ++++ .../networkpolicies/allow-to-public-web.yaml | 23 +++++ .../deny-by-default-egress.yaml | 9 ++ ...ault.yaml => deny-by-default-ingress.yaml} | 0 13 files changed, 381 insertions(+), 1 deletion(-) create mode 100644 apps/base/shields/kustomization.yaml create mode 100644 apps/base/shields/namespace.yaml create mode 100644 apps/base/shields/networkpolicy.yaml create mode 100644 apps/base/shields/shields.yaml create mode 100644 apps/k8s01/shields/certificate.yaml create mode 100644 apps/k8s01/shields/egress-policy.yaml create mode 100644 apps/k8s01/shields/ingress.yaml create mode 100644 apps/k8s01/shields/kustomization.yaml create mode 100644 shared/networkpolicies/allow-to-kubedns.yaml create mode 100644 shared/networkpolicies/allow-to-public-web.yaml create mode 100644 shared/networkpolicies/deny-by-default-egress.yaml rename shared/networkpolicies/{deny-by-default.yaml => deny-by-default-ingress.yaml} (100%) diff --git a/apps/base/shields/kustomization.yaml b/apps/base/shields/kustomization.yaml new file mode 100644 index 000000000..53822a389 --- /dev/null +++ b/apps/base/shields/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: shields +resources: + - namespace.yaml + - monitoring.yaml + - ../../../shared/networkpolicies/allow-from-ingress.yaml + - ../../../shared/networkpolicies/allow-from-monitoring.yaml +patchesStrategicMerge: + - networkpolicy.yaml diff --git a/apps/base/shields/namespace.yaml b/apps/base/shields/namespace.yaml new file mode 100644 index 000000000..e2e5d8d30 --- /dev/null +++ b/apps/base/shields/namespace.yaml @@ -0,0 +1,31 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: shields + labels: + pod-security.kubernetes.io/audit: restricted + pod-security.kubernetes.io/enforce: baseline + pod-security.kubernetes.io/warn: restricted + pod-security.kubernetes.io/audit-version: v1.23 + pod-security.kubernetes.io/enforce-version: v1.23 + pod-security.kubernetes.io/warn-version: v1.23 +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: flux-reconciler + namespace: shields +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: flux-reconciler + namespace: shields +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: admin +subjects: + - kind: ServiceAccount + name: flux-reconciler + namespace: shields diff --git a/apps/base/shields/networkpolicy.yaml b/apps/base/shields/networkpolicy.yaml new file mode 100644 index 000000000..bfd39a5bf --- /dev/null +++ b/apps/base/shields/networkpolicy.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-from-ingress +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: shields +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-from-monitoring +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: shields diff --git a/apps/base/shields/shields.yaml b/apps/base/shields/shields.yaml new file mode 100644 index 000000000..580d41a92 --- /dev/null +++ b/apps/base/shields/shields.yaml @@ -0,0 +1,96 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app.kubernetes.io/name: shields + name: shields + namespace: shields +spec: + replicas: 2 + selector: + matchLabels: + app.kubernetes.io/name: shields + template: + metadata: + labels: + app.kubernetes.io/name: shields + spec: + affinity: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchLabels: + app.kubernetes.io/name: shields + topologyKey: kubernetes.io/hostname + containers: + - image: docker.io/shieldsio/shields:next + name: shields + resources: + requests: + memory: 128Mi + cpu: 70m + limits: + memory: 256Mi + cpu: 100m + env: + - name: METRICS_PROMETHEUS_ENABLED + value: "true" + - name: METRICS_PROMETHEUS_ENDPOINT_ENABLED + value: "true" + - name: PORT + value: "8080" + ports: + - containerPort: 8080 + name: http + securityContext: + runAsUser: 937 + runAsGroup: 937 + readOnlyRootFilesystem: true + runAsNonRoot: true + allowPrivilegeEscalation: false +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/name: shields + name: shields + namespace: shields +spec: + ports: + - name: http + port: 80 + protocol: TCP + targetPort: http + selector: + app.kubernetes.io/name: shields + type: ClusterIP +--- +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: shields + namespace: shields + labels: + app.kubernetes.io/name: shields +spec: + endpoints: + - path: /metrics + port: http + scheme: http + selector: + matchLabels: + app.kubernetes.io/name: shields +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: shields + namespace: shields + labels: + app.kubernetes.io/name: shields +spec: + minAvailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: shields \ No newline at end of file diff --git a/apps/k8s01/shields/certificate.yaml b/apps/k8s01/shields/certificate.yaml new file mode 100644 index 000000000..529494eac --- /dev/null +++ b/apps/k8s01/shields/certificate.yaml @@ -0,0 +1,64 @@ +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: shields-tls + namespace: shields +spec: + dnsNames: + - ENC[AES256_GCM,data:7f28/ffW5slUxv094Lv7k5ud257I3siDQvnd,iv:/sw9Q6lykDfv8ZJVS36wjSY9zjMsI2oR/56SL8dYI/Q=,tag:ixERn0tc+ru01ptWvtPsZQ==,type:str] + issuerRef: + name: letsencrypt + kind: ClusterIssuer + secretName: ingress-shields-tls +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2022-11-21T23:24:16Z" + mac: ENC[AES256_GCM,data:1xL3S+gHIgDSiLUpH/CSjLssgjdRbOJWkODjpI4M3r1P4RxKFpG2Mdua6+RJ15n64SThvFPueu57w0pF+wiKZYIqZK8mbPeYgFnluEJxCn99kuU6Zh0/MTGkmtN1i7d7u0xtgXXTsrtTJRmpunv0bhDfvXb1pV8SXQq8KnZG95Q=,iv:6r9d1Fi7r7hnVjmPOGx6jf6JgDOYWBe7AmxOY73bpfw=,tag:QGziqBtRte8NRFHX8cHUBA==,type:str] + pgp: + - created_at: "2022-01-21T18:13:48Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + wcFMA7kpg2bgzVHcAQ//U+q9DXB4HdBgO6bn7G8+MrAvTrfjyrLkuMmtzGIreuf0 + DKUr5P3U+8c3i5zwF7vD6i6qMhfFus//Ix1MHmbOk0H2ZSDzkN6gk6KdPEyKGpG8 + IpiMGu7qdGiR2pQ1UrwA3FDvRttkKADyjx/L+RvYlPZrRWZkWw16OCIdYwBPxfqp + q9uuVd41TZ1LpCRPEVCUH8iY61VLhgAx9JUx7ojX4bc45186u8jySDZMkjv/xXwN + bS6SqgqlD68Wq4dBiJwVbILo98WNMDyGFGia9EO3VfAdXHG4REvWr+uXPf4nDznp + mg1oQcrvc41s/M/Nc5QvWdc4gRDJZaXUwzjsrGtsM67s7zLzYq/diUFcA70mjmjr + cGzHz6FSpV4APuj3aVhxtKQGnxQRRH66O1tFs6MjOsImSXONDHXHeCw4QoYWABHS + 6n2KOojyzmeug/ya8FUTsCyVZ3PDFd+UOxxdtKl3nzBwUocBmRfvWFeBvyo/QOfB + A0pNBL8Q/yA3p2XIKuibiL8OMNuxfiMF3SHei4KMGP2Zk6dKss5N13TBzQ4oYBIq + gqQQjYXSq8b6OkojSHja3OO77qIAzzMD7ztxUwAtq7a0/dZOU1ZXCNczfS0Y7Bun + Ay8ELsdhZ0IQY02RMsMxCy8f0aemEOAAGiZ+LR9LE7QS5lVL86bk8SlKUsLKbfTS + UQE87XjR8vATk5CDPZ357fl4rcrND1TehqrByB5p/TqJVe+9rbvE56AJgK0vEYzp + XKO5Sp201jBInr8WmUWTQ5paFNU9lZwhEpb1fqTvgt55Mw== + =64L4 + -----END PGP MESSAGE----- + fp: 286791FB6648539775DB31B8FCB98C2A3EC6F601 + - created_at: "2022-01-21T18:13:48Z" + enc: | + -----BEGIN PGP MESSAGE----- + + hQIMA4oYbIHZIrAPAQ//XcmHPgwByh+6vS7cVnsVCt6HQDk834gW5t3MaKtqFjJW + GY0MPkV1eZ+PIBm9oP2SZ8sVcNWzIgbtuzGht0BhsJlssf0rxvVJJSNC+iQ6ZGsD + L+cp5EPmepIFMGVda5OKT18Q5N9RKYT58MTjgzIYGTfIe4rdVddmAZq3KDLG6pV/ + SOTw6Zq5TshE/zyFPu3ndFKCS3twCb1AmPx1YMFLpPFiGYXXemdMtoOlQJE2qdRX + DciGW97ZLrj8WMKQl3zU35N2oQeoCcxYLVrYq0+qGhFujhRYCgOL+O3d4+XmCvEO + ZpZR7KWfrSn7976EDMRqRYxz++mMgD+V6RIJdbw2AmIYwX7HwkTwjwK89H0lh9R5 + kkYhxgKaVsoGTa3NBATuf0au+qHMzlkRjSToaZ+QKCPnwxSoURQgyNAkR1aQ2Pgi + muTu915RrAy7f9A3Jtt8zZSYXPXkSW44YXOCjhHy7ayFh9nQ4zeklfM4eMAU1Ni/ + hKXmN35tlCorstv+i6Zyc+bBkL8an2v4YHWZQ8BKYkqAWRJDalKErbaP+Y/i/Bwf + +PjS+umpm9OkZ5lPTdxhzLxRMOWuGfbhrwAwuNV+jF3iDAGvPBR2EsJ/Ga2aXdqZ + TGeeMKs5sozba2d2fJyPADL/6DyKHS58b9VJt6r368xy7cVOD/UCaYOm01Ygn2vU + ZgEJAhDyxv/+3B3ju7Y1YlyM3WwbVEH6Vq1ZkOCy9uF7r24u5L01zx2QcGufpC1L + +XWTKormP0iD0q8PV7n/PU9F8Feb5bCmJTzIoWfT3+9X8cYwBaGUZqA3fNWJxOAu + vC+njoUTFA== + =5FJc + -----END PGP MESSAGE----- + fp: B137EE1549DFAF960DD1E2B15147025FB9F09E07 + encrypted_regex: ^(data|stringData|email|dnsZones?|dnsNames?|hosts?|tang|externalURL)$ + version: 3.7.3 diff --git a/apps/k8s01/shields/egress-policy.yaml b/apps/k8s01/shields/egress-policy.yaml new file mode 100644 index 000000000..4d7be916a --- /dev/null +++ b/apps/k8s01/shields/egress-policy.yaml @@ -0,0 +1,18 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-to-public-web +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: shields +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-to-kubedns +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: shields \ No newline at end of file diff --git a/apps/k8s01/shields/ingress.yaml b/apps/k8s01/shields/ingress.yaml new file mode 100644 index 000000000..34be67e14 --- /dev/null +++ b/apps/k8s01/shields/ingress.yaml @@ -0,0 +1,75 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: shields + namespace: shields + labels: + app.kubernetes.io/name: shields +spec: + rules: + - host: ENC[AES256_GCM,data:Ls6Wg25JUEowgV8YTOfGp1daaimJC5yFg8uq,iv:khJiOaFri7CCjdilB7R7FSUanMAwAP7X9ETn5XXi2ZA=,tag:cSPvZNtTkMX4jXuXXbIaEw==,type:str] + http: + paths: + - backend: + service: + name: shields + port: + name: http + path: / + pathType: Prefix + tls: + - hosts: + - ENC[AES256_GCM,data:TrW9Zg/zjwIVzqCAeVX72ye5ZEeWgD6mypRH,iv:VTiUhXKSPBy+lH3EpjipQyxYI/+kRPbot9X4xiVft8A=,tag:ZhPJ6DfKnSmWbNnrf1ABHA==,type:str] + secretName: ingress-shields-tls +sops: + kms: [] + gcp_kms: [] + azure_kv: [] + hc_vault: [] + age: [] + lastmodified: "2022-11-21T23:23:18Z" + mac: ENC[AES256_GCM,data:x+8H6dF5IcvYPur32fIXoMUjpd31bRzrSRW3w0q1Af9qcskKjxYGTdy5QgpFXRxs07tdV8ALYha4z3y/QNLAFvxEYYPLOI3Qw9FFIfMIWe1cUVrInO8JpogwIbIMyXg3KYGXREPXhmNn6lxA9NrLqo6cHrNqX6V5ZT9yY4FqreI=,iv:mOQdfZk8joW/vZTzUYrYDfwYihCT136zOJz5n6qBjaE=,tag:8wJdEIaShlgTFpxiWEPA0w==,type:str] + pgp: + - created_at: "2022-11-21T23:23:18Z" + enc: |- + -----BEGIN PGP MESSAGE----- + + wcFMA7kpg2bgzVHcAQ//Qv9uHg5XLGXazE73KPUoH4fqOH9uO85CwtJvizNN8fyR + 7j9BgCfjEze/OTzy9h+Zmd6W9WV47/I61j3tkzkq+UE7mFsSmZrzsWgHcNJ8qg3w + dXezL9GzMv/B/p7DlZjUtwCLHLVcSKFZh01rO0r3q4v8hXpl4UO38gNOoZg1R5rp + KP+eE3JVvBw5QqZaWUS3EGM4Vy3drNmz1vlEYnh6xfd9TaHaNk+xzIgarpUlVKd9 + 38lxwTxZJRjQp3b2CkKJ09RWkxhyJV+YjOcsCsZW4slWlV9RLRqWqHcy3BPAPRN5 + WLBzx57dEENX3Hi1WU2yFfBnN0h3rujv6SBJL77UTgjjUEMJNXkwfk5BuQHVWrMd + yCa5J6oXAAV5ii2UNnfW+UTSlsSpO0QreP6KmzNBGOmb/C6BlV3znYX+jPsDyl7l + 4toIhjSSAFjlSBLa6DZZAiTtEcSPHrELvBmFvQSkoXd4ZcXN16VUGN1bh2REYdzO + H7NNFAToxFT3Z5Y6CU2rCb7kQYtY2JIYzfLRt56HReTDlfNSN+/PFmGHZWq/M8bq + Cjm6sViN3GajWqITgrt9QAUvb8BDg7QlaChN1Z1U2yHtYN6ndPHyisgsNVTE2V+3 + A8iRJE0NDPNs+UpW2+VQS2vh0US0gjdZUIorkcDZ0LFouP7i09H+pyZ+ZBrPym/S + UQFTiGiV4NO5SA/2tbmuoJho91+F8uL+ieOIxkS+yzrene3wj911gnkwsPOmy6/Q + 4YzqYyNKWA60xo/jEPaIVYyBDH+PBJsQ9SyUcHEcukoVlw== + =5/Nx + -----END PGP MESSAGE----- + fp: 286791FB6648539775DB31B8FCB98C2A3EC6F601 + - created_at: "2022-11-21T23:23:18Z" + enc: | + -----BEGIN PGP MESSAGE----- + + hQIMA4oYbIHZIrAPAQ//WqQj3zETMqQoFyeM00Q6VwIu8aZDwSsEWR07twfLk6is + hlRAWdmOdSiItYxxGgAEbK6DiBmqoyw56EfvJbOfAAF+BXnl+pgV0ozeJdl+LJar + xVfQNwxIaP7LnqbmHXINDln0+0AfckNhXUcjP+HhwCr+aRz/pdiJkHsr7VTLH8nM + bf+Z2nKFhzdTbOjxC3O/yxh9baPBSRAq2fNx2SQk1JPzqBCIsqGkArF3shjtDzKc + MkU4BiRv+8fqX31bZXU00+TBZnMIkZT8OM4Kk4gWDCgVtWO3b+tl2xA4nO389/FP + nRWpeUHZU2BPIstW41FEIv/ID62s6D8WZZ5Ewh7NpLb7Xb29ABKh3wMOxe4lUEMx + b95XjT7jPwwcTNkea4v3nlQRkcsVzn1wr2zwZtr8FX4m8KHMc8OYCZoAS+C0FQbn + te35wyN7CBV3G9Xg+PijI+OxXXZR7wtrwtVMMAF7bNO6ySWOChuLzzkARUIF1SYc + soB5FShuNcwUFB9BnI5QpjOWT1tMiMYlXC7LgMbkh2pMjs8pnFxmjbx8g+WezdVF + eDiKP2rLLWeewA5+wGow2b1jlWpzveZXghFkKPHt0EvpjwFj9yk+f3E8SCqE6BFJ + LvhI41HtckenpQxvHa6I35RrP67ANGOahV4X4zTT8lu50hLOgFE4A3GD0yOlTczU + ZgEJAhATTbyPPq6mLTW9dBlVcO8NYYUdNzWBRVUeISx3A69AU185TUBMVYfJVXSg + 0poGps/+ASmRFsuTDfNd7FXF8feEHzFafEC0uJ0xeZGrgPRyVC+5WH6vdLuqyyKU + iJ9te6Wzmw== + =1rfI + -----END PGP MESSAGE----- + fp: B137EE1549DFAF960DD1E2B15147025FB9F09E07 + encrypted_regex: ^(data|stringData|email|dnsZones?|dnsNames?|.*(H|h)osts?|tang|externalURL|.*-secret|.*-url|.*Secrets?|.*-domain|password|subjects|node|apiURL|.*(S|s)erverNames?|.*SecretKey|externalName)$ + version: 3.7.3 diff --git a/apps/k8s01/shields/kustomization.yaml b/apps/k8s01/shields/kustomization.yaml new file mode 100644 index 000000000..6db555b8f --- /dev/null +++ b/apps/k8s01/shields/kustomization.yaml @@ -0,0 +1,15 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: shields +resources: + - ../../base/shields + - ../../../shared/resourcequotas/default.yaml + - egress-policy.yaml + - certificate.yaml + - ingress.yaml + - ../../../shared/networkpolicies/deny-by-default-ingress.yaml + - ../../../shared/networkpolicies/deny-by-default-egress.yaml + - ../../../shared/networkpolicies/allow-to-kubedns.yaml + - ../../../shared/networkpolicies/allow-to-web.yaml +patchesStrategicMerge: + - networkpolicy.yaml \ No newline at end of file diff --git a/infrastructure/drivers/kustomization.yaml b/infrastructure/drivers/kustomization.yaml index 22ea8ec76..a38892b4e 100644 --- a/infrastructure/drivers/kustomization.yaml +++ b/infrastructure/drivers/kustomization.yaml @@ -4,4 +4,4 @@ namespace: drivers-system resources: - namespace.yaml - amd-gpu.yaml - - ../../shared/networkpolicies/deny-by-default.yaml + - ../../shared/networkpolicies/deny-by-default-ingress.yaml diff --git a/shared/networkpolicies/allow-to-kubedns.yaml b/shared/networkpolicies/allow-to-kubedns.yaml new file mode 100644 index 000000000..0edb8bfb4 --- /dev/null +++ b/shared/networkpolicies/allow-to-kubedns.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-to-kubedns +spec: + egress: + - to: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: kube-system + podSelector: + matchLabels: + k8s-app: kube-dns + ports: + - port: 53 + protocol: UDP + - port: 53 + protocol: TCP + policyTypes: + - Egress \ No newline at end of file diff --git a/shared/networkpolicies/allow-to-public-web.yaml b/shared/networkpolicies/allow-to-public-web.yaml new file mode 100644 index 000000000..7c9277a65 --- /dev/null +++ b/shared/networkpolicies/allow-to-public-web.yaml @@ -0,0 +1,23 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-to-public-web +spec: + egress: + - to: + - ipBlock: + except: + - "192.168.0.0/16" + - "172.16.0.0/12" + - "10.0.0.0/8" + - "169.254.0.0/16" + - "100.64.0.0/10" + cidr: 0.0.0.0/0 + ports: + - protocol: TCP + port: 80 + - protocol: TCP + port: 443 + policyTypes: + - Egress \ No newline at end of file diff --git a/shared/networkpolicies/deny-by-default-egress.yaml b/shared/networkpolicies/deny-by-default-egress.yaml new file mode 100644 index 000000000..a4659e141 --- /dev/null +++ b/shared/networkpolicies/deny-by-default-egress.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: default-deny-egress +spec: + podSelector: {} + policyTypes: + - Egress diff --git a/shared/networkpolicies/deny-by-default.yaml b/shared/networkpolicies/deny-by-default-ingress.yaml similarity index 100% rename from shared/networkpolicies/deny-by-default.yaml rename to shared/networkpolicies/deny-by-default-ingress.yaml -- GitLab