From ac16fabbd5f8dcea9ce72a80d23c100d4457ee82 Mon Sep 17 00:00:00 2001 From: Vojtech Mares Date: Sat, 3 Sep 2022 16:06:25 +0200 Subject: [PATCH 01/10] feat(monitoring): add blackbox-exporter --- monitoring/Chart.lock | 7 +++++-- monitoring/Chart.yaml | 3 +++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/monitoring/Chart.lock b/monitoring/Chart.lock index 8956be1..4fbfe82 100644 --- a/monitoring/Chart.lock +++ b/monitoring/Chart.lock @@ -2,5 +2,8 @@ dependencies: - name: kube-prometheus-stack repository: https://prometheus-community.github.io/helm-charts version: 39.11.0 -digest: sha256:2000f95ea7c9e6ac6ec0cc0ed3f08ee6adebf5e3ad383a0e8d89d80ab61439eb -generated: "2022-09-03T10:54:33.34106+02:00" +- name: prometheus-blackbox-exporter + repository: https://prometheus-community.github.io/helm-charts + version: 7.0.0 +digest: sha256:7a9382529ff259e31426aa23cf9eea9a0dc5fe7d6950339133bc78bcb310fa30 +generated: "2022-09-03T16:06:15.239791+02:00" diff --git a/monitoring/Chart.yaml b/monitoring/Chart.yaml index 00605fb..c22dfc7 100644 --- a/monitoring/Chart.yaml +++ b/monitoring/Chart.yaml @@ -9,3 +9,6 @@ dependencies: - name: kube-prometheus-stack version: 39.11.0 repository: https://prometheus-community.github.io/helm-charts +- name: prometheus-blackbox-exporter + version: 7.0.0 + repository: https://prometheus-community.github.io/helm-charts From 4c114a5010c8d3b0c86deb58fdf3bce3aed5c2a9 Mon Sep 17 00:00:00 2001 From: Vojtech Mares Date: Sat, 3 Sep 2022 16:08:05 +0200 Subject: [PATCH 02/10] docs: update README.md --- README.md | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 5dd5977..5417fbd 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,28 @@ Underlying Kubernetes cluster components for [KISSJ](https://github.com/skautdev - namespaces - `kissj-db` + - `kissj-dev` - `kissj-prod` - `kissj-staging` + - `kissj-monitoring` - PostgreSQL instance +- monitoring + - Prometheus + - AlertManager + - Grafana + - blackbox exporter ## Security - Users -- ServiceAccounts + - lung + - majkl + +## Monitoring + +As mentione before in [components](#components), the monitoring consists of: +- [Prometheus](https://prometheus.monitoring.kissj.net) +- [AlertManager](https://alertmanager.monitoring.kissj.net) +- [Grafana](https://grafana.monitoring.kissj.net) + +To monitor everything, we also deploy [prometheus/blackbox_exporter](https://github.com/prometheus/blackbox_exporter) to monitor the overall availability of the site. From 43f2caa08718e88494fc1228c032cdeb1f10df08 Mon Sep 17 00:00:00 2001 From: Vojtech Mares Date: Sat, 3 Sep 2022 16:54:38 +0200 Subject: [PATCH 03/10] refactor(monitoring): add fullnameOverride for blackbox exporter --- monitoring/values.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/monitoring/values.yaml b/monitoring/values.yaml index 3618e5b..28e6d80 100644 --- a/monitoring/values.yaml +++ b/monitoring/values.yaml @@ -127,3 +127,6 @@ kube-prometheus-stack: defaultRules: create: false + +prometheus-blackbox-exporter: + fullnameOverride: blackbox-exporter From 1b290d7cb2eb998f67a4bece8649cf90db8bbdc8 Mon Sep 17 00:00:00 2001 From: Vojtech Mares Date: Sat, 3 Sep 2022 16:58:48 +0200 Subject: [PATCH 04/10] refactor(monitoring): prefix blackbox exporter with kissj- --- monitoring/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/values.yaml b/monitoring/values.yaml index 28e6d80..538cbdf 100644 --- a/monitoring/values.yaml +++ b/monitoring/values.yaml @@ -129,4 +129,4 @@ kube-prometheus-stack: create: false prometheus-blackbox-exporter: - fullnameOverride: blackbox-exporter + fullnameOverride: kissj-blackbox-exporter From b192ffe85d6e82968a29fd4b0b46888ab925b0b6 Mon Sep 17 00:00:00 2001 From: Vojtech Mares Date: Mon, 5 Sep 2022 21:02:01 +0200 Subject: [PATCH 05/10] refactor(namespaces): delete kissj-dev --- namespaces/kissj-dev.yml | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 namespaces/kissj-dev.yml diff --git a/namespaces/kissj-dev.yml b/namespaces/kissj-dev.yml deleted file mode 100644 index 42385a7..0000000 --- a/namespaces/kissj-dev.yml +++ /dev/null @@ -1,6 +0,0 @@ -apiVersion: v1 -kind: Namespace -metadata: - name: kissj-dev - labels: - prometheus: kissj From e7d20864a9bd1c7bdcab5f9013e00c7f829a3fac Mon Sep 17 00:00:00 2001 From: Vojtech Mares Date: Mon, 5 Sep 2022 21:03:15 +0200 Subject: [PATCH 06/10] refactor(namespaces): kissj-prod -> kissj-production --- namespaces/{kissj-prod.yml => kissj-production.yml} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename namespaces/{kissj-prod.yml => kissj-production.yml} (74%) diff --git a/namespaces/kissj-prod.yml b/namespaces/kissj-production.yml similarity index 74% rename from namespaces/kissj-prod.yml rename to namespaces/kissj-production.yml index b948144..f513b48 100644 --- a/namespaces/kissj-prod.yml +++ b/namespaces/kissj-production.yml @@ -1,6 +1,6 @@ apiVersion: v1 kind: Namespace metadata: - name: kissj-prod + name: kissj-production labels: prometheus: kissj From 4f0927c6bfbb3271daefba9233964f8f449cbbd0 Mon Sep 17 00:00:00 2001 From: Vojtech Mares Date: Mon, 5 Sep 2022 21:27:08 +0200 Subject: [PATCH 07/10] feat: add quoats --- _apps/kissj-production-quotas.yml | 18 ++++++++++++++++++ _apps/kissj-staging-quotas.yml | 18 ++++++++++++++++++ quotas/production/compute-resources.yml | 10 ++++++++++ quotas/production/kube-objects.yml | 15 +++++++++++++++ quotas/staging/compute-resources.yml | 10 ++++++++++ quotas/staging/kube-objects.yml | 15 +++++++++++++++ 6 files changed, 86 insertions(+) create mode 100644 _apps/kissj-production-quotas.yml create mode 100644 _apps/kissj-staging-quotas.yml create mode 100644 quotas/production/compute-resources.yml create mode 100644 quotas/production/kube-objects.yml create mode 100644 quotas/staging/compute-resources.yml create mode 100644 quotas/staging/kube-objects.yml diff --git a/_apps/kissj-production-quotas.yml b/_apps/kissj-production-quotas.yml new file mode 100644 index 0000000..7f16afc --- /dev/null +++ b/_apps/kissj-production-quotas.yml @@ -0,0 +1,18 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: kissj-production-quotas + namespace: argocd +spec: + destination: + namespace: kissj-production + server: https://kubernetes.default.svc + project: kissj + source: + path: quotas/production + repoURL: https://gitlab.mareshq.com/gitops/skautdevs/kissj.git + targetRevision: HEAD + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/_apps/kissj-staging-quotas.yml b/_apps/kissj-staging-quotas.yml new file mode 100644 index 0000000..c0b7b2b --- /dev/null +++ b/_apps/kissj-staging-quotas.yml @@ -0,0 +1,18 @@ +apiVersion: argoproj.io/v1alpha1 +kind: Application +metadata: + name: kissj-staging-quotas + namespace: argocd +spec: + destination: + namespace: kissj-staging + server: https://kubernetes.default.svc + project: kissj + source: + path: quotas/staging + repoURL: https://gitlab.mareshq.com/gitops/skautdevs/kissj.git + targetRevision: HEAD + syncPolicy: + automated: + prune: true + selfHeal: true diff --git a/quotas/production/compute-resources.yml b/quotas/production/compute-resources.yml new file mode 100644 index 0000000..522ccaf --- /dev/null +++ b/quotas/production/compute-resources.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ResourceQuota +metadata: + name: compute-resources +spec: + hard: + requests.cpu: 6000m + requests.memory: 16384Mi + limits.cpu: 8000m + limits.memory: 20480Mi diff --git a/quotas/production/kube-objects.yml b/quotas/production/kube-objects.yml new file mode 100644 index 0000000..a97b3c2 --- /dev/null +++ b/quotas/production/kube-objects.yml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ResourceQuota +metadata: + name: kube-objects +spec: + hard: + services.loadbalancers: "0" + + count/deployments.apps: "5" + count/statefulsets.apps: "0" + count/cronjobs.batch: "0" + count/jobs.batch: "5" + count/pods: "50" + + count/persistentvolumeclaims: "1" diff --git a/quotas/staging/compute-resources.yml b/quotas/staging/compute-resources.yml new file mode 100644 index 0000000..d2e4c38 --- /dev/null +++ b/quotas/staging/compute-resources.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ResourceQuota +metadata: + name: compute-resources +spec: + hard: + requests.cpu: 800m + requests.memory: 1536Mi + limits.cpu: 1200m + limits.memory: 2048Mi diff --git a/quotas/staging/kube-objects.yml b/quotas/staging/kube-objects.yml new file mode 100644 index 0000000..db6542b --- /dev/null +++ b/quotas/staging/kube-objects.yml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ResourceQuota +metadata: + name: kube-objects +spec: + hard: + services.loadbalancers: "0" + + count/deployments.apps: "5" + count/statefulsets.apps: "0" + count/cronjobs.batch: "0" + count/jobs.batch: "5" + count/pods: "20" + + count/persistentvolumeclaims: "1" From f16a7aa3657333d2f7c71695935979727d4426d4 Mon Sep 17 00:00:00 2001 From: Vojtech Mares Date: Tue, 6 Sep 2022 11:41:19 +0200 Subject: [PATCH 08/10] feat(quotas): add storage quotas --- quotas/production/storage.yml | 10 ++++++++++ quotas/staging/storage.yml | 10 ++++++++++ 2 files changed, 20 insertions(+) create mode 100644 quotas/production/storage.yml create mode 100644 quotas/staging/storage.yml diff --git a/quotas/production/storage.yml b/quotas/production/storage.yml new file mode 100644 index 0000000..21165c4 --- /dev/null +++ b/quotas/production/storage.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ResourceQuota +metadata: + name: storage +spec: + hard: + requests.storage: 1Gi + persistentvolumeclaims: "3" + local-path.storageclass.storage.k8s.io/requests.storage: 0Mi + local-path.storageclass.storage.k8s.io/persistentvolumeclaims: "0" diff --git a/quotas/staging/storage.yml b/quotas/staging/storage.yml new file mode 100644 index 0000000..21165c4 --- /dev/null +++ b/quotas/staging/storage.yml @@ -0,0 +1,10 @@ +apiVersion: v1 +kind: ResourceQuota +metadata: + name: storage +spec: + hard: + requests.storage: 1Gi + persistentvolumeclaims: "3" + local-path.storageclass.storage.k8s.io/requests.storage: 0Mi + local-path.storageclass.storage.k8s.io/persistentvolumeclaims: "0" From 7734c642e16c40be5700a7398de64437bebb3912 Mon Sep 17 00:00:00 2001 From: Vojtech Mares Date: Tue, 6 Sep 2022 11:43:45 +0200 Subject: [PATCH 09/10] feat(quotas): add service.nodeport and separate service.loadbalancer to standalone quota --- quotas/production/kube-objects.yml | 2 -- quotas/production/network.yml | 9 +++++++++ quotas/staging/kube-objects.yml | 2 -- quotas/staging/network.yml | 9 +++++++++ 4 files changed, 18 insertions(+), 4 deletions(-) create mode 100644 quotas/production/network.yml create mode 100644 quotas/staging/network.yml diff --git a/quotas/production/kube-objects.yml b/quotas/production/kube-objects.yml index a97b3c2..5fb9c8b 100644 --- a/quotas/production/kube-objects.yml +++ b/quotas/production/kube-objects.yml @@ -4,8 +4,6 @@ metadata: name: kube-objects spec: hard: - services.loadbalancers: "0" - count/deployments.apps: "5" count/statefulsets.apps: "0" count/cronjobs.batch: "0" diff --git a/quotas/production/network.yml b/quotas/production/network.yml new file mode 100644 index 0000000..63cbb06 --- /dev/null +++ b/quotas/production/network.yml @@ -0,0 +1,9 @@ + +apiVersion: v1 +kind: ResourceQuota +metadata: + name: network +spec: + hard: + services.loadbalancers: "0" + services.nodeports: "0" diff --git a/quotas/staging/kube-objects.yml b/quotas/staging/kube-objects.yml index db6542b..dbb9720 100644 --- a/quotas/staging/kube-objects.yml +++ b/quotas/staging/kube-objects.yml @@ -4,8 +4,6 @@ metadata: name: kube-objects spec: hard: - services.loadbalancers: "0" - count/deployments.apps: "5" count/statefulsets.apps: "0" count/cronjobs.batch: "0" diff --git a/quotas/staging/network.yml b/quotas/staging/network.yml new file mode 100644 index 0000000..63cbb06 --- /dev/null +++ b/quotas/staging/network.yml @@ -0,0 +1,9 @@ + +apiVersion: v1 +kind: ResourceQuota +metadata: + name: network +spec: + hard: + services.loadbalancers: "0" + services.nodeports: "0" From f09c1ec86bc3c8cd752f2cd9f1a2e26d931266ee Mon Sep 17 00:00:00 2001 From: Vojtech Mares Date: Tue, 6 Sep 2022 17:36:51 +0200 Subject: [PATCH 10/10] refactor(monitoring): do not monitor cluster components --- monitoring/values.yaml | 53 +++++++++++++++++------------------------- 1 file changed, 21 insertions(+), 32 deletions(-) diff --git a/monitoring/values.yaml b/monitoring/values.yaml index 538cbdf..bea8d7f 100644 --- a/monitoring/values.yaml +++ b/monitoring/values.yaml @@ -13,6 +13,27 @@ kube-prometheus-stack: kubeStateMetrics: enabled: false + kubeApiServer: + enabled: false + + kubelet: + enabled: false + + kubeControllerManager: + enabled: false + + coreDns: + enabled: false + + kubeScheduler: + enabled: false + + kubeProxy: + enabled: false + + defaultRules: + create: false + prometheus: prometheusSpec: serviceMonitorSelectorNilUsesHelmValues: false @@ -96,37 +117,5 @@ kube-prometheus-stack: serviceMonitor: selfMonitor: true - # This AM is for KISSJ, cluster components are monitored by different instance - kubeApiServer: - enabled: false - - kubelet: - enabled: false - - kubeControllerManager: - enabled: false - - coreDns: - enabled: false - - kubeScheduler: - enabled: false - - kubeProxy: - enabled: false - - kubeStateMetrics: - enabled: false - - nodeExporter: - enabled: false - - prometheusOperator: - serviceMonitor: - selfMonitor: false - - defaultRules: - create: false - prometheus-blackbox-exporter: fullnameOverride: kissj-blackbox-exporter