From da638e642c65ef57259b898c7c915b321ffc4357 Mon Sep 17 00:00:00 2001 From: Dario Mapelli Date: Wed, 24 Apr 2024 14:40:52 +0200 Subject: [PATCH 1/3] crabserver helm - added values for test2 and test11 --- helm/crabserver/values-test11.yaml | 14 +++ helm/crabserver/values-test2.yaml | 173 +---------------------------- 2 files changed, 18 insertions(+), 169 deletions(-) create mode 100644 helm/crabserver/values-test11.yaml diff --git a/helm/crabserver/values-test11.yaml b/helm/crabserver/values-test11.yaml new file mode 100644 index 000000000..062b3059f --- /dev/null +++ b/helm/crabserver/values-test11.yaml @@ -0,0 +1,14 @@ +environment: "test" + +image: + tag: "v3.240325" + +livenessProbeTest: + failureThreshold: 3 + initialDelaySeconds: 120 + periodSeconds: 600 + timeoutSeconds: 60 + +readinessProbeTest: + periodSeconds: 600 + timeoutSeconds: 60 diff --git a/helm/crabserver/values-test2.yaml b/helm/crabserver/values-test2.yaml index 5a4206f71..062b3059f 100644 --- a/helm/crabserver/values-test2.yaml +++ b/helm/crabserver/values-test2.yaml @@ -1,179 +1,14 @@ -# Default values for crabserver. -# This is a YAML-formatted file. -# Declare variables to be passed into your templates. - -replicaCount: - prod: 8 - test: 1 +environment: "test" image: - path: registry.cern.ch/cmsweb/crabserver - pullPolicy: Always - # Overrides the image tag whose default is the chart appVersion. - tag: "v3.240219" - command: - - /bin/bash - - /opt/setup-certs-and-run/setup-certs-and-run.sh - env: - - name: CRABSERVER_LOGSTDOUT - value: "t" - -environment: - -imagePullSecrets: [] -nameOverride: "" -fullnameOverride: "" - -serviceAccount: - # Specifies whether a service account should be created - create: false - # Annotations to add to the service account - annotations: {} - # The name of the service account to use. - # If not set and create is true, a name is generated using the fullname template - name: "" - -podAnnotations: - prometheus.io/scrape: 'true' - prometheus.io/port: "18270" - -podSecurityContext: - runAsUser: 1000 - runAsGroup: 1000 - fsGroup: 2000 - -imageFilebeatSecurityContext: - allowPrivilegeEscalation: false - -securityContext: - privileged: true - -serviceMon: - name: crabserver-mon - port: 18720 - protocol: TCP - -ingress: - enabled: false - annotations: {} - # kubernetes.io/ingress.class: nginx - # kubernetes.io/tls-acme: "true" - -resources: - limits: - cpu: 1500m - memory: 3Gi - requests: - cpu: 200m - memory: 256Mi - - -deamonset: - name: filebeat - path: docker.elastic.co/beats/filebeat:8.5.1 - policy: IfNotPresent - args: - - bash - - -c - - filebeat -c /etc/filebeat.yml --path.data /data/filebeat/${MY_NODE_NAME}/data -e - env: - - name: MY_NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: NODE_NAME - valueFrom: - fieldRef: - fieldPath: spec.nodeName - - name: MY_NAMESPACE - valueFrom: - fieldRef: - apiVersion: v1 - fieldPath: metadata.namespace - resources: - requests: - memory: "128Mi" - cpu: "200m" - limits: - cpu: "1" - memory: "1Gi" - - -livenessProbe: - exec: - command: - - /bin/sh - - -c - - | - cmsweb-ping --url=http://localhost:8270/crabserver/prod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]" - failureThreshold: 3 - initialDelaySeconds: 120 - periodSeconds: 60 - timeoutSeconds: 60 - -readinessProbe: - exec: - command: - - /bin/sh - - -c - - | - cmsweb-ping --url=http://localhost:8270/crabserver/prod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]" - periodSeconds: 60 - timeoutSeconds: 60 - -livenessProbePreProd: - exec: - command: - - /bin/sh - - -c - - | - cmsweb-ping --url=http://localhost:8270/crabserver/preprod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]" - failureThreshold: 3 - initialDelaySeconds: 120 - periodSeconds: 60 - timeoutSeconds: 60 - -readinessProbePreProd: - exec: - command: - - /bin/sh - - -c - - | - cmsweb-ping --url=http://localhost:8270/crabserver/preprod/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]" - periodSeconds: 60 - timeoutSeconds: 60 + tag: "v3.240325" livenessProbeTest: - exec: - command: - - /bin/sh - - -c - - | - cmsweb-ping --url=http://localhost:8270/crabserver/dev/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]" failureThreshold: 3 initialDelaySeconds: 120 - periodSeconds: 60 + periodSeconds: 600 timeoutSeconds: 60 readinessProbeTest: - exec: - command: - - /bin/sh - - -c - - | - cmsweb-ping --url=http://localhost:8270/crabserver/dev/info --authz=/etc/hmac/hmac -verbose 0 | egrep "^[2-4]" - periodSeconds: 60 + periodSeconds: 600 timeoutSeconds: 60 - -autoscaling: - enabled: false - minReplicas: 1 - maxReplicas: 100 - targetCPUUtilizationPercentage: 80 - # targetMemoryUtilizationPercentage: 80 - -nodeSelector: {} - -tolerations: [] - -affinity: {} From 021aa7abce85d3b292758d36dca748eeda0f0f29 Mon Sep 17 00:00:00 2001 From: Dario Mapelli Date: Tue, 30 Apr 2024 18:44:30 +0200 Subject: [PATCH 2/3] crabserver helm - added values for prod and preprod --- helm/crabserver/README.md | 33 +++++++++++++++++++++++ helm/crabserver/templates/deployment.yaml | 4 ++- helm/crabserver/values-prod.yaml | 4 +++ helm/crabserver/values-test2.yaml | 10 ------- helm/crabserver/values-testbed.yaml | 4 +++ helm/crabserver/values.yaml | 3 ++- 6 files changed, 46 insertions(+), 12 deletions(-) create mode 100644 helm/crabserver/README.md create mode 100644 helm/crabserver/values-prod.yaml create mode 100644 helm/crabserver/values-testbed.yaml diff --git a/helm/crabserver/README.md b/helm/crabserver/README.md new file mode 100644 index 000000000..40a0e5668 --- /dev/null +++ b/helm/crabserver/README.md @@ -0,0 +1,33 @@ +## Deploy CRAB + +At the moment CRAB team is transitioning to using helm. However, we do noy like +to use helm for deployment. See this +[comment](https://github.com/dmwm/CRABServer/issues/7843#issuecomment-2025085120) +for some context. + +We like to use helm for templating, using the helm charts to generate the +manifest file, then applying it with `kubectl apply`. + +We therefore ask cmsweb operators to deploy crab with the following procedure: + +Generate the manifest with + +```bash +# testbed +helm template crabserver . -f values.yaml -f values-testbed.yaml > ../../kubernetes/cmsweb/services/crabserver.yaml + +# prod +helm template crabserver . -f values.yaml -f values-prod.yaml > ../../kubernetes/cmsweb/services/crabserver.yaml +``` + +Then to deploy it with the usual `deploy-srv.sh` script + +```bash +# testbed +./scripts/deploy-srv.sh crabserver v3.231006 preprod + +# prod +./scripts/deploy-srv.sh crabserver v3.231006 prod +``` + +Changes to `../../kubernetes/cmsweb/services/crabserver.yaml` should not be committed. diff --git a/helm/crabserver/templates/deployment.yaml b/helm/crabserver/templates/deployment.yaml index 3c156eb06..4efe1fd0d 100644 --- a/helm/crabserver/templates/deployment.yaml +++ b/helm/crabserver/templates/deployment.yaml @@ -8,8 +8,10 @@ metadata: namespace: crab spec: {{- if not .Values.autoscaling.enabled }} - {{- if or (eq (toString $environment) "prod") (eq (toString $environment) "preprod") }} + {{- if (eq (toString $environment) "prod") }} replicas: {{ .Values.replicaCount.prod }} + {{- else if (eq (toString $environment) "preprod")}} + replicas: {{ .Values.replicaCount.preprod }} {{- else }} replicas: {{ .Values.replicaCount.test }} {{- end }} diff --git a/helm/crabserver/values-prod.yaml b/helm/crabserver/values-prod.yaml new file mode 100644 index 000000000..5d5e637ee --- /dev/null +++ b/helm/crabserver/values-prod.yaml @@ -0,0 +1,4 @@ +environment: "prod" + +image: + tag: "v3.240426" diff --git a/helm/crabserver/values-test2.yaml b/helm/crabserver/values-test2.yaml index 062b3059f..0e4629d92 100644 --- a/helm/crabserver/values-test2.yaml +++ b/helm/crabserver/values-test2.yaml @@ -2,13 +2,3 @@ environment: "test" image: tag: "v3.240325" - -livenessProbeTest: - failureThreshold: 3 - initialDelaySeconds: 120 - periodSeconds: 600 - timeoutSeconds: 60 - -readinessProbeTest: - periodSeconds: 600 - timeoutSeconds: 60 diff --git a/helm/crabserver/values-testbed.yaml b/helm/crabserver/values-testbed.yaml new file mode 100644 index 000000000..930744e19 --- /dev/null +++ b/helm/crabserver/values-testbed.yaml @@ -0,0 +1,4 @@ +environment: "preprod" + +image: + tag: "v3.240426" diff --git a/helm/crabserver/values.yaml b/helm/crabserver/values.yaml index 38af26e38..9e53abfeb 100644 --- a/helm/crabserver/values.yaml +++ b/helm/crabserver/values.yaml @@ -3,7 +3,8 @@ # Declare variables to be passed into your templates. replicaCount: - prod: 8 + prod: 15 + preprod: 1 test: 1 image: From 8cf744aaf4d1d025a8617b62321d3b61d723ca82 Mon Sep 17 00:00:00 2001 From: Dario Mapelli Date: Mon, 13 May 2024 13:49:05 +0200 Subject: [PATCH 3/3] crabserver helm - new version in preprod and prod --- helm/crabserver/values-prod.yaml | 3 ++- helm/crabserver/values-testbed.yaml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/helm/crabserver/values-prod.yaml b/helm/crabserver/values-prod.yaml index 5d5e637ee..1316b5411 100644 --- a/helm/crabserver/values-prod.yaml +++ b/helm/crabserver/values-prod.yaml @@ -1,4 +1,5 @@ environment: "prod" image: - tag: "v3.240426" + tag: "v3.240508" + diff --git a/helm/crabserver/values-testbed.yaml b/helm/crabserver/values-testbed.yaml index 930744e19..38058d38a 100644 --- a/helm/crabserver/values-testbed.yaml +++ b/helm/crabserver/values-testbed.yaml @@ -1,4 +1,4 @@ environment: "preprod" image: - tag: "v3.240426" + tag: "v3.240508"