diff --git a/.github/workflows/run_tests.yaml b/.github/workflows/run_tests.yaml index bb52caa0f..94defc7b3 100644 --- a/.github/workflows/run_tests.yaml +++ b/.github/workflows/run_tests.yaml @@ -74,7 +74,7 @@ jobs: echo " test mode: ${{ github.event.inputs.test_mode }}" echo " test mask: ${{ github.event.inputs.test_mask }}" echo - + source ~/venv/qa/bin/activate set -x set +e # disable the "exit on failure" @@ -121,7 +121,7 @@ jobs: retention-days: 90 - name: Test Failed - if: ${{ steps.vars.outputs.test_result != '0' }} + if: ${{ failure() }} uses: actions/github-script@v3 with: script: | diff --git a/.gitignore b/.gitignore index 1894072b9..3e2b68c7a 100644 --- a/.gitignore +++ b/.gitignore @@ -19,6 +19,7 @@ # Skip logs and caches log *.log +*.log.txt __pycache__ # Skip dev runtime-produced diff --git a/Vagrantfile b/Vagrantfile index c3e26a46e..a5e28b618 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -18,7 +18,7 @@ end Vagrant.configure(2) do |config| - config.vm.box = "generic/ubuntu2004" + config.vm.box = "generic/ubuntu2404" config.vm.box_check_update = false if get_provider == "hyperv" @@ -111,7 +111,7 @@ Vagrant.configure(2) do |config| apt-get install --no-install-recommends -y clickhouse-client # golang - export GOLANG_VERSION=1.19 + export GOLANG_VERSION=1.23 apt-key adv --keyserver keyserver.ubuntu.com --recv-keys F6BC817356A3D45E add-apt-repository ppa:longsleep/golang-backports apt-get install --no-install-recommends -y golang-${GOLANG_VERSION}-go @@ -148,7 +148,8 @@ Vagrant.configure(2) do |config| # MINIKUBE_VERSION=1.19.0 # MINIKUBE_VERSION=1.20.0 # MINIKUBE_VERSION=1.23.2 - MINIKUBE_VERSION=1.28.0 + # MINIKUBE_VERSION=1.28.0 + MINIKUBE_VERSION=1.34.0 wget -c --progress=bar:force:noscroll -O /usr/local/bin/minikube https://github.com/kubernetes/minikube/releases/download/v${MINIKUBE_VERSION}/minikube-linux-amd64 chmod +x /usr/local/bin/minikube # required for k8s 1.18+ @@ -168,7 +169,8 @@ Vagrant.configure(2) do |config| # K8S_VERSION=${K8S_VERSION:-1.22.5} # K8S_VERSION=${K8S_VERSION:-1.23.1} # K8S_VERSION=${K8S_VERSION:-1.24.8} - K8S_VERSION=${K8S_VERSION:-1.25.4} +# K8S_VERSION=${K8S_VERSION:-1.25.4} + K8S_VERSION=${K8S_VERSION:-1.31.1} export VALIDATE_YAML=true killall kubectl || true diff --git a/config/chk/keeper_config.d/01-keeper-01-default-config.xml b/config/chk/keeper_config.d/01-keeper-01-default-config.xml index a7574bc21..9c86b7cea 100644 --- a/config/chk/keeper_config.d/01-keeper-01-default-config.xml +++ b/config/chk/keeper_config.d/01-keeper-01-default-config.xml @@ -17,7 +17,6 @@ /var/lib/clickhouse-keeper/coordination/snapshots /var/lib/clickhouse-keeper 2181 - true :: 0.0.0.0 diff --git a/config/chk/keeper_config.d/01-keeper-03-enable-reconfig.xml b/config/chk/keeper_config.d/01-keeper-03-enable-reconfig.xml new file mode 100644 index 000000000..e06915c8a --- /dev/null +++ b/config/chk/keeper_config.d/01-keeper-03-enable-reconfig.xml @@ -0,0 +1,11 @@ + + + + + + + + + false + + diff --git a/config/config-dev.yaml b/config/config-dev.yaml index 008daf9d8..0c00a69c9 100644 --- a/config/config-dev.yaml +++ b/config/config-dev.yaml @@ -351,6 +351,27 @@ label: # LabelClusterScopeCycleOffset appendScope: "no" +################################################ +## +## Metrics management section +## +################################################ +metrics: + labels: + exclude: [] + +################################################ +## +## Status management section +## +################################################ +status: + fields: + action: false + actions: false + error: false + errors: false + ################################################ ## ## StatefulSet management section diff --git a/config/config.yaml b/config/config.yaml index e69851e57..cab6b4c96 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -147,7 +147,6 @@ clickhouse: # These credentials are used for: # 1. Metrics requests # 2. Schema maintenance - # 3. DROP DNS CACHE # User with these credentials can be specified in additional ClickHouse .xml config files, # located in 'clickhouse.configuration.file.path.user' folder username: "" @@ -351,6 +350,27 @@ label: # LabelClusterScopeCycleOffset appendScope: "no" +################################################ +## +## Metrics management section +## +################################################ +metrics: + labels: + exclude: [] + +################################################ +## +## Status management section +## +################################################ +status: + fields: + action: false + actions: false + error: false + errors: false + ################################################ ## ## StatefulSet management section diff --git a/deploy/builder/templates-config/chk/keeper_config.d/01-keeper-01-default-config.xml b/deploy/builder/templates-config/chk/keeper_config.d/01-keeper-01-default-config.xml index c8a6026a5..e22385d46 100644 --- a/deploy/builder/templates-config/chk/keeper_config.d/01-keeper-01-default-config.xml +++ b/deploy/builder/templates-config/chk/keeper_config.d/01-keeper-01-default-config.xml @@ -11,7 +11,6 @@ /var/lib/clickhouse-keeper/coordination/snapshots /var/lib/clickhouse-keeper 2181 - true :: 0.0.0.0 diff --git a/deploy/builder/templates-config/chk/keeper_config.d/01-keeper-03-enable-reconfig.xml b/deploy/builder/templates-config/chk/keeper_config.d/01-keeper-03-enable-reconfig.xml new file mode 100644 index 000000000..e5d8fb8fd --- /dev/null +++ b/deploy/builder/templates-config/chk/keeper_config.d/01-keeper-03-enable-reconfig.xml @@ -0,0 +1,5 @@ + + + false + + diff --git a/deploy/builder/templates-config/config.yaml b/deploy/builder/templates-config/config.yaml index 8111097d7..10099f37f 100644 --- a/deploy/builder/templates-config/config.yaml +++ b/deploy/builder/templates-config/config.yaml @@ -141,7 +141,6 @@ clickhouse: # These credentials are used for: # 1. Metrics requests # 2. Schema maintenance - # 3. DROP DNS CACHE # User with these credentials can be specified in additional ClickHouse .xml config files, # located in 'clickhouse.configuration.file.path.user' folder username: "${CH_USERNAME_PLAIN}" @@ -345,6 +344,27 @@ label: # LabelClusterScopeCycleOffset appendScope: "no" +################################################ +## +## Metrics management section +## +################################################ +metrics: + labels: + exclude: [] + +################################################ +## +## Status management section +## +################################################ +status: + fields: + action: false + actions: false + error: false + errors: false + ################################################ ## ## StatefulSet management section diff --git a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-01-chi-chit.yaml b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-01-chi-chit.yaml index 823f978a3..2ef4e8bfc 100644 --- a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-01-chi-chit.yaml +++ b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-01-chi-chit.yaml @@ -1087,7 +1087,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1235,7 +1235,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml index af357342e..799e703eb 100644 --- a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml +++ b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-02-chopconf.yaml @@ -374,6 +374,40 @@ spec: - "LabelClusterScopeCycleSize" - "LabelClusterScopeCycleIndex" - "LabelClusterScopeCycleOffset" + metrics: + type: object + description: "defines metrics exporter options" + properties: + labels: + type: object + description: "defines metric labels options" + properties: + exclude: + type: array + description: | + When adding labels to a metric exclude labels with names from the following list + items: + type: string + status: + type: object + description: "defines status options" + properties: + fields: + type: object + description: "defines status fields options" + properties: + action: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'action'" + actions: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'actions'" + error: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'error'" + errors: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'errors'" statefulSet: type: object description: "define StatefulSet-specific parameters" diff --git a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-03-chk.yaml b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-03-chk.yaml index edb28e13d..a6aa1616b 100644 --- a/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-03-chk.yaml +++ b/deploy/builder/templates-install-bundle/clickhouse-operator-install-yaml-template-01-section-crd-03-chk.yaml @@ -671,7 +671,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -819,7 +819,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/helm/clickhouse-operator/Chart.yaml b/deploy/helm/clickhouse-operator/Chart.yaml index 8065eb462..9d0828a99 100644 --- a/deploy/helm/clickhouse-operator/Chart.yaml +++ b/deploy/helm/clickhouse-operator/Chart.yaml @@ -10,10 +10,11 @@ description: |- kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml + kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml ``` type: application -version: 0.24.0 -appVersion: 0.24.0 +version: 0.24.1 +appVersion: 0.24.1 home: https://github.com/Altinity/clickhouse-operator icon: https://logosandtypes.com/wp-content/uploads/2020/12/altinity.svg maintainers: diff --git a/deploy/helm/clickhouse-operator/README.md b/deploy/helm/clickhouse-operator/README.md index 2a12b8b2d..7f5bd7da0 100644 --- a/deploy/helm/clickhouse-operator/README.md +++ b/deploy/helm/clickhouse-operator/README.md @@ -1,6 +1,6 @@ # altinity-clickhouse-operator -![Version: 0.24.0](https://img.shields.io/badge/Version-0.24.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.24.0](https://img.shields.io/badge/AppVersion-0.24.0-informational?style=flat-square) +![Version: 0.24.1](https://img.shields.io/badge/Version-0.24.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 0.24.1](https://img.shields.io/badge/AppVersion-0.24.1-informational?style=flat-square) Helm chart to deploy [altinity-clickhouse-operator](https://github.com/Altinity/clickhouse-operator). @@ -11,6 +11,7 @@ For upgrade please install CRDs separately: kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml + kubectl apply -f https://github.com/Altinity/clickhouse-operator/raw/master/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml ``` **Homepage:** @@ -25,31 +26,31 @@ For upgrade please install CRDs separately: | Key | Type | Default | Description | |-----|------|---------|-------------| -| additionalResources | list | `[]` | list of additional resources to create (are processed via `tpl` function), useful for create ClickHouse clusters together with clickhouse-operator, look `kubectl explain chi` for details | -| affinity | object | `{}` | affinity for scheduler pod assignment, look `kubectl explain pod.spec.affinity` for details | -| configs | object | `{"confdFiles":null,"configdFiles":{"01-clickhouse-01-listen.xml":"\n\n\n\n\n\n\n \n ::\n 0.0.0.0\n 1\n\n","01-clickhouse-02-logger.xml":"\n\n\n\n\n\n\n \n \n debug\n /var/log/clickhouse-server/clickhouse-server.log\n /var/log/clickhouse-server/clickhouse-server.err.log\n 1000M\n 10\n \n 1\n \n\n","01-clickhouse-03-query_log.xml":"\n\n\n\n\n\n\n \n system\n query_log
\n Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day\n 7500\n \n \n\n","01-clickhouse-04-part_log.xml":"\n\n\n\n\n\n\n \n system\n part_log
\n Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day\n 7500\n \n\n","01-clickhouse-05-trace_log.xml":"\n\n\n\n\n\n\n \n system\n trace_log
\n Engine = MergeTree PARTITION BY event_date ORDER BY event_time TTL event_date + interval 30 day\n 7500\n \n"},"files":{"config.yaml":{"annotation":{"exclude":[],"include":[]},"clickhouse":{"access":{"password":"","port":8123,"rootCA":"","scheme":"auto","secret":{"name":"{{ include \"altinity-clickhouse-operator.fullname\" . }}","namespace":""},"timeouts":{"connect":1,"query":4},"username":""},"configuration":{"file":{"path":{"common":"chi/config.d","host":"chi/conf.d","user":"chi/users.d"}},"network":{"hostRegexpTemplate":"(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$"},"user":{"default":{"networksIP":["::1","127.0.0.1"],"password":"default","profile":"default","quota":"default"}}},"configurationRestartPolicy":{"rules":[{"rules":[{"settings/*":"yes"},{"settings/access_control_path":"no"},{"settings/dictionaries_config":"no"},{"settings/max_server_memory_*":"no"},{"settings/max_*_to_drop":"no"},{"settings/max_concurrent_queries":"no"},{"settings/models_config":"no"},{"settings/user_defined_executable_functions_config":"no"},{"settings/logger/*":"no"},{"settings/macros/*":"no"},{"settings/remote_servers/*":"no"},{"settings/user_directories/*":"no"},{"zookeeper/*":"yes"},{"files/*.xml":"yes"},{"files/config.d/*.xml":"yes"},{"files/config.d/*dict*.xml":"no"},{"profiles/default/background_*_pool_size":"yes"},{"profiles/default/max_*_for_server":"yes"}],"version":"*"},{"rules":[{"settings/logger":"yes"}],"version":"21.*"}]},"metrics":{"timeouts":{"collect":9}}},"keeper":{"configuration":{"file":{"path":{"common":"chk/keeper_config.d","host":"chk/conf.d","user":"chk/users.d"}}}},"label":{"appendScope":"no","exclude":[],"include":[]},"logger":{"alsologtostderr":"false","log_backtrace_at":"","logtostderr":"true","stderrthreshold":"","v":"1","vmodule":""},"pod":{"terminationGracePeriod":30},"reconcile":{"host":{"wait":{"exclude":true,"include":false,"queries":true}},"runtime":{"reconcileCHIsThreadsNumber":10,"reconcileShardsMaxConcurrencyPercent":50,"reconcileShardsThreadsNumber":5},"statefulSet":{"create":{"onFailure":"ignore"},"update":{"onFailure":"abort","pollInterval":5,"timeout":300}}},"statefulSet":{"revisionHistoryLimit":0},"template":{"chi":{"path":"chi/templates.d","policy":"ApplyOnNextReconcile"},"chk":{"path":"chk/templates.d","policy":"ApplyOnNextReconcile"}},"watch":{"namespaces":[]}}},"keeperConfdFiles":null,"keeperConfigdFiles":{"01-keeper-01-default-config.xml":"\n\n\n\n\n\n\n \n \n 10000\n 10000\n information\n 100000\n \n true\n /var/lib/clickhouse-keeper/coordination/logs\n /var/lib/clickhouse-keeper/coordination/snapshots\n /var/lib/clickhouse-keeper\n 2181\n true\n \n ::\n 0.0.0.0\n 1\n \n 1\n information\n \n 4096\n \n \n true\n /etc/clickhouse-keeper/server.crt\n /etc/clickhouse-keeper/dhparam.pem\n sslv2,sslv3\n true\n true\n /etc/clickhouse-keeper/server.key\n none\n \n \n\n","01-keeper-02-readiness.xml":"\n\n\n\n\n\n\n \n \n 9182\n \n /ready\n \n \n \n"},"keeperTemplatesdFiles":{"readme":"Templates in this folder are packaged with an operator and available via 'useTemplate'"},"keeperUsersdFiles":null,"templatesdFiles":{"001-templates.json.example":"{\n \"apiVersion\": \"clickhouse.altinity.com/v1\",\n \"kind\": \"ClickHouseInstallationTemplate\",\n \"metadata\": {\n \"name\": \"01-default-volumeclaimtemplate\"\n },\n \"spec\": {\n \"templates\": {\n \"volumeClaimTemplates\": [\n {\n \"name\": \"chi-default-volume-claim-template\",\n \"spec\": {\n \"accessModes\": [\n \"ReadWriteOnce\"\n ],\n \"resources\": {\n \"requests\": {\n \"storage\": \"2Gi\"\n }\n }\n }\n }\n ],\n \"podTemplates\": [\n {\n \"name\": \"chi-default-oneperhost-pod-template\",\n \"distribution\": \"OnePerHost\",\n \"spec\": {\n \"containers\" : [\n {\n \"name\": \"clickhouse\",\n \"image\": \"clickhouse/clickhouse-server:23.8\",\n \"ports\": [\n {\n \"name\": \"http\",\n \"containerPort\": 8123\n },\n {\n \"name\": \"client\",\n \"containerPort\": 9000\n },\n {\n \"name\": \"interserver\",\n \"containerPort\": 9009\n }\n ]\n }\n ]\n }\n }\n ]\n }\n }\n}\n","default-pod-template.yaml.example":"apiVersion: \"clickhouse.altinity.com/v1\"\nkind: \"ClickHouseInstallationTemplate\"\nmetadata:\n name: \"default-oneperhost-pod-template\"\nspec:\n templates:\n podTemplates:\n - name: default-oneperhost-pod-template\n distribution: \"OnePerHost\"\n","default-storage-template.yaml.example":"apiVersion: \"clickhouse.altinity.com/v1\"\nkind: \"ClickHouseInstallationTemplate\"\nmetadata:\n name: \"default-storage-template-2Gi\"\nspec:\n templates:\n volumeClaimTemplates:\n - name: default-storage-template-2Gi\n spec:\n accessModes:\n - ReadWriteOnce\n resources:\n requests:\n storage: 2Gi\n","readme":"Templates in this folder are packaged with an operator and available via 'useTemplate'"},"usersdFiles":{"01-clickhouse-operator-profile.xml":"\n\n\n\n\n\n\n\n \n \n \n 0\n 1\n 10\n 0\n 0\n \n \n\n","02-clickhouse-default-profile.xml":"\n\n\n\n\n\n\n \n \n 2\n 1\n 1000\n 1\n 1\n 1\n nearest_hostname\n 0\n \n \n \n"}}` | clickhouse-operator configs | +| additionalResources | list | `[]` | list of additional resources to create (processed via `tpl` function), useful for create ClickHouse clusters together with clickhouse-operator. check `kubectl explain chi` for details | +| affinity | object | `{}` | affinity for scheduler pod assignment, check `kubectl explain pod.spec.affinity` for details | +| configs | object | check the `values.yaml` file for the config content (auto-generated from latest operator release) | clickhouse operator configs | | dashboards.additionalLabels | object | `{"grafana_dashboard":""}` | labels to add to a secret with dashboards | | dashboards.annotations | object | `{}` | annotations to add to a secret with dashboards | | dashboards.enabled | bool | `false` | provision grafana dashboards as configMaps (can be synced by grafana dashboards sidecar https://github.com/grafana/helm-charts/blob/grafana-8.3.4/charts/grafana/values.yaml#L778 ) | | dashboards.grafana_folder | string | `"clickhouse"` | | | fullnameOverride | string | `""` | full name of the chart. | -| imagePullSecrets | list | `[]` | image pull secret for private images in clickhouse-operator pod | +| imagePullSecrets | list | `[]` | image pull secret for private images in clickhouse-operator pod possible value format `[{"name":"your-secret-name"}]`, check `kubectl explain pod.spec.imagePullSecrets` for details | | metrics.containerSecurityContext | object | `{}` | | | metrics.enabled | bool | `true` | | -| metrics.env | list | `[]` | additional environment variables for the deployment of metrics-exporter containers | +| metrics.env | list | `[]` | additional environment variables for the deployment of metrics-exporter containers possible format value `[{"name": "SAMPLE", "value": "text"}]` | | metrics.image.pullPolicy | string | `"IfNotPresent"` | image pull policy | | metrics.image.repository | string | `"altinity/metrics-exporter"` | image repository | | metrics.image.tag | string | `""` | image tag (chart's appVersion value will be used if not set) | | metrics.resources | object | `{}` | custom resource configuration | | nameOverride | string | `""` | override name of the chart | -| nodeSelector | object | `{}` | node for scheduler pod assignment, look `kubectl explain pod.spec.nodeSelector` for details | +| nodeSelector | object | `{}` | node for scheduler pod assignment, check `kubectl explain pod.spec.nodeSelector` for details | | operator.containerSecurityContext | object | `{}` | | -| operator.env | list | `[]` | additional environment variables for the clickhouse-operator container in deployment | +| operator.env | list | `[]` | additional environment variables for the clickhouse-operator container in deployment possible format value `[{"name": "SAMPLE", "value": "text"}]` | | operator.image.pullPolicy | string | `"IfNotPresent"` | image pull policy | | operator.image.repository | string | `"altinity/clickhouse-operator"` | image repository | | operator.image.tag | string | `""` | image tag (chart's appVersion value will be used if not set) | -| operator.resources | object | `{}` | custom resource configuration, look `kubectl explain pod.spec.containers.resources` for details | -| podAnnotations | object | `{"clickhouse-operator-metrics/port":"9999","clickhouse-operator-metrics/scrape":"true","prometheus.io/port":"8888","prometheus.io/scrape":"true"}` | annotations to add to the clickhouse-operator pod, look `kubectl explain pod.spec.annotations` for details | +| operator.resources | object | `{}` | custom resource configuration, check `kubectl explain pod.spec.containers.resources` for details | +| podAnnotations | object | check the `values.yaml` file | annotations to add to the clickhouse-operator pod, check `kubectl explain pod.spec.annotations` for details | | podLabels | object | `{}` | labels to add to the clickhouse-operator pod | | podSecurityContext | object | `{}` | | | rbac.create | bool | `true` | specifies whether cluster roles and cluster role bindings should be created | @@ -60,6 +61,7 @@ For upgrade please install CRDs separately: | serviceAccount.create | bool | `true` | specifies whether a service account should be created | | serviceAccount.name | string | `nil` | the name of the service account to use; if not set and create is true, a name is generated using the fullname template | | serviceMonitor.additionalLabels | object | `{}` | additional labels for service monitor | -| serviceMonitor.enabled | bool | `false` | ServiceMonitor Custom resource is created for a (prometheus-operator)[https://github.com/prometheus-operator/prometheus-operator] | -| tolerations | list | `[]` | tolerations for scheduler pod assignment, look `kubectl explain pod.spec.tolerations` for details | +| serviceMonitor.enabled | bool | `false` | ServiceMonitor Custom resource is created for a [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) | +| tolerations | list | `[]` | tolerations for scheduler pod assignment, check `kubectl explain pod.spec.tolerations` for details | | topologySpreadConstraints | list | `[]` | | + diff --git a/deploy/helm/clickhouse-operator/README.md.gotmpl b/deploy/helm/clickhouse-operator/README.md.gotmpl new file mode 100644 index 000000000..8dff37845 --- /dev/null +++ b/deploy/helm/clickhouse-operator/README.md.gotmpl @@ -0,0 +1,17 @@ +{{ template "chart.header" . }} +{{ template "chart.deprecationWarning" . }} + +{{ template "chart.badgesSection" . }} + +{{ template "chart.description" . }} + +{{ template "chart.homepageLine" . }} + +{{ template "chart.maintainersSection" . }} + +{{ template "chart.sourcesSection" . }} + +{{ template "chart.requirementsSection" . }} + +{{ template "chart.valuesSection" . }} + diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml index fb1d6c9e7..5abf359fa 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallations.clickhouse.altinity.com.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1083,7 +1083,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1230,7 +1230,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml index 503b9063a..ead66666a 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseinstallationtemplates.clickhouse.altinity.com.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1083,7 +1083,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1230,7 +1230,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml index b3e19dc37..1662c4051 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhousekeeperinstallations.clickhouse-keeper.altinity.com.yaml @@ -1,13 +1,13 @@ # Template Parameters: # -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.24.0 + clickhouse-keeper.altinity.com/chop: 0.24.1 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -669,7 +669,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -816,7 +816,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml index 57e944890..057ee8b2e 100644 --- a/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml +++ b/deploy/helm/clickhouse-operator/crds/CustomResourceDefinition-clickhouseoperatorconfigurations.clickhouse.altinity.com.yaml @@ -7,7 +7,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -373,6 +373,40 @@ spec: - "LabelClusterScopeCycleSize" - "LabelClusterScopeCycleIndex" - "LabelClusterScopeCycleOffset" + metrics: + type: object + description: "defines metrics exporter options" + properties: + labels: + type: object + description: "defines metric labels options" + properties: + exclude: + type: array + description: | + When adding labels to a metric exclude labels with names from the following list + items: + type: string + status: + type: object + description: "defines status options" + properties: + fields: + type: object + description: "defines status fields options" + properties: + action: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'action'" + actions: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'actions'" + error: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'error'" + errors: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'errors'" statefulSet: type: object description: "define StatefulSet-specific parameters" diff --git a/deploy/helm/clickhouse-operator/templates/dashboards-configmap.yaml b/deploy/helm/clickhouse-operator/templates/dashboards-configmap.yaml index 091cc1cb6..8e8ad08ee 100644 --- a/deploy/helm/clickhouse-operator/templates/dashboards-configmap.yaml +++ b/deploy/helm/clickhouse-operator/templates/dashboards-configmap.yaml @@ -15,6 +15,6 @@ metadata: {{- end }} data: {{- range $path, $_ := .Files.Glob "files/*.json" }} - {{ $path | trimPrefix "files/" }}: {{ $.Files.Get $path | b64enc -}} + {{ $path | trimPrefix "files/" }}: |- {{ $.Files.Get $path | nindent 4 -}} {{ end }} {{- end }} diff --git a/deploy/helm/clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml b/deploy/helm/clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml index d1176734a..7fa28f547 100644 --- a/deploy/helm/clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml +++ b/deploy/helm/clickhouse-operator/templates/generated/Deployment-clickhouse-operator.yaml @@ -2,9 +2,9 @@ # # NAMESPACE=kube-system # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.24.0 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.24.1 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.24.0 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.24.1 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator diff --git a/deploy/helm/clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml b/deploy/helm/clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml index b8f05e68a..9c4e3bb0c 100644 --- a/deploy/helm/clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml +++ b/deploy/helm/clickhouse-operator/templates/generated/Secret-clickhouse-operator.yaml @@ -3,7 +3,7 @@ # Template parameters available: # NAMESPACE=kube-system # COMMENT= -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # diff --git a/deploy/helm/clickhouse-operator/templates/generated/ServiceAccount-clickhouse-operator.yaml b/deploy/helm/clickhouse-operator/templates/generated/ServiceAccount-clickhouse-operator.yaml index 803619f97..3bc8d89af 100644 --- a/deploy/helm/clickhouse-operator/templates/generated/ServiceAccount-clickhouse-operator.yaml +++ b/deploy/helm/clickhouse-operator/templates/generated/ServiceAccount-clickhouse-operator.yaml @@ -13,6 +13,7 @@ metadata: namespace: {{ .Release.Namespace }} labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }} annotations: {{ toYaml .Values.serviceAccount.annotations | nindent 4 }} + # Template Parameters: # # NAMESPACE=kube-system diff --git a/deploy/helm/clickhouse-operator/values.yaml b/deploy/helm/clickhouse-operator/values.yaml index 406496dbc..4eb72790b 100644 --- a/deploy/helm/clickhouse-operator/values.yaml +++ b/deploy/helm/clickhouse-operator/values.yaml @@ -3,30 +3,21 @@ operator: # operator.image.repository -- image repository repository: altinity/clickhouse-operator # operator.image.tag -- image tag (chart's appVersion value will be used if not set) - tag: "" # operator.image.pullPolicy -- image pull policy - pullPolicy: IfNotPresent containerSecurityContext: {} - # operator.resources -- custom resource configuration, look `kubectl explain pod.spec.containers.resources` for details - + # operator.resources -- custom resource configuration, check `kubectl explain pod.spec.containers.resources` for details resources: {} # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi # operator.env -- additional environment variables for the clickhouse-operator container in deployment - - # possible format value [{"name": "SAMPLE", "value": "text"}] + # possible format value `[{"name": "SAMPLE", "value": "text"}]` env: [] metrics: enabled: true @@ -34,61 +25,45 @@ metrics: # metrics.image.repository -- image repository repository: altinity/metrics-exporter # metrics.image.tag -- image tag (chart's appVersion value will be used if not set) - tag: "" # metrics.image.pullPolicy -- image pull policy - pullPolicy: IfNotPresent containerSecurityContext: {} # metrics.resources -- custom resource configuration - resources: {} # limits: - # cpu: 100m - # memory: 128Mi - # requests: - # cpu: 100m - # memory: 128Mi # metrics.env -- additional environment variables for the deployment of metrics-exporter containers - - # possible format value [{"name": "SAMPLE", "value": "text"}] + # possible format value `[{"name": "SAMPLE", "value": "text"}]` env: [] # imagePullSecrets -- image pull secret for private images in clickhouse-operator pod - -# possible value format [{"name":"your-secret-name"}] - -# look `kubectl explain pod.spec.imagePullSecrets` for details +# possible value format `[{"name":"your-secret-name"}]`, +# check `kubectl explain pod.spec.imagePullSecrets` for details imagePullSecrets: [] # podLabels -- labels to add to the clickhouse-operator pod - podLabels: {} -# podAnnotations -- annotations to add to the clickhouse-operator pod, look `kubectl explain pod.spec.annotations` for details - +# podAnnotations -- annotations to add to the clickhouse-operator pod, check `kubectl explain pod.spec.annotations` for details +# @default -- check the `values.yaml` file podAnnotations: prometheus.io/port: '8888' prometheus.io/scrape: 'true' clickhouse-operator-metrics/port: '9999' clickhouse-operator-metrics/scrape: 'true' # nameOverride -- override name of the chart - nameOverride: "" # fullnameOverride -- full name of the chart. - fullnameOverride: "" serviceAccount: # serviceAccount.create -- specifies whether a service account should be created create: true # serviceAccount.annotations -- annotations to add to the service account - annotations: {} # serviceAccount.name -- the name of the service account to use; if not set and create is true, a name is generated using the fullname template - name: rbac: # rbac.create -- specifies whether cluster roles and cluster role bindings should be created @@ -97,35 +72,26 @@ secret: # secret.create -- create a secret with operator credentials create: true # secret.username -- operator credentials username - username: clickhouse_operator # secret.password -- operator credentials password - password: clickhouse_operator_password -# nodeSelector -- node for scheduler pod assignment, look `kubectl explain pod.spec.nodeSelector` for details - +# nodeSelector -- node for scheduler pod assignment, check `kubectl explain pod.spec.nodeSelector` for details nodeSelector: {} -# tolerations -- tolerations for scheduler pod assignment, look `kubectl explain pod.spec.tolerations` for details - +# tolerations -- tolerations for scheduler pod assignment, check `kubectl explain pod.spec.tolerations` for details tolerations: [] -# affinity -- affinity for scheduler pod assignment, look `kubectl explain pod.spec.affinity` for details - +# affinity -- affinity for scheduler pod assignment, check `kubectl explain pod.spec.affinity` for details affinity: {} -# podSecurityContext - operator deployment SecurityContext, look `kubectl explain pod.spec.securityContext` for details - +# podSecurityContext - operator deployment SecurityContext, check `kubectl explain pod.spec.securityContext` for details podSecurityContext: {} -# topologySpreadConstraints - topologySpreadConstraints affinity for scheduler pod assignment, look `kubectl explain pod.spec.topologySpreadConstraints` for details - +# topologySpreadConstraints - topologySpreadConstraints affinity for scheduler pod assignment, check `kubectl explain pod.spec.topologySpreadConstraints` for details topologySpreadConstraints: [] serviceMonitor: - # serviceMonitor.enabled -- ServiceMonitor Custom resource is created for a (prometheus-operator)[https://github.com/prometheus-operator/prometheus-operator] + # serviceMonitor.enabled -- ServiceMonitor Custom resource is created for a [prometheus-operator](https://github.com/prometheus-operator/prometheus-operator) enabled: false # serviceMonitor.additionalLabels -- additional labels for service monitor - additionalLabels: {} -# configs -- clickhouse-operator configs - -# @default -- check the values.yaml file for the config content, auto-generated from latest operator release +# configs -- clickhouse operator configs +# @default -- check the `values.yaml` file for the config content (auto-generated from latest operator release) configs: confdFiles: null configdFiles: @@ -349,7 +315,6 @@ configs: # These credentials are used for: # 1. Metrics requests # 2. Schema maintenance - # 3. DROP DNS CACHE # User with these credentials can be specified in additional ClickHouse .xml config files, # located in 'clickhouse.configuration.file.path.user' folder username: "" @@ -539,6 +504,25 @@ configs: appendScope: "no" ################################################ ## + ## Metrics management section + ## + ################################################ + metrics: + labels: + exclude: [] + ################################################ + ## + ## Status management section + ## + ################################################ + status: + fields: + action: false + actions: false + error: false + errors: false + ################################################ + ## ## StatefulSet management section ## ################################################ @@ -719,7 +703,6 @@ configs: /var/lib/clickhouse-keeper/coordination/snapshots /var/lib/clickhouse-keeper 2181 - true :: 0.0.0.0 @@ -742,7 +725,7 @@ configs: - 01-keeper-02-readiness.xml: |- + 01-keeper-02-readiness.xml: | @@ -759,72 +742,59 @@ configs: + 01-keeper-03-enable-reconfig.xml: |- + + + + + + + + + false + + keeperTemplatesdFiles: readme: |- Templates in this folder are packaged with an operator and available via 'useTemplate' keeperUsersdFiles: null -# additionalResources -- list of additional resources to create (are processed via `tpl` function), useful for create ClickHouse clusters together with clickhouse-operator, look `kubectl explain chi` for details - +# additionalResources -- list of additional resources to create (processed via `tpl` function), +# useful for create ClickHouse clusters together with clickhouse-operator. +# check `kubectl explain chi` for details additionalResources: [] # - | - # apiVersion: v1 - # kind: ConfigMap - # metadata: - # name: {{ include "altinity-clickhouse-operator.fullname" . }}-cm - # namespace: {{ .Release.Namespace }} - # - | - # apiVersion: v1 - # kind: Secret - # metadata: - # name: {{ include "altinity-clickhouse-operator.fullname" . }}-s - # namespace: {{ .Release.Namespace }} - # stringData: - # mykey: my-value - # - | - # apiVersion: clickhouse.altinity.com/v1 - # kind: ClickHouseInstallation - # metadata: - # name: {{ include "altinity-clickhouse-operator.fullname" . }}-chi - # namespace: {{ .Release.Namespace }} - # spec: - # configuration: - # clusters: - # - name: default - # layout: - # shardsCount: 1 + dashboards: # dashboards.enabled -- provision grafana dashboards as configMaps (can be synced by grafana dashboards sidecar https://github.com/grafana/helm-charts/blob/grafana-8.3.4/charts/grafana/values.yaml#L778 ) enabled: false # dashboards.additionalLabels -- labels to add to a secret with dashboards - additionalLabels: grafana_dashboard: "" # dashboards.annotations -- annotations to add to a secret with dashboards - annotations: {} grafana_folder: clickhouse diff --git a/deploy/openebs/lvm/README b/deploy/openebs/lvm/README new file mode 100644 index 000000000..890dadd2a --- /dev/null +++ b/deploy/openebs/lvm/README @@ -0,0 +1,123 @@ +## OpenEBS - LocalPV-LVM CSI Driver with ClickHouse +LocalPV-LVM CSI Driver became GA in August 2021 (with the release v0.8.0). It is now a very mature product and a core component of the OpenEBS storage platform. +Due to the major adoption of LocalPV-LVM (+50,000 users), this Data-Engine is now being unified and integrated into the core OpenEBS Storage platform; instead of being maintained as an external Data-Engine within our project. + +## Setup + +Find the disk which you want to use for the LocalPV-LVM. Note: For testing you can use the loopback device. + +``` +truncate -s 1024G /tmp/disk.img +sudo losetup -f /tmp/disk.img --show +``` + +> [!NOTE] +> - This is the old maual config process
+> - LocalPV-LVM will num dynamically provision the VG fro you
+> - The PV, VG and LV names will be dynamically provisioned by OpenEBS LocalPV-LVM as K8s unique entities (for safety, you cannot provide your own PV, VG or LV names) + +Create the Volume group on all the nodes, which will be used by the LVM2 Driver for provisioning the volumes + +``` +sudo pvcreate /dev/loop0 +sudo vgcreate vg-test /dev/loop0 ## here lvmvg is the volume group name to be created +``` + +Display the Volume Group + +``` +vgdisplay +``` + +## Installation + +Install the latest release of OpenEBS LVM2 LocalPV-LVM driver by running the following command. Note: All nodes must be running the same version of LocalPV-LVM, LMV2, device-mapper & dm-snapshot. + +Create a variable containing the K8s namesapce + +```bash +OPENEBS_NAMESPACE=openebs +``` + +Install the OpenEBS component without the distributed component (MayaStor) and ZFS + +```bash +helm repo add openebs https://openebs.github.io/openebs +helm repo update +helm install openebs --namespace ${OPENEBS_NAMESPACE} openebs/openebs --set engines.replicated.mayastor.enabled=false --set engines.local.zfs.enabled=false --create-namespace --version 4.1.1 +``` + +List the OpenEBS chart and check that it's deployed + +```bash +helm list -n ${OPENEBS_NAMESPACE} +NAME NAMESPACE REVISION UPDATED STATUS CHART APP VERSION +openebs openebs 1 2024-06-01 12:28:49.358189 +0200 CEST deployed openebs-4.0.1 4.0.1 +``` + +You can uninstall the OpenEBS componenent if there is any issues or for testing different version + +```bash +helm uninstall openebs --namespace ${OPENEBS_NAMESPACE} +``` + +Create a new Storage Class using the test volume group (!! for testing only !!) +```yaml +apiVersion: storage.k8s.io/v1 +kind: StorageClass +allowVolumeExpansion: true +metadata: + name: openebs-lvm-test +parameters: + fsType: xfs + storage: lvm + vgpattern: vg-test +provisioner: local.csi.openebs.io +reclaimPolicy: Retain +volumeBindingMode: Immediate +``` + +Install a test installation of Clickhouse with a ``volumeClaimTemplates`` +```bash +kubecl apply -f clickhouse-installation-with-openebs.yaml -n ch-test +``` + +Connect to the the Clickhouse pod and check the mounted volume name +```bash +# df -k +Filesystem 1K-blocks Used Available Use% Mounted on +overlay 73334784 10300660 63034124 15% / +tmpfs 65536 0 65536 0% /dev +/dev/mapper/rhel_vg-root 73334784 10300660 63034124 15% /etc/hosts +shm 65536 0 65536 0% /dev/shm +/dev/mapper/vg--test-pvc--d2c6219b--755d--4540--8382--a8959c5f1eb5 983040 41684 941356 5% /var/lib/clickhouse +tmpfs 15804764 12 15804752 1% /run/secrets/kubernetes.io/serviceaccount +tmpfs 7902380 0 7902380 0% /proc/asound +tmpfs 7902380 0 7902380 0% /proc/acpi +tmpfs 7902380 0 7902380 0% /proc/scsi +tmpfs 7902380 0 7902380 0% /sys/firmware +tmpfs 7902380 0 7902380 0% /sys/devices/virtual/powercap +``` + +You can find it on the host with the following LVM command +```bash +# lvdisplay + + --- Logical volume --- + LV Path /dev/vg-test/pvc-d2c6219b-755d-4540-8382-a8959c5f1eb5 + LV Name pvc-d2c6219b-755d-4540-8382-a8959c5f1eb5 + VG Name vg-test + LV UUID KqDQ6f-70gM-fbyN-gePw-iROE-XL5E-6zTn5d + LV Write Access read/write + LV Creation host, time openebs-lvm-localpv-node-zfg4x, 2024-10-11 08:52:06 +0200 + LV Status available + # open 1 + LV Size 1.00 GiB + Current LE 256 + Segments 1 + Allocation inherit + Read ahead sectors auto + - currently set to 8192 + Block device 253:4 +``` + diff --git a/deploy/openebs/lvm/clickhouse-installation-with-openebs.yaml b/deploy/openebs/lvm/clickhouse-installation-with-openebs.yaml new file mode 100644 index 000000000..d477fa686 --- /dev/null +++ b/deploy/openebs/lvm/clickhouse-installation-with-openebs.yaml @@ -0,0 +1,41 @@ +apiVersion: clickhouse.altinity.com/v1 +kind: ClickHouseInstallation +metadata: + name: clickhouse-openebs +spec: + configuration: + clusters: + - layout: + replicasCount: 1 + shardsCount: 1 + name: simple + defaults: + templates: + dataVolumeClaimTemplate: openebs + podTemplate: clickhouse:24.3 + templates: + podTemplates: + - metadata: + name: clickhouse:24.3 + spec: + containers: + - image: clickhouse/clickhouse-server:24.3 + name: clickhouse + ports: + - containerPort: 8123 + name: http + - containerPort: 9000 + name: client + - containerPort: 9009 + name: interserver + - containerPort: 9363 + name: metrics + volumeClaimTemplates: + - name: openebs + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: openebs-lvm-test \ No newline at end of file diff --git a/deploy/openebs/lvm/delete-openebs-lvm.sh b/deploy/openebs/lvm/delete-openebs-lvm.sh new file mode 100755 index 000000000..fb8262156 --- /dev/null +++ b/deploy/openebs/lvm/delete-openebs-lvm.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +OPENEBS_NAMESPACE="${OPENEBS_NAMESPACE:-openebs}" +CLICKHOUSE_NAMESPACE="${CLICKHOUSE_NAMESPACE:-ch-test}" + +echo "Delete Test ClickHouse installation" +kubectl delete --namespace="${CLICKHOUSE_NAMESPACE}" ClickhouseInstallation clickhouse-openebs + +echo "Delete OpenEBS namespace ${OPENEBS_NAMESPACE}" +LVMVOLUMNE_TO_DELETE=$(kubectl get LVMVolume --namespace "${OPENEBS_NAMESPACE}" | tail -1 | cut -f1 -d ' ') +kubectl delete LVMVolume $LVMVOLUMNE_TO_DELETE --namespace "${OPENEBS_NAMESPACE}" +helm uninstall openebs --namespace ${OPENEBS_NAMESPACE} +kubectl delete namespace "${OPENEBS_NAMESPACE}" diff --git a/deploy/openebs/lvm/install-openebs-lvm.sh b/deploy/openebs/lvm/install-openebs-lvm.sh new file mode 100755 index 000000000..702507b0f --- /dev/null +++ b/deploy/openebs/lvm/install-openebs-lvm.sh @@ -0,0 +1,105 @@ +#!/bin/bash + +function ensure_namespace() { + local namespace="${1}" + if kubectl get namespace "${namespace}" 1>/dev/null 2>/dev/null; then + echo "Namespace '${namespace}' already exists." + else + echo "No '${namespace}' namespace found. Going to create." + kubectl create namespace "${namespace}" + fi +} + + +echo "External value for \$OPENEBS_NAMESPACE=$OPENEBS_NAMESPACE" +echo "External value for \$OPENEBS_OPERATOR_VERSION=$OPENEBS_OPERATOR_VERSION" +echo "External value for \$VALIDATE_YAML=$VALIDATE_YAML" +echo "External value for \$CLICKHOUSE_NAMESPACE=$CLICKHOUSE_NAMESPACE" + +OPENEBS_NAMESPACE="${OPENEBS_NAMESPACE:-openebs}" +OPENEBS_OPERATOR_VERSION="${OPENEBS_OPERATOR_VERSION:-v4.1.3}" +VALIDATE_YAML="${VALIDATE_YAML:-"true"}" +CLICKHOUSE_NAMESPACE="${CLICKHOUSE_NAMESPACE:-ch-test}" + +echo "Setup OpenEBS" +echo "OPTIONS" +echo "\$OPENEBS_NAMESPACE=${OPENEBS_NAMESPACE}" +echo "\$OPENEBS_OPERATOR_VERSION=${OPENEBS_OPERATOR_VERSION}" +echo "\$VALIDATE_YAML=${VALIDATE_YAML}" +echo "\$CLICKHOUSE_NAMESPACE=${CLICKHOUSE_NAMESPACE}" +echo "" +echo "!!! IMPORTANT !!!" +echo "If you do not agree with specified options, press ctrl-c now" +if [[ "" == "${NO_WAIT}" ]]; then + sleep 10 +fi +echo "Apply options now..." + +## +## +## +function clean_dir() { + DIR="$1" + + echo "##############################" + echo "Clean dir $DIR ..." + rm -rf "$DIR" + echo "...DONE" +} + +############################## +## ## +## Install openebs.io operator ## +## ## +############################## + +# Download openebs-operator sources into temp dir and run all installation scripts from there + +TMP_DIR=$(mktemp -d) + +# Ensure temp dir in place +mkdir -p "${OPENEBS_OPERATOR_DIR}" + +# Temp dir must not contain any data +if [[ -n "$(ls -A "${OPENEBS_OPERATOR_DIR}")" ]]; then + echo "${OPENEBS_OPERATOR_DIR} is not empty. Abort" + exit 1 +fi + +# Temp dir is empty, will clear it upon script termination +trap 'clean_dir ${TMP_DIR}' SIGHUP SIGINT SIGQUIT SIGFPE SIGALRM SIGTERM + +# Continue with installing help repo +helm repo add openebs https://openebs.github.io/openebs +helm repo update + +echo "Setup OpenEBS operator ${OPENEBS_OPERATOR_VERSION} into ${OPENEBS_NAMESPACE} namespace" + +# Let's setup all OpenEBS-related stuff into dedicated namespace +## TODO: need to refactor after next OPENEBS-operator release +kubectl delete crd volumesnapshotclasses.snapshot.storage.k8s.io +kubectl delete crd volumesnapshotcontents.snapshot.storage.k8s.io +kubectl delete crd volumesnapshots.snapshot.storage.k8s.io + +# Setup OPENEBS-operator into dedicated namespace via kustomize +helm install openebs --namespace ${OPENEBS_NAMESPACE} openebs/openebs --set engines.replicated.mayastor.enabled=false --set engines.local.zfs.enabled=false --create-namespace --version 4.1.1 + +echo -n "Waiting '${OPENEBS_NAMESPACE}/openebs-lvm-localpv-controller' deployment to start" +# Check grafana deployment have all pods ready +while [[ $(kubectl --namespace="${OPENEBS_NAMESPACE}" get deployments | grep "openebs-lvm-localpv-controller" | grep -c "1/1") == "0" ]]; do + printf "." + sleep 1 +done +echo "...DONE" + +# Install the test storage class +kubectl apply -f openebs-lvm-storageclass.yaml -n ${OPENEBS_NAMESPACE} + +# Install a simple Clickhouse instance using openebs +echo "Setup simple Clickhouse into ${OPENEBS_NAMESPACE} namespace using OpenEBS" +ensure_namespace "${CLICKHOUSE_NAMESPACE}" +kubectl apply --validate="${VALIDATE_YAML}" --namespace="${CLICKHOUSE_NAMESPACE}" -f clickhouse-installation-with-openebs.yaml + +# Remove downloaded sources +clean_dir "${TMP_DIR}" + diff --git a/deploy/openebs/lvm/openebs-lvm-pvc-test.yaml b/deploy/openebs/lvm/openebs-lvm-pvc-test.yaml new file mode 100644 index 000000000..212c23964 --- /dev/null +++ b/deploy/openebs/lvm/openebs-lvm-pvc-test.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: pvc-test + namespace: openebs +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + storageClassName: openebs-lvm-test \ No newline at end of file diff --git a/deploy/openebs/lvm/openebs-lvm-storageclass.yaml b/deploy/openebs/lvm/openebs-lvm-storageclass.yaml new file mode 100644 index 000000000..4d9d182a1 --- /dev/null +++ b/deploy/openebs/lvm/openebs-lvm-storageclass.yaml @@ -0,0 +1,12 @@ +apiVersion: storage.k8s.io/v1 +kind: StorageClass +allowVolumeExpansion: true +metadata: + name: openebs-lvm-test +parameters: + fsType: xfs + storage: lvm + vgpattern: vg-test +provisioner: local.csi.openebs.io +reclaimPolicy: Delete +volumeBindingMode: Immediate diff --git a/deploy/operator/clickhouse-operator-install-ansible.yaml b/deploy/operator/clickhouse-operator-install-ansible.yaml index 2025a63e7..34db98648 100644 --- a/deploy/operator/clickhouse-operator-install-ansible.yaml +++ b/deploy/operator/clickhouse-operator-install-ansible.yaml @@ -11,14 +11,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1094,7 +1094,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1242,7 +1242,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -1291,14 +1291,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2374,7 +2374,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -2522,7 +2522,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -2574,7 +2574,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2941,6 +2941,40 @@ spec: - "LabelClusterScopeCycleSize" - "LabelClusterScopeCycleIndex" - "LabelClusterScopeCycleOffset" + metrics: + type: object + description: "defines metrics exporter options" + properties: + labels: + type: object + description: "defines metric labels options" + properties: + exclude: + type: array + description: | + When adding labels to a metric exclude labels with names from the following list + items: + type: string + status: + type: object + description: "defines status options" + properties: + fields: + type: object + description: "defines status fields options" + properties: + action: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'action'" + actions: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'actions'" + error: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'error'" + errors: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'errors'" statefulSet: type: object description: "define StatefulSet-specific parameters" @@ -2989,14 +3023,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.24.0 + clickhouse-keeper.altinity.com/chop: 0.24.1 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -3660,7 +3694,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -3808,7 +3842,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -3840,7 +3874,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 --- # Template Parameters: # @@ -3866,7 +3900,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 rules: # @@ -4085,7 +4119,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -4107,7 +4141,7 @@ metadata: name: etc-clickhouse-operator-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: config.yaml: | @@ -4260,7 +4294,6 @@ data: # These credentials are used for: # 1. Metrics requests # 2. Schema maintenance - # 3. DROP DNS CACHE # User with these credentials can be specified in additional ClickHouse .xml config files, # located in 'clickhouse.configuration.file.path.user' folder username: "" @@ -4464,6 +4497,27 @@ data: # LabelClusterScopeCycleOffset appendScope: "no" + ################################################ + ## + ## Metrics management section + ## + ################################################ + metrics: + labels: + exclude: [] + + ################################################ + ## + ## Status management section + ## + ################################################ + status: + fields: + action: false + actions: false + error: false + errors: false + ################################################ ## ## StatefulSet management section @@ -4510,7 +4564,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: --- @@ -4526,7 +4580,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -4625,7 +4679,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 001-templates.json.example: | @@ -4725,7 +4779,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -4788,7 +4842,7 @@ metadata: name: etc-keeper-operator-confd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: --- @@ -4804,7 +4858,7 @@ metadata: name: etc-keeper-operator-configd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 01-keeper-01-default-config.xml: | @@ -4827,7 +4881,6 @@ data: /var/lib/clickhouse-keeper/coordination/snapshots /var/lib/clickhouse-keeper 2181 - true :: 0.0.0.0 @@ -4869,6 +4922,19 @@ data: + 01-keeper-03-enable-reconfig.xml: | + + + + + + + + + false + + + --- # Template Parameters: # @@ -4882,7 +4948,7 @@ metadata: name: etc-keeper-operator-templatesd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: readme: | @@ -4900,7 +4966,7 @@ metadata: name: etc-keeper-operator-usersd-files namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: --- @@ -4908,7 +4974,7 @@ data: # Template parameters available: # NAMESPACE={{ namespace }} # COMMENT= -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN={{ password }} # @@ -4918,7 +4984,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator type: Opaque stringData: @@ -4929,9 +4995,9 @@ stringData: # # NAMESPACE={{ namespace }} # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.24.0 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.24.1 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.24.0 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.24.1 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -4942,7 +5008,7 @@ metadata: name: clickhouse-operator namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator spec: replicas: 1 @@ -4990,7 +5056,7 @@ spec: name: etc-keeper-operator-usersd-files containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.24.0 + image: altinity/clickhouse-operator:0.24.1 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -5066,7 +5132,7 @@ spec: name: metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.24.0 + image: altinity/metrics-exporter:0.24.1 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -5157,7 +5223,7 @@ metadata: name: clickhouse-operator-metrics namespace: {{ namespace }} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml index cec7c1b79..685c20712 100644 --- a/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1079,7 +1079,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1226,7 +1226,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -1275,14 +1275,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2348,7 +2348,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -2495,7 +2495,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -2547,7 +2547,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2909,6 +2909,40 @@ spec: - "LabelClusterScopeCycleSize" - "LabelClusterScopeCycleIndex" - "LabelClusterScopeCycleOffset" + metrics: + type: object + description: "defines metrics exporter options" + properties: + labels: + type: object + description: "defines metric labels options" + properties: + exclude: + type: array + description: | + When adding labels to a metric exclude labels with names from the following list + items: + type: string + status: + type: object + description: "defines status options" + properties: + fields: + type: object + description: "defines status fields options" + properties: + action: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'action'" + actions: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'actions'" + error: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'error'" + errors: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'errors'" statefulSet: type: object description: "define StatefulSet-specific parameters" @@ -2952,14 +2986,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.24.0 + clickhouse-keeper.altinity.com/chop: 0.24.1 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -3621,7 +3655,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -3768,7 +3802,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -3800,7 +3834,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 # Template Parameters: # @@ -3825,7 +3859,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 rules: # # Core API group @@ -4034,7 +4068,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -4056,7 +4090,7 @@ metadata: name: etc-clickhouse-operator-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: config.yaml: | @@ -4209,7 +4243,6 @@ data: # These credentials are used for: # 1. Metrics requests # 2. Schema maintenance - # 3. DROP DNS CACHE # User with these credentials can be specified in additional ClickHouse .xml config files, # located in 'clickhouse.configuration.file.path.user' folder username: "" @@ -4413,6 +4446,27 @@ data: # LabelClusterScopeCycleOffset appendScope: "no" + ################################################ + ## + ## Metrics management section + ## + ################################################ + metrics: + labels: + exclude: [] + + ################################################ + ## + ## Status management section + ## + ################################################ + status: + fields: + action: false + actions: false + error: false + errors: false + ################################################ ## ## StatefulSet management section @@ -4458,7 +4512,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: --- @@ -4474,7 +4528,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -4568,7 +4622,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 001-templates.json.example: | @@ -4666,7 +4720,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -4728,7 +4782,7 @@ metadata: name: etc-keeper-operator-confd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: --- @@ -4744,7 +4798,7 @@ metadata: name: etc-keeper-operator-configd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 01-keeper-01-default-config.xml: | @@ -4767,7 +4821,6 @@ data: /var/lib/clickhouse-keeper/coordination/snapshots /var/lib/clickhouse-keeper 2181 - true :: 0.0.0.0 @@ -4807,6 +4860,18 @@ data: + 01-keeper-03-enable-reconfig.xml: | + + + + + + + + + false + + --- # Template Parameters: # @@ -4820,7 +4885,7 @@ metadata: name: etc-keeper-operator-templatesd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: readme: | @@ -4838,7 +4903,7 @@ metadata: name: etc-keeper-operator-usersd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: --- @@ -4846,7 +4911,7 @@ data: # Template parameters available: # NAMESPACE=kube-system # COMMENT= -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -4856,7 +4921,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator type: Opaque stringData: @@ -4867,9 +4932,9 @@ stringData: # # NAMESPACE=kube-system # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.24.0 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.24.1 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.24.0 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.24.1 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -4880,7 +4945,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator spec: replicas: 1 @@ -4928,7 +4993,7 @@ spec: name: etc-keeper-operator-usersd-files containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.24.0 + image: altinity/clickhouse-operator:0.24.1 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -5002,7 +5067,7 @@ spec: - containerPort: 9999 name: metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.24.0 + image: altinity/metrics-exporter:0.24.1 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -5092,7 +5157,7 @@ metadata: name: clickhouse-operator-metrics namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-bundle.yaml b/deploy/operator/clickhouse-operator-install-bundle.yaml index 762ce24b0..5180e0606 100644 --- a/deploy/operator/clickhouse-operator-install-bundle.yaml +++ b/deploy/operator/clickhouse-operator-install-bundle.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1087,7 +1087,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1235,7 +1235,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -1284,14 +1284,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2367,7 +2367,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -2515,7 +2515,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -2567,7 +2567,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2934,6 +2934,40 @@ spec: - "LabelClusterScopeCycleSize" - "LabelClusterScopeCycleIndex" - "LabelClusterScopeCycleOffset" + metrics: + type: object + description: "defines metrics exporter options" + properties: + labels: + type: object + description: "defines metric labels options" + properties: + exclude: + type: array + description: | + When adding labels to a metric exclude labels with names from the following list + items: + type: string + status: + type: object + description: "defines status options" + properties: + fields: + type: object + description: "defines status fields options" + properties: + action: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'action'" + actions: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'actions'" + error: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'error'" + errors: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'errors'" statefulSet: type: object description: "define StatefulSet-specific parameters" @@ -2982,14 +3016,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.24.0 + clickhouse-keeper.altinity.com/chop: 0.24.1 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -3653,7 +3687,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -3801,7 +3835,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -3833,7 +3867,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 --- # Template Parameters: # @@ -3859,7 +3893,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 rules: # @@ -4078,7 +4112,7 @@ metadata: name: clickhouse-operator-kube-system #namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -4100,7 +4134,7 @@ metadata: name: etc-clickhouse-operator-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: config.yaml: | @@ -4253,7 +4287,6 @@ data: # These credentials are used for: # 1. Metrics requests # 2. Schema maintenance - # 3. DROP DNS CACHE # User with these credentials can be specified in additional ClickHouse .xml config files, # located in 'clickhouse.configuration.file.path.user' folder username: "" @@ -4457,6 +4490,27 @@ data: # LabelClusterScopeCycleOffset appendScope: "no" + ################################################ + ## + ## Metrics management section + ## + ################################################ + metrics: + labels: + exclude: [] + + ################################################ + ## + ## Status management section + ## + ################################################ + status: + fields: + action: false + actions: false + error: false + errors: false + ################################################ ## ## StatefulSet management section @@ -4503,7 +4557,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: --- @@ -4519,7 +4573,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -4618,7 +4672,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 001-templates.json.example: | @@ -4718,7 +4772,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -4781,7 +4835,7 @@ metadata: name: etc-keeper-operator-confd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: --- @@ -4797,7 +4851,7 @@ metadata: name: etc-keeper-operator-configd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 01-keeper-01-default-config.xml: | @@ -4820,7 +4874,6 @@ data: /var/lib/clickhouse-keeper/coordination/snapshots /var/lib/clickhouse-keeper 2181 - true :: 0.0.0.0 @@ -4862,6 +4915,19 @@ data: + 01-keeper-03-enable-reconfig.xml: | + + + + + + + + + false + + + --- # Template Parameters: # @@ -4875,7 +4941,7 @@ metadata: name: etc-keeper-operator-templatesd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: readme: | @@ -4893,7 +4959,7 @@ metadata: name: etc-keeper-operator-usersd-files namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: --- @@ -4901,7 +4967,7 @@ data: # Template parameters available: # NAMESPACE=kube-system # COMMENT= -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -4911,7 +4977,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator type: Opaque stringData: @@ -4922,9 +4988,9 @@ stringData: # # NAMESPACE=kube-system # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.24.0 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.24.1 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.24.0 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.24.1 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -4935,7 +5001,7 @@ metadata: name: clickhouse-operator namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator spec: replicas: 1 @@ -4983,7 +5049,7 @@ spec: name: etc-keeper-operator-usersd-files containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.24.0 + image: altinity/clickhouse-operator:0.24.1 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -5059,7 +5125,7 @@ spec: name: metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.24.0 + image: altinity/metrics-exporter:0.24.1 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -5150,7 +5216,7 @@ metadata: name: clickhouse-operator-metrics namespace: kube-system labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml index c6204ec06..3ba53a09b 100644 --- a/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml +++ b/deploy/operator/clickhouse-operator-install-template-v1beta1.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1079,7 +1079,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1226,7 +1226,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -1275,14 +1275,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1beta1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2348,7 +2348,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -2495,7 +2495,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -2547,7 +2547,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2909,6 +2909,40 @@ spec: - "LabelClusterScopeCycleSize" - "LabelClusterScopeCycleIndex" - "LabelClusterScopeCycleOffset" + metrics: + type: object + description: "defines metrics exporter options" + properties: + labels: + type: object + description: "defines metric labels options" + properties: + exclude: + type: array + description: | + When adding labels to a metric exclude labels with names from the following list + items: + type: string + status: + type: object + description: "defines status options" + properties: + fields: + type: object + description: "defines status fields options" + properties: + action: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'action'" + actions: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'actions'" + error: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'error'" + errors: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'errors'" statefulSet: type: object description: "define StatefulSet-specific parameters" @@ -2952,14 +2986,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.24.0 + clickhouse-keeper.altinity.com/chop: 0.24.1 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -3621,7 +3655,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -3768,7 +3802,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -3800,7 +3834,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 # Template Parameters: # @@ -3825,7 +3859,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 rules: # # Core API group @@ -4034,7 +4068,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -4056,7 +4090,7 @@ metadata: name: etc-clickhouse-operator-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: config.yaml: | @@ -4209,7 +4243,6 @@ data: # These credentials are used for: # 1. Metrics requests # 2. Schema maintenance - # 3. DROP DNS CACHE # User with these credentials can be specified in additional ClickHouse .xml config files, # located in 'clickhouse.configuration.file.path.user' folder username: "" @@ -4413,6 +4446,27 @@ data: # LabelClusterScopeCycleOffset appendScope: "no" + ################################################ + ## + ## Metrics management section + ## + ################################################ + metrics: + labels: + exclude: [] + + ################################################ + ## + ## Status management section + ## + ################################################ + status: + fields: + action: false + actions: false + error: false + errors: false + ################################################ ## ## StatefulSet management section @@ -4458,7 +4512,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: --- @@ -4474,7 +4528,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -4568,7 +4622,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 001-templates.json.example: | @@ -4666,7 +4720,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -4728,7 +4782,7 @@ metadata: name: etc-keeper-operator-confd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: --- @@ -4744,7 +4798,7 @@ metadata: name: etc-keeper-operator-configd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 01-keeper-01-default-config.xml: | @@ -4767,7 +4821,6 @@ data: /var/lib/clickhouse-keeper/coordination/snapshots /var/lib/clickhouse-keeper 2181 - true :: 0.0.0.0 @@ -4807,6 +4860,18 @@ data: + 01-keeper-03-enable-reconfig.xml: | + + + + + + + + + false + + --- # Template Parameters: # @@ -4820,7 +4885,7 @@ metadata: name: etc-keeper-operator-templatesd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: readme: | @@ -4838,7 +4903,7 @@ metadata: name: etc-keeper-operator-usersd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: --- @@ -4846,7 +4911,7 @@ data: # Template parameters available: # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -4856,7 +4921,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator type: Opaque stringData: @@ -4880,7 +4945,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator spec: replicas: 1 @@ -5092,7 +5157,7 @@ metadata: name: clickhouse-operator-metrics namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-template.yaml b/deploy/operator/clickhouse-operator-install-template.yaml index b91794ef2..16c0727dc 100644 --- a/deploy/operator/clickhouse-operator-install-template.yaml +++ b/deploy/operator/clickhouse-operator-install-template.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1087,7 +1087,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1235,7 +1235,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -1284,14 +1284,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2367,7 +2367,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -2515,7 +2515,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -2567,7 +2567,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2934,6 +2934,40 @@ spec: - "LabelClusterScopeCycleSize" - "LabelClusterScopeCycleIndex" - "LabelClusterScopeCycleOffset" + metrics: + type: object + description: "defines metrics exporter options" + properties: + labels: + type: object + description: "defines metric labels options" + properties: + exclude: + type: array + description: | + When adding labels to a metric exclude labels with names from the following list + items: + type: string + status: + type: object + description: "defines status options" + properties: + fields: + type: object + description: "defines status fields options" + properties: + action: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'action'" + actions: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'actions'" + error: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'error'" + errors: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'errors'" statefulSet: type: object description: "define StatefulSet-specific parameters" @@ -2982,14 +3016,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.24.0 + clickhouse-keeper.altinity.com/chop: 0.24.1 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -3653,7 +3687,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -3801,7 +3835,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -3833,7 +3867,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 --- # Template Parameters: # @@ -3859,7 +3893,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 rules: # @@ -4078,7 +4112,7 @@ metadata: name: clickhouse-operator-${OPERATOR_NAMESPACE} #namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 roleRef: apiGroup: rbac.authorization.k8s.io kind: ClusterRole @@ -4100,7 +4134,7 @@ metadata: name: etc-clickhouse-operator-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: config.yaml: | @@ -4253,7 +4287,6 @@ data: # These credentials are used for: # 1. Metrics requests # 2. Schema maintenance - # 3. DROP DNS CACHE # User with these credentials can be specified in additional ClickHouse .xml config files, # located in 'clickhouse.configuration.file.path.user' folder username: "" @@ -4457,6 +4490,27 @@ data: # LabelClusterScopeCycleOffset appendScope: "no" + ################################################ + ## + ## Metrics management section + ## + ################################################ + metrics: + labels: + exclude: [] + + ################################################ + ## + ## Status management section + ## + ################################################ + status: + fields: + action: false + actions: false + error: false + errors: false + ################################################ ## ## StatefulSet management section @@ -4503,7 +4557,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: --- @@ -4519,7 +4573,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -4618,7 +4672,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 001-templates.json.example: | @@ -4718,7 +4772,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -4781,7 +4835,7 @@ metadata: name: etc-keeper-operator-confd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: --- @@ -4797,7 +4851,7 @@ metadata: name: etc-keeper-operator-configd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 01-keeper-01-default-config.xml: | @@ -4820,7 +4874,6 @@ data: /var/lib/clickhouse-keeper/coordination/snapshots /var/lib/clickhouse-keeper 2181 - true :: 0.0.0.0 @@ -4862,6 +4915,19 @@ data: + 01-keeper-03-enable-reconfig.xml: | + + + + + + + + + false + + + --- # Template Parameters: # @@ -4875,7 +4941,7 @@ metadata: name: etc-keeper-operator-templatesd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: readme: | @@ -4893,7 +4959,7 @@ metadata: name: etc-keeper-operator-usersd-files namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: --- @@ -4901,7 +4967,7 @@ data: # Template parameters available: # NAMESPACE=${OPERATOR_NAMESPACE} # COMMENT= -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=clickhouse_operator_password # @@ -4911,7 +4977,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator type: Opaque stringData: @@ -4935,7 +5001,7 @@ metadata: name: clickhouse-operator namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator spec: replicas: 1 @@ -5150,7 +5216,7 @@ metadata: name: clickhouse-operator-metrics namespace: ${OPERATOR_NAMESPACE} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/clickhouse-operator-install-tf.yaml b/deploy/operator/clickhouse-operator-install-tf.yaml index 0fb8e9f9d..4795ecee3 100644 --- a/deploy/operator/clickhouse-operator-install-tf.yaml +++ b/deploy/operator/clickhouse-operator-install-tf.yaml @@ -11,14 +11,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1094,7 +1094,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1242,7 +1242,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -1291,14 +1291,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2374,7 +2374,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -2522,7 +2522,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -2574,7 +2574,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -2941,6 +2941,40 @@ spec: - "LabelClusterScopeCycleSize" - "LabelClusterScopeCycleIndex" - "LabelClusterScopeCycleOffset" + metrics: + type: object + description: "defines metrics exporter options" + properties: + labels: + type: object + description: "defines metric labels options" + properties: + exclude: + type: array + description: | + When adding labels to a metric exclude labels with names from the following list + items: + type: string + status: + type: object + description: "defines status options" + properties: + fields: + type: object + description: "defines status fields options" + properties: + action: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'action'" + actions: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'actions'" + error: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'error'" + errors: + <<: *TypeStringBool + description: "Whether the operator should fill status field 'errors'" statefulSet: type: object description: "define StatefulSet-specific parameters" @@ -2989,14 +3023,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.24.0 + clickhouse-keeper.altinity.com/chop: 0.24.1 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -3660,7 +3694,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -3808,7 +3842,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -3840,7 +3874,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 --- # Template Parameters: # @@ -3866,7 +3900,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 rules: # @@ -4085,7 +4119,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 roleRef: apiGroup: rbac.authorization.k8s.io kind: Role @@ -4107,7 +4141,7 @@ metadata: name: etc-clickhouse-operator-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: config.yaml: | @@ -4260,7 +4294,6 @@ data: # These credentials are used for: # 1. Metrics requests # 2. Schema maintenance - # 3. DROP DNS CACHE # User with these credentials can be specified in additional ClickHouse .xml config files, # located in 'clickhouse.configuration.file.path.user' folder username: "" @@ -4464,6 +4497,27 @@ data: # LabelClusterScopeCycleOffset appendScope: "no" + ################################################ + ## + ## Metrics management section + ## + ################################################ + metrics: + labels: + exclude: [] + + ################################################ + ## + ## Status management section + ## + ################################################ + status: + fields: + action: false + actions: false + error: false + errors: false + ################################################ ## ## StatefulSet management section @@ -4510,7 +4564,7 @@ metadata: name: etc-clickhouse-operator-confd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: --- @@ -4526,7 +4580,7 @@ metadata: name: etc-clickhouse-operator-configd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 01-clickhouse-01-listen.xml: | @@ -4625,7 +4679,7 @@ metadata: name: etc-clickhouse-operator-templatesd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 001-templates.json.example: | @@ -4725,7 +4779,7 @@ metadata: name: etc-clickhouse-operator-usersd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 01-clickhouse-operator-profile.xml: | @@ -4788,7 +4842,7 @@ metadata: name: etc-keeper-operator-confd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: --- @@ -4804,7 +4858,7 @@ metadata: name: etc-keeper-operator-configd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: 01-keeper-01-default-config.xml: | @@ -4827,7 +4881,6 @@ data: /var/lib/clickhouse-keeper/coordination/snapshots /var/lib/clickhouse-keeper 2181 - true :: 0.0.0.0 @@ -4869,6 +4922,19 @@ data: + 01-keeper-03-enable-reconfig.xml: | + + + + + + + + + false + + + --- # Template Parameters: # @@ -4882,7 +4948,7 @@ metadata: name: etc-keeper-operator-templatesd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: readme: | @@ -4900,7 +4966,7 @@ metadata: name: etc-keeper-operator-usersd-files namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator data: --- @@ -4908,7 +4974,7 @@ data: # Template parameters available: # NAMESPACE=${namespace} # COMMENT= -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # CH_USERNAME_SECRET_PLAIN=clickhouse_operator # CH_PASSWORD_SECRET_PLAIN=${password} # @@ -4918,7 +4984,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator type: Opaque stringData: @@ -4929,9 +4995,9 @@ stringData: # # NAMESPACE=${namespace} # COMMENT= -# OPERATOR_IMAGE=altinity/clickhouse-operator:0.24.0 +# OPERATOR_IMAGE=altinity/clickhouse-operator:0.24.1 # OPERATOR_IMAGE_PULL_POLICY=Always -# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.24.0 +# METRICS_EXPORTER_IMAGE=altinity/metrics-exporter:0.24.1 # METRICS_EXPORTER_IMAGE_PULL_POLICY=Always # # Setup Deployment for clickhouse-operator @@ -4942,7 +5008,7 @@ metadata: name: clickhouse-operator namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator spec: replicas: 1 @@ -4990,7 +5056,7 @@ spec: name: etc-keeper-operator-usersd-files containers: - name: clickhouse-operator - image: altinity/clickhouse-operator:0.24.0 + image: altinity/clickhouse-operator:0.24.1 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -5066,7 +5132,7 @@ spec: name: metrics - name: metrics-exporter - image: altinity/metrics-exporter:0.24.0 + image: altinity/metrics-exporter:0.24.1 imagePullPolicy: Always volumeMounts: - name: etc-clickhouse-operator-folder @@ -5157,7 +5223,7 @@ metadata: name: clickhouse-operator-metrics namespace: ${namespace} labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 app: clickhouse-operator spec: ports: diff --git a/deploy/operator/parts/crd.yaml b/deploy/operator/parts/crd.yaml index 12dd8e5c7..bbccf18d3 100644 --- a/deploy/operator/parts/crd.yaml +++ b/deploy/operator/parts/crd.yaml @@ -4,14 +4,14 @@ # SINGULAR=clickhouseinstallation # PLURAL=clickhouseinstallations # SHORT=chi -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -1699,7 +1699,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1860,7 +1860,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -1909,14 +1909,14 @@ spec: # SINGULAR=clickhouseinstallationtemplate # PLURAL=clickhouseinstallationtemplates # SHORT=chit -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhouseinstallationtemplates.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -3604,7 +3604,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -3765,7 +3765,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object @@ -3817,7 +3817,7 @@ kind: CustomResourceDefinition metadata: name: clickhouseoperatorconfigurations.clickhouse.altinity.com labels: - clickhouse.altinity.com/chop: 0.24.0 + clickhouse.altinity.com/chop: 0.24.1 spec: group: clickhouse.altinity.com scope: Namespaced @@ -4258,6 +4258,140 @@ spec: - "LabelClusterScopeCycleSize" - "LabelClusterScopeCycleIndex" - "LabelClusterScopeCycleOffset" + metrics: + type: object + description: "defines metrics exporter options" + properties: + labels: + type: object + description: "defines metric labels options" + properties: + exclude: + type: array + description: | + When adding labels to a metric exclude labels with names from the following list + items: + type: string + status: + type: object + description: "defines status options" + properties: + fields: + type: object + description: "defines status fields options" + properties: + action: + type: string + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + description: "Whether the operator should fill status field 'action'" + actions: + type: string + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + description: "Whether the operator should fill status field 'actions'" + error: + type: string + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + description: "Whether the operator should fill status field 'error'" + errors: + type: string + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + description: "Whether the operator should fill status field 'errors'" statefulSet: type: object description: "define StatefulSet-specific parameters" @@ -4301,14 +4435,14 @@ spec: --- # Template Parameters: # -# OPERATOR_VERSION=0.24.0 +# OPERATOR_VERSION=0.24.1 # apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com labels: - clickhouse-keeper.altinity.com/chop: 0.24.0 + clickhouse-keeper.altinity.com/chop: 0.24.1 spec: group: clickhouse-keeper.altinity.com scope: Namespaced @@ -5161,7 +5295,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -5322,7 +5456,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.18.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.18.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index 8846c528c..4ba236761 100644 --- a/deploy/operatorhub/0.18.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.18.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -1130,7 +1130,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1276,7 +1276,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.18.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.18.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index fb8c6a5c4..bdffd61aa 100644 --- a/deploy/operatorhub/0.18.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.18.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -1130,7 +1130,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1276,7 +1276,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.18.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.18.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index f426484cd..07293a392 100644 --- a/deploy/operatorhub/0.18.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.18.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -1135,7 +1135,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1281,7 +1281,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.18.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.18.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index 549d3b2d0..17b4a76d6 100644 --- a/deploy/operatorhub/0.18.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.18.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -1135,7 +1135,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1281,7 +1281,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.18.4/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.18.4/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index 355b6a81a..d829ec791 100644 --- a/deploy/operatorhub/0.18.4/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.18.4/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -1138,7 +1138,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1284,7 +1284,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.18.4/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.18.4/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index 2db6b2c5f..f00a2495b 100644 --- a/deploy/operatorhub/0.18.4/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.18.4/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -1138,7 +1138,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1284,7 +1284,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.18.5/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.18.5/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index f0a5c3dec..dcaa29466 100644 --- a/deploy/operatorhub/0.18.5/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.18.5/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -1138,7 +1138,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1284,7 +1284,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.18.5/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.18.5/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index 7c27ec118..68cdb6b04 100644 --- a/deploy/operatorhub/0.18.5/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.18.5/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -1138,7 +1138,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1284,7 +1284,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.19.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.19.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index 8b69d12b8..59e6b8aba 100644 --- a/deploy/operatorhub/0.19.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.19.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -1166,7 +1166,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1312,7 +1312,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.19.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.19.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index afe349614..8f222c23f 100644 --- a/deploy/operatorhub/0.19.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.19.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -1166,7 +1166,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1312,7 +1312,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.19.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.19.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index 97430f0b5..e2e707b3d 100644 --- a/deploy/operatorhub/0.19.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.19.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -1166,7 +1166,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1312,7 +1312,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.19.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.19.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index 65ceef223..3892696b6 100644 --- a/deploy/operatorhub/0.19.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.19.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -1166,7 +1166,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1312,7 +1312,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.19.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.19.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index 7e31a6c67..ec1789cd0 100644 --- a/deploy/operatorhub/0.19.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.19.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -1166,7 +1166,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1312,7 +1312,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.19.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.19.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index cce07c0cd..7195cfa02 100644 --- a/deploy/operatorhub/0.19.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.19.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -1166,7 +1166,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1312,7 +1312,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.19.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.19.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index ed7229600..1f98f08da 100644 --- a/deploy/operatorhub/0.19.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.19.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -1170,7 +1170,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1316,7 +1316,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.19.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.19.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index 173b4a8cb..a4157bca9 100644 --- a/deploy/operatorhub/0.19.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.19.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -1170,7 +1170,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1316,7 +1316,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.20.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.20.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index 9791a3e5f..242331f82 100644 --- a/deploy/operatorhub/0.20.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.20.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -918,7 +918,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1059,7 +1059,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.20.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.20.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index 8aec7404d..df819e90b 100644 --- a/deploy/operatorhub/0.20.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.20.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -918,7 +918,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1059,7 +1059,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.20.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.20.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index 13a048b0b..073256ded 100644 --- a/deploy/operatorhub/0.20.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.20.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -927,7 +927,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1068,7 +1068,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.20.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.20.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index 2fd153449..f45b2db71 100644 --- a/deploy/operatorhub/0.20.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.20.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -927,7 +927,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1068,7 +1068,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.20.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.20.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index e1398e81c..22d878b17 100644 --- a/deploy/operatorhub/0.20.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.20.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -930,7 +930,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1071,7 +1071,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.20.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.20.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index c990e231e..5aa019a9e 100644 --- a/deploy/operatorhub/0.20.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.20.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -930,7 +930,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1071,7 +1071,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.20.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.20.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index 6a0c2edc3..bd947f758 100644 --- a/deploy/operatorhub/0.20.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.20.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -930,7 +930,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1071,7 +1071,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.20.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.20.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index eb1f9d6e4..8b1f146e6 100644 --- a/deploy/operatorhub/0.20.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.20.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -930,7 +930,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1071,7 +1071,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.21.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.21.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index 6faab92aa..c54dc3495 100644 --- a/deploy/operatorhub/0.21.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.21.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -970,7 +970,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1111,7 +1111,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.21.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.21.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index eac22ae7b..c8a418f2e 100644 --- a/deploy/operatorhub/0.21.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.21.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -970,7 +970,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1111,7 +1111,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.21.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.21.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index aaf5cdbab..4b0c0583e 100644 --- a/deploy/operatorhub/0.21.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.21.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -970,7 +970,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1111,7 +1111,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.21.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.21.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index 503968d7f..05ca09df1 100644 --- a/deploy/operatorhub/0.21.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.21.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -970,7 +970,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1111,7 +1111,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.21.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.21.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index 517a9e8cf..e0874c1fb 100644 --- a/deploy/operatorhub/0.21.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.21.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -970,7 +970,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1111,7 +1111,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.21.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.21.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index 5be3ae2f2..d29ae7061 100644 --- a/deploy/operatorhub/0.21.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.21.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -970,7 +970,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1111,7 +1111,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.21.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.21.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index b47c5b99c..d2a286eda 100644 --- a/deploy/operatorhub/0.21.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.21.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -983,7 +983,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1124,7 +1124,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.21.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.21.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index f434f0cad..5327df3d7 100644 --- a/deploy/operatorhub/0.21.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.21.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -983,7 +983,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1124,7 +1124,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.22.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.22.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index f84abd64e..c4fa3f332 100644 --- a/deploy/operatorhub/0.22.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.22.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -997,7 +997,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1138,7 +1138,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.22.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.22.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index 43dd55fa3..dcc88563d 100644 --- a/deploy/operatorhub/0.22.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.22.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -997,7 +997,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1138,7 +1138,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.22.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.22.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index f80c0f8de..0ec6a96fc 100644 --- a/deploy/operatorhub/0.22.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.22.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -1010,7 +1010,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1151,7 +1151,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.22.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.22.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index 659dd9849..c59af4a20 100644 --- a/deploy/operatorhub/0.22.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.22.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -1010,7 +1010,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1151,7 +1151,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.22.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.22.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index 55d6db79f..72c7f07d7 100644 --- a/deploy/operatorhub/0.22.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.22.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -1010,7 +1010,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1151,7 +1151,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.22.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.22.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index 4e3c34f74..27aab81e3 100644 --- a/deploy/operatorhub/0.22.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.22.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -1010,7 +1010,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1151,7 +1151,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.23.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index 7ed41338b..2f075ad20 100644 --- a/deploy/operatorhub/0.23.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.23.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -1027,7 +1027,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1168,7 +1168,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.23.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index 44af53f6b..8b364855a 100644 --- a/deploy/operatorhub/0.23.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.23.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -1027,7 +1027,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1168,7 +1168,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.23.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index f9f6b42f6..3503c01b2 100644 --- a/deploy/operatorhub/0.23.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.23.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -1027,7 +1027,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1168,7 +1168,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.23.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index 7175710cc..78260fbdd 100644 --- a/deploy/operatorhub/0.23.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.23.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -1027,7 +1027,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1168,7 +1168,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.23.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index f16c17d15..9286b5951 100644 --- a/deploy/operatorhub/0.23.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.23.2/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -1027,7 +1027,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1168,7 +1168,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.23.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index 29451b1ba..88f050d10 100644 --- a/deploy/operatorhub/0.23.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.23.2/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -1027,7 +1027,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1168,7 +1168,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.23.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index 093e55fc9..082f421f3 100644 --- a/deploy/operatorhub/0.23.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.23.3/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -1036,7 +1036,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1177,7 +1177,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.23.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index 837b2f286..964be1b6a 100644 --- a/deploy/operatorhub/0.23.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.23.3/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -1036,7 +1036,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1177,7 +1177,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.23.4/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.4/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index f3eff2751..d97ee2c0e 100644 --- a/deploy/operatorhub/0.23.4/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.23.4/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -1036,7 +1036,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1177,7 +1177,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.23.4/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.4/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index d8ef8ba5e..d854c33f3 100644 --- a/deploy/operatorhub/0.23.4/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.23.4/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -1036,7 +1036,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1177,7 +1177,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.23.5/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.5/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index 70beee4cb..1ab6e6a00 100644 --- a/deploy/operatorhub/0.23.5/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.23.5/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -1036,7 +1036,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1177,7 +1177,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.23.5/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.5/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index fa5b50c4c..372998a59 100644 --- a/deploy/operatorhub/0.23.5/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.23.5/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -1036,7 +1036,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1177,7 +1177,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.23.6/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.6/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index 9821c68c8..ba8c45e41 100644 --- a/deploy/operatorhub/0.23.6/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.23.6/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -1036,7 +1036,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1177,7 +1177,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.23.6/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.6/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index a115defc1..9c19bcb51 100644 --- a/deploy/operatorhub/0.23.6/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.23.6/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -1036,7 +1036,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1177,7 +1177,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.23.7/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.7/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index 7e3da2898..2d50369b5 100644 --- a/deploy/operatorhub/0.23.7/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.23.7/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -1036,7 +1036,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1177,7 +1177,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.23.7/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.23.7/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index e144c73d0..ec3dccad6 100644 --- a/deploy/operatorhub/0.23.7/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.23.7/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -1036,7 +1036,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1177,7 +1177,7 @@ spec: replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` generateName: type: string - description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.24.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.24.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml index a49e1a848..17eb6fee4 100644 --- a/deploy/operatorhub/0.24.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.24.0/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -1056,7 +1056,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1203,7 +1203,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.24.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.24.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml index 35bc793cd..767c09052 100644 --- a/deploy/operatorhub/0.24.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.24.0/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -1056,7 +1056,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -1203,7 +1203,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.24.0/clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml b/deploy/operatorhub/0.24.0/clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml index b3e19dc37..0f003fe28 100644 --- a/deploy/operatorhub/0.24.0/clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml +++ b/deploy/operatorhub/0.24.0/clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml @@ -669,7 +669,7 @@ spec: description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" generateName: type: string - description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about aviailable template variables" + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" zone: type: object description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" @@ -816,7 +816,7 @@ spec: description: | allows define format for generated `Service` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates - for details about aviailable template variables" + for details about available template variables" metadata: # TODO specify ObjectMeta type: object diff --git a/deploy/operatorhub/0.24.1/clickhouse-operator.v0.24.1.clusterserviceversion.yaml b/deploy/operatorhub/0.24.1/clickhouse-operator.v0.24.1.clusterserviceversion.yaml new file mode 100644 index 000000000..480e0eba3 --- /dev/null +++ b/deploy/operatorhub/0.24.1/clickhouse-operator.v0.24.1.clusterserviceversion.yaml @@ -0,0 +1,1636 @@ +apiVersion: operators.coreos.com/v1alpha1 +kind: ClusterServiceVersion +metadata: + name: clickhouse-operator.v0.24.1 + namespace: placeholder + annotations: + capabilities: Full Lifecycle + categories: Database + containerImage: docker.io/altinity/clickhouse-operator:0.24.1 + createdAt: '2024-12-02T12:39:09Z' + support: Altinity Ltd. https://altinity.com + description: ClickHouse Operator manages full lifecycle of ClickHouse clusters. + repository: https://github.com/altinity/clickhouse-operator + certified: 'false' + alm-examples: | + [ + { + "apiVersion": "clickhouse.altinity.com/v1", + "kind": "ClickHouseInstallation", + "metadata": { + "name": "simple-01" + }, + "spec": { + "configuration": { + "users": { + "test_user/password_sha256_hex": "10a6e6cc8311a3e2bcc09bf6c199adecd5dd59408c343e926b129c4914f3cb01", + "test_user/password": "test_password", + "test_user/networks/ip": [ + "0.0.0.0/0" + ] + }, + "clusters": [ + { + "name": "simple" + } + ] + } + } + }, + { + "apiVersion": "clickhouse.altinity.com/v1", + "kind": "ClickHouseInstallation", + "metadata": { + "name": "use-templates-all", + "labels": { + "target-chi-label-manual": "target-chi-label-manual-value", + "target-chi-label-auto": "target-chi-label-auto-value" + } + }, + "spec": { + "useTemplates": [ + { + "name": "chit-01" + }, + { + "name": "chit-02" + } + ], + "configuration": { + "clusters": [ + { + "name": "c1" + } + ] + } + } + }, + { + "apiVersion": "clickhouse.altinity.com/v1", + "kind": "ClickHouseOperatorConfiguration", + "metadata": { + "name": "chop-config-01" + }, + "spec": { + "watch": { + "namespaces": [] + }, + "clickhouse": { + "configuration": { + "file": { + "path": { + "common": "config.d", + "host": "conf.d", + "user": "users.d" + } + }, + "user": { + "default": { + "profile": "default", + "quota": "default", + "networksIP": [ + "::1", + "127.0.0.1" + ], + "password": "default" + } + }, + "network": { + "hostRegexpTemplate": "(chi-{chi}-[^.]+\\d+-\\d+|clickhouse\\-{chi})\\.{namespace}\\.svc\\.cluster\\.local$" + } + }, + "access": { + "username": "clickhouse_operator", + "password": "clickhouse_operator_password", + "secret": { + "namespace": "", + "name": "" + }, + "port": 8123 + } + }, + "template": { + "chi": { + "path": "templates.d" + } + }, + "reconcile": { + "runtime": { + "reconcileCHIsThreadsNumber": 10, + "reconcileShardsThreadsNumber": 1, + "reconcileShardsMaxConcurrencyPercent": 50 + }, + "statefulSet": { + "create": { + "onFailure": "ignore" + }, + "update": { + "timeout": 300, + "pollInterval": 5, + "onFailure": "rollback" + } + }, + "host": { + "wait": { + "exclude": "true", + "include": "false" + } + } + }, + "annotation": { + "include": [], + "exclude": [] + }, + "label": { + "include": [], + "exclude": [], + "appendScope": "no" + }, + "statefulSet": { + "revisionHistoryLimit": 0 + }, + "pod": { + "terminationGracePeriod": 30 + }, + "logger": { + "logtostderr": "true", + "alsologtostderr": "false", + "v": "1", + "stderrthreshold": "", + "vmodule": "", + "log_backtrace_at": "" + } + } + } + ] +spec: + version: 0.24.1 + minKubeVersion: 1.12.6 + maturity: alpha + replaces: clickhouse-operator.v0.24.0 + maintainers: + - email: support@altinity.com + name: Altinity + provider: + name: Altinity + displayName: Altinity Operator for ClickHouse + keywords: + - "clickhouse" + - "database" + - "oltp" + - "timeseries" + - "time series" + - "altinity" + customresourcedefinitions: + owned: + - description: ClickHouse Installation - set of ClickHouse Clusters + displayName: ClickHouseInstallation + group: clickhouse.altinity.com + kind: ClickHouseInstallation + name: clickhouseinstallations.clickhouse.altinity.com + version: v1 + resources: + - kind: Service + name: '' + version: v1 + - kind: Endpoint + name: '' + version: v1 + - kind: Pod + name: '' + version: v1 + - kind: StatefulSet + name: '' + version: v1 + - kind: ConfigMap + name: '' + version: v1 + - kind: Event + name: '' + version: v1 + - kind: PersistentVolumeClaim + name: '' + version: v1 + - description: ClickHouse Installation Template - template for ClickHouse Installation + displayName: ClickHouseInstallationTemplate + group: clickhouse.altinity.com + kind: ClickHouseInstallationTemplate + name: clickhouseinstallationtemplates.clickhouse.altinity.com + version: v1 + resources: + - kind: Service + name: '' + version: v1 + - kind: Endpoint + name: '' + version: v1 + - kind: Pod + name: '' + version: v1 + - kind: StatefulSet + name: '' + version: v1 + - kind: ConfigMap + name: '' + version: v1 + - kind: Event + name: '' + version: v1 + - kind: PersistentVolumeClaim + name: '' + version: v1 + - description: ClickHouse Operator Configuration - configuration of ClickHouse operator + displayName: ClickHouseOperatorConfiguration + group: clickhouse.altinity.com + kind: ClickHouseOperatorConfiguration + name: clickhouseoperatorconfigurations.clickhouse.altinity.com + version: v1 + resources: + - kind: Service + name: '' + version: v1 + - kind: Endpoint + name: '' + version: v1 + - kind: Pod + name: '' + version: v1 + - kind: StatefulSet + name: '' + version: v1 + - kind: ConfigMap + name: '' + version: v1 + - kind: Event + name: '' + version: v1 + - kind: PersistentVolumeClaim + name: '' + version: v1 + - description: ClickHouse Keeper Installation - ClickHouse Keeper cluster instance + displayName: ClickHouseKeeperInstallation + group: clickhouse-keeper.altinity.com + kind: ClickHouseKeeperInstallation + name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com + version: v1 + resources: + - kind: Service + name: '' + version: v1 + - kind: Endpoint + name: '' + version: v1 + - kind: Pod + name: '' + version: v1 + - kind: StatefulSet + name: '' + version: v1 + - kind: ConfigMap + name: '' + version: v1 + - kind: Event + name: '' + version: v1 + - kind: PersistentVolumeClaim + name: '' + version: v1 + description: |- + ## ClickHouse + [ClickHouse](https://clickhouse.yandex) is an open source column-oriented database management system capable of real time generation of analytical data reports. + Check [ClickHouse documentation](https://clickhouse.yandex/docs/en) for more complete details. + ## The Altinity Operator for ClickHouse + The [Altinity Operator for ClickHouse](https://github.com/altinity/clickhouse-operator) automates the creation, alteration, or deletion of nodes in your ClickHouse cluster environment. + Check [operator documentation](https://github.com/Altinity/clickhouse-operator/tree/master/docs) for complete details and examples. + links: + - name: Altinity + url: https://altinity.com/ + - name: Operator homepage + url: https://www.altinity.com/kubernetes-operator + - name: Github + url: https://github.com/altinity/clickhouse-operator + - name: Documentation + url: https://github.com/Altinity/clickhouse-operator/tree/master/docs + icon: + - mediatype: image/png + base64data: |- + iVBORw0KGgoAAAANSUhEUgAAASwAAAEsCAYAAAB5fY51AAAAAXNSR0IArs4c6QAAQABJREFUeAHs + vQmgZ2lVH3j/r6p676abpSNLE2TrRlwSQBoVtVFHiUGFLKOEKCQTTUzGmTExhmhiSJBoTMZEs2KQ + BsWNREWdyCTOGOMEQZbI0i0NyCaIxAB203tXvfef33LO+c697/+qqqu3V1Xvvq773fOd3/md5Tvf + 9+57/erVajq4DiqwoQLnvf4PH3Pe4ekx29P0oNVqfTEgl6xWE8eLV6uVxgnjCjLHab0TI+ZW00PW + 6zWgq0+up/XNkG+edjiubt6Zppu31tOnd1brm7fwvL1e3by1Bf165+b1+vDN02r7Jhh+6MZnXfYh + jAfXQQVmFUC/HVxnbQV+aX3Bg875xJOOracrD21NV00705U4XK6aVtMTVqut81WX6BAcNpOOIIye + x4iJFMfDhmry8CIwRh5mZBHfppH61XT7znp672pavQeH3g07W1s3rLbX77npQQ+6YXra6rYNXg6m + zoIKRPedBZmerSmu16vzfvnGRx9a3XXlar2+CscEDqXVVTgSrtzaWj2SZRmHURRJh0uf9/EycNE2 + cVpxPg+jLLMOJczPRiiPd0j1Q634eMgtr/X693CIvWe9OnQD+G/goXbXzs4Nt3/FZR8BxwaDJcGB + fLpW4ODAOl1Xbq+4cUBd+Mv/4/Om9bFrcF5cs1qvvgSvM5fpMIDN8tCxzDcaHjoCiFkyn+psur/f + sOaHneLfdHgxRs7zcNxZfwrjryOPX5u2pl+78VmXvgsyvgo9uM6UChwcWKf7Sq7XWxf+8u997s72 + 1jWHpvU169X6S/Dl3GVKi4cQrjp8JO11aGnPGxGHlw+ztPeh5jOtTjHhfdj50AgX8zcrHib8Mg9K + 2W8a49DJw2c2Jmkf85Aib/LnGPxw9ik8/joyODjAeu1O4+eDA+t0WzwcUBf84keeslof4uF0DRbw + mTgJ8I1xHArIRYdHjD4c4piAntdm3JnxhjU75BaHF7PH98Q+hfHgAFMnnJ43d/HpGfvZEXUcUOtt + fHm3tb4Gp9Izp62tBzF5HU4+pVQLnkn90AIg5ufLvPnQIp/gfgDRHHf6vWExHR/abWxvcsgIB9hO + HGBbB19CxvLv5yFbdD/HeFbGdu7PffSJ07T9l7ZWqxegAI/YdJioMFLsPkzqsMkvxNrh1Q81486O + N6xNh5fyzy8rd3Y+tl5NP74znfPKm7/ikveelY23z5M+OLD20wL9Xx++7Nw7tl+wNW29EBvnadxM + vOrwydVq8/FKFbiDN6xT+l6Zqje/gectWINXr849/JM3ffGlfzjXHkgPVAVyCzxQ/g/8vnZ9zjmr + D/0JLMQ34bh5ztbW1jkqSjuU/EYUpeI8JvIw89dxB29YqkP7co/yyRxeszcs2vcLMip7Fwr+Szs7 + 61fffM5DXz89a3WsQw6e798KHBxY92+9y9uRf//Bq7d2dl6IDfH1+HmoB0uhw+g4h0+uVjvMDt6w + ol44XOrwQTF3ffmHOZaPh9iuw03FX9wC1w89PP8BiH9ie3341bc++7J3LCwOxPuhArkF7gdXBy6m + 137wM86Zdl6EHfNN+N7Uk+JVaVaY+ZuT36S0+XKlDt6wZvWSsOkQupfesDa+qcEfjsd3Y0lefezI + zqtvfdblH98d1MHMfVGB3Ab3BfcBZ1bgtR958Dnru/43NP//ii9UHqI3AehYfB9GsQwHb1h8Bbr7 + b0B5OOWYdd00ngp/8GBwfHrwbWe988nVtPXPp/U5//zTz34Qf+7r4LoPK3BwYN2HxZ1+9n2POnJs + 62+gyN+MQ+pCHk/jsIrjiodUuw7esHgmtC/v4hCqL+Narepx0yF0koeX1qP5K04+BG//slCrxvn6 + dBO4abp1Z1r/yLHtwz94+1c/5KM0P7ju/QrMd8u9z39WMp772g9cubOz/bfwd9z+PPr6SB1C0eTj + 0OIR5i/7VCgeXrl52nzhc7XikBOvCYZ5s9Mm77JQ9tf9buQHYMxrmy5kEYvRccggPGw+dMwytvpM + jsMhD4nZWKztoR8meTjlCJjy2zRu8tNo67HzB490nG+XDzP+0C4OWczvrNdHkeGPT+vVD9z8Jx72 + ngY9eLwXKsAaH1z3UgWO/NT78aMI67+Nln4uvkeF357SN7G3Zx0C+Rk6Dp8MQZufQjuUtPlypTgv + 2pgQrr25Le0Wfsr/DGd78na/iqnczH+SXrhZehmgrOa3xSGx642FbvFH7jkCrzjbaH9EbLgW/HnY + nZKfTh+8u3g4XxHjMeQ8tCjiz860Wv/89tbW99/2lQ97a6c9eD71Chyny06d9GyzPPxT7//y1Wr7 + b+OM+vI8o9zS8Zk3Dods8jo0UCjhUs8RnV762aFSZ0k9EDc/ZFKMZW32fU1Oih+BzXG749IhAmLH + IYNys+nQYVSuy4aRuzzy3zUWa3sI/L3ip9HWY+fHJOPWxfl2+TAbb1iUkQj+GBc0/w9+LOL7bvnq + z/jVZnrweAoViM4+Bcuz3eQl661DV33guVvrHRxU09O8yWLPoTb4chB3NG0cGtnEdQjs0rug2vx8 + bIeNtkCulDY11TGhcfdhspefmp/x296niXkH/4jLcTS/s/QyQONn99i19+jNR3kzgg3Xgv8e+en0 + wetDqR2ynM/1Iz7k/oZlmog34Ov1zlumaev7bn725b+ABTz4LRIu0t26H6fL7hbPWQU+8tPv+Tz8 + FpeX48u+q3OvqADaVD5r3KMHb1izNyAUqW91Nl/JWchN46buCtyMH/XfdbjA9oR+TsQfcQpGv+2y + vxO+YUVcjg8B/cb26tC33fbsh/23RnXweBIVODiwTqJIBcGPJxzZvvNl+Ez5Lfhhz63abQRsOKy0 + HTmvy9um3nByG5U+UCnHWPiiwQP2zHgDWvAu7RZ+Bp8JLR+8YakOi8Nozzc14Vx3rVrIJ37DQp3x + wUMOF355xPrlq3Mu/Lv4e4uf9Oof3E9UAXftiVBnux5f/h258n3fggZ7GX4h3oN5JujNAA/svTgj + Nh5aauIBQCXbl2+SFocPCDcfKgs/sCUuAtEKDTGWNfwKJ4RvJ8Ufh2LmOYs78+n8s0IAnXn0Ee7F + t2lM+01ji70eA3ev+CnS9tD5Mc24dXG+Xaf4hsUCgQXrtLPzyenQ9F03P/vhr8CCHnyZ2Gq76TE6 + e5PqYI4VOPJT770azfVyNBN+i6cPiTqEoudUqTgtYtBnUrV5bm42JwjqsAgZEzLPWx0uMV/4hIWD + Oa7xLu0WfgafCS3b3qfJmHdejmxpp7hVj4h8kUfmozE2f57u3uQ+BOgty1gj8PLXRvsjYsO14L9H + fjp98O6Kl/NZV+JDVl+kyHllFgMSrcMNeC1j8mDE7zb7b9s7h77t9uf8kd+Q6cFtYwXcnRtVZ/nk + j/3O5UcOb/9jvLd/I75XxXbjadWH2FSeVrWWejR1HW4G4N4OF0k+BId905MP1zgsJJbDDEtxCafw + hBey2YdlTDOu4Xcjv9LtuN1xDb+sS9QnHGlzwv9shE5+N41pv2kMztkQuBl/+tvEjzlWk3jF3ccZ + cQidn3aJ4Xy75D/XGfPib4fZcIP6EacJAXHLutFOpFCvf2xaHX7xrX/y4K/7qCKLm3fEYvKsFv/z + +vDWx977bfghqpegOJd4U2aTe5PXIcQmywrycBgwTFMREyqo5TocdulddR1CfGyHjdzs8hMTwu0+ + TPbyU/Mzftv7NDHviGPE5Tia31l6GaDxs/vYtcqLm5Zo8W0aqUd8wsVYh8yMOIQFv3Z/2m/ix5z8 + b/LT+YN3V7ycrwzowPI9ecMindyRblp/Gs9/79YLH/4vpoPfDtFXRFWfTZzNwuGffN+XTjvb/wab + Bf+qTG4a7rHYXhxjk6pFtSm0B122pR7lTZ4AYAhePAVr8HPCXavNKpEI+7c/ieVQcTVFuJ/zhX1Y + ajgpfuXJ+O1/Fjcd8YrRccjA87j3w0b+sAMrX+pp3kftVsxsGoHbdQXuXvGzixwTnZ9iYjjfLh9m + sc6YpzwyKxrXg/0gXgGNC7mm05Pd/Pb2tP7m25/zyIMvE1EtXtF5Fs7a+2vwd/5W2z+Ipvmz3sw+ + VGK3oizonjgNduujaqV3cx+8YbVu0m4ch5E3edZpwwh8HXKoqzd52Dfaelzww0DrdUp+ihQPe/Fw + vo7bwPEwwgc3lNQYnVkMCp9656N2SR75CXeCDx54orODLxNRBF6s79l9/eT1Tz60fegX0ECPc7ex + 16P5tFksq2/UZZTdRd5UllXEpZ7NySbmvAG4W+4tX3rZN33YOZ6FHzDJTkTmD/fDX7O3f98HX9ox + zgU/Jua43XEBIELHIYNyo8MC+tkIrfxsGrVpwbdpLNb2ELgZf/rbxI85Rku84u5jo63Hzk+7VHC+ + XfKf64x58WcjSU53qB9x4g0FcSHXdHoqN3yA3c76evxWiK+/5Wsffj0mztoL36o5e69DP/7ubzm0 + c+gt6PPHae+hN7xJvTnZO9qMflDXzjax9FE/EoSMQc3JCdsTo+0S/KlP/uBA1ya+j+KjOa/0o2YP + WX7kfre9cGTwNYsfU5bpF4JgmYcdaj7ycBwRSMVBO2gMtJPgpaA8FnmJB7rZGPYzfLNb8qe8F955 + wf/J+Mk4Mdal/LweGSd18idQ1scedBht1GNS5eEnhVjfkEPRB8SbvHJCRhGstlZPXk/bb7nglz72 + zak5G0dX52zL/LXXX3ToztWPo/me610TZXBvqCmzubwXSzFvLjT1bK+qydnUgqn5ksclNs+uzUQD + XjJsmyTmCx8w4QRPR1aU386XOPLHNfjSjpvJ7gUJojlud1zzOLQL04XeJGQfh47fRLIuG8Ys5Kax + WNtD4MSLeGcjYMpn03gq/MEj77Rvl/OKwwjzlPOQIWy4Q/3wIT3LnjgBAsdpRa4H3HiZz36sB8/r + brnoyDdOz7r8FmPOnrs79uzJd5p+4t1P3dpe/wx+Uv1x7oXctNE0bH58cLMNPZpmtom7PopX+mwx + dWU/BQBsmz4+c+amzyWQXwrk08B4SpzFEQAMjXdpt/AzP4RItylfz5tf98D1edcn3DkuQ3ffx64V + bmw+iED3LS4ZeMXVRtWDPJuuBX+eEqfkp/MH7y4exZGRwyBk9U2K4ol4I0Hz+NBi3SirAvJjGrMi + /108MghW6d+/s9r6+tue84i3afIsuZ09XxLiL/9t/fi7//rWsZ034ueqHpe94marnlMT9c+Eu5sq + mwnNNnqoui16D5vQzWh7dtOQsyk1q0Ci23h4hNzHWfOGA/GXgnax+Zf29FunCsNs8TMqybbXLhHe + 846P99hkgedMOmRWmw6THj/1XXb+ES/NRScm4xKfI31EXnzU1fNMXI4AVJ54nvnbZBd48eaNuOBL + e6oyDzwJaRn54aPnMfSAQeF4og4hh6IP4rEf0eNGP61+8kMZ33Pd3n7j+b/40W9P5NkwssZn/vUT + H77s0M5tP41V/srxhoG01QsuAXqTTYAejebTBDAcCQx5tz7KV/pssWYniOXyn9tI/MGBgXrDPRY+ + pnscA4fNAjuns7Bb+NmMs/28HhlHhjPnH3FYLzkedw2x+aPAGGJzA0gviruP1DOfNtofkRuuBf89 + 8tPpg3dXvJyvyPEY8ji0kiTiDXg/tLjMlMUjP6ZxPaI+YV4VEp4S9a4PPUF+3W1bF//F6Tln/j9H + Fl2ZBT4Dxx+7/plb6+knsQGuyB4bm1arzVPCibtbvFnYFNo0VJdCvaNmg8XQR91CUXo2VfB0B+Uf + k2pZ8YsQE+FXouMqfIQJx6JTXCYIeQNf4xEo5O53Iz8AY975z2URidJxyCBdaDMKn/lwhFZ+N43c + jCrchrFY20Pg+6FW/jbxY07+N/lptPXY+YNPOs63qw4hrjPm6xBSplrOSIv6OGQMBBoP8lMDrIK/ + 3PAhDm/y46N4IOeF37f1kdXhQ887079EPHO/JOSXgK+6/nu2ptWvoZGv8KbinuAhwhG3ehiy9WgK + bR7jBWyyzdhsYceuKX3QshnZXHIkQMmylz75qceF5k18H1uYww/tS0G7FOl38LK5tSk063mbDZwP + VQCoCN7hn3OOq9ulQ7HE5iYyr2Fv/10WD4CzMeokXNYhR5JWHcPDCfDkmfGnv012onc9gt3+wn/y + UJd5qE4loz74EK7izPoCBIXjifUNORR9EI/98M6LPGEniX6GTAQvfE/2iunYMX+JiN737Jl3PzMT + ++X3nbv1B3e+arW19Q3+DBZpeu2jedhEWFB3mVd2pndTeC+WYt5c3BRqvmgMEoYsWjVxby7z7NpM + 2eSyD7+gzM1ReJrzCrz0Lf5wX3YznC3DfL65NvIrj47bHdeMf1YIlCE3ex/h3XXZMKb9prHFXo+B + u1f8FGl76PyYZty6ON+u/fKG5Tqw7jzi1j9965ErXjR99erOFuoZ8Zhb4IxIRkn82DsuXK0P/RJ+ + XdWz+mmiTYnVnG1O7R6XIM6K0MchU3p2AXAauIm7PkpX+tyM5A07QSyX//jMmZs+WOTf8IwrD41A + lB/rbd/zWtgt/JR/8uCy3PMZ8wboHjjjPQ/cLD3bGb24x+bP9fAmzzptGHOd2qg8ybPpWvDfIz+d + P3h3xas4mG/EE7K/XMNshZl6QFkuKPLLOS1j8nCUXgNYiVvyBEBeBw/Ecc15/vOt5x35mumrPuPW + ATj9n86sLwnxGxZWO4d+FV8GPovN0ZvccjZNNEPritE7rRnUI2y6aJaQbTaaTG0wbxY1He3k1wDx + uGnhHx+8hp7qgc/5whvuwBkH7R0IDbVJKq7Gaxw1vgbf8O9NBL1g5h3+aee4On86FIt3nx3EfdiT + b56XeMTKPOx2I77ZyV/3oAWL+iUuR+Aqz+TPcZNd4DGMS+vZ8g5NxunIwz/rg49ZfZQZ9TCEwvFE + HUIORR/EQ1cRJp9EkH7tZ9STWF7Si7fq+azzbz/6q9PPffxyI86MO2t8ZlyvfNcVq0MrHlaP1xKj + ebXqHHHpTQJdMN4wMGlg6BMezSd7GRoY8u43EpnTgfjKzO3reXsQT/nfpTcP9Ya3uDGR08NPx/W8 + FnYLP+U/CHfnM+wdR4bDuriOngcOE7O4DN19j82f6+HNG+UHWrx9zHVqo/MmcsO14L9Hfjp98O6K + l/NZV+JDHodWkkS8ATePDxvWTYcMeeSnBtQjD7/OEzi6k373obWLx/H/znq99WW3P/eRH0m203k8 + M96wrn3356AB3oA3hcdXc3mx3AVc5GqK1gzoAS1yDtFE+druXqpuU4/O+cvcD6P31FQ0cFOyRcxT + 9pJpFk1NCJJIuY8tzOGH9qWgXYrmK3vhCPTFeZsN3MEb1mIdWCoV1OuherX6RSU1uM7A4aPXFQSh + xwCF6x7rG3Io+iAeGo62IE/YcV5+hkwsL8UhXtGFrInHT9P2G8553Uc+x8jT+46MTvPr1dc/fbXe + +Y/4ntWltZhISW8q0XTMUG8S3Kw8FCTjVg/Uu0k2v3EAKEDwsGlCVpfagfjGNEsbdtS7nYf/kMUr + vW/iFdxLM4s350UbS0eHEY/TWdgt/Ay+juv5jHlFZHHELX/QRKIpLvOQbd5yHWKsQ5M0+KO4+5jr + 1Eb7I3LDteDXbkdgp+Sn0+8VL+crcjyGPA6tJIl4A94PLZUveeTHNK5HHn6dBxrhWS/qdx9azrvx + AK/1lt36xqPrra86+rwr3pysp+N4er9hvfpdz8Fh9atYvEv1GSYWUYuuJmiLLJmL2ZrBQK2b1lTw + 1gwhqzl32aMLB311ScDUVAQ4Lrpw15Z/yY6nGic2mdAKaBGvFfZL+4qfflKkn8HL5u6HSfmf8dte + fMGbcRvmPOzO/OlQknef/OZt2Nt/l8UD4Gxs8TBe4XMkaeiTP+UZruE5P+NPf5v8iD7ySgfEBV/y + UJV54ElIy4gXH8JVnKkHDArHk3kljxShz3o0O3sQQfrd67CSftA5zlk9pksPTzu/et7PfvRZoj1N + b6fvgXXtdd+wWk8/j8PqQtbebyZoGi5SytF0pYdi6DFrINXsTcnWm4f65BVgZs/uCLskCBkDetSE + thdADsq/9MlPPS7Em/g+io90vBgo/dC+FLRL0cCyF45AX+VfCdu/NgHNBDNR2YtuUVdSVRy0Y0D2 + SxWvYU++eV4KG5jZ2OIpfLNb8qcsP4nLMfzP+NPfJj8tXjz6Un4t75rOPD06T+DwIX9Vh9TDEAri + XGfWI+ujh5CzHs1OPim3+snPkCMs13vQhTxwsR4XrraOvf78n/vgN6Td6TaengfWq971HVj+n0ST + HM6C12c6Ni0m/RmHzYFVLNnz1mOyHgImOO3BTruQ46Fkm6Ve9CYQPmjZlckjyJBlL73jCQbAwy8m + nI9H+wtUxqWuFxA32oXfni/VwpHBF3nNx3iS3/ZSBO/wT5Tj6nbpUCyxuYnMa9gzsHle4gFwNva4 + E58jSUOf/CnLT+JyFDzyxPMJ/QS+uPlAf8GX9p523VjZISM/fAhXcaYeKChc96hDyKHog3jMK3rc + yNPqJz9DLpTijbBp1eInxv75sDoXP1L9k+f93IdPy19Tc9odWFuvetc/xv8J/Mf4jMH+qMsimoZN + hlnJsWgEWfZoPSbrgfOWbW8e6mXnB+Gt73xkx0UC4YM2tontBcCtxSd98lOPC/Emvo8tzOFHuwA2 + EXi4320vHBl8zeLHlGX6hSBY5sGJoO9xCxgK5ktQqzNFXj3+ZV7KB5jZGLyyyzrkaELex3UCfOUJ + ixP6ASbjLQfkD/9pT93AZX04on74EC7r48pUXR1PrC+A5tGDApQ78nc7OpTc+gJyvalJ75v4Bp35 + W/3sn37xBz8Jj+FHzvvZD31XozgtHk+rA2vr2uu+H2vwHaosHvqlzyhcTC4SFPkZhk3Hy7JH6zFZ + D5y3bHvzUC87PwhvfecjOy4SCB+0sU1sLwBuLT7pk596XGijxPexhTn80L4UtEuRiQzegzesqAvL + 2+uigu2uN2unq+llV9OuLyocMI7gwUfnH3qpta7Hf3PPOJM3HMpPi1N+hlyoZf+1+IkZfQuBxxV5 + VtPLzvvZD34/9afLxchPj+va6/4P/FPL/9TF3h1yfQZT83BNkFosGtHSQ+6fadwLLsGAR/NpQoa4 + oYlCLj+lJzuu0guNKNwUmjcA9+Z/l14gxWd4xsV4TF/zCsd68zdexiF82C38jPw7zvbdEXEmigGy + 48h5jFEW42Le8Pk916FtIqLFt2nMdWqj60uLDdeCP9e9NilMNvpr/KoLefoVvLt4hEtGGIQ8Dq0k + Cb5I1DyoI/Bql+ThCLoYUBfok1ZURRAS9eZJT1YseORn4CqP9KsV4D9Bvf72O//0Z/6zGdc+FViX + /X+96rrno6qv4T8P78XO1V2E7y7QptcSh+xNjTTVBdz8ufkwZ6BrMNN7E6uJyEOg+GowT0yLYKln + U+GjNn8EUP7LfeQh+/ALXdoVPmCKI/Ut/nBfdjOcAvRt8JlwyAGqPLJOzn/gWrwVRy8EN1/Wr43A + KtxNowtNwyhwGyOs2RC4e8XPjDiEzo8pxq2L8+3yIRDrjHnKSIBPQgWN68F+0ISAxoVc0+mp3JjP + fsiKeiaPPPgmPdwWjx5iQu5oZ/14oAz+Nf796Wn9wrv+zGNf0yj35SNz2N/Xte+6Bl9z/wr+IvNh + r8bmkL2ZYjGREeWOlx6LMzYdQOoF8w14O2RKj4fZJjaP7aN8pQ9ad4XtBKGf5n+X3jyKm4/k05CH + hsRZHAHA0HiXdgs/I//O3/MZ8+a336XdqIf1GW9I86FvHsQ3Nh/SAVJl7mOuUxvtj8gN14I/1/2U + /HT64N3Fw/mKHI8hb34zot5w80T+SJyyeOTHNK4H1iNpMTaCkKg3jybytuSBrHWTH/sz7zx+x7E+ + Nq22vuqOP/WYX026/Tju7+9hXXvd52Oxf5GHlYoaza5CxiJkUXPxOdaicPO2xaJi6GFp4KArfTRD + yPOm6vxF7wfhs5XdFI6LLoYst5IdjwLgreWXdrN4iWE+9EN7ESVvipQHr3EE+hp8A+fmh14w8w7/ + tGuHTNa94qCaAZmPaF7Dnnxjc5V/YBR+jmEvu8TnaELex3UC/N3yA9aMtxwov5Z3KAZu1E/1wXoo + n6pD6mGo8lB/vL7KejQ7+aTc6ic/Q46wHH8sQ8XR6lf1EJ0QZke8lEB7eL2z87ojr/3A05NzP46K + dT8GNr3ynY9Fo78Zv874IdoMKH6NGwL2G0A0BfSUOz4/0xhHPUC1eJbdo2gGNoXsRWRgyLbveqh5 + lT5o1QZwIEcC4GY7t2AEUHpiCOe8HmJgPJ2GeS1xjTfsK89FHDU/w/V87H/EoTAUl+No8SkO60ee + IfchNn+uR20eYMgm3j5yEyE+4WIUP3k2XQv+e+Sn8wfvrngVR0YOg5BP5zcsZaM8pk+uDx2++s7n + Pfr9vRT75Xl/vmH96A2PQIF+BcE9hM3CZp2NrJ6LW3WUHu1fzRV2ibM9zbAZ0rwequdCn/6M17YS + X7dnXGGX8YQs2tiGjksA3Fp80ic/9bgyTzym3SxeYjIOnVohi1fhzOwEF44R+Rp83vyWGRf0gmVe + qaddi1tATFUcVEfihMbV41/m5fpEvMBLDl7ZZR1yJGf6Df6U98I7L5glf46b/Ije+Sa982t5hyLz + MjPDsodxWCWPx1ALt5/fsCqP9fSQ6ejR/zj9wu9yD+67Sy26r6L60Rsunrbuwj8UsfVkNV1uhj5u + CNhvDjyM0GTQ6w2B3UK7kJPPekzWg2GG0375xhE8GpZ60QdBDYpDDsI/NHLoOClZzviCxXFT6HEP + sRwov8LFG4lwLV/KCz/lf8a/zJduzCPz4HW5ch5jlIVhZLx6Xt5yHWLUOgAjvk0jcIqzjeKn/aYr + ePOQyfGU/HT+4N3FozhYh4gn5HFoJUnqIbNcwOWhxfJSDkUfwEoc9Z1HBpqw3odpIqyAQcBsP+pI + feWRfiP++TyBwl5357mXfOH0dQ+7Wdz75La/3rCu/eB509bRX8QPhvqwiiKzWbW4ObJ4YzVVylz8 + WfHdFUOvxWzN4FUNPYbSpz8vcu+COX8Lg/HIXoOajk+Oiy7YfZbltjULtbpafmk3/AUm/dC+4idv + ivSTcdMrFQT6GnwD500EvWAmGv5p1+LOulccVMMw5wnHNeztv8sKG5jZGPbCZR1yNCHv4zoBnjwz + flhK3mQHXcZXDogL/8lD3cCN+pF5HDKeB1JUcgcCx+M6MhDz6EGByR35u50ZcA+70OehJ3XcxDfo + zN/qZ//0SzoAy0/uBymsJ+dq+uxz7/z06ybuyX107Z8D6yVr/JjVLf8OAV3jmrIJWFuvwmxkAVX0 + UUm/EcRnlGaXONubb7ZmwZNrSJyajxMAJq94Qh5xtTAK71bINxvbK2DcWnzsWlxDT7X1fd7xbPCj + rhcBDTO84ktexaEuJav9Vfwl2575AqEEyl5htrhVKBkaRxPtNudDkdewJ988L/kHZjYGr+wSn6MJ + eR/XCfDkmfHDUvImO+gy3nJAXPhPHuoGzvlaRn746Pyqo/C4QeF4og4hh6IP4rEf3nnRT6uf/AxZ + EKIUL0fRhTxw9m+91ku8thtxE29G9+/0ZedcuP0zE/fmPrn2TSDTFe/8l1ur6TkqHoozPiNodSWz + mv7MJMCshJ63XkuMZvPqkdF8XKPBi0kDQ2/Z+vATeAHF1+3JT1nmfghZtG7fEW+Th958waB4M78+ + Cr/0o10ASwXAeDMdA8teODL4cn7Dr2Xbu4dNVPaiW9SVVFkPPmuX2C9FXsOefFHPmHf+ES/mnJ/t + ZZf4HE3I+7ii8HvhOX/SfsCa8ZYD5dfyDsXAtXjh6Ux7w9KbHgqI/7723Cd94IerLg/wA9f0gb+u + fddfwcn5r9kC1WTRrP7M4ab3Jtkccn0GU/OAB/YdLz2bmLz0Mxwp/wGP5tNEA4ZcfkovcxNmmDTL + TIjTZYflf5c+UImPsfBFgwf5iQnhel6eL7uFn5qf8dt+FCbqx5DKTdatJiIOx+2CxvNyiM2f67Hn + YRLupEd8fRQ/eTZdC/575KfzB++ueBUH6xDxhDwOrSRJPWTAzZN5WQ5FH8Aah22lywcRiNh686Qn + K4ALmKJDXFpvxZf+m9+Iv/IL3EgrA8BPlq5X33r06x/3b2b+HgCBeT2w1yuv/yz8RsS3IJALajMh + olgij4vmJS6bchZ8zBfPEqdVxJqSD4ZUjwdOWLY+F9t4AcVnO9uzaYIHpt5UNYCOfuIwoD4clH/M + JI/Vzqv0CpB8LV7RMFDP9/gjPOEDMHB68m3wDZ6qByGVZ/p1YZZ2wgnuuFMmRT9sZMfNw3niN41Z + yE0j8LuuwN0rfnaRY6LzU0wM59tVhxAQykv6zLBoXA8giA8gWIALuabTU7nhg+3MSj/Bs4xj0IWf + mADOcdod3crvbH63H9ErH/71nem21aH10+/6M0+4XvMP0O2B/ZLw5R+7YDXt/Ds09O7Dqjapi86m + Z5E1sliU2+V561X6wCfO9jTLTQhjA8UiOsjWh5+QBRRf6j3SXnZkKH3QuivEJwdNllvJwWPALD/n + Y73wmW76oX0pGG+KBpa9cAT6GvkPnA9V6AUzUdkLtqgrqSoO2sGQcruGPfmintCXfzwr/BzDXnaJ + z5G8C/6U98LfLT+in8cv/vCfcTqMxHmUf2TiT07Ojzg8+c5B5WHfRR1CDkUfxCPrdCOeVj/IxSMP + vrkOLpPi1fo0O8ieZzx6kmHVSfUlPvjolzD5x797uJouwM/C/8z0Sx+7wIgH5v7AHlhHPvGvUBW8 + YbGGKBZH/NGYMovLYvfRBrzXJT0siyfwXhzzk3joYWpH4iB86NNf2FnR9J0vQpA/zgdtZOK4iLGD + 8u9uUDzBAMPwS7QC8tjCLAds2hE/7VKkn2YvHBl8lf8Zv+3FF7zDP+0cl+MwfzqUxC4PvnAzi3+Z + l3jECrMcWzyFb/VY8qesOBOXIzgrz+TPcZOfwGMYF3HBl/FSmXVx5CmjPviY1UeZUQ8jlYd61zHl + UPRBPPbDOy8TpF/7CR4DjFK89ldxbKqH6EZGVacIVIO8Mn9S08C8+CHuJx+55Rbs2QfueuAOrGvf + +SLU44X+DIXasLioQ5ayZMxz1YXLkfXifLvMY5zXxHaJs33zMxyJRXRcI8WR/ixr0RRH6mNe+Aii + 9F5iNiefHBcxQ3aelJNfjyPPNu94iAtM+iGfE4WCflKc8yqO7MLgtdnA+TO2aIgQUcZtv4u6ApUO + xUL+CpDKaTrv0Gr6zsecO7356RdPz3vYYai7P3nZtd60Ey7XOUcreB9X49vUH1U3WLjeG+rT+DO+ + ckD+0Kc9dQM38iHzmf6GpXIgz62trRce+an3vqjqdD8/uOr3s9OJ37daH3sL/o7gBfoUtKE5skk0 + Qq/PBG0sux77kifk2kzMFoTVzCFv1scmDXwY9sE80kcQ8kd+waKJzWOEHZZ/hyMe6TfFD0Xhac6L + OA3pKOXwG/oZTha+Db60i08WFiuBOW6PQ6viGIX42ocdmf7BY8+drjhvfD78zZuOTX/zvbdP1926 + wyNxfohQxuGgeDeNDnt+D9yyL+qNAeiT9jNnttT5MYPoxnw+Y7Q/1o+HVuRRnilnWtS7jwMINCwE + qEE8oh8OhRt5NR4BfXMdGs9wbDrIjk90BuKxeImP+Fy3CEDzc15obju6Pvzk6fmf+SEY3a8XY7t/ + r5e/9ch05Jy34/Xys1zk2CyIwiWLEZsu9W3VvfobIvbmisWEnnLZhZx88uNVKb4Bj+bThAxxg0XI + 5af0UPMqfeYRDjivy3IdAm6f8h8gx00h7ApfNHhQODEhHPLG6Lw8X/LCT83P+G0fBI429DJXOHP+ + nm8YTE84f2v6oSvPm65+UP3maqnyhm/cTq/5+F3TSz9wx/Spo9ziSBN/do3cXMynjfZH5IaLm4rx + LkbZA76LH3Py2/jT34w9+HbxcL4ipwPL49BKFs7jCrh5Mi+alUJ0cic4+zhoaT8IQqLePJrIWxAU + Dx56XpUHAS3++TxUGbZwJPdEjx/Pbz566OgXT//zk+8i4v66xqfA+8vjkSP89cY6rNhkLAKvKhqe + q5livnAN7yahpS/zRDNgSnI2ccnNT61Z+gdIPd+aIeTeTRUn45be/hVPyBiwxHbguIgZ8tBHnEFR + eRK9rIvDpMJ+ySei5E3RwLIXjkBfs/gxZZl1gyCYectedIu6kqrimKZLjmxNP/D4c6c3fP5Fex5W + NME3bqdvevg509uuvnj61ivOnY5ATrfDH/1HX+SY/jjm1eqzCV95As+0juuH+uBLeufX8g7FwKkw + YQccPuSneFIPQ+aJ+TpkQg5FH8RDV0UTkadf+4n6REzGs262qzha/ew/eLkvxEu5x01ek6p/CSsc + 6Wd+n37k2JEfMPr+uyuk+83dK9/5dXizep2KFMXUZwAEkCWsselVRRaZ1VSxd0dsnmgK8i3w0nNx + yCs9bvVgWtNTb9zQh18NS33EIn/BQ/7MqOJ1ZuV/l948ipuPYVd4mvMqPzEhXM/L82W38FPzM/5l + vnST/Ok265bzq4mf7f7CI8+d/ja+V/Vgnj5383r/bdvTd+DLxP9y43ZG6THXqY3Omwu24fLCjf4I + uTYjTBidlruPjV91oV2/9uIRLhlJaOZxaCVJ8IVjx4M6Aq9lTB6OoIvB/Ze0oiqCkGCPD/LMriUP + 5J5X1UN2I/75PBiDdvB7wrjmFzzb0/pPbT//yp+fxXEfCoz6/rmuffdjpvVdv4VCX1rN51WLTRjF + RTRZyhqBU7Ha6NVdhB98s02pRQxcEA49nKkXNulzsRleGQovN4xT8Th8FTEUpWdT4cP2RJin/GPG + 7tM/xmwyokkUYw8Tiprv8Zff0M9wsvCt/DeeXfyYmOPa+sDu6Zccmv7ZVRdMV114qDGf2uOvfPLo + 9F2/c/v0/tvhNNerj5toQ7/sC2+qqCvslFcfO68KRsSGq/NDXSjOt6s2MRBcFcq5zoQNd9THZjfQ + OAECR3x6KjfmG3k1HjqIS3rwBp3jaPmVfYVnB2N+tx9RK58FLxRph1+vfOOx1Tl//P76fpY7P7O+ + r0Z+3+qcc9+A3xz6+doEKMJshF+tYR9RbC9CrkIbN8TpzRWLSZ5cLI4hJ5+WphymPhaFftl8spch + brAIufyUnuy4Si+0mrPsDMA98iY8M6Zdu+SXcszbX4nNT9gJ13iXdgs/gy/z3pQv3SS/g0u7R553 + aPrex583Pffyc1rU9/wR39KafuSjd04/8KE7p5uP4Rvz8F/rpV2IeGLz7PLW9Yw75NxU3opajayG + R+B2+enke/FwvpjwGPLZ+IbFcu3srN987H1P/ILpJasdyvfldf98D4vft8JhxUTYRNyMszHmuUWq + FYCTvAfeTUJGX+KDRTVp2CXO/uzfvLCrh+q5sM/4Il4CxdftmUfYMYTSZys7E8clgBxWfMrUfNTq + yrpASLvCk45X+tEhGrLyznQMLHvhmKivwTdw/swPvWCYV16pp91qOhdzL/7M86e3PuPie/2wogd+ + RfnX8H2tt+L7W9/4iHMjz1gHAph3v0JWnlm3HAWP/sEz06K1xk12gccwLtW59VNosq5mZFhmHodV + xukx1ML1N6y0q/UE3HE2O/mkPOpgP0POgF0Hl6ny3FQP0WVFHH/h5ceM9KOAVLnkHX6zDopna/X0 + w0947/dkLPflyFjv2+tH3/W0abXzptVq6xAXZ/kZLT9zZwlrjGJT70Vt44aIzcMmRVGhn9mFzCKX + v+FIbMMN7Y0LIui5ePRv3rle5k0vmOIoO3sQT/lXNwRvUHBQ3H7QbOEZL68WR8kRb+UtWNRh4Wfw + mdDyMt8exzR9Hd6mXvqEC2Y/piDf9+Htulu2p2+/4bbpv92Cfx6Buz4XaJPPrk8cRq038KrLpjH7 + oY0z+uDdxcP5rCsNQlZfpMh5eY6B7UM/+NAYsnjkxzRclc08MiBp6M2jibwteegv6iA7+XccPf7K + j/YCxpBy5NHjN854zuNp+9g0ffb05668wdb3zf1+eMNa/2sskg4rpsDk2HyzMea9WFFK4CTvgQfB + rCLiy2ZofhJnf/ZvXoDqgfOWicumoj55BQjZZswj7BiJ4qxBTUWA7QWQA/MD15qAWl1ZFwhpV/hM + N/3Q3oEATT8pGlj2whHoa/ANnPOFXjATEcfvT73+qRdPr/qci+7Xw4qRfvZFh6ZfedrF07990vnT + I85FmzLBfoWsPLNuOQJXeeJZZcpxk13gMYyLuOBLeyrlT6hRP3oYh0zGmXqAQeB4su+TR4rQaxCP + /cgJbuQZfWQ/Qy6U4iVv8LT4ial6iG5kNObTjxnVn4TJf/IOv1kH49hB06HD6+lf2Pq+uyuk+4z+ + R/HT7KvpWvHjgdXME382AsBAXLIYs1nCjs3j1dgccr0pcHHJt8Dv8lcOzTfgtHecQeTIBDDvXA81 + r9JnHuGA87osO07mu9QHKvExFr5o8IAElR9NhIu6SjSw7BZ+an7GP8/3ksOr6Xsef8H0Fx513oQf + WH/Ar9u319MPfeSu6Yc+fMd0J3+Ya3nF5qz+aJuV4c/6qssb+nFGvRcP57OuNAhZfZEi5+U5Bi4b + /eFDY8jikR/TOF7igpY0mYH8UBo8UudtyUN/WOc6XCiTTTz2RFPHlfOc4GyTY8K4wQfDBY5/SXr1 + F7Zf8MRXSXEf3Bj1fXPxVx1Pd34ABXvoLgfcLEx2MdZmgkFviSx6H8u+kwdf8XR+4mKNhj4dRRlm + +lxshlkKBSZa0uEh0yC986kB7qDHh+0FwC14CZdkHmpt3/Tya73qEWEKJzgmmqLHVXyJ04RvI38T + pnwILzJ/8VHnT9/9uAumy07hxxSai/vk8aN37Ezf8/47ptf9AX5WMQqvTaR1yPXCCO+z/ulyLlgf + N0Xb+cNeMM63qzYx1xXzlMfKVpiaz8MqgMbJT+Bor8j1gBsv89mP9cVjgFHg0foDXnHkBFmo14hb + PfT53X6CuIbiFU/WOfJV3vpHWT+xfdH02OnrrrpPfrXyffcl4erOl2IjjMOKxcOlzaviQs4x5iN1 + LxGb8Dj43LTk5CVeWOTmSz+Js2yceWFUD5y3bHvzUC87PwhvfcxLT++41Bw1oCdMaHsBcGvxSZ/8 + 1ONa1INTwx8lXOmH9hU/eVOk38GrOKKZct5mA8dN8oxLD09veMal0z+56sJ9eVgx9kfhr/q88skX + TK9/ykXTlfHjFKpv1i3HyF954nk2aqGjPgs8fdSlgvb1sma+nlln4PDR6wqNDOQOCq+j+Qg0jx4U + YK1ftzMD7mGHJ/sZsiCcD4Li0cPA2T9xpFOkMh3zUlhPPsZBWIunf/J1/IkTUHz4OcuHHr5leqmE + ++Dmqt7bxPxG+7Ttb7Rv4o5iqjosXitulnI2Qq/PEG1U0WnXryVPyG0VojmyuWAsR8HDQeFQ78V2 + eKUIfZoxLoevMOQv0sHE7uYyTzVJuU//GEFY+shvyPJSDtU0Lf5wL/uKh3EET1jP+B+BQ+D7rrxo + eu5n4P/MnUYXvzL88d+/c3rp+2+fPoXv9s76A3moLJvGXLA+bso79PVmkhjOt8v66BfMU+bKOwLK + Xq7CaUJA40KuaUVe5vHAPkNfSOJoGWJd0rt9jBNhTABV9hWe8xjzVpRccSQusiJv58tKt3mot4+t + VvfJN+DvozcsfKOd/1ewX23zcRW92WIErjYlnlU64HPchIdBZzcfLIqH+ly04Cfh0Jcj8Yiu9Bmf + 8YpIfN2e/JQjjNJH/NFeytMe5LD8S5/8wZF1gZh2hV/6ob0LFLwpGlj2whHoK/nOx9d/L378hdNv + ffGDT7vDipnwr/m8ED/+8LYveND0rY/CX/PhxKJ+Kg+wszEWTPVZ4F2huGs9Wz/VtOuLFdKM6wwc + PrwcSz1gULjux+urjDN5w6H8hB2mdn8SNM750E/wtPiJsH/rtS/E2+fpl36Cj3omVDiKLY4AMh7z + hZ3nDx3eWd8n34CP8OzsXrnnN9rbYbGLN4rp6mo1lXQVFQZa/ByB92eQMapIUbTiD97i6X4IYrbV + PLl4nI8yzPReHKdRirCP+GA3S1P+TOf42cRjkTOAis/hqJnwGIbhV6LjKnyEmfFy3oVKXIoBjLyE + kwPfnvvw86eXXXnhdMX5888pDXLaPfKv+fwN/jWfTx3NZd485oL1cVO2oR9vHAHifLusj3XGPOVc + Z8KGG6wrPqTn8iROgMBxWguqB9x4mW/E0XgMMAo8agfATc8HPMkPB9qFWA99frefIK6heMUTfBHf + bj/IZDXhG/BPepUI7qUbY7j3rpe/56HT4TvejQ0yvneV7FE8bT4V14snGRgVo4/Aq8hZ9D4mZxvN + E4tJngV+5ld63LRGLsGAR/NpogFDLj+lB4ZX6YM2M+K8Lmdoe+YbGZc+UCnHWPiiwYN6MSaEY7My + 7sg7Rrfg8HMVflTghz/7kukZl927P6XuyPfH/T/hr/m8+L23TR+6AxsmNynrgT+uRxuXfQh5dlFm + XZc8wiUjic2sT04piij4rA6e7GualUIBUjRrO1yKBxrhGX/0eciC8BYExYMH9UXgKg/JI/75PHnM + 6PhErAnjHH9M1Lwe4pZ8+Gs7n9i+c/XY6X+5974Bf+9+SXjkzu9FzD6sokiVSCtaNkGNAGWSsYRD + pl00TY0kXfC7uK25wi5x0mvNWzNozbw6oit9LErIvZsqTvG3MJos2tgejksB49bii64YeqpHM+T8 + 8EcOXOmH9hU/7VLMfGIEjr9N4Z981sXTG5/50DP6sGJ5vvIhR6Y3Xf2g6e8/7rzpYvxMBqvg9dhQ + nw31JocuFbSvV067rliIgNnDOKyWesCij/obltdXitBnnMlrf/bT+gJ+iychGMU36EJudshHdVC4 + epL16K/II8JnPjKoPCnO+UhgHPl8dT7MPnTrnJ179Rvww1N6PNXxJL/Rnif+bITPLGGN+ZkNo3dj + GzfE6DeMWEzydbuQVUzySsatHohPN9SDRxMNGHL5KT0wvEoftJkR53VxNK/c7tIHKvEx2p/pTcNA + Iz9OCNd4mx0f/9KjL5z+zhP5f/7u3c9NjnZ/3//grp3pe/G7t17zsTsVqOue64ORmxhF6uMsow2H + llaR87l+NAh5HFrJQhyugHszpz/7F4/8mMZdgrhoFuaNIOioN48m8rbkgZz5EWL/Hnv883kCTcj5 + eNLQ44+JmjfO984Hhu3tra177Rvw92IX7+gb7ZVkJRuphCx9NAk3W+IrScCzKXIsXMO31ZQD80Qz + YCb9JM5yX7RyFPaWHUfEhQCSVzwhj7iol7kfpM9WoGLkB0DJso+uMH9wtPxy3vFs8EN7BxK8KTqg + Z1x2RG9U/+eTLz4rDytW9PJztqYfxm+U+LXPv2S6Gj+2cR6+Md/rWn0FbM7TThcXNtbD65XTrq/X + M+3Aiw8vx1IPOyi8jsfrq1i/6gv7s58Wt/wMuVCKl35aHyz6abTLyGj0F+Mmrxn95sRnT5h3+M16 + GUc+X3M+zK2nQ4e2t++1b8APT+nxVMZXvPO5q63p512tPShRPOrzxJ+N8EkrlyzGbJawY/Mcj198 + uZjkW+B3+SuHfCA+6bEo5NFEKBhZyOWn9DJv+swjHBCny7LtAc+MSx+olGMsfNHgQeHEhHBRV1D8 + 0QsOTy97Ev6CMr6xfnDNK/CxO3emv48fg3gtfuupVmNDP84stEu9SYWHMu3iyXDioBmHVrJwHpfV + aF8easEHIm96PYhO7gQnjnoa8yqCkAaPJvIWBMWDB/VPEM0PE3kwH3F4cjx8MGHJMWF7xx+GMVSg + Jc/5WJlDz9v+xitfZ+ZTv9/zN6w1/l/A1vrvKjlsnkoyilShtaJx8y/xLkYtjfSV9AY8AEXNB/uN + ZkiZm3nm1zjzAlQPARO8NUPIvZsqTvJKT++4mizaaDLHJQBuLb7WBNTqyjwhpN3wF5j0Q3s5Yh1W + 0/l4e/h7V148ve2ahx0cVlGq5cC/k/jyz7oQP3h68XTlBWj9DfUuG9W5r5c1uS5YIU1YBg4fXg7P + Dz1g6pMT9VUsZ/FmJF7f9Gs/jisRHKWXn+Bp8ad+tIueZE47z6cfTSsfKVo8edgmn0bquc/imvMx + LiqgX2+/ODH3ZByeTpXl2nc8G78F5/UKOoq0kSoOjzzxZyMMVLQ+RjMR58OgjRsciA/FU1HJ0+1C + VjHJKxm3enDNHX40n+wbMOTyU3pgeJU+aDMjzutyhrZnvpb7YhOmuP1gK9i3MJsf8xL/9fjrNC99 + 0iXTw/G7qg6uk6sA/nri9GO/d8f0vR/E75fH97q0Lt5dgyD6uTYhNFo14WL9iA55HFpJwZXDpQUk + bBxaapfk4Qi6GADPQ8TmjSDoBk8irFjw0B/7R37Sv8fIJMzSX4sXmrSz/7QffAp4hhOd7EadBMBt + NeHne581feOTfs2oU7vf8zesnenF3HRKLkfGEkWqsFrRNuG9mC5NJpvjJvyS38V1HN0ucY4vix7h + GagQFR5kx5H5WO7dVHHSQPjIsMmijS51XMSwGVp8kpM/OFr90m74C0z6gf0fu/TI9Otf/NDpFX/8 + soPDKspzsgP/Ujf/cvfbnvGg6ZuvOA+/7jk2axKozn29rMh18Xrm+gGHD6077XR5lKg+aYdMyNVA + kiWJh+ZFo7iiHzkvP0OWK+Hhr/O0+ImZ9RH6jEzzecrk1bT8KKHCUTv8Zh0YD/dnXsNP8lMDO3wc + Wq9fkrhTHYenU2H4t++6Bv989X9Wlgw6irSRKvR54s9GGGQJawReyXfe4/CLD0U5G96wHoZvJr/s + yZdOf+6KC1S3jfU+mLxbFXjvrfz9W7dOv3Ej3gPyapu++hI6b9ac0QRu3pSajU2fhwL3NBfKmzn7 + uvHID+XcB3n4wU5XEYTUDr9AWGGC4sFD7jPq54fJiH8+T6DYIk9ZaqLHHxM1r4e47eZz/JzfXq3u + 0VvWPXvD2tp5MYM4mTcgrQYS2gtfSQLjRc9FOzl+8bJpuEjNz9xvX7RyhAfOW7a9ebLJotti6Pxh + lwRqyog/utRxCSAHFV90xdBTHX7xmPOFR3z86yd/4wkXT9d9xcOnFxwcVizqvXY9EX+Z+j889ZLp + 1fm7v1pfj6091gVP8u11wrrho/rOmtBjUF9Qf7y+6n3T+kp+Wl/Iz5DlBDfF0fuvxU9M7yO/EY34 + R9zkNaPfnGSpCdFl/MFnLQz4UhHX8JP8VICXeQB26B5+L2t4So8nO77yHU9BDG/FCe4s+5vQJo7Q + 54k/G5WSW0DFoyxaLHLnjUXYTO+inKlvWF/7iPOnf4i3qsdeuPnf/NtUk4O5U6sAf+fWD+N3b/3g + B2+f8APzY7ODTv2pXZ2dismQx6GVfr1puVdp6M3M7cK+thyKPmhz20/nkYEm7Mc8ibDCh4K2CSbs + Z+Dmh4k8hNnisM2wlRchnujxh2HZ6yFucz+YzPpgxP+iW++st54+fdNVb+02J/t86m9Y+NVESBm/ + qp1FiqLkSO+VbIQS8l74ShJwL1YrYvLmuIFfvLAsnogr47DfWMQMz44UoMKDbPvMx3Lvpjl/S1P+ + LIs2ulR+7QH3Fl9rAql5a/ml3RPwmf8/PvNh008//aEHh1UV6r59OBdvsn+Tv7/+Cy+dnodfD+31 + tM9cF6y0JixjXfHhdordXnrATqqvBBMPiWO78EkE6Xevw0p6+Qme6P+yg+z4SKcnPNBPzqcfTTsO + wloe9YYYdtbCTnyUlnyUOev6MAAcGPj1w9t/h7Oncimku23It6tp9VY6V7BRHEXXgp/xcp7Fwagi + 9RFABuKSxdj0xZt+ZsQWxAuGM+UN65JzDk0veRK+IfzYi/fFb/3cUPKzZupNNx6dvv3dt07vxve5 + 1KfahdmxKEPI49DK0mi3VmP7cMj+p1l0vPraNGbNQ6TzQCM86aLPQ06U90njgT73GzHLw4lMu+cJ + 1HTEJ4QmevwxUfN6iNvcDyYVp/NmAXiYnWQAAEAASURBVKHnW9ZV0wuf9N5udzLPp/aGtbP+Tnhl + bZ1UHC48vBisrhwtRdB74ytJ4LVoLDaexZe8OZJzwW+/0QzNLnHmMZ95AaqHoIPsOGbFNZD+Sm+c + Zah5lT4PX9ah1UNd0OJrzSJ73pAf/2/VX37cxdO7v/IR01/BuB9+RXHFd5Y+PAP/N5a/3PCf4pcb + 4gfm43KfV99h3dxO0f+1voBX3xyvr3rfRD/K07yP9jqsRn8Hj/px9J/7OnjRZ3gyO3AjbuI1DS0e + qCgcxTmftcQJKMPhJ/k5Dbvksz/8nPn2d8ngbt6Gp5M1vPa3nzBt3/Xbq62tw96koIji1LiJi0kx + WIxKqo/AMxCmWGPTF2/62cB/JrxhPfOh507/4ikPna68+MiGDA+m9kMFbsQ/oPh9H7htesVH7pi2 + 2Y+8NHpTqn9j2h1NPf6o/Xk4ZP/TrBShz/7PQ4TkvBpO0uCROm/aH+QNHjzkfiNkfpjkTlvOE2hC + xydLTdje8cdEzeshbnM/mIw8xRcFwgF2bGd96Ml39y3r7r9hbR/9bhThcCbjICKJOGQii55DBO3i + 8ESe2SmnWCQ8O6dcNFZ/N95FGC4cj3FpT7vE2Z/9R83SkUiipoormyqbTMDoAvIM/qL3Q7iTPprM + cdGFm67soysoPxp/nea1X3D59Ctf+vCDw2os6b58uhS/6/4f4XeJvRG/OPCL8Lrl9UXf4aP6QpF7 + 1598X0U7Vl9k+qNvOGM/sR8Swvnqz2zrtm9C7/ggcF+Unx43eaGSlvZ+0l3i8Ou8HY/5iHIcvQ7m + c32iQAGcDm9NO3/dwsnfFdJJw1/+2w+fDt31uwjwME9uZbdp3EQYuDzxZyPwSrKPwLMoG/1s4Bef + mobF4Zq0+EJOPusxWQ9eQ605/ZJH9jI0MOTyU3qoeZU+aDMjzutyhrZnvquJ/8PvxU+6bPq2J1wy + 8Ru9B9fpV4Ff+oM7p+96z63TR27Hv5/IvkEKuendCZzAHyjUf3hwH1oORR/24BGBCrTXoSXHAXMc + 7mP642X/za8CW84TKLjw8aShxx8TNW+c73M/mJN/590LtLPeuX197Mjj8fuyPtbtj/d8996wDh39 + W9iY+EegMukYsSlVlBzpcaya/beicXMv8ZUk0Mmf4yb8kl982Qxyj6rzsJj5dbzmLUeKL2qquO7r + NywW/QWPuWi67tlXTN9x5YMODiutwOl5+5rLz53e8kWXTd/9+POnC/j7t6rv85BAXmpDHmbZ97Fv + rNh1WLESRROnXfLudVhJH+3u/nb/lx0IR9/rSQWnvvCKT9M6NKWoQ41hjvyKl3rus7jmfJkH7IQD + KBIjbmvaOn9ra/s70/ZkxuHpROiXv/XIdPjIR2FwOZdi9gbDIPJwaMHPKEOvNwwWCbKS40g+/JmN + Ta8kw74Xp/P7zSWagnwL/C5/5dAlGHAX1/YicmQCmFdNE7IXNXBZBoqZEXG67PApl+H7VE996MTx + 4DqzKvAx/DNkf/e9t07//vfviMTY0biisb2Zs++5d0uhDaBtJHgeIrLuBJrY69DyPvGZoG5r+4yG + 9t/8KrDlfLgLPO2A8J18+HDcnB7zAsRt7idxYefAhBTPev3fd+6884rpLz/taOfY6/nk37AOHfqT + 8OXDSjG0YLEp5TxHeotkynFPLnE5Bp+3tNZOfFl0HlIn4pc+itntMg7b98WBUwMVosKDTFwtSsi9 + m6w3jvaVJh9CxoAlZn3G4n4G/nWaa59++fSGr3jkwWGlip95N/4LRD/6uRdP/+/Vl+JfzfbWOvm+ + inaswyHrM++jvQ6r0d/Bo34c/Tf6lm3pDqWHMZ9+7Ff9S1iLp/ZF2FnLvheQ4oKPMmcRB3nkVhOB + 48Tqj0znXPAniTqZ6+QPrGn1DXQlnxwjyHyTmY303JKgmPIMh2w6z5JfMnkSl2PjE7foHRn5ut3c + r+O2Hkb1EOFBtn3EFbKAiiP1HrUGdMur9EEblToP/zrN38L3qa7/E4+env9HL9KsDQ7uZ2oFnoYf + g3jjFz1k+iH8WuoH4+99uk+8ad3/0T/ZQOqz3jduJ9dn9DVlHRqwy32TNRy8wdP3De1qX0DQKeLG + HfPpx4z0w/DoUXeJw2/6N07AwLX9R2uZwy75PBHxkA//AtK0/Q0yPombozkR8F9ef9F0ztHfX22t + LmJoNJqNLEYcJjmqKBHcLnrOs2iLsYp3qvzBVzydn0FE4EOfjqIMM70Xx2GWQomLlnR4yDRI73xq + mJ73qIum7/+8B0+PufDgxxRUn7PwdhN+DOIfvv/W6Uc+fCv/GXftE/dh7CA1UGwH1Mdv5nqIahlX + b0I8rPBBuV/ed40nGzNwZR9u1bh0A726W7jYx5zXDhdAbkQXeE6UnXBg2OXH8ZXdLr6RBzA3r3cO + P/Jk/rGKk3vDOm/7z+w6rLhrcXHTMtjZaIX0dTsBvg4RGKimPAzwPONNPyQNPj7yEg4WxRNxJc48 + xpkXRvUQdJBtn/kMXvGUPuYly70JQr7qkiPTr1zziOmnv/CPHBxWUZ6zdXgQfwziqoum38Qb1xde + dlj9xb7Lfh19Fe2o48Pt5JrpOAk85nlYRZ/3mo7+zrZu+xJA93Xw8hQpP22fiTe90p7P9G+77tfx + U2s/AgnX+WxHouO9YWGrXjxtbf/p5DjeeHIH1rR+0ThRnQJlXhp5uFDO0Qrp63YC/CZ+lmrGexx+ + 4VCW4ol4MKEQzGM+82K6Hjhv2faZT/jPRdHasPgZV9jRAwgefO6h6Yfwg59v+6pHT19y+cGvKGZZ + Di5X4IkXHZ5ef/WDp9f88UumKy44pD5VA6pP3UfqK58S2bYwdmO6vyl586ec9R39nW0934/ua/vR + IVl+Wj9r/5hRb1gMqHDkjX3B2dxX1GNf5jX8MO7wRzvhakL25hMMf8dw56S+LByebLf7/mPvuHw6 + On0cwJVL5xR8okewKjoQOe5mcVLQ66RfjsAzkCW/kmcxkjfHDfz+DBKLSb5uF3LyyU855EOF5/gQ + ie1DoWIzDvPWZ5oMOAg+9DV/dHr4+fV3Nzh7Vl//7HfvnF7z8aPT33z0OdPXf8aZ+28h3t1F/oWP + 3zF909tvik2f/R1tDjIfSpbNHY3G/i/9ODyMoYINWoP4c7/JDnp2uw8bPcl0eciwz3kZp6eS+6El + fzOcYPbb5xX3eAnodo5Hvo6uj01XTN/8Of/dLJvvJ37DunPnz4FUv5UhU7STKJ6KlMG0IirI5jRk + FSHflHKM5Jb85SdxOZJ2wZ+LMCs+8ImzXy+CecFRDwETPA495WV874I5f9Hr4ZIjB7+imEvznz55 + bHrKb948vfSDd04fwL/I/Fffc8f0rLfePL395m2qz/prHCLRn2PQYcUCjfbmPhv7yofZkLOYo7+z + rXl4DdzoW9Kp8WU65tOPGeuNKE4vb4c5H5HGkc/XnC/zgB155JZ+OJ/7zHZQ4cemVs+3tPf9xAfW + 1upFSoXJgydT1RsI5TgUZiP9qSh8iCvkGY5Bt/klf/rl6s3sSLngN49x3S5xtne81oOjHoIOMnFq + CvKHHA8l2yz18/xCOiuHD+GnvZ/3jlun51932/Rh/iIpXLlu77x1PX3F226ZvvWG26eP41+vOduv + 7Ff1p/os2lE7LPpRRfKOMx7z+Kj+bEV0nW03+nPD/hKdEGZXv8c6gTm3Ff2w/+lRd4lzPmupEDBw + cU4EkQfYJV/NZx4yU+B4K3pRSHsOxz+wrr3+jyHoz2PIdXLiOWWycp5ZzkYreB9XJDXDpR1Qm/jL + T+JyJGsrkkUvQvFEXImz3+bH8OIRnWrv4na8MhZft2feZd4eGM3Zdd2Cf9HhJfgHS5/xllumX7/R + b1GqH8rgOro/WPKfwT+x9bQ33zL9U/yCvLP53HJ9ooHGoMOI3TPae/S15nmo4CPryzlerrPtQDer + e+o9Dwn7CAhOC1d48WracVBROLb78Jv+7603LATCsD5vuvadVzmCzffjH1jbR7+BJ7dKlmOkUCc+ + k5ezNtIX5/sVsj8TMLg5vvzAZuav49IPeRf8jse8ac+4Emd/FDOfckQ2wwSnPuMzXhEpjm7P+MOu + CER11txY55/GAfRUfPn3zz9y14T/g1+X1yPq19aN87fhgPtefLn49DffPP2H/3FSP+BcvGfKg+sT + DTQGHQrMcbQ391H0I+d5aDSZWF7i6zzq12YHmcsjXu4L8PCinefTj6YdBxWFYxRzPmu5DwSU4Zwv + /NGOPHbUcOSTmED8TNbquN983/vA8i/n+7M8SZVKjuBPma58siNpBs3kM/gcI55MaoZreM6Lt/GX + n8TlaMfJrNF+7b/bzf06XuvLUdhbdhyZT+SnYjO/bp+yzKmIh7NjePvNx6Yvw/el/iq+xPsEz5xF + /l6PqF9bt77OH8VfZfnG62+bvua3bpn4j0CcTVf2q+qmvop9pV3dy8m+GvtKh0aTs2bi6zxcj011 + F512gExrPbR+9GNG+mG/I5LAUWxxBNA4AQM3zgtZyxx2yZd2GM0ns0p4Z71+AYIYhKHOYe8D6xXX + fSkMH9tPTPrOVHWiU0ZRmOVsJDvn+xXyDJd2gjOpOb/kk+R3PI6j22Uc9ut4rYezeohwIRPH4na8 + gIoj9R5pH2m1h570mff8P/Bv+H3bDbdNX/62W6d33Ix/z48psgmrEM7Z65F1inpC5fpipBll/HnD + TdvTM996y/Sd77tj+sP+mgbdmXq5PlG3MaAe3jejnK5U1RP66s9WHOk7j/p1Q91Fl5Vv6yGHxJu0 + 3ohaPN1vj0frH7FwXuxB5AG85LFCSOOGv3S8tVo9fnrVO69uqc0e9z6wVseeS5J+Art00WxsUlw+ + 2Y1LfCg01O0E+PIDA9eUSZ48v+KARfHkJpr5NZ95y5FCFAwK22c+4V/FRr6lz7g4yrw9hHyGDTxH + /tXv3jE95Y2fnn7i9++KdUfazJNdWYVw4l4PTrNuUU+oan1oRjlG/Da36RW/d+f0tN/8tEb+Q6dn + 8uX6RN3GgHo48VFOV6jqCb3fTOYFcp29DKrrXnUXXVa+rYcccp1cdcVBWIun++3xaP1jsThf/mkt + PvCSxwohjRv+0nHkv+eXhcc5sFbXkKSfmMo1UqgTNppVcuAVEef7FfIM1/DlBzbywybH8174XiS6 + EQ4WxRNxJc48xpm3HNHcMChsbx7VWHFLocDm/GFXBKI6427/5VNHpy94003Td//O7dOt+T/4VF/k + z2zZlarTSF31hui6Rz1DVv1pRjnHsOc/C/id7719+qK33Dy9of8bgcCdSZfr0/sq6+F9M8rpClU9 + UTFu6pSzJq6zl0F11foMHPWehwXXyysX/R7rJF4z1htR4RjfnI9I48jna/jpecCOPBWA/ZmvDPUg + 3A7Onj2uzQfWtb91Kcg/l03YT0yXzqnWCRvNKjnw8qWiNK8hz3ANX35gIj/A55hx1EjaBb/jmcdL + fOLsl2LyliOyGSa4i9vxioj+Sm8eyzIvPyGdEcMHb9+env+OW6bn4ntM78fPU9V6MLusB59bnSny + 8npknbwuOS8eCLMx1tN1X+l7Wl/z9lunb7ru1unD+HGJM+1yfdRQvb10KLhOmbEPl6onDw18pFyo + XA+3qfWb9lcdGnzw+ox1Ja8Z9aZDhU4Z4hjm8Jv+jRNQhpwffLajZfGFA+OGv3QcuM+ZXnH9gx3J + /L75wNqersFJKbZ+YirXSIHzvDQyyGjanFcTCxG3E+Bpt+SX3HnTjx13dvvnYiZP2GUcjs/xmhfm + 9cB5y7bPfCI/AsXX7WEQ0wpEBLOQTluB/xfvH7z/9unq37hp+r8/cZcKM6srM8t68JlNuMhf9RaM + dYp6hqyy04xyjmHvdcr6r6f/8Ilj0zPe/Gn9X0XGdaZcrk/UbQyoByvSy+kKVT1j86ec9XDdalmw + HHvUXXRZeeJj36n+rLsZFYfK7QnRYcXSb42Ml+sf15wv84CdcACFA+OGv5o3cms6tP0lydnHzQfW + tHWNTlCQ9xNTudIn/kifY+DoNOfxAG27Qu68HV9+YJL8ORbuOPz2O49XRZj5ddzmLUcKUjAoHEfk + EbIiIqD0kb/kyDH8hHTaDq/9/Tunp/zGjdMP4h9a4PetnNairswu68FnrMtyvb0eWafRF7vWmeai + c7/ILtc5Rv681g9+6PbpqW+6WT/HtegsRnDaXa5P1G0MOBKyDpkS5VY/HhpNLlSuB+BVz6wjQFV3 + 0Qkh0zGffsxYb0Qtnu7X8YOXeq5/XHO+bAvELxxAsU+MY15lqIeGuyY0s2GPA2vnGp2gYOsnplNi + kIzRQXYcvec8HmaOUt4Lz/klf/lJ3hzJvOC333m8qsYsTsdtXnDUQ9BBdhyRR8gCkqf0xlmONBfx + xOxpM7wLP6bw5b954/Qt190yfRw/buB6Rl2QaK1P5pn1YIZYl83rkXUafVE8NMMfdonG4JXfXOcc + gSH/f8f/ofzWd982fRm+v8UfqzidL9c36jYGHUaRbqTnChnPennzp5w1cN1UplHPVj/qXWdYcL1U + eeJzPv2Ysd6ICsd1mq8jkcaRz9ecz/HQsvjaOpuvDPVQuGnz97F2H1j8/tVq63N1gjIZJg2qGvGc + Mj10HJtKshW8j0tF2hu/ib/8JG+OZA2+dGC/83i1iWZ+Wx65ZqUHExw6jsgj5FA0febRwljEk3Ht + 9/ETOAT+99++ZfriN944ve3TPARQmMo781vUlUkxX+L4jHXZvB6cJm70xa51prnoxDTHN7vO/45b + tqcvx6H113B4/QHiPx2v7FflFeVTHVzRli7r0uoXm9/2I3PXuZZlXkfAqu6isydaj/n0Y85602nx + 8DBJvzVSz/WPa87neBS/cADVfvMhVtsm5ws3bfw+1u4Da/vQNSDFQYkgQNJPTKeEafiVPsfAJV6x + tyS63Hk7vvwAnPw5Fi79kHDB73jm8dIucfbruM1bjshmmOBYFHx0vCJSPbo961P07UF0+/52DLH/ + 6w/fNj3lv/7h9KqP3o74ETILw4eogyTORz2odp31YJzUMuBTXYlzHaOe0FIWD55nY6znDH+c9UaH + Tj+Fn7J/Cr5M/CH8NZ/T7dxyfaJuY0D1VXD3o6rphTGeq9P6s6od69J51K8b6i66rHxbD9WfeJO2 + Nx1HgfnaF5jp8WifRSycF3sQeQAv87Ii+DKPMvR84PAvreJs2v19rN0H1rR9DaPWCRqjc6TT1mQ8 + DCjHoTAbrZC+bifA18kMg5m/k+SXf1gWT9hlMR2f4zV/OVKICg8K22f+kZ+KiHxLn3lzjAzrIeR9 + PPzXT901Xf1fPzm9+IZbp5uO8oc/mS8CZmHYVZGnJOW3qGsqiOMzu3KRv9eD0+SLegLq+tqO7miv + Mexn+Ga35E/51mM709/H32O8+jdvml6vH7kH4WlwuT5RtzGgHqpopodMXCHjKXnzp5ypum7Qs9y0 + 2qvuohNCpsQVHk+xDPIjhfwn73wdSaB4uf5xzflsx4iMo0Hml3mUoR4GDuLW+prQ1rD7wFqNn78i + eT8xXToGyR50kBoDl3ixtyS6vBe+/ACc/DkWb/oh4YLf8czjVXFmcTpu85YjspkOCsdhHgaSvAKU + PuYlyzwI4nmfDr+LH1P487910/TVb75x+p3b/KXUyBdBIx8kotF5Z1qLugoWOD6zCTeuB6eJi3qS + HTLdsHtm42ydsv7DbsmfcvLzt0O84J34EYy3nx5/zcf1jbqNAXVhZXo5XSnjWTdv/pQFFp51tp3q + ulfdRSeETMlTeDzFMjgOKlo8Oiz7OklLvwJu4Ms8wEseO2q44S8dDxxg6xMdWIufvyJJPzGVq4Ik + l4PUGLjEK6KWRJf3wpefxu/aws9J8DueebwqwixOx23ecjTCY02BV1PQLuR4KHnERbzM20PI+2i4 + HT8O8LL33To99dc/Of0ifnlcxY8YR74QqFDXe16S8lvUNRWsD5/ZxFUITnR71tH2OS//EGZj2Gsd + E5+jDXkf1x54/pDrF+HHIF78vtunG/l17z69sl9Vtyif6+GYRzkpt/rxUGlypue6eRnEQ4JWP69z + LBPm8STTMZ9+zFhvOoVjFC2OrD/14gs7zJd/epEb2AlXE/O+o+mMjzInt3Z9H2v+hrX4+SuS9BPY + KZkrT3iNgUs8XfUkurwXvvwALD8stmiQ/knwO555vCpCFNN+GVbylqMRHhxabx7VWPZSKLBhz7ha + muFHZPvo9rP4N/Ke8l8+Mf2j37kFv85lUR/EOfKFwIKz3aIOkth9mJjlnQripO6F4IR5a8z1i3m6 + od1snK1T1j9GE/E+ruPg+eNaP/JR/HjGG2+arsVf99mPP77Fekah+4C6qKKod6ZqnPGsmzd/yoWi + QSyD6ip51G+sH91m5b1Ohdc6m1FxUNHiub/fsFar9a7vY80PrGm6xic1isKkkHQ/gV06pyC9ch+4 + xCtlFUVPvoXceTu+/AAtP8DnWLiIZxO/45nHSzsvDgfG6dG85WjQlT7zDzsbLuyTT+blJ6QHfHj3 + zUen/+mNn5xe9Fs3Tr/XfvFU1bmthz9zImQWRl2feWdai7oKFvnzudWZIi+vR4xt3co/MFqHHFs8 + J7Pe83XN9Yox/N+IHyT76++5bXrmm2+a3rTP/pqP66OGi77KeqCuil8Dn/Cn5cVDpcmFqv4OHsnN + DrLqLbqsvNfH8+nHjPVGJP/EkXfOR6RxZPBV60sDXB5gR54KIP2Sz3b5MHCcl8E1gdCw68DSyQ2W + PpKzTmg8p0yGjqNTyVbwPi42Na698Jv4y0/y5mgi8eXNfu2/28GhIPbb8lAtoCo9nmHoODL/iNeK + po954SMC8dDzA3+95Q/xTfX/75PTm/7wqJosm4GRVZ0rbzfTCB+FiToYz/uirqkgTupeCE5k3bJO + oy/KPzCs1lgGMSm+6qPjrPdYN8ab67XZzw34C5DPx/e39sulTZ0F5xjlcz2yDhkt5ZYXKsZNrX5O + CEbJnUe8A1d1F11W3naSIh4N5OPKUKEVIo7inM9aKgSkqDgGn+0Uf/KFA8dDPpklcPjVPJj4PfV2 + jQOL37+aVrOfvyJbPzHJoWA4RpAaA5d48bckurwXvvwALD9swvRzEvyOZx6vqjGL03GbtxyN8KBw + HOZRjWUvhQKrODkf0zMCCQ/s7SZ878b14Hpp0SugWfyYHflCYGGE97wkLjgUSztMOH+peyE40e3t + v+KBHd2QdjaqzmF3Eust/+lnA77ibX7wuC8ubVblG3Ubg9cLUUY5+IQ/0Y+SWL8hY0qX6tt5SJB1 + AaLqIbqsfJ9PP8EXfWD/xDGK4bfWUzjyhR2AYo8EPMAu+Wo+8yhDPQwcRSU0+z7WOLCOHf5jRLCY + jK6PzpFOReGROBvswodCQ91OgNcikg9/Zv42xCPO4Et+xQvL4gk7Lhov5+PR/JisB85btn3mH3YE + iq/bwyCmyZ9+9LwPbq4HwvKiV0Sz+ihsN1PmTwvnlXWj6aKunMp6SN0LwQmq0558Uc+YV9nxPBs3 + 4ZvdrvqeAF95hh8M++pyfaJuY/B6IdJIj0/40+oHmZs665tJSe48JGj1q3qILivvdZIkh+Q14355 + w8K/h7o1rY89MfMcB9bWscdw0ic1ioJkGf04mZEM9fjjnPm0GR8KDXUjH67Om/w5v+QvPxFHx4NI + fHkTLxeTi9T8JM5+7d96gOqB85Ztn/lHvAQSoKHzhx2DWMTDqQfycj0QFlcsuxABzepTMvOFwHoI + b5wkzkOxtJMB6yF11IfPcZV/1S3qCV3x4Jnu0u1GfK47OZf1DVl2ictR8Hm/kmI/Xc436jYG1EMV + bem6QlUf6PubTubkOrhMqutedRddVr6th+oZfQDSetNp8XS/PZ7j9xcjBC95HJhCpr35JNb6DlzY + Ma6tVf2e93FgrTzpkxpkbHKSsgnoMkc8pyzKhks853sSXe68Hb+Jv/xEHB2/5BcvIiueiCtx9tvy + yDUjLsOFQ9tn/paVsfhSH/PCy5wT8bA/BtcDYbFLuOhxzeqDuZEvBMGIj/ykp+GirpzKekjdC8GJ + bm//FQ/s6IbVmo1RP+FOYr2z3nvhndfww5j20+V6RN3G4PVCoKOdXCnjmY83f8qZk+TOo/WJPhZf + 7mMI6gfy0k/Opx9NOw4uEJ50x8DDJP3WSP1x+4vWsBMOj22dzUc9rpwvHCfD32pnw4G1vdakT2ok + wSCYDJuHpjmSO2QMM1ziOQ+FhrqF3Hk7fhN/+Yk4On7JL15EVjz0xyLM/LY8GJ4dKETBBHdxHWfk + R6D4uj35PT0IKtsH/MH1QHz4UB0ioll9MGeZdYPAegjveUmch2JpV/WQuheCE93e/iseOKKbdFej + Agi7k1hvB7w3vuKFL6XFoPbR5XpE3cbg9UKcUQ4+4Y/rz/C1yZvMOV7i6zwkyDqGnnUQL/eFeG3n + +fQDlbS095PuElscuV7kER9RS77wx3iFE6DhyCcxgQ3H+fC3s+kNa2v1GEHoHCw+sT2Ss05iPKe8 + F57zPYkud97yIziTYoiDv/ws4ul8eqZdLELFGXlkHPbb8hiOBh0c2j7zH7ziKX3MS44I2qJlTA/k + 6HogTla0uiLzy7xSZr6IlgUXvus5N+8DzlQ9+Ez+Rf7lP9ahy3ST7moMe+FOYr3T3154znc/DHM/ + Xa5H1G0MXi8EOsrpChnPujGv6M+WkOtgO+W9V91Fl5UhPuokh9EH9I8PFZAjZYnDb4/n+P1Fa9gl + n/yk3+EvEx64sCN+tX4MJV7+kvC160PTzvQ4TvikRhLRhBwZco14TnkvPOd7El3uvAxSsuDhB8/J + n2PhGn7Jbx7zdbvE2W/LI9eMecq/HROnplD+UQ9G1OTB39IMHpHtg1vVlV0XTcKwnF/mlTLrBiUT + E77rObeoq2DkFdr8i/zLv+p2nHUWe/fneGV/nPWer2vwN3zlGfwMeT9drk/vK5dfbyIIdJTTC1P1 + jM2fcuYkedBpnWvfiC/3MQT1A3lddy57xpN+642I/SA945uvo+ZP2F9EwU64INKQ+4x6XOF44DhZ + /j5z4hmFywfWrW+/At/YukgQJgNjn9geVTI2gyig5qikY1zgyeOi6Mm3E+DrpAc6+XNcxiPC4EsP + jmcer4ow8+t4zVuOBh0UjiPzj/xUbNYl9TEvOSJYxJNxPVBjrQ8WPZuBsVSdW13YJBJZGOEjP+E5 + t6grp2jA/KXuheBEt7f/igd2dEO72djiOZn1dsDhZ0P/VZ7hB8O+ulyPqNsYUBdVNNNDzK5U1Q+y + 18u4TEr6zqP1iT4mS9ZddFn5Pp9+zFhvOi2e7rfHc/z+Ih/iII/cOm7HE31HSK5/4cJO86tLJp5R + uHxgbW9dlU59UsfJxiTZDADWiOeUSbAJz/nk03OT98Jv4i8/EQeTkn3jS37Pz+NVERC/4VzNlkeu + WemBKn3mH/lZ0fSZd0szeDKeB3rMOvkzFivpq+pcebuZJAo26kQLwxZ1TQXrxWc21SL/8s/5tm7l + n2b4M5ZBTF7fxOdIHwv+lOUncTkKHn2LZ6VFjn10uT5RtzGgHlmHDNYVqnpC3990CqU6u0yq6151 + F50QMq31UH25zmZ03/B5xNP99ni0/jbT+pV/WsscvOSxQkj7Hf7S8cARNvpuOjY9hjM+sNb4LjwW + m5dPahSFMrz5JPQ8fctn4nIMXOIxTQMNdTsOv+FMas4vucVxPH7FC4Yer4ow89vyGIkoRMFYUzyw + aOILWYumOFLvkQEHfXuojB/QB9cDYXHFsgtZX+XnkQGOfCGw4MJ3PecWdRWMvEKbvwpBZbe3/4on + /QNDd2MZ+BR290E/iXwf3VyPaKAxeL1UhwzWFar6AVH9mRDhWWfWL+qqh+jj0KveotOTrL3+uV7E + m7TeiLRCyTvnI/LE/UUU7MhTAdif86AeVzgeOE7aH83yRxt8YK2mx2RT6+SEcR+dI52GT46xCTqO + TnM++ehL1wnwtFvyp9/iPQ6//dp/t8s4HKfjth5R1QPnLTuOzD/zBJAADRFnyU4v/YT0gA+uB8Jm + l6BueTm/zMsjmyTzp0XmSRvNS9/yTgXrwWfyG0hJV/lXnUZflH+gGBXtNYa97HKdcyTjgj/lvfBL + P6TYT5fidqGj3lkPVTTTQ8iukPGUuA6jnpmT6+AyVT1b/aoeosvKE9/XNfpAXgEkTP6Td/jt8Ry/ + v8gBO/I4ME6E3+EvEx44ouzPYfinGOINC//bkE1HCEcm0UbnSKfhM3E5LvCYJpGGuh2H3/Dd/Ol3 + GY84F/z+DOS4u13G4Xycn/VgqYcIFzJxagryhxwPJdss9ZHhIp6YfcAG1wP5sEvaZnd+zpPBjXwh + MDHhu55zi7oKFvlLrULxqa7yrzpGP0Fb/vGsOuYY9ZPdfdBPFdg+eXB9om5j8HohxtFOqHPUn6Hr + sGoy53i5brZTXfequ+iy8rYrvHiDL/qAHs3PKObrqHnhyOCr1rfWk/OwS76az31WhnoYuLADXuwr + fNsKV7xh4fSKpvZJDRBlgtk8ANaI55RJ0HGJ5zwUGup2HH7Dww+E5M+xeCMecS74FQcsK07qWZyZ + 35YHw7ODQSe4i+u8Ij8Cxdftye/pQaCnfXFzPRAfl5t1iGtWH8xZZt0gCJZ5cSLzW9Q1Fcyfz63O + FHmV/1iHLtNNuqtxtk72V+tuQt7HdQJ85QmLkf0wf6CfXA8WEBUYg9cLwUV6fMKfqIek1p+Q8xJf + 5xFvs4PMOohX/UBeyjmffjTtOGgg/7bTYSkC21nL+AWU4ZzPdoqfPBWA7c0nswQOvwpv9N3Eb1vh + 2pr+5fX8v4OPSqebPsMpFQSlEeCUSbAJz/nk03OT98LXyQxs8ufIKs/sGl/ySw/L4slNFMW0veM1 + bzkShWBQ2D79RX6MSHypj/mYHgQZzQM/uh6Ik10STcaoZvUpmflCYGGEj/yk59yirpzKekjdC8GJ + bm//FQ/s6Cbd1agAwu4k1tsB742vPOFLaTGofXS5HlG3MXi9EGeUg0/4E/0oifUbMqZ0ia/zaH0G + ruohOlaED66fJDkkXtOOg4rCMYo5n7UwOG5/EQU78tgRJ+An85DICc8XjqL9OYzVo/ijDVvTkbse + D/ShdOqTGmQMgqRsHprmiOeURdlwied88um5yZ234zfxl5+Io+OX/OJFZMUTcSXOflseKt6Ik3DV + Snlm/saHouljHgHKruXHx/1wuR6Ij10SzcC4ZvUpmflCYMGFN04S55d1TQXzl7oXghPd3v4rHtXX + dnSXbru+1jnX3YS8jysKL7vE5QhU5YlnpTUs98WT8426jQH1UEVHX0WFqj6Q+5tOJuM6MG/nazn6 + GCDKqrcKrieZjnmvRJTVcRDW4ul+xS8tHQq4gc/xqH/IUwFkPNF3ckP/9JY4So7//2fvXYB2za6y + wPc/nU5CEjAoXoay8IIZCAm5QBDRGhFrlKmxykJriDrKCASJiKMU4zhT5eCoU17GKRwZLEdrnBKm + RDOiI1rlBS1jB+SWEGIIBJAAIUAg5Nb3TtLd55/nsp611/ue7z//6aS7z59071Pn23vt9axnrfXs + /b3f13+fPl1l3LG9/y0vwP+b4tqvoitJ/aT2k43dryczgkXhFoRTGOgK1/Pg41KjmjrFT3/nwVp5 + ePlqv3mTxwF87eF69vUybt/XyCPxEN51YS0486Z/48sx/LW/6JunC7rNiz4f3hLqUKN17r7TLwCC + EZ++09ZBV3Ixnjiuh840OTq/cKVn7TMN43bzqOdWznudG+vIeV2cB+mu1Mh9bR0tJ3SRomkPNVup + 1hN238/Rkfw6t9L1It1FF+V9TrKkP/UzaX8jGvXMvLOem98v8oGXPE6kBIw3n/Ml8cJVnHB1n+44 + /7X4Gdb1ZyukLvXuyUwwLwMAPWMdW5R1WXdxdvB1jZvwE3SKv/NUHWxKeRywuCuelTVP1YUN4Vzf + yJMzaz9g1BQ2RZv4cgy/ccZXGcVT1m2fVD+q4KXIZWBR7q/qb5v9wqDgwk8/9w66CkZeoc1/6L/z + c3+cW+cXq+P7nLGnuOAzJx/njMp3Ef6YJ2FXZVbduUC6d5LT54Uil5w+GOOp17ifoxnr4LjWc+jX + eohOCEWv/eQxqe8N19wPb70vZNc+/ciTsedzHDtrvnFufp9VZPbFxwTcP96768/GA+uaH1iV1E9q + iEIbJH4SsiYmJYW55Kc9cMFjmw5N/VL2RfhT/Mp3i/yuZ1+vTn2Xd/SxGlGJgkl7i+s6qz+JSD1m + fOzqsPJ0v7d5YT1QL09s3f51jkMXvQmih/DpmzMbOejKLTqoh9wShqsenV+4uk/wch9oxe1mJ5K/ + 71HdP5GWfySobdZR/JnJf8jTcVdkYX1KtzVBFykqeV0q7aEf/D4v49KO+CYP9Tqlh+ii/NBJ+jJP + sjKea2+IbtZRwMvvFznASx6lDV/6oB9jx0ebm+7bYdy4hgfW9XP8TaP0qTpMmBE8Z0L7yYl1bIfd + iOd++LQe9uTtPIKzKZa4+DvPoR7SHfnFC4aus/oIznlHHyvRokNCx6d/26pIfPHXvvAK50YtrsZk + PVAWFa3LwMp2+rTNfmFQcOGNk6W2DrrGwf65Jv+h/84v3UpPQDs/w2hnrnjF3cJ5J99F+GMepLlS + Q3VTAenTE/SQomkPNVsh42n5zR87TcledNL55PtLdFGeeet9J/3rHigrgISNevjQSd6e6b/p/SIH + 4oQjHQtI3pWv9xtHlPOpWsbhWXUNe35gVVI/qf1kI0l/UvESicIt+BOCtVql3axc6pYrj5vwE9B5 + sFae5LtFftezr1ci7PKOPFJBiZnemiOx60j/tlWR6oi/9oVXODdqcTUm64Gy8CuXgZW1zkMXvQmi + h/DVn/CMOujKregh9xSCGzPe+bsexOl8gdnNo57cu55NyNc1LsF3n4hgnqs2rEfptiafF4qt9rjC + 77qPsqjfsrGlIb7JQ4J6/xLQeoiOinAx95NH266DsMaxipXX9dPrPI468tGmB3HCkW7mJV9FZr9x + FYd9VUs/nlXrgTWefGTxE9szOftJzJxli5JxBzz3KdZu3ITfcDbF1hZ/57kFfj/x9/VKjV3e0cdK + pDIFQ0L3mf5tqyIC2l/7shXOjVpcjcl6oCwqysOu0ec4dPEnJwCCrT4ZYthB1zjYP9fkP/Tf+bmf + 8xPf4ZwZXvuYpH/jR9yRP7byBJe5eMSLNeerNqxP6bYmnxeKXXJS4bqP3Mcvn5eU77asg+PU90W6 + i04IxTKu8cpjSt8brp1HdLOOKvDy+0UO1E8eJ+KGzrnvnTe83ziah3unB9b29DcsSVNn6E8iHCIu + v7Tj7EXbPvP4GY0hnJdX4fXWPwF9mdKmbhXvVvXjyXp03+m37qDeXYf+Vzx1Kj0lE/Mpy37ufAM/ + 4m7Q9xI88888V+FMZg3WhwKue+R6YWNUe1zh99Cv3vzRl1gO2Yuu7BEXPUQXZRwnSwmJLz7mpUP5 + iaO557OXDgEV2LoXkSfEha/3x70TkRMvHDedb9THH7qfP/1vCSmNVOFscdcnVjt0drtPpHlW49Ao + 9e0eqh9FXP4JmH4B1p1Bv+qLfZcu2Nj1HQdxXOs2G0+To/NTF/inzTRE7+bST7jgM5uQr2tcgu96 + EcE8V21YDwpIfXryeaHYao8r/B76wX7KfsPazvDAOj97+t8S8krUu8efEH6y8674k0wL3Z3dJ0ht + I7wItLoSL66bV73eDVXVrn6VPT7p+N4QPn2nLetBd3glGPtniN50WtHSCE4z/NMWD1C7WQdQ/MFn + JmP5RT7sHf/Ac3/yd9wVWVgPCsjz6cnnhRpXu9R16AfE/KaTdqzD4BHviIseoosyxJdOSki8Gfub + jk+4ytzzEXn5/SIKceRRWidw3pUviReu4mZ95/wZVv6REIctCGeCxuwembRycr4Jvog09csl+OMn + Ytujjq6LpMUXftfjulOvVZ59uW77EdmLooPtvOnftoCqI/7aF74qONSTum7XbD1QJ29JbiGK2ena + NvuFQT2Er/7k595BV25FD7mnENyY8eM+1T7TJF3PpZ/qPtw/8rlArfxyCb77BFptjdCrsPT5lG5r + 8nmhwGqPK/yu+yjLb37HY6OGdXOc9NX5jDjY3keA7gN5ic9+8mjbdTBA+cO757MXcTe9X0Qhjjxd + QPKSj36MWiwcN52v+8Gzav3QvZL6SY0maLMZzGolM7lJdRM8U80mpj15w2945YER/syNq3omn9Z4 + cT37eiXCrk7jzNuJFh0c7jf9L1710/7al10VVJ6ybvvU58Nb0rci/aWv2OwXJVMY4aefewddBSOv + 0OY/9N/5uT/Ozfo6jumS9iR+xLlAJq5R+RQXXGZAjnkSdlVm91sXaE3Qg4qw/lRqhVof+J+y37B2 + P3THYXP4Se0nG1WbT+C+XMFlLlzw5Jlvkmmf4jccbwqG4bdmXj7arOsW+IXjYR7iUod5zGfeToRF + lQuH49N/5WclqiP+2q/tRaDVlXixHqhTTxV27LHTB1urXxiCUe/qT37GHXTlVvSQewrBjRk/zq/2 + mabPGWvZ5OO+eKN/zXbI3y+X4LtPBJD/qg31aaFL79JBylhe12yljKdufF8MXaox6+a41jPvG2Ba + D9HlBOZ+8iQrbAnH/fCuvLOe8XQdeVYcidY3p+ynD+fTfWIe/FJewZyv+9E3LPxzoULQHMepTyzG + Hj+x+onPOFyeXZyJ+LrGTfgJOsWfvLfC73pcx4wDsWpwfSOPVFDi8mNCoOtIP7bLMfy1L7zCuVGL + qzFZD5TF0683Nytzf1V/2+wXBoUTfvq5d9BVMPIKbf5D/52f+7kfZIfNNEnXc8UrLvjMycc54xL8 + MU/CrspsfeoCrcnnhSKrPa7wu+6jLL/5HY+NGtbNcdKXBEO/1kN0OQHi6zyUsO4BOPtnScof3lFH + FXj5/WKBiCOPC1PFzrvypeGFq7h9fU//W8K8mXG2PCWYFnd9YrVj+I0zXvpzoxZXY7r1T8D06/51 + q0oHduK2bviks6PuoC7bof/Oz31euvJzRtiND63hb/yIu0HfS/DHPOzlKg3rIaHrXmmCLr5H1R5K + 9v1r/WDzTR07PcledPYP/eiX7qLLCfAYs588ZuxvOqOemTf5jSNfxe34fE3YWfNVY87LPjpQi4Wj + 6T5dN4D4F4T4ofvT/5ZQ0tQZric/NqAR7Vq0raPlvvyMxhDOy6vw6rp55KyTFXu4v/TlmZckbTJi + 9U0/4+jnpVtxcrB/uacQ3Fg41cFLXPo0DzDiyzz8rPcYp3wkzrgEf8yTsKsyW4/SbU0+LxRZ7XGF + 30M/2D4vKd/tWC/HSVcSREfx5fxIF+V9To1XHlP2Nx2fsOqZeV0/q3OeFNK69/nQg/qFw7L300dF + Zr9xFYf9VZ//LaEj1AR7gZugMUsy2JqZE7/7CTtwHadcTDMGcRiTd+K5f+TvPId6ikhTXlyP655x + SFhw9uX89mO7F9y37TrSf/pU4CE+fFVB5Snrtk/WA/XzuOsysKjWeejCy5T+GRGdjOfrQdc4KIvc + pQ/XNTo/iXN+8HV+rCV/5lFP40ecCyxyTpfgj3lG5JVYWp95r6KHFE17bBS/6z7K8ps/+qYZ2YtO + OreOjINe0lt0UX7uJ48ZfW+4XvX4nsSuWfeFfB4rT3DcR/3CYTnOre8dIdlvHDdvvHf8Yw1305VL + 7Sc1muMlZ5O8NHLXjHVshy1c8NwPn9bDnrwT33mADX/mxlU94mR9Y4gXkc1T9acO52VZ6QfBTiAW + 0cG2P/3bFlB88de+8FWECEZBt3mpflGDP9nYqMfq3/qtfuEXjOeZvjkz7qArt6KH3FMIbsx48pWe + tc80pN3NpZ/qDj4zsFUIVx6X4LtPoJnnqg31aaFVYMnp80Kx1R5X+D30gz2/6aQv6+Y46SrCEQfb + +6SL8sRnP3nM6HvDNffDu+fTPv3iE+zA5zjVL1wRaUofjkvDnVdpna/7wbMKf6zh/AMKqaR+UoOM + NpvBrFYyMxl+y5+5cMFPPq35chN+uysPjPBnbt7kGXxccriefb0SYZfXOPMiqBdVHmz3m/4Xr+pv + f+3LVvoiqPUVmPp88qboMtd5cmv1C4N6CF/9yc+9g67coq7sX+4pBDdm/LpH2Wcaxu3m3TlF/5od + yNc1LsG7r5VnBV6Nlc+ndFsTdJGikteVWql5nv5mYly6kX/y6HyWfq2H6KK8z0mW9CQ+WbGgY9Qz + 88569D6rQlYeE5kPvORxIiGNW/mSeOEIc/1dH55V/BmWH1h4GAjCGVn8xPbM1P0kxjr2RXjuzyam + PXk7j+BsiiUu/s5zqGfyac041b+vVyLs+hp9rESLDgndZ/pfvOqn/bUvuyqoPKnnds/WA3VS0dxC + FOX+0lds9gsnBRd++rl30FUw8gpt/kP/nZ/7OT+yw2aapOu54hUXfObk45xxCf6YJ2FXZbY+dYHW + 5PNCkdUeV/hd91EW9Vs2tjSsm+OkLwmGfq2H6HICxNd5KGHdAzD2Nx3lD+/K6/qDI5/Hns9xqp88 + LkxA41a+NNx5WWf12f3gWbX+kRDNCcKZTYzZPYJcFCySWlyMLyJN/XIJvp/MCAh/5mM9p/hdj+ue + cTw0Dvfj2X5s9oL7tl1H+k+fABKgqXRom+wYlcfG7X+1HiiLt0SX0TW5v/Tl2Z+c8FMP4aefewdd + BSs95C59uK7R+aVT6Qlf58ea6Up27TNUcYf7x/0b9C29L8If84jjCr2obl8oCVHXyee1a9cKGU+9 + eP+WnmnJOlgm6XqR7qKL8ta78eI1Y3/T0QmFd+Wd9dz8fpEPceRxIiVgfN877uQ8G8dN51v18R8J + z5/+GZakqTNcT35sUOP65NGhlO0zj5/RGOOh4I3b++q6eeSskxV7uL/05VlvguqfEatv+hmHy8WH + CFbhlYN6yM2FVrQ0gtPMy1l+zuIBajcPP+s9xh35Y+9wiQP3MY+rujqvqpsKsO81+bxUf2qlrkM/ + IHxeJ/SePOIdcdFddFF+6CT9iXde3xuuvSG6WUcBL79f5AAveZQ2fOmDfowdH21uun6HKR4PrO3p + n2FJmjrD9eTHBjXSm12Ltqml9mub8fOhIPs2v7hulMVbkluoMnl50pdnXqa0yYjVN/1shP59nBzs + X+4pBDfM2zPydz3hgVN1ZHYi44LPbCK+rnEJvutFBPNctWE9Src1+bxQbLXHFX4P/WD7vKR8tyW+ + yUOCoV/rIToq4vi1L0fn7W9EjWMVo47oTz/yZOz50gfihGPamZd8FZn9xnHf+VSt/XxgPf0zLElT + Z+hPZj/ZfRbt0Bn3JzcPaZ7VOLQ6gts6+RPcl2zdCt4tXh7P7tuXSeXr8qQv9k0cX63HjJOD/cvN + hfE0OTo/93HZpi0eYHZzxQsXfGYT8nWNS/DdJyKY56oN61G6rUkPBda65KSuQz++iYedvqyb46Tr + RbqLTgiFtk5KyDxm7G9EPmHtz7yuH/no98PkBF/6AK9wgFQC5135er9xpHPf3Y9+6H5W/0hYSf2k + 9pONJPOJaelYJGskTc2FC74cmvrlEnznQUD4Mzdv8pB0iGTTh9A89FOcXV7Xa95OxHDDBLe46q9s + VSS+GU/+ph8L0d32lz4fHLp0qIp2+qjv9AuDwgjvPmVRVjiOcRKM/cs9heDGjHf+rgc6Mg3jdvPu + nPb3j3w5R62HLd7ci8yC7/N03BVZWI/SbU3QRYqOdq1U6wf//KaTdqyDZZKuuq+l49RDdFHe59R4 + nbMZ+xvRqGfmnfXc/H6RD3WQx4mUgPHmc740vHAVl/ui+zF/6M43N4af1CCrN7ufhN63dHXZboIv + Ik39cgm+8yDAmrLJ0/WIs/jCr3oR0TxVf8SUn5ph37ydaNG1P/1XfkaIb8ZDr9reEaSg2zi/9BPu + 3H7nr6i/5oy3RIftglb/67x5SSQnhRE+fbttNnqMaz0YQn4R0PCQ3lhq5qUrf/PAx3SsQvPwk+8Y + d+SPvcMlrvLu+LF3VYberBGcc8nnenMuqdYKtX5QzOdlXKOOPLJP6C66KOPzkVX1aAJpfyPSCRHH + Mvd8zG0cGTz6fIvIE+LI40QCGke+DvR+42g636pv/tCdl44QzmCZs3tk0soZXOYDHtsk0tQvN+E3 + /Eb+5D3WI84Dv+pFhboMAKT+1GHb++YFqBdVLmzHp//ikYjUJf7al61quFGEZd/G6Zc/69r2jz/n + E7dvfcUnbr/mOfynfjbqsdMHW6tfGIKtPhlhmQ+6xsH+uSb/yfMwf5+f+A7nzPDax6R6Gp97ZQdf + 16h8PtecV81AdZ9Yi39F3vaV3qy5L5xLPtcpRYectEdffBMPO81YBx9D6zn0az1EtxRZ+8ljxv6m + 4xNWPTOv8gFqHPk89nyuR/WTx4UJaBz76kDvN46m++5+8C8I1w/d61LrSQ6WObtHBIuCRYLqJnim + 4qXbjUvwOkSG4Xf4M7OrWY94D/yux7gZlzoc77rt70SLDg7XkXy2VRHztb/2ZSucG7W4OtN/8Sue + tb3xt37S9uf+0+duH3eH63N/6cuzPzlRN4XhCVSfshR20DUO4rjWm878NDl8HjXn/GqfaYjezZfc + jxv0vQTffVYeTFdqWB8JXXpHD+u4rpOVaj2h3Pymk6bkX3TW/5TuoovyPh9ZSshzNmN/I/IJa3/m + nfWsp86Rjzb5wEseJ1ICxve9404lXjhuHu6dHljn9UN3XjpC6vLN2T0yaeUMLjOTzzjszyZoxt7h + Eif3jfzJy2Z2cYOPSw75USHnGbfPa5z9COpFlQfb8cm3eMXT/tqXrfRFUOsrND3z2tn2tZ/6vO1N + v/WXbV/8yc/e64M6V78wqAdvVfUpS9fioGscxHHNy1b3hyaHz6PmnF/tMw3jdnPFKy74zMAe+WNf + hHdfKw8prtKwPqXbmqCLFE17KNlKGU+L97vu52jIOlgm6Xp4P7YeoovyxI/3i3hNqjoIG/XMvLOe + PGyE3vG5Hp50841zNp/zpeGF4/7h3t3BP+l+7ezp/5aQ0tQZric/NnC4/iTRom2fefwluAhqfQWn + T372Hdv//dJfsr32837p9mnPu6P6cn96E1T/vCSr79IFG9YlepRglIW91ptjtm3dCs9LXPo0D8Pw + O2mnn3yyM5P4qO/gO4U/5iHFVRrud96r6CFFR7tWqPXhm7jOY/ZjvSzTup8ndBddlPf5NF68Zu1v + Oj5h1TPzznrysGEk9xdfjg11kMcOJTCO9TlfFgvHfdfffOfXnv6T7hGb7zlpyjcJfq1PrHYMP6Rk + AM+Abo5e2Lyqr694/p3b9/6WX7Z9/Quft33infhzw90vKtblSV9uzG0dPunYHB11B32bI4Q7709g + 4UpPhVFfhOP3bnYi1UM+67vilM/Ufr0E775Wnhl6FdbWhwKWjp6gi3Ws9lCqlWo9Yff9HI1Yr6a7 + Qb/WQ3RRnvg6DyWk3iZVHYSNembeWU8eNkLv+FwPT7r5KoHzrnxJvHBk8/mrWsfhZ1jXr/8CXUnq + JzWa4KOPyccT09K5BfkVtnDBTz6t+VKP0sk78Z0HUGvKJhl2a/yuZ1+vRNjlNZ95OxEWVR4criP9 + V35Wojrir/3aXgRaXfkX/FPi9hWf8nHbm/6zT8T8HLTGflE2heFjpHSQxX1s9PmUnq2H3FMIbpQ+ + meseZZ9pkq7n3TlF/5odyNc1LsF3vYhQWyvySqxyX1tHyw5dJLjPQ5VaIeOpm9/8sdOM7DoG6Ut9 + TukuupyAz6nxOmczqg46Rj18mCRvz/T7YaJA7i8+8nMbccKRThviMZ/C1n7jKm7y4Vl1bXvO834S + hI8mqZ6cBLGImt0jkzI1tjlXkRMXPFOFT+thX4Tn/pG/8xzqEecQyfSurHmq/tThvK7bvIjqRZUL + 2/Hp37aA4ou/9oVXNdyoxUfPxG9YX//C527f/XmfuH3WJzzDevCESwd24rash+WqPqMHQTifY//S + W/HforHqAABAAElEQVTkKz3LFg/DaGcu/XxO0X/FHfljX4T3OS5+pLlSw/qUbmuCHtZ3XScrZDz7 + 8Zs/dpqSPXl0Pks/+qW36LRS6NpPHjOqDsJGPXpYznOSF3E8/xp7PkSTFpmbb8SbrwO1WLiK67qv + P7o9/My3Xdv+0Avuhetnk9RPajRXl3A+Md1SXYIqcuJYnWzlWk2okkvwnQdg5QE+c/PehN95nX/G + 7fuituHtRKs8BNqf/m2rItbf/tqXrXBu1OKjb3rh856xvfZzn7/9nRd//ParnnVH98lO3NZB1zjY + P9e8hIf+fR7cpm7rXrT+DMNvxmuu+B1+xB35Y1+EP+ZBmis1VLcvlASQTKhQ30Q493WyQsbbr4fW + Aqgv6+C41nPo13qILsoTn/dD8lim/kbkE1Y9M++sR+fvsANf+sD5k8eFdb3m60DvN47mundg+tnt + q190P/5Yg/bfnqR+UqOJuoTziemWEMoQ+jOz6YEvh6Z+uQTfeRAQ/sxU6zJ+12PcjENglYMZjs4j + 8eBqvxPbn3zpU4GH+PCJvnnK+qic/qv/5FnbD/yW529f++s/bnuGblfaOujK7nTemLjmm6d0pMnh + 86g551f7Oh+sd3OfA3WN/jWbkK9rXILvc0YE81y1YX1KtzVBTyk65KS9dNCbfNjpS3yTR+cz4mBT + B8nG8+o82U8eM6oOwhrHKvZ89vq8HOXzdp7ZB+LI0wUER76KrMXCcd/5XMb527njB9bZ2Y/q0hFS + l2/O5OwnMdaxsTyJ5374tB725GW1sgVnUwjD7/BnbtzAH/nNs57IyROc7dHHSsTqDENC4nQppINt + VTTsVVfFNYGoPqpfnoM/r/VnP/U52/f95udvX/hJzyz5DrqmX+rFNS8b9RnD51H6jXOzvo6TjgzH + 75P4EXfkj6244DIX3+QfpV2Jpfst3dYEPa3jktMXtfWBv+/n6MQ6+Bhaz1N6iC7KWPfGgzl5+xvR + qGfmnfWsp86Rz/XwhJuvEjDefNVE9pmvy1v3DoX9KJH1Dev8R5PUT2qQoVlWz9k91oyg2CSYuOC5 + Hz6th30RvvMAG/7MzVv1iJP1jSFeRDZP1Z86nNf1mhfBvahyYTs+/dsWUHzx177wVcShnlHaR+Xy + 1z/nju01L3ve9m0v//jt1+CPROx0ZUfRg2tetkP/0luwdY8IbR6G0c5c8YrLOWcG5sgf+yL8MQ8p + rtJQ3VQgOlImFMg3MceS0woZb//8piOw8CYoOsTTrntcfvGLLsozz3p/s4LkVR2EjXpm3lmPzp9Q + jD0fbe6ClzxdQHArXxIvXMVVffCPB9bZtae/YdUZric/NnTm7WjbZx4/hcXgm/ZjcHz+L71ze/3n + /ZLtz/+G5+hPy+eT1Q8p30FdtkP/wWnmpSs/Z+kHrXbz8JPvGHeDvpfgj3mu2tFYD12wulfRw/eo + 2kPZvn/G0/KbP3b6kr3obtCPfuktuijPa5v95DFjfyNSfuJY3/4ciTSOfB57PscxsvmqMePI14Fa + LBxN5xM7n1EY/oZ1x/Wnv2HVGfoTAofIjwYeEmcv2tbRtp8yYgjn5cfa6zNxS/7Er3n29sbf9Anb + F//KO91e+qel26xr1a1bt9IP/mlLP4bhd6m785PP+q+4G/QtvXe4xJEX/snfhV2RhfVAhdHR1w16 + WMdqrxUynnr5zR877chedDfoR7/0kOBRZuikhNTbjP1NZ9TDh0ny9kw/z7/GyjP7QJxwAFUC41a+ + 3m8cCZ1P7HxGYfiB9dyX/Qx8jwrC5GxuzO6RScclqyInruNMxNc1LsH3kxkRu3yjjpvxqw5ENk/F + RUzXSW2rj5xZ1+XE9qd/41WR+GY8daJdLfZitfyxtvpV+I+q//ZnPHf715/9vO1F+NPy6p9N8hIe + +pfecFn30rPs3T1ieHCZD/cP2zfwJ9+OP3HFM/OI4wq9WJ/SbU14G/pCLTl9UVtPvonxK3Zasg6W + qfU8pYfooozPp/HiNWN/0xn1zLzJbxwZKg6FL74cG+oljx0CMt58Hej9xtF0n2jj/o3PKAw/sF55 + hj+Htb1NEHh5GfQErdk9Mmnl5Exc5gO+HJr65RJ8P5kRsMt3op5T/K7HdSdel3qX13Xb34kWHRyu + I/2nTzlUWNepuuhX+FiU/TE8vQJ/Zut1r/j47a992sdtz78T2uD8lxBu3OdR+uV+wNX6YY0o3+Ha + xyT/8f5x/8gfW3nCn7l4Jr84rtCL9Snd1qQ3Mcvse1UKtZ6w+eaPnZasg+PUt+7nwtHvfUTwvMRL + fPb1NOm8/Y2ocTyvPR9zG0c+jz2f62Fk81VjxpGvA7VYOJqV7+z8JzY+ozD8wOJq80/h/aRGE2yK + zfAS0JsZ69iKGrjguW9RtPILcRiTd+JP8XeeqmPij/ziRWXNU3UF57yjj5xZ14XikNDx6b/qtWP4 + 0wdntTUWZX+MT/zT8l/6yc/c3vi5H7+9CvM1bozh84hOpSf8fT5Y63wz9zmA5xbOO8L7XHNeF+cZ + pV2JpfXRhat7FT2sY98rvGnpaT3rzR87zVgHX8Mb3jcA0e990kX5uZ88ZuxvRMof3lFHzot+8VVc + 55l9IE44FSKg6yGf47JYOO5XvvOztxdqPrD8U3g++Rg8Z7XCSyQKuDlXkRPXcWQfTdCMfRGe+0f+ + znOoZ/JpLXofQvMwP9XY1em6zYugXhRMcNaR/o0XUHwznvwVxyIqD5dPpfH8Z5xtf/UFz96+87Of + u/3GT8A/JtbQOWO9O++yJTvWu3l3TtG/ZnIe9b0E3/eg8pDiKg3rUxdoTXiL8h7Pdte91j7fxLmf + QvrFOjtOulKfvG/EV+8v0UV5n0/jxVt8rIOOUc/M6/rpdR5HHflo04NzFI502tC9MF9FZr9xFcf9 + a342cWd9w6qfwvtJjebqzb6ezEw6Lhn9tAeOxci2Q/5+uQTfeRCgPMBnbt6b8Duv8884HhqH6/Rs + fycqv23Xkf4rTiKCB4Fdp/qmrfCxKPspNn36c+/Y/uXLn7v9nRd+3PYrn4mruNN93YvWz3LrPdHn + hT3F5ZwzU8sWmgbGKf6BP+Zx0NV5tT66UHWv6t77KZH22Ch+D/3qze/41Y91syyt5yk9RCeEglsn + 6ck85uTDRG/AUQ8fOsnbs3DkqzgQdH5smQ9x4asEzrvyJfHCka/ynfvfEHJnPbCuP/p2bvhJjaRo + liSc3WPNwMS+CM99PmR2o+zJG37Db+TvPFXHxB/5xYvKZr0SYZfX/ZkXWXtR5cJ2fPq3LSB52l/7 + sqvLY7+75p86xu/5FXfiHxOft33tpzxrw8/opec8tz6fyJ95d07Rv2bKd9T3EvwxDymu0mB9+3tV + 19FPidGuccZDBr6J657PfuSv+4jpYt1FJ4TCW6eqJzL3N6JRz8w769H7rIrZ8+XYcI7kcWEjL8+3 + A73fOJp1/tdPfcN6xjP1rw315ATLnN0jk0ZUzONJyawTr8zl15ovl+AZf+RP3lvhdz2uY8bt87pu + +1FTL6o82K4j/dgWkPW3v/ZlV4fw11/qWRtP3enj8POsP/PrnrV962fyb4KgbqUnJLnhnLHnY+Bt + jq7Rf8XlHAUyUMsd/yV5OvY2L/Smpi6+UHPSw4jlya06jVOf3OebGL9iCyI8dXZc63lKD9EJYXYk + arx4zdjfdJCPQ8c48ia/cWTw4P7iSx+olzx2CGgc++hA7zeOZvX5DP+RBu6sb1hf9iL8NTPnb9aT + k0nRLNnmE1O9Ikgz/RgTF3w5NPXLJfjOg4DwZ27e1EXS4gu/6kBk81T9wblO12veTiQK0cHh+PRf + /UlE6hF/7cuuCkDwqCVJSU/Z+QEI8b/81Ae3L37Lg9Kzzw+K9PlgrXPIXOfpc4r+NVPJ8nOpcQn+ + VJ6E3u5Zb1bVXxdoTXiL5n2VKmkvHfSwGnajyDd5ZI842NJbdFoptHWqejTB09+IRj16WBZA59Q4 + 8nns+XJsqIM8XUDuAevrQC0Wjib919+8fdnL/ZeMYmc9sATf7tKTEyxzdo9MWjk516Nx4pg9+1iY + Ma+X4Bl35E/e5r0Jv/M6/4xLHa7TdduPwnrBfduuI/2nTwAJ0FR1tl0NiiDNPjVnyvkP3/Xw9jmv + v3/76+/44PbB66XfOLcbzhkxPgbfF59T9H/87tNVOxH16QtV96p00Lu67qOK9sU0Hvt8E+NX7PRl + 3fqa2n9Kd9FJcbPj3i79yWvG/kY06pl5k984MlTcji99gJc8TiQg483Xgd5vHE0EnJ3dJUe93PDA + 0pOTZGiW1c8npqXDNqnoz3zAl0NTv1yC7zwICH/m1NEzSYsv/K5nXy/xwckvk+LVdi+W7TrSP/el + sgG7eOpTcSfqSV1PlfmH7n90+50/8MD2VT/60PauD+Eqnjpv6V76Yy35M5/C515RxMN5x/a55rxq + FnyfhxRXadx4r6JH3lep1vev9cSbWA+tgx7Woa+p9R/60d/Xne8LPT2Iz37yOC8fJgpoHM29vkQa + Rz6PPZ/rYWTzVd3Gka8DtVg4muC9frMH1p3P/k6ArpPFT2zPaoXNm8Kzmgaa8wGvzOXXmi+X4P3E + VYnNn7y3wq86ENk8Vdc+r+s1L2rqRZUH2/Hp37aA4ou/9oWvDo/91vbH+vQePJz+5I89tP22Nz6w + /cB9+rN9atnnEZ1KT3j6fLCW/JkvuR85R5Hz5RL8MU/HXZGF9akLtCY9FEZ7bBS/h3715o++aUf2 + 5NF9HXGwpbfotFJo6yQ9iTdjfyNSfsvNh0ny9kx/P3XG+RaRJ8QJB+7e90Ms+Xq/cewaxHc8epcr + 8uv+G9YffuF78Uh7C4P5BMzsHpl0XLIqcuKCF/VoYtoX4f3E3fMnb/OmLhIe+MWLCpun6g/OeRlW + feTMikeTtLe4Ew8hnK/95sn27E/rp8DLw9Dvb/7MB7fP+r77tr/3Cw9L19m29MOGdaz7VLbOFevd + 3Oew7l2fO4nLz6XGJfg+Z4CZ56oN66MLNa8X36QqdbXri9p68qGBX7HTl+xFd7HuoovyPh9ZSkhe + M/Y3nVHPzJv8xpGh4kCw+HJs4CWPHQIy3nwd6P3GwTw/+8H58ysC9g8shVy7i1X7ie3ZPTLpuGTj + SXnEi6YfnbIQ6KYmb8eRlw+jA3/yNi51kfLA7yf+vl7GBee8I4/EWzyiE9ziTjyUM0/7zZNtlpM8 + Wn+Mv7zufQ9vn/f6+7b/6W0f2PgD9qlzWvd5RKe6T5LpcM7YA4POn7HWfX//uH+Dvo/xPonjCr1Y + n3mvSge9q2e7vqitJ/x888dOS9bNca3neL/Q731E8H3RebKfPGbsb0SNY30rb/IbR76K6zx5v3Mf + ceTpAlhn+uhALRYO5tn5XeXt6cQD6/pdvBx+YntWK2weYWm1n7D1UJh4sUuUzoNAN7XDJQ95T/An + 77GeU/yuZ18v4/Z5R57VyKIT3OK6TuN1uOpzxoNg0Xee0fHH3PInH3p0+wM/eP/2e978wPaTD+If + /9g/uxw6p2mfR+l3s3NmOH6fxI+4nGP4Y/uc9veVGO6LF2vOV224Xwq47pHrlaJpD2XTrv5kjfs5 + mrIOTaf++33DuOghuigz95PHpP2NyCdcZY46WDd56ef511h5yq8JccIxIPvpowO1WDiYh59fEXDj + A4s/x7p+ft1PbJDy0gDYM9axSTBxLEa2HXxdo5q6CH+Kv/OENzNZh0g2fQjNQz/F2eUdfeTM2g8W + wS2u66z+7Bj+2l/0nYe1fKwNfov68z/x0Pabvvfe7dvf+4jbk75om9bQOb1LPxjWcd2LPh+G0Z+5 + z8HndozLOQLucQn+mCdhV2VWf7t7FT2k6GjXChlPvcb9HM1YL+pdPFqc0F10Ud7nI0t6El/y8mTo + 0AmFd89nL3ECKrB1LyJPiAtf76cPhTmBsjnvOd2Hn18ReeMDiz/Hura9xU9sRPEhAWDPIrVNgolj + t7Lt4Osa1dRF+FP8ydu8N+F33n29Pj2L6byjD50Syuu6sJb2FnfiyzH86bvDx2K1/NG+okSv+fkP + bi//7nu3v/72h7ZHsGGd4aBuvluYJdyu3eCs47oXtHWuQO/mPgfyFj4zmcvfSS7BH/N03BVZWJ/S + bU14l1L12S7toR/8emgd9LDOfSw+p6Ff6yG6KE/8en87j9K7DsJGPTOv66fX5+WoIx9telC/cKTT + RuVlXxWZ/eCubzf8/IrIGx9Y3D0/v8tPbDTDprHVM91lY9I+s04897GhqV/K3uESJ3jlwTr8mW+F + X7yI7DqZjyLs8o4+cmbtd2LHp5/qjxWJb8aT39vqsXi0/hh4+Q/3PrJ9wffds33VWx/Y3v2hR/e6 + sr/owfXQmSaHz6Pmm50zsH3OiQs+swn5ukafm89Z+Qa+7wEiyH/VhvWZ96p00GN83CvZdR/RxFPl + Gxb+g+e7Tp3Z6QfWdsddfmLzCQixENmzRLNNwonjJZZtB1/XAA/HRfhT/MnbvDfhd959vX5Tzbyj + D247QdVl23VUH3ovFJD1l73qIp/Cx6Lsj9LpFz90fftqPKT4sHoz/piC+zvoyt6iB9d6eEUIbljn + nse53XDOALWeiQs+s4n4ukYJr3MPLjNQxzwr8GqsVLcvVN2r0qEer32v/DhXP6yc31TmN510Yx36 + WIw/pYeusxRXaOukhDxnM/Y3olHPzOv6XY/OvwrZ87kedtZ849zM14Guh/lY3qP7P39VqAu+Yd15 + 5+7PY7lHJo2omCEGx+6TDcVkfzZRwJviGXfkT16quMvjxOLLi/MaN+NSh+Ndr/2I7AX3bbuO5Euf + ABKgqepsuyoQQar56Jv5xxS+8e0Pbi/793dv3/JzHyhBMh10ZXvpn2tewkP/Pg9uU7fSU2GHc2Z4 + 7WPa40fckT/2jn/gfY7gK35yX6VhfUq3NelhxDqXnO7AePZD/Zae6ck6OK71PKWH6IRQaOukhOQ1 + I/PoYDhj6BhH3llPP+WEG++PiiNR81UC5135krgeYid/fsU6Tn/DOvx5LPfIpHW5OEMMDs0oYjfb + IX+/XILvJzMCdvkYdwv8rsd1JN4qzzpdr/2dSCWqPDhcR/pJn3KosK5TddFfHfai7I+i6XXv/dD2 + G7/rfdvX/fiD24OP8r+nSd/p76Are0v/XOs2RwhuzPhxfrUv/bHezaWfzvEWzjvCX4Tvc6o8mK7U + UN0WuvSOHrmvKZd23UesnhLfsLbTP7+iIqcfWPRs689jSTJeIuz2JeMlpV2XdTfbIX+/XIL3E3fP + n7y8nJfxy1+fADNOb6au0/Xaj81ecN+260i+6o9A9TnjEVDb6rH60/qj5OUn8EcTft+b7t1+9xvv + 2X7qITyoqIfeROnbbdPR55M+owdD9PDyfaDJ4fOoOedX+5Id691cvIoLPrMJ+brGJfiuFxFqa0Ve + iZX1QWXR0dcJ6lvHag+10q77KIvvw2WnGevWdNZ/6Nd6iC7K+3xkKSF5zdjfiEY9M6/rZ3UIQJ6M + lcdE5gOvcEBVAuNWvt4nbjv98yvmuPiBdX7920jSn1RsnlT4rbmKvOgTbjbBRLEvwneewd95qo7U + M/m0Fr0rax7WR3F2ddKsPlYji05wi+s6jVfH4pvx5G/6sUhFV3e+H/+67+t+7IHtc//9+7Z/9Ysf + RBs8Z9RLwXnCpYMs7ssf3bThfomTewrBDbqNs46+R9lnGnp38yl8zt2BfF3jEnyfMyKY56oN61O6 + rQm6RLdUbKVaT/h9XsY1inpMHtkndBcdFXF86yQ96x7Ia76FI/2ej7lVL99nNfZ88CsN4phPaWfe + la+Ahbv2beE7zhc/sL7iZa/bzq+/bT4xmSqt9hO2HgqyUV329bCY2aqpHW7guX/kl32L/M7r/DMu + dTgv6k+e1YiqVHkItL/6KFuHpjri96wzyFmJYDZ89dbU5e//3EPbS7/jvdv/gZ9X8edWq184CeAJ + d9/0c++gq2CFk1sBXPWQ3rA03+ycgWHak/gRV4U0f+wd/8BzX7zFvwKvxsr9lm5rgvq+UOs60V7v + Kz00hp1urIPPq/U8pYfooox1b7x4zag66Bj1PBnfsJDwp7YvffHrXMWNrxc/sM7OzrdrZ/94PjEt + nVvoJz5E4eWRnZl5uD9H2TvcwHcexFhTHBLWF+GP/MIhonmqruDMYz7zdiJVqfLgcHz6qfysRHzx + 135tLwKtruTLm+55ePv873rv9kd/8N7tPfg3gdYr/bBflE1h+DYvHWTpGA+6xkEc13x3iYCGx+Tv + +wGX9XUc0yXtSfy4H0f+2IoLLvOJPK7q6ry639JtTdBDiqa9Vqj1gX9+00lH1sHHIF11X+sekwW2 + 92HwvDpP9n0SOcb+RtQ4Xos9H3MbRz6PlWf2gTjydAGph3wdqAVw/w/qy24513TxA4uYR85f059U + vAzYSqv+hGDv2KEYc2asROGiRtk7XOIA4f6RX/bkHfgjv3jB0DwVF5zzjjyrERWo8pDQ8enHtg5X + fPHXvvD7/sq6MtO78JdS/dEfvGf7bd/9vo1/tmqnD6pc/cKg4Dzh0kEWdTrqGgdxck8huFH6ZB7n + 1vnhY7p1DGJSPU/EfUKqKzWoQwk9J+gRHVKuccZTL7/5Yzeq72fpKrvuMUCtu+ii/NxPHjOqDsJG + PXzoJG/P9ON8M1Ye8pGfr4gTrjeqHvLRj7EWr/HG6debP7C+8uX/gX/jHzn7yUnusknpJzuflCga + STXbwdc1qqkdbuC5L15EhD9z8w48Ei1urJzX+WdccM47+mC4geIRHWzXkX4Wr3jaX/uyq4xDPbV7 + 26YPXT/f/vpP3L+95K534x8DP6CHA4tpnave1S+c1IPI6lOWZD7oGgdxXPOyHfqX3nBpHufW+RlG + f+ZRz62cd/Lt+G+SB2mu1LA+pduaoIcUTXuo2QoZT8tv/thpyjpQ79JVi/V+pF96iy7KE5/95DGj + 6iBs1KOHJXm5m5l+nn+NPZ/rYUXNlzjlZX0dyDX+dtGX6K9qD99xvvkDi+iza9+kVngZaOJ3bLlZ + LJPP2Q6+rlFN7XCJA6qfzFiHP/Ot8IsXkc1T9URM5x15ViOqUeUhoePTj21VJL74a1/4ajEEZd6u + 6UH8d39/9x0Pbq/4jvfgB+v3b/jvlXU+qWenDzZXv8QRxfOs/uTn3kFXbkUPuacQ3Jjx437UPtO0 + /FjLln4Vl3uRGZicI5cal+C7T4DJfy/+RcOrfviB7R0fwL8Nvc1Db+rcl+ho2aELlZntWin2o334 + 9dAqW5vCm6DodK79vim/dYYBXX0C1nvpz3OGS17zLRx1rPcF/aMe8ylM+4uPOO4jjn11Acm78hF4 + fnb2TWa5+PXyB9b1a/+AGfvJCYM10ObQzGS0M9shf79cgj/F33nCm/kEv+txHTMOhakE1+d67cd2 + L7hv23Wkn/QJoPqb8ey34roeEt6ewfK/5Wce3F702ndtf+It+GMKD/g/UvYn26qrdR66+JMTBIKl + LzKmv4OucbB/rnEu0Zkmh8+j5nFunR8YpmO85lFP36MRd+SPrTzBZa78O37s/ZN34c+bfc8921/4 + yfprcbB3O4YepuvCSYC6XtBj6s7qrFDrCXt+00n91oF6Dz1P6SG6KOPzWfrznM3oe5P84a33Basq + 4OX3ixyIYx9OxA3F972jff06/lK1Z32LnDd5ufyB9Uc+812I/47jJ1Y/YSEKu5SdmQm5P0fZO9zA + n+K3trfG73pcx4xLHc7LsiheldeLZduffoxHgwZoSjzrqjj2eex39v4Er99094e23/qd795e/ea7 + 8QN1NpW6MfOW1OXKPhHWyzMvk8pXaPrCLBxfD7rGwf7l5sJ4mhyTv+9H7Ss/1ru54hWXe5HZhHxd + 4xK8z9H1zTyU53/Hf8j9Wd9z7/YP8B9203c7hvVBdvaxJp8XClpyUte6j9zHL5/XCb0nj3hHHGz2 + Kl7dB8e3TnZ0Xt8bBPiEq8w9n72sn8wee770gTjydAHcTx8JvPba7ctf8O7wXDRf/sBS5Nk38YnK + FpWTcxXpJzuS02YRKT5zMl+CZ9yRXzbjwpuZnAd+53X+GRec63Pd9oOjF0UH23WkH9sCqo74a1/4 + avBQT+0+odMvfODR7Svf9H48rN6zvenuh6sdnlDqxswTg24ZrXPVu/oFQjDiR7zoDrqSLHpwTf5D + /9JbMOefNtOQdjePem7lvJNPvLkXmSvvjj/5Ks8vfvDR7Y/9yIPbF7z+nu3776m/MgeYJ2tYj9Jt + TT4vFLHktFKtHxB6aC2ASrYOfSyIv0B30UUZ4ut9Jz6esxXob0Q6qfDW+0L1GXj5/SIf4sijtBWn + vDPf9Zv+sN1V3ewPjgah+Zn/GJfoPqZKq/6EgF2XdTczhvtzlL3Dseixf+SXfYv85jHfjEsdzut6 + 7UdxvahyYRNHcSdeQNURv2edQdqsPmbLT9SaP1D/az9+3/bSf/uu7e//7ENVb9pxQdYDdfLEcgtR + kPur+ttmvzCoh/DTz72DroKRV2jzH/rv/NKt9CQ7bMkuVsfLrnjF5V5kTj7OGZfgbzXPD95/ffsd + 33/v9pX4D77fyf/Nz5M0rA8FLB09+bxQQ7XHFX4P/WD3/Ry1Wremk848d+fhfukuupzA3E8ek/Y3 + IuUP756PyMvvF1GII48PmhtVD/m4ff7Adsf5t8pxycutfcN61affh7/U71+4JRaJJPUm2D3JKUrt + zzeJargE3096gMOfmV3t8pAweUSeeoybccE53jj7O5EYRAeH60i+xSue9te+7C6gFk/s9C9+Af84 + g59Tfd2P3LM9gAeX+2O9SzdWoH4585bwVtRwf8OPQF6m9M8IHkDHw+TGMU4BxMk9hXCiFe/802Y1 + jNvNKqDy3sJ5u+CL8V3vreQB5lvxj4evwM+3/upPPbR9gLo+wcN6lG5rgi7OXXKgCivV+sH2ee1r + lH/ykCA6kgW29BadVupw7SePG/e94dp5RFf3QLtV4OX3i2jcH/J0Aamn7u352T/a/puXPkDkZePW + HlhkObv2GrdUEtabwE92PimtVp7o802iIi7B9ycAwMpDsZl28lL04jnye9/+GRececxnfyda5cFB + HMWdeFWkOuJPXZwVPhZlP87T2+5/ePtd3/WL2xe//r36gfr+E4v1Lt2Y2npg5i2py5X91qdw7hcG + HcKPePV30FUwJ7RbwnG3R+dXYaUnvNZXWXy+2DvWw3qt/4pTg80uIlk7XOLkrvuDtfgz14Ht4gr/ + EP4N61/+yYe2z/7ue7b/Dz+gfyKH8rOyOrg+P72rve38VHjpoIfVsFOj+2m6G/Sj3zqTLooQn/3k + SVafLxDacH2jDm5gXH6/iEIcebqA5CUf/ibk7eyW/nGQTLf+wHr0Q/8cCd+VVvuJz+bZ9JxVI5Fj + SCRqdRrfT3qEUIq2L8Bb9MUvXkQe44KTn5qBz/ydSCQqr/3pp+qV2Kx7xseuGkSw6nm8Vvc+fH37 + H95y9/ZZ//YXtrve80EW4Dp4+qOR2lZ/zG09MAtHoMfqH/EYttkvDMGKv/rxdNDVgVUH4yQMd3t0 + fhVWesLb+bFW+Zk7H/NH/xXnApueRDKU5wT+MeUBU+olKf/R8FVvuW/7z99wz/aW+56Yn285X+m2 + Jp+X6mElHD6Y1KcPFygX25iqf/JcpLvoorzjZElP8prR9yb5ieN5rbzJf/n9IgfidA+xrASMJ9/1 + 6+fv3j700L8l6lbGrT+wXv2Kh0H4mrTqTwjeLau0m5mZ+3OUvcOx6LHvo0FPCmeTt85vHvPNuNTh + vOazH+S9qHJhE6dLwbrKrkXbDou/mqw+yvqIJ/5Tyd99+/3bi//Nz2/f+BP3bf1//EtdOOxVP/uO + iQVG6yocK/Zwf8OPQPcLv2DpKzyMO+jKra6DbgTSHqPzC3eTc2Y4fp/Ej/tx5I+tuOAyF594w5+5 + 6tzFjfxYegD3Rvww/vNff+/2x/HzLf7Fho/ncL+l25r0JmaeJSd1HfoB0fdzFOR++lis5yk9RBdl + rLssJfQ9Uv7cL84YdM+8rh/7wpHBg/uLz3GqP3zVmHHId+3a39/8bAnFTedbf2CR5vr538ST8RH3 + 7CL1pGWREIddaSaW9hxl73ADz33xIib8mZt34I/8zuv8My4452VZlUeqIlnX5cT2px/jVRFxIF7x + savJ4inrI5re8L4Pbr/pte/cvvpN79veg3+b5X5GHtbBa9EO1hsTCwz1y1k4Aj129WNr9QtDsPQV + HsYddOVW9JCbBRlPk6PzC1d61r7Kxno3V7zics6ZTcjXNS7Bd5+IuDQPMKm3E1Td1zF/yzs/sL3s + u+7evgF/HOLxem45X+m2Jp+X6kkl1HXoB8T8ptMo1dvH4n6Gfq2H6KKI+5YlPX2PyNn3i/eHNqaZ + N3pdfr8YDV7yOBE3ovcj16+d/a/auMWXx/bA+sqX/Udk/Rb3zOywIAq72c12yN8vxGHscImrffFi + Hf7Mt8IvXkRynnHYYNrK69n+TlR+245PPxUnsdnnjI+tcDpq8eFPP/fQI9uXvuE92+ff9QvbD+G/ + ++NY/RQv87AOnv5opLbdZ8UpXjgCPRaf6139wi9Y8bdujDvoyq2ug24WZD66OMjb883OGSC3cQI/ + 4o78sZUnuMyVX7zhz3yTugBZQ/2tvvnzrf/5bQ9un/s9d2//4hc/8p9vWZ/SbU0+V1Sx5KQudR+5 + j19880ffFGwdHNd6ntJDdFHG59R48Zqx7xfvD4bkGHmT3zgyeHB/8TlO9ZPHDgEr/lu2L/mMn0/s + rcyP7YFFxuvnfwmveMC6SM0sknZm4srPpcYleMb7aECDgLYn7034lR+Rx7jUIT81Sx6Jp0SrvPan + H+NVkeqY8eyXtsLHouzHMH2AP+z9kbu3l/zrd27/7zvur3pMvKuXntTB07dQ2GS9MVec4MIR6LH4 + Fk5vApqCmUh6Ycv9HXQlVdfBuCkEnXQv/nkvOj8wKj/zKfxNzrsKc57gMlf+Hf/N8hQe0xrqb/Rd + np968JHtv37zvdvvesPd23+s/5pgBd36yvqUbmvCaUW3cPlgWk/4fV7GNSrngW31PeonpnUXXZSZ + +8ljxv5GNOqZeWc9Ov8qZOVxfT5W6EgeFyYkvrniD7df+4sVdsvTY39g8VvW+fbP+glbl1U2qsv+ + bELVEIexww089y0ZRCxcZl7OXZyJ+NrDeY2bcanD8c5vP0J7wX3briP5ql4C1eeMR0Btq4jqrwu6 + xcW3/dwD20v/9c9tf+Gtd28PPoJ/X1I8cx5lrjp4+u1gvTHZSOrGLByBHu5v+BHoT2z4BUtf4WHc + QVduVUKheCurbro4Zv19frWvsrHezRWvuFs47+S7CN993kqeUS+WHupv9N3b1uW77n50+zx82/pT + P/rA9j78i5HHOqxP6bYmnxfIlpzMV/eR+/jl83IdyWsdHCddR/3EtB6ii/JzP3nM6HujSG2IbtZR + BV5+vxiO+vELkwvEdO3s7J9tX/YZP07vYxmP/YFF9vPtr/QTti6rbDSRfV7S3Sh7hxt47lsy9MQU + sRkXXGYSF19yOK/zJ16nXjjnnbyINFAUgsF23uqjbAFVR/yedQZp81BP6rpo/rF7P7T9jrt+fvv9 + 3/OL2zvwqa0x+nM/M18xpQ6eftfPemO6oI4XjkAP91f1Y2v1C0MwE3W86A66kqrrYJyE4m6PFU++ + 0hPezo+1ys9c+iku+MxkLT+XGpfgH1MeEKbeYq/+Rt/lWLjzDV+Mt/8L/6H5y77z/dvf+mn/fxs7 + /pKFeUq3NeG0JPhol/bQD349tA56iG/y0D/0o196i04rVbj2k8eF9zeiUc/M6/ohE/3Ik7Hny7Gh + fuGAqrofffTsryTmscwf3gPrj7z0e1Hk65jIT3aIwaIpSorPnGrK3uEGnvuWjCKYN3PzDjwShVmz + 8zr/jAvOeScvwgyseNuuI/1UfxKb/c342AqnoxY3n96Pn9p+zX947/bZ/+ad279/zweWXgwb/bmf + ma94mYd18Pp1/aw3puvoeOEI9HB/5uXO6heGYMVf/Xg66OrAqoNxLMh56eLo/Cqs9Kx9lY31bu58 + zB/9V9yRP7bynMB3n7eSZ9SLpceoO3XSkb6wEo72vfhm/D/ir5z+HPyPPPg/9LiVYZ7SbU0+V+UJ + C/MsHfTQGHajVK+PQfWO+olpPUS3Olr7yWPGvl/dJ6sYdZCfvPTz/Gvs+VyP6heOAYp7Hb5dfW9i + Hsv84T2wmOH6uZ6QetKiiN1M/2iCZuwdLnFyQwzO+K2Zl7D22eQuDvvh45JDfkRwnnHBOd44+xHU + i6KD7fjkW7ziaX/ty1b6Iqj1iYmfxn/rbfduL/qX79j+NuZHSh/XXQHpE2b2Xc+gZxzzUinM7o/1 + xsRixgtHoMfiWzi9CWgKZqKVn3EHXbnVddDNgsxHF8eKJ1/pWfsqG+vdXPGKCz6zCfm6xiX47hMR + l+YBJvV2AvU3+i7Hwrlf28Dh10/hf+rxu7//7u2VP3DP9rYH+g+iNOVcJK51tOziIW7JyTxDPyB8 + Xs4fTvHVMajfUb/58r4gXRRx341XHjP2/UI+x7OKUUf0p198FYf9xZc+rE858L8cvPbnjH7srx/+ + A+tVL/l2lPr9+qRgkSw6M+sYTaissne4ge8nM8CUqO3JO/BHfvHyMIFPvC8D5TOftY0fmwaW37bj + 00/FOVD4Pf9o89ivWP3yHe9+aHv5t//s9jVves/2fvxF6kqr0wt/gUd/7sf+UaYTYsOfbIhTXtab + dka/dDMPeDN29Svcl6loFBGdGOO2DrrGoTpgkN9AejRm/fNedH6gWBWr1VzxiosOmcl44I99Ef4x + 5RG9dWMqDear/KmT+8pnQMHcgd7M5f/2d39o+1x82/oz+PvI+PdwnRrmgU95evJ5iSdRxR99gOAN + WnUYJ3vR2T/0o199iE4rBa795Ck+3RuuuZ/6Vt7kv/x+MRpxi+/7ti954V3c/XDGh//A4t+7fL59 + vZ/sEAPiUHx/cqAU2nOUvcMNPPctmSVqe/IO/JHfeZ3fZ+J6gnNellV5cmZdF4pFoP3px3Y5hr/2 + ha8mi2e2/NP4t0i//7vftf1O/KzqP97Hf1QY9en6hL+iRn/uJ/VwHnmYl/FutHhjGtjxwhHosfpf + OL3ZaApm3o4XbNSdQjirDsZxIWBlcd00xHPoi2mI3s0Vv8OPuCN/7Ivw3N/xJ9+pPKkTcw/iKn94 + 6FM+gdyvbeiDX8IVP79B/w38jz74P/z4pp95CP9A0syOFg4RytOTeAgoGq7wu+6jLOZZNrY0rMPg + Ee/CtR6iWx2t/eQpPuYlTPnDu+ezlzgBFbjnc5zqL75Hz7YP62dXIsfLh//AIsOnvuQf4RX/1hAi + sujM9I0maMbe4Qa+n/SAWlOIo7DBO/DhIzWHn/iuY8YF57zG2Y+gXlR5sF1H+lm84ml/7ctW+iLw + mn/r51/4ofdvL8E//vHfAnrwUEd9vg1Vd0FGf+4n9Qx66Qyb8V0/eWMyj+M0C0egh/sbfgT6Ext+ + wUy08jNu1J1z7TronkIQv+ef96LzA6PyMxev8kaHzCbk6xqX4B9THrCm306g/kbf5Vi4qTNw+KV+ + qi4wKoL/BvFP/vB92+fhfwDyve9fP98yT+m2JvEwsGnE4zq0rzzLVhLhc26l66hfcbBdHyyeV9XX + OikheYmm13wLR96VNzoYRz6PPV/6QBx+nV/ffnz7Qy/8p8F+OPNH9sD6gjP8662zv8wu/YSvmZVI + lFFS2Ttc4gRnUyX2tBkXXGbSHvjFS1GwL56KC855GRY/OAwkm2E8I/nTj20BxRd/7QuvcG6QZfuH + +HNUn/HP37H9xbe+b+PfVqK8gviSdH5dn+kHaPSXuMbnTqQOxnf9rDemgR0vHIEei2/hdJloCmai + jhfsoCupug7GTSHoXH2J59CXygZmN0u/igs+swn5usYl+O4TEZfmASb9dgL1N/oux8It/Zjh+A0L + jIpQmSjgR+5/ZPvC77t7+5I33bP9NP6tsHngUJ6exMPAao8r/HYd2oft8zI/9zjEt+jKHnEglA6i + 06rjvJ882nYddCi/65l5owP71vk7THkXn+Nan7OzvwTsvvCKu9VJJd0q+CTu350/Y/vJN//02dm1 + T5bKvLynBvcpWl3C3Qy8mpzzwDWvDvc0v5/sdZjkqXwR84Z8ndB8C16XTxsiwksdiqajH24O4D/n + lz57+/73+X/4wMPtOANku072e/QL5Lq5ZH5NzNemF1VHATCVroXn/kV5en/H7/iZiDiNTLBdR28c + 2qt9R+1fc24187ITLb5TM/3MN2bpwfhTg/us9zB/WHkmf/HdwKM60gECyl4PrZBUvdWoedIXw9qB + +qt80uGX2CucOw1ov3mSSbPqHTywoyP93Ufyive4T6DYqj5FamPWXxu9r0W97PNgE/lQyc8/+qs/ + /VM2fcmZ6Me2/si+YTGXv2X9bxK/LplKkCijmLJ3uIHvJhHiw8qhQb3gMpP2wC9eRDYP/cAH57w0 + w9uJyGaY4PTXZSi7L0v7zcNCuwws3sCHFdOST6c+LxVvwaivboXrZgUYo7/s7+olphKIX4nCW3mr + oI5nHupQY/Exzn24XxiCEe99+/k66i7+VQfdCiCwR+dXvUuHzg8k07n6mc/1Kn7osYSuFLPP4DKT + F/4df/Kdiit8MXsadYeHjvTlymOjP/wSrviXH0FwuJ7SoexyzEk8zsNXDiuUvM6z9DSm6hBv6Trq + Fwts1weL5yXe1FXxQKT8vl+NI+/KO+u5+f1idsX91Y/0YUWmj/yBRZYHnvN/oqgfY7frk5mijCGR + qJVV3c2A6ZOAM37riHj5ar95b8LvvM4/4yKm8408KxGycB8vCHQd1UfZ5Rh+44xXuAmEr/qrE9el + DEpgfsDlL56i6D5hJ67xrI9D+lW8G8Um6628amTEM09uocKHrm07XoILP+KV96Ar4pLQ7mqc+zVm + /ce+VDZwu3nWnXPOTM7yF33byhNcZsGrT6wvzVP45uZCgo6+y5m+ANCObeDwS3m6zvgBg4M4PWzE + a7sccxIPiZtGeVyH9pVn2dzjsA6O6zpO6cGysA8kw6quikeFyav7SVjjWObKGx2ME5DgAx9t7p7/ + 2PVPuhPPiI98PD4PrD/xAvxt/udfw+ry5LUoo0CJRK3QXHCZAeO+tMTampZ9Af7I77zOn3iptcs7 + 8uTM2u/ErqP6AFF4la/sxU9/9ag6bcuPw2Unjidm2csf/uI46KEo8DpfYZKHfO1gnpjMs3j9SUmg + h/sbfvE7XnzFm7rJmz6czvxJaDc8BlaWPX+fN7ydX6ywM1e88kaHzGQ98Me+CP+Y8oi++mIuDuar + /Orbu6rfS+OVn+eMXzt91FmVLXnor/tQtk5MeSod0844JbJCzmN/81RNnKzD4Bn1x+/6YKEvMu33 + k0fbroOwxrHaqp+7dR6X3y9yXPua7b/EM+JxGI/PA4uFfMVL/9X59fN/mievRRkVSiRqpdPaz4Bx + 35JZorYvwB/5nReXIjwVF5z8SLD8SKqEOhXD2m8e+sMrQPvTB+fqUflsixaHywSOJ2bZy188RcE3 + SfBzFv6Yh3ztYFxMAzteOAI9uO+whdObjaZgJup4wVzXjEtCu+FpIVYersRz6Es8lS5pVz7mLx0y + m4iva1S+Hf/Ac/+W84A1+TsB+YsvPPQtnDovG/Xil3CtQ/wIgsP1pK/wyFF+TeJxHr5ykKfiZDHP + songUF2LruyFc37iAEZf5nWcLDvslxdAOpTfcTNvdGDf5iP2yMeN83/6yJd8+r+S83F4efweWCzm + 2rX/Fk/e+1SXRBkVlu1PAohOGyL1kxq2jwbbCON+5sYN/BSJWcxjvhkXnPNNXgQZyHDDYDtv6lu8 + ArS/9mUr3ARlY9KlYgLXpQx4GfX5Ngw/3QufONdjerJ0HYxXIl23bDdfxwtHoMfi020U3p+c8Atm + 3o4XbNRd55iEdiMw+yMPl+I59KWyKx3jZVf8Dj/ijvyxL8Jz/5bzpE7MPVhP5Q8PfconkDovG/rg + 1+wDSKOqQddT5wugebSQAEpH/hnXeca9UJ5lC8K4ImieUT8xzs8ZBvpiJo6170Lllxc2YY2jufK6 + fnqJE5DgHR989z+63fnH5XicXh7fB9aXf+bPoK4/q9pGE9P2JwEOl36o009q2JbMEnFf9sQN/BSJ + /OYx34wLzvmMsx9BvSjNYTtv6lu84ml/7ctWdyYoG5MOlwlcFzG+JOaPn2HrsI96KKp1oIUhPSpe + icKbdswX3ss+AV0P6wQ3+Vin+ggP9w66ClY4uRXAVY/Or3qXDs6nLEqXtCfxNzlvF0w5WEfxZ0YV + jylP4bt4LgavZC5n6nQHlZ/64JdwElIEipBZeuqhJt4V5zyVjmmpP2dP2on+2leepSf3OKzD4FGe + hWs9JPjqaO3L0Xl9b8Rc/Kxiz2cv4qB7xuTD7tdtf+gFPxvf4zE/vg8sVvTxL/lG1K8fwO8KrKb8 + SVBPaojaT2peNgREyt2TP7jMJB4i2XTkMS4452VY5VmJGG6YtKc/9RnPayRA+2u/tptgwXS4jFNe + A/BqG7Dyh18A3bbg5yw86+Vg38xDpdpB3pgGdrxwBHpw32EL537hF8xEHS/YqJuJOLoOrOvNof16 + WfHkWzp0fobhN9mO9TR+xClfcWuqOpQnuMzkTZ/hz3wqrvCY1lB/o+/ypC9X7jzsQG9mYG70YxMN + up7SoexyzEk8TFVlciWC8DrP0pNYDvnFK7qyF875ixc6mddxspSQeNG5DjqU33H9vuBuAX0PBVTg + ynP+xuvPfOE3avNxfHn8H1ivPMN/9Xntq/pRnWIlEsTkjGZ3MzD9ZMZaR8TLV/un8Ed+8SGieSpP + cM438uiUlAAv3McLEjo+9dkux/DXvvAKN0HZqrvejq5LGZSg65M//MURXWAmrvG5E+oLfr0LBCze + pRvZOl44VuSx+Exom/3CLxgW6iN+xh105VbXQbcCuNuj8wtXesLb+bFmuqQ9iR96uMCmd/7iO3U/ + HlOe8Ax69zf6Ll/qdOXuR/qgE/UjIQke+pWeethEN+FKtzUhasQppxVKXj006jyqJKOat3SVfUJ3 + 0anSjpNV9WiCp+/XqKfrp7+AxpHBo3U/21696VkQz+MzP/4PLNb1qhf/O/wA/pt3JfJSY/iToJ78 + aLqf1Lyc9OO3NS273gy7OBPxtYd5zJf4XDqCHO/Zfmz2gvu2idOlUN6KI3DYi7/inEB8BUMfJnRd + AiiB+REnf/jpxzjowa3GWz4nZDmM7/rZd0wDk9c4Aj0W38K5X/gFM1HHC3bQlVTRg2teXtpjrHjy + 3eScGY7fJ/Ej7sgfW3HBZS4+8YY/c9W5ixv5sfRQf6Pv3k6fnl03cPg1++AJcSid5LnsXpUOM84M + eB36Kc+yBVEe6ux8XccpPVgW9ld9s27ymrHv16in3xfJJxbmJZ8H9Tg7P//mh//gC9+YvcdzfmIe + WKzw0Wf/KRT/ni62mtKTmU3RzgxQP5mxtqa8BN5v3MBPkZhDfDxM8M644Jxv5MmZdV1O7PjUt3jF + o7OZ/PQzO4b66Ql3xwlclwB4GfXJH376MUZ/iXM9J/LodiJGBZB36UaqjheOingsPhdu2/ESrng7 + XrBRdxpOv6RF3a6jkmBa8SzM8fR2fqx1TpmLV3HBZ3YgX9e4BP+Y8oA19XYC9Tf6LsfCLf3Yid7M + O574sdn3pnQouxxzEg9TVXtciSB5nWfpSSyH/HUM0nXUH7/3YfG8xOu4xrMPl+066Ggcy1x5Zz3m + I1bjvQ8/uv13MR7v+Yl7YL360/CwuvZ1XbBEolZQAarsZoD0ZOaM3zoiXtbaP4U/iGQ+RDRP5QnO + +UaelQhZuI8XJHR86rNdjuGvfeEVboKyVXd1oryCOEHXJ3/4iyO6wExc4xnOob4wMV6JwhvTwI4X + jkCPxbdw/uSEXzDzdrxgB11J1XUwDoESkA6PFU++0hOuzo+1ys9c8YoLPjMpD/yxL8I/pjyitx5M + paH+Rt+9HZxn5UcnT3/DKtm28z+1/eEXvrfketyndZMfd2oQ/sPzO7b73vK9uHevyKXmk1mXac6A + 7i4v7eHX5YTd84lahce12X3S1aUjfPLxqpGO72UvPBlel0/54gew7M7TfmA42l+06Yj7Gk7oeMBv + 8Bcq+Job3zRYqJzaEM6ffG7H+x13yNP7O/765NzpEf5V1+Sf/bq9whu+f805nHgIMEq8cwZOdY7Z + +Yg8MQ78uScXPrRAobyDP/l27MV7Aw/3oysDyl4PrbBUvQU3T+4/w9ohOqUjHX65vsmDHeHjN08Q + mougebCYfXUf4onyriN6mMesro9r9zHrN27tl/3Gh//gp79C6yfo5Yn7hsWC9UO3s6/CATzaYkvN + EhtvmojSYiIs4mXmZRUuM7klOhce5jFuxgXn+Hk4iDNQBDlD15F8xs/bZH/tI77LUF+2RVtvQ9fF + FL6cHT8ugQrgy+gvcY333egEvNSrftYb08COF44VeSy+hXv6GxZltR7RyYKO+zT0KyU1OQ44/PJ1 + Ck/0BUz3hP6b3as6P54XxiqHdsVxX3mWTSyH6lCecQ8O98n1AYx9MnHs7wN5ta08rHvhyLvyRi/f + Q+zjh0Db9oxXO/qJe31iH1is+1Wf+f14/RsWiVpZ1d0MgD4JOOO3johi1z5VPOLDB4iG/IhonsoT + nONHnpWo4jEhoeOTz3Y5hr/2hXd+5SlbdVcnrosYJ+z6ZIe/ONIn0bpUnsXHcA71hYnx7WC9MQ3s + eOEI9Oj8O37Hi694O150B11J1XVgzVtefM5Cc9Rx6EtlM4w0mU/hR9yRP7byBJeZvODb8d8sT+Ex + rcF6ii88dKYvVx4b+uCXcNXH8iMIDtdjHWOXY07icR6+clih5HWe4jHAKNXLPKJznaf0EN3qqHVS + 3eQ1ad8v5Q/vyjvr8VPu/G88/Ad/wxPyg/bR5uP0Hz9PxlPr5z3jT+N/QvZ6uvxJgMOFmFSnn9QU + l378tqZlT9zAWyQyepjHfIn36ZExeT3bj81ecN82eXQplLfiCBz24q84JxBfwdCHCdMfkEpgflqj + LsZzjP4S13jDVx2M7/rZd8w9bz4BnYDhQ1ds2na8+Ip35VdhN8QlobLxlrOAMVY8C7vJOSOm9cRa + ccFnJu+BP/ZFeO6LN/yZi2cXJ/p9/eKv/OEBzPVxsTs/9Idfs4/lBxQO11M6lF2OOYlH7F0OF0M/ + 5Vk2sRzuh7Poyl641kN0q6O1nzzFhzxqiLP4ae75tM+Kz7Y3PHLfp/33Aj7BL0/8Nyw28MoXfWi7 + 9szfh9bu1pOZlwmXger2k5qXA9BIyX3ZEzfwfBPMYR7zzbjgnA/8zYtoA0UjOtj2pz7bAqqO+Gtf + +Kqi/UVbnbguYtxZ55cd/uIY/SWu8Wk3eRjf9bPemAZ2vHAEeiy+hdObjaZgJup4wQ66kqrrYNwU + gs7Vl3gOfalsYHYz+RIXfGY75O+XS/DdJwIuzQNM+t3xV/7E07dwo15k8DefU34EgcD13Oxepc7w + phLaFcf8+OXzMq5ROQ9sq17ZIw6290m3OmqdpCfxZmQeBXDGEB020n9mPK3ufmS7/srt1WcPO/KJ + fX1yHljs4cte+Pbt/NqX+5MA4lE0iijxqCHEACxStj1xA2/RlzjmMZ94Ki44+XkGybMSiYRwnZH8 + qc/4cgx/7YuvalA+7guGPkzouohZtvukHX4tEbjXI373U5jkIV87GBdzz6s6eNs6vHRmAIb1cLzv + polSt2Guy+kcl4SyyF98lUa84T/2JR44d/Oop/FDjyN/bNUZXObuK6qP+VSewrPeHsQVX+qkL7qY + MTb0wS/hWgfrJBOOpTN5V5z6kB09RpyKoW39lV95li0IXqwD5+IZ9SsOtuuDhb6A5HbVVfHKo231 + o4DGkXfljQ7Xr13/0u0P4L39JI0n74HFhl71mf8ET+ZvoKp6QmeGnLO7BAAAKHpJREFUi7aPxlK2 + TXGDy0wuic6Fh/jAcIwLTn4eZvLkzIpHU/tTn/GqiID2177sLmDCdLjccF3EOGHnlx3+4hj9Ja7x + DOdIHYxHfvfHPDEN7HjhCPRYfAvnT2z4BTNRxwt20JVUXQfjphB0rr7Ec+hLZQOzm8mXuOAz2yF/ + v1yC7z4RcGkeYNLvjr/yJ56+hRv1IsNT8RsW5PmGR3//p39Ef0d7632Liyf3gYWirj/32p8+Pzt7 + vT8R9k9sXoFcDn8i8b2AHVzO3czmuD+G/Lg2x7jgHG8+5VmJxCI6OByffJU/b/r2177sKkJ1ch/l + Yuvpb1g+H+t+k3OGVtKrznOHz7lT4vJzqXEJ3ufIc8h53CQPMMprZr/qIMd9Kt/Cjf547/Br9uHM + VbbuST3UxJt8cqiw2q57U3HK6Q6S13mWnilZ/kXnfoZ+9Ls+ROhhv+r3fvKYUfeXDinoeuY3rOvn + 19/w8H0veFJ+buWK/KqS5saTsv67P/Jrz7aH3wQtnn+zT0KJTHHr8vR8okjzHD7pEge8/Dw0HqJs + vPTCZ2h4XT7lHcCyO0/7geFof9H6etTlEAAvI/8NfmJcZy3aHmWOPHV0zDt5ZZvHV9B+X9Kxv8O5 + rr0e4VcZQ7fsY0aCohkL43evOYea+80DUFW3n3NOY7a+7OjEOPDnnnxYeSb9RfVyvyvGsuz10ApJ + 1Vtw1+OHDXWjLR7lMY314D0uWlENHPd53vjleAH8cuSBnXtPQOuRvDrw4z6BoavFDneG/2XZ+d0P + nz/j5dsf+HVvN/LJe33Sv2GpNfw86/xR/wfSEb3FBMCHlUODaDhd4TKTRKKLTS/mqcsgt+OCc/w8 + HIDGrcgZuo7kM35/qWZdowwSgK8mHDEPe16qZSvtuARqgC+jP/dzqJeY5GF81888MZnHcZqFI9DD + /Q0/An354RfMRCs/4w66cqvroBuBtMdY8eRbOnR+huE3ozRXvOKCz0zeA3/si/CPKY/o9/W7v9E3 + a8BQPq+GDRx+zT7cWZUteaIz9QiPFmVHD9ex2rVCyXvRw8o6OF/XMfRrPUQnRNffeBSWvLq/dOiE + wms98D+Dws+tnvyHFau5PQ8sZv6KF78GIn4zPwE4/M0lhzZs+qGicJkdoLi8mMc4n4nj9GYqfmrf + eZi2D4/7tu1PPuMFVB0znvwVxyLaX7S+vspHNwD4PeqTbT65+TL6cz/2jzJXHsa3g7wxmWfx8mHU + t7D2HbZwehPQpKN4V37ujbolFGHkFdr82SccY8U7/7SVH5jdXPHCRYfMJuTrGpfgybPjR6TsU3Hw + pb5OQFzlDw99C0fBYkMf/Jr88JQfExyuxzrGLsecxGNeheOFPBUni3mW3SjVyzyic51Dv9ZDdKuj + tZ88ZvS94Zr74VVm/NzqBU/qz61UQL3cvgcWC3j4k/4Y/ln4h7nsTwCsrSkvgfd5Cv4EqdkBfO0h + PyKaB4fl0+PhhGfkyZkRJz9ekNDxyVdxdgz/5FM4N8qvSZeKK9elDErQ9QGh3covI33CSFzjDV95 + GG+BgGaemHtef1IS6LH4Fu7pb1hL7+hkQcd9GvqVkpp8TsDhV99Xe8qP6ZbuVZ1f3wuF44XntO6R + 8yy7Ucf7J3vh1rmTTpUqdO0njxl9b7jOPdH+Wx++51Of9J9buSK/3t4H1qs/+cHt2jN+H0R7sJ/0 + JVHb+sioTxQcAvc1MtuqffslfcX5cHhGvDWe7e9Ei679yVdxDjzEh68LKL+PWJ9QiOt6fZ1lK7/s + 8BfH6C9xnLtewtKH3gVlK0/lpV/bNQtHBo/Ft3B6E9AUDAvMKz/jDrpyq+ugWwHc7bHiybd06PxA + Ml3SnsSPOOVrdgYysuoMLnPt7/ixJ/tUXHgw91B/o+9ypE5XXvmpD35N/uVHIBzuu3QouxxzEg9T + VZlciSB5nWfpSSyH/OIdfZ7SQ3SqtONkKSF5te066FB+TQ9i+cVP1p+3chU3vt7eBxbr+dIX/TCE + +Op+0mPLmvISQHyIThV3M+O4P4b8iGieigvO8ebbnVnxaILD8clX+VmJ+OKv/dpWGe2v+n19xecy + dS2KH/HyF0/6SJ+w3Y/nrpe45NG7oGz1vXQzzPr4k5IMHjt9sLX6hSEY9Z75GXfQlVtdB90K4G6P + WX+fH7ydH2umsyozH/NH/5rJynxzlK08J/CPKY/oT/AXb+pk+vTlymOjTvwSrus0n0zJQ3/6WnGt + I9smvxShn9k4uFg6OM+yBSGKAcpTPLIXrvUQnTMxdu0njxl9b4TQBrxf/aFXvuCt9t6+19v/wGLv + X/aSb8InxDdYsjoiXha4/MnBJ79PQzNjaI/hfeNmXHCON5/9CO5F0cEmTpdC+WwLOOzFX3Gso/1F + W9ev6x224mWHvxrBJQt+zqPMlUe3s/Ki4kp/Y7xwZPBwfyuvbcdLj+Jd+Rl30JVb6Vdu8NMeY8Xv + z63zA2sdaq54xUWHzOQ98Me+CP+Y8oh+X7/7G32zBgzl82rYwOGXzyk8nlW25LnsXkWPEdd5XAfN + 2/ENCz9k/4aHX/kbvknl3OaXq/HAggjXf/rFX3t2vv29vsS8rNj3JwcOGzYvkWaKRnsM79s/44Jz + vPnsR3Avig42cTf/JJx1jTJUn23R1tux6x328jtftzH6S5zrOZGHfF0/+45pXTpeOAI9Ft/CuV/4 + BTNRxwt20JVU6Zdrvitpj7HiybfOrfMzDL8ZpbniFRd8ZvIe+GNfhH9MeUS/r9/9jb5ZA4byeTVs + 4PBr9uHOqmzJc9m9ih6uY7VrhZL3yf6GdX5+/e89/NZP/Vo1ewVeqPHVGf/u/Blnb3/Lt+Pofrs+ + IXFq+QTtNwXfHCeGP1HrUsC/iytbl5hvAtl46QXxvlzNo40BLPtGPzAc7S/avB25r8HZ/fgK2lZc + ITipbi+063ymrw0l2OMGb+XruEMdvb/DRWcV0Hm90KvqslzVz+i3Agw89cp3n/CeL3yYIFaq1Ln3 + ec34W+A/9bAptaPGxXkmf+W9oV7uNxOWZa+HVkiIwyi4eeohiIJoi0d5TOM6eY+LlvGLoCz6zaON + vBx5YOu8lcf5zLuvv/srnNIx6/n5az/4y3/9Fz4e/4v5lPiRzlfmG5Ya+YKzR86vP/OLsNafhG9x + cdl1uJkJjrgK9GHw8Hfi500iOA4JhMuPzXErRNf+5Fu8ytf+2pfdBcy7hzP3pVDdgixbaetWLD/r + WZcw+7t6ycNCmZfxIgpvTNqpOzgCPRbfwj39DWvpFZ2s87hPQ79SUpPPCTj88nFYVyrPcev3qs5v + xpkBr+NeKM+yBVEe5OO98ISZi4Vb5046VarQtc96idc2/3Do6z/4rE/4oqv0sGJl6ya7zqvx+s0/ + 8svOzh/9Pqj3qfomUeL7NE6X7G8O9cnDxvKw0uHY1uHwEOXHSy/or8OW359Myw+gAMXDS1N2K9j+ + opWj4qSqL4nrBM8NfoHMyyX5NI16vaEEyh+76nE7h7hDns6/45/9rHjSK5wT8JNf9e3ac5xiji+H + 8+s3SdGLF+uegVe+MTsfESfGgT/35MPKM+mL9wYe7kdX4steD62QVL0FNw8fCuyPYe0QHU2qeJpH + ASK23zzJZIcJmkd5Fq77SF4p7jqUV/uwr5//xAef8azP3X7vr37C/qrjXd2Pwbha37BSOP5OaBzK + F+JE36lD5ZtFp7DEx0bQmnP4u0PxrVj+uiQ+HGz3godm2/HJ58Oct2nPX3HMoPp60qVjnOsSQAk6 + flwWejXSJ4zENT7tJg/ju37miWlgxwtHoMfiW7inv2EtvaOTBfX5SeahXympyToDh18+DusKxvJj + gsO6130ouxxzEg8DdR/NIIJ5nqf+cVB+8Y57cLhPro90WpkdibzPelHf9e2d2513fuFVfFixYNZ6 + dcffeeuLz+64/t2Q8eP7E1Zi31iyvwFQfF4engla07vYLe4+seXHi4EiW3DG8xOQ8QNYdudpv8J9 + CYBf28xbG84g2/EU/ug3j/JySSJN6UdmJ9jjXK/LPcQd8nT+Hf+xX6Yxj8J3dWQf86692q8yd1PO + oWa/eUteAFX3nPkmQn7harawRJ4YB/6c+4eVZ9JfVC/3oyvxZa+HVkiq3oK7nvTFsHaITulIh19U + U25RDVz7zSN3XoqgebCIjoS0HiJWBkVmH3+I+77za3f+5g/93k/5oVBetflqfsOKSl/xGT90fn72 + RfgbDR/R4eby0i/RA/Rh8NQjfvDB2Z6HhthxK3KGjq/LAL/iCCSgbIfFrhraX7T1NnQ8Mb505qdF + O/xaInBdwsQ13vBVB+NdCIIZF3PPqzzgzVh8C/f0N6zDOVAsCTruUwmYcwFAO7aBw6++F/aUHxMc + 1r3Ot+xyzEk8DGR6Dy4qDivnWXaBxO88oit74da5k06VKjT716+ff+Ds2h1fdJUfViz4aj+wWOGX + v+i1+Jr6ZVD5Ok9xffJT9DW8X58o/397VxtraVWdzzn3zgxQgQEKSP1ATSWhlaaJ/dNUrYxNTEYL + CRoxVRtpy4/GtJo2TfnRxFT7o/5ppLRNjKlaCtoMatKG4h/LxJg2/eE/EezwVQlWKylcYGBm7syc + t896nrX2Xvs97713mLngPTPvvtyz9seznrX22muv+56Zwx1Mc+xJZyiNJZkCcWZ+meMMDceksAkA + gxedMpZ+jN2Hsk4YckcGpE8P8JL843rwO0faX+jJH8P17DA7MccF43W7Diz6xJnHapVPhBpL3/YL + wrJPjghLfocjblDLDJQM+GuxT5z0xWfxpZVWZr8jDiGl6MwutsCXfQLe2BvSAyb8LUaS36FvaxVX + 48dzRdyIc37t0PBQwoL88Tj42BeyAEvwhiciCLv84QLCGBcU/ZW94keKn+yHP0RQ1ebn3WSOSnDz + 0ZvecH/w7VS58wuWRe53rrPPZ/2xnb5+omEOh5Gb5rXOI7Z1HqJwXMdC/EShuoCkibHW3Y7jQSR7 + jb7xJzdor8CQUmY3+ZvGUPN1w8s/OpH2F/ONvwYKO8ZHorATQ/EVfeIMqFb5Km58wuqdg4WKcU75 + lOLnkaRQnIHDl45DcQWBr0NgQXHfLK/8/LKeGEiQz1PnFXYIUh7RTsqDXj7JP6NjT+zMv+kfHnvf + m/5FTDv7dTkKlsXwlrd8Fn8i+Jnyk4VBr8HVvH7y2FFy7ElnKI0ltY7J0qlnaLjxCcvjxbhZ9Hpx + tSmLP+JnqR+X27rRdB7Og3PIY4YdwEb6eeqcHJ/0aC/ITW6B1znKvy3tkI47qRa4v7RvX4l9+M59 + X8Dhi3bcr7oORSzIn9iXxr6QBXnMVKFhhF3P5mmnjg1rTXGTXvEjxa/Eg7UqIkLVPz/2/jf8LXtL + 8NI7pSXw+AsP/NlsNvn00CXhKfsh5UMrp8/D0uFqHfstHfSbdSUF6JQMyro2uZBVWve4MclT0iwk + lwyU5Cnm/Rio73bNHc/agndY7IfryX83X/QanLtoovKJsI4dVPaRL+GiXw1/EwiEKc4hS9DT3SEZ + +kPS3WqE47bFTkPsg8yPKfObzeZTK09OQPB0uc5egetYbd3yBfoCYh0dH5fpsFTMWEd6Yk08fT8q + ndvxCeDkp8yFfTB98uhNV38q0ez4rsVg+doXH7gVSfA5OF/816Xzw8SOdJktOQThOrKiuZzMhVj3 + 3AE+P2FJ33ko+usePmalzJFWWVHsowNgsr+wLh76bd3sdx0WAy0u8fb1enbq/mPf/f3UeXqkYYpb + mbDthJupQ632hbcRei7L5bF94VvxSjLOKUkaMv2h5rzBH/K07GR+513goR/hORR8XN8WBon76xsU + jxcfhqMsMBA0Z3SWJ0FLqoQr6+Lhcrw4QeFBh+dtE2ixD3woFBOz3zt60+u/EKrLIi0uy9m++MAH + 4fyXcD/3lA2waPQuL0/Pt8kssLtll9TvWOnYBL4x1noctsa+kIUng/OYE7RfBOhSMbB1N1DsY0bm + wz/ISDJDG5/L7CbtxHpacPNFr8GRSS/F/mb8JQ7hRy+uRpX0o0iEGV4OrDcSi3R3SMY5DckgzdJx + DX/YG+LHHI/3dPidj+ZNPzXaj3PGvI3dElHVHOJnOE4QKJyPyzQjZOtU947HkaPEExCbB4EdR+Fh + xydiHRLF6hj8+MiRm66+J6kvTVc3Ymnc7Tn6Dw9cP+26r+MA/HfD+2ECxsseh+ZjHaoVEb9rpaOx + 4LZuh2+HnYA+1mXP68BYK+uupuvhhgjAi/RIu7BuGIP7kbiUvUwjv1pc4u3r9exUPtnROO+nzssh + vtIvhSP5h4lwt3aEb17jHFzyHABQHAYkcPQrSfKb/lBzXt1Wi48cOy07mX8jHvphcXB/fDz8ZOQw + uoV94asWF48A7bjbhBtOYwjr4dv3xVHlsdXS+jwYRxwNg3/M+NlusnLj0Zte962is2Qdi8tyty8+ + dN10euKbSIQrIqmbS8lD9G2aYC4PFy1liYpGFC2pF0XX9xTCbc30sq9LrBRLxY9RFk/xT+4wqbSM + 9UgyTBjOWsFrKAOxTkOBq365ootQ5DDxhV6Kh0HMbhMnw3nyc9n5kn9tIGwb9bLQfxuTBfRDMgI5 + JIFfaI7bFjsL5JjI/DYMjM2npqLo54x5G9cdFhrFAzvnOgPgONpxnOmHpWJGONnRehS95Ib4wet0 + bscnjHc+f+rk6vT69Ruu5m/4zbrL1F+evyXcKKq3XPvdbmX2a7hkj9hx1ssdp2fZYXfQxpJMAZsu + HZvXWPriCbwr9vSDz9jRCr/T+rWkXQHwmvzjuvvFdVt2u+iGnvxx/wwXdky/+G96MbSNJH3iDKhW + +SpOxRnrhImo2je95DcDhanihy1DMeYNjlb1ja/dF90GppGuT73AhxShvda2Bb7sExpb2gEm/C0G + uL+0b1+ouBo/s8Ai0vDEOiYZnvTDy8e+kAV5zFQNp/Gk+AERP0wNF01xkx73m/3vJo/Op7NfXfZi + ZXtVVGPXyyzvfOSK6fwYnrQ6PHHZoXoS5suks2dyah2g0jEFjaXvyYp1JSk7bXKZHZ+GpgHb9YXk + koHin8w5f+inS2J8nI79cCg7Pp/9d/MtX+Bc1USxvxm/7xvC+Rb9wgJZGZ8mEAiDF5tGAk2+IRn6 + Q5JWei+Oa/jhT30Sgd+naqdHzWHmd54yn/CyZ+eD+BjO9IplG1v8PB6WD5wgUDgfl2lGyNbxzSa+ + ui+z4zwBgeQ67bh1N4z57xzZvWv/ZP9VTyX40naX/wkrQv/bP/+TrrvgHUiOgzxiZYmyBRhdKkmt + Y7J0HIaxLrMup61TTx3ite7zXHcHaE88pPWklb5hdH2KPsfB7xxIssBnST5TtxZ2TL8smF4MBSz6 + xBlQrdg3BTSNpU8+5y36hMkvmZNeGNQyVpxPVuq+yNPbF3kAbGTyx255X6/PH+MGF3plX/DjVOw4 + HqI2BjTt21ciLmKOfQKHryY+tGzrUGR4vKiRt+oRwPWIh+Lr4YCyCMKu7Hh8qregAS7zYIzfuvCt + Iyurv362FCvbrqKTNr703fse3jP7ybEv4QQ/GMmsrPHdluSxpMEcxsPrnqyOz9nguQE1+0lX1cnj + Y9IyiXNy1eTTepj3YyCx26Vbmqed5Gb4G0laxwN8iQddtsq3CT/3kS/hol/VLniaQNgw4pckrDf7 + zuPQH5Jyu3113LbYaZk1yvyYMb/ZbD412o9zxryNkRHWI8ppFA/DcYJA4XxcpsNSMSM+2TFWxDN4 + aEEvXI9jwBT+R+Z7jux+/Ucm+6fHEmzpu8rYpd9GbwP4P6Zn//jQ5/A3iLfmS6VDjUsIHeaCQuC1 + AnBb1yWr6wASYKK/7rbLutNG0to8m0m/vOhZ0hFZ1gkiP3s+X4uL1rMfFZd4+3o9O5Uv9t3fT50X + v+z29aofyS/vLoh6axnHevksDhGvJIGnvSRlr9zi1kSPH1Xh9O1k5o14bL54bhvQuL4tDBL31+Ha + N+INvB2TSfKY5DhYsR60pEo4m7c8wpf0CdCL8+CjC589csPr/ghGTPGsahaXs7at3Png7+JQ70Dy + n89NMguQFMgWpoCPmT0GaNbj0gjP68QsszFH4rFcMz1r/XVLKkuuCgDIeQ3Okak5AfXTus83/kIn + DFKvbKSYb/kIDwdNWfakpvlBfu4r4iRPKy7563y8tLEPzOlS6lJRzy6pzRt+SPKyYWVIAr/QHLct + dhbIMZH5bRgYm0+N9uOcMW/jusNCo3gYjrwECufjMh2WihnxyY75YXF0HjBEk93JEfyDER8/euPr + Ph/zZ5v0zDvbtpX2c9dD181Odl+bzaZv5qHjUjEFyq1RCLxW8LKXIpOBBMRlV1ERkdsq60ruxSco + GSyXHklHpOmlZutsLgs+YMVOxm1S5Hp2Kp/0Nc77qfPyI9yJuCW7iE+4WzvCN6+8jdBzWS9fvdrG + qqtpMN9PkuQ3/aHmvMEf8rTsZH7nXeChH+ExFHysYuJD8ri/vjHxqNjwGIOHdqQnVuw/aAsPZoiH + wFe/aIH70RPT1RvXb7hqqT+2wO1u8nL2/KH7Rpv88LXfnZ+YvRWfQ/mqZUFJvno7lBIDSRV4XquS + VJFMUKg5VLLNYUwqA5g9NRko9nk95U9xHVkc+Cyb5A0/TL8smF4MZa/oE2dAtWLf/dJY+uRz3qJP + Ovklc+IPg1rGStlntWM98vT2RR6sNTL5Y8Wpr9fnj3GDCz232/CHvSE74SdkaYZzvuCxtYgLeoRq + DH/xRVyJQ6wDhgXDlSLjY1/IgjyyQ3q8GI/Hg6PEE5DJ5K4Xj+355bO9WNl2LcbnTLO3iEjCO7Dh + 85GLngsKgY2Vo558nAAmgD5efCLx8JX1SDE3QH3DaFyecHwsfucwVOBdFrzclD+8Sz5BHJIYkqnd + 1+vZGcZJ3wnoTPVDvvX16Df9cN/dro9a4ZffA+yXN+I0IO1y236SlD3b4UDr8Z+RnUzvvCo2yU+b + j7ga3se1aAWJ++vwXLQsXDYmD+2IRlliRclpSZVwNo8vK374g/Wj+F1Wt+It4F2EnQMvnvXnwE5j + i3iLuNp1X8ORv5l3jLngYVC2+OWPSxNFBEBmWRF+qTQmfX8dScXkoiFDyEC5/JiR+bAPieQt665X + x7RSDNq8E3ChDhOfWS32pV/5hKvjxM/t5iK46BfjEfy8dG4Xc7nYkN/2ZfOGH5KhPySBX2iO2xY7 + C+SYyPw2DIzNp1aKEBDcF9djh4VG8QCCRUpAsKBDO0WAxfmLGet48UavFKv5/Jx4C5hCze7Z/5aw + v2O8RTyxPnsrcuBLzI1SFJgNXix0OW1dl50dyxZf93mfpgmvFqVoWDJCQfqGqGOoYWRj52EPL0je + wGdJvOCmID9MvyyYXgwFLPrEGVDN5qVWcSqqWCdMREWfMPmV9cKglrFiDqRW9Y2v3Rd53JxpZV7q + BT6k8fb4Y7wR3uZP2Q7pW/+1v7Rv8wEt9oVeGgOHr7yPug4YFuSPx8HHvpAFeWSH9HhRhMKuv628 + 68WLzo23gBGFkL1TiulzQ67c/dBH8cth/w7JMPC3iJ6smyWXXQque7zsUvkYwpPYk5SQmnxaJ7xe + Auq7XdO3sUvi47TSvN8Sxw3wuT4B/qLLswU/95Ev4aJfcNDtQjaBsGHEL0mgm33ncegPSfe7EY7b + FjsNsQ8yP6bMbzabT+2VfMKCF0dPTrtbj7733HkLmELNblyB/vy5M7a3iJPJ13CJ8beIusTlbRxv + l4WICy7sEusSMosjgiw2pq/k1hOU65VQp0vvPznj0kfAo0jFfC0ujih23DCLRuLlOPYB2bNT+aSv + cd5PnS9uo9PXi426OW08NtGXfvmjqOmSR5wGJPC0l6Ts2YEMtB7/GdnJ9M674K/NR1wN72O9XfMh + edxfh5fiBjyPMXhMWt5IKL+C1nnw0cLvTbvpzYfP8r8F5HY3efGs3wRxLiz9/fcvXNkz/Wt8zu6j + VoTyJWYSeVFQluluMgcxr3UPErMwrSMLS3EjxMKdigtHsqdlZW2/ONQxUTKArs2bv/TLRfhFJP1O + /JzUOOMG+QGo8z2/nbfald8xNjO8nMA10ubxLbaejEAOSSPsN8c1/GEP2Jdkp89t48zvfITZfGql + CNm5Yt7GdWeFRnGwfCAvgcL5uEzTc1vHNxsz6K4XfuaqW/GvMB+N2XNVWozH5hFYvfP7+6az6R3d + tPuFthgge+ySUqQiFLlp+mVdudZ/sokkLkUgrq3ppUa7Nvb5gg9YseMTxKUi2Nfr2al80tdY+rwk + SZ9uFTO2b3OrTHg8iCr++qgVvI3Qc6lLHnEakMDRryQVX/NgoPX4z8hOpt/IX5uPuBrexy/DE9aD + k9nsDw7vf/WO/9dscthezr5n38tpYsm4D3aru/7nvz6Gvy7+FD5sepFy1sKEJLXLWoVfqnRX++v2 + ExVf5ZJ7kqtI1JQv69TfpPjEaXnRoB7vjhbcfLWXcekYiv20nmjSPnORWvSL8QCv/PDAuB0WJfA3 + UlEsV92jqrEXhyg2jUy+l67jG/6w91LtFNLUyfzOx1WbT03F188Z8zauJ6taZmEuOPISKJyPy7Ql + 2GTyHMafxFPV3+Cp6oRNjE0RiCswxqMfgQOPv3rX+vrtk9n0A7lK6bJ7EYrcNF1mZRG4hH4dbZ5N + 41IsFtYdFXiXBV9o0IFdFglTIW6xmBS9np0y3/Dn/chQ5a9+cbvZP/qhdfnh/b7gbTS/paDLy22E + d60Ejn4mSX7TH2rOG/whT8tO5nfeBR76YXFyf3y8HU9YID2A3wr68Rfec8WPsytjXxFQdo7R2DAC + q1/G28RudscEbxN1aVJRwuWNu0ICu8x+iS2VrWiNT1ip+CAmisuAjEAOyaHTcRyLCc/hDOxsxe9+ + E2Z2U1Mx25YnrAeRX+PbvxTboe5YsIai0p+zt4n/+/DHupP+NhHXjk8gcfsMP1CsSvUin34ilyec + eKYwvdTqk43mCz5gxY5PUN8vK3hCv+j17JR5t6tx3k+1S7eKGbuUlT/vV7hwkKP2pVeEFp5YjBff + Ec6hIiR7bbEoRnr88VPktOwUUnMI9hCnBR4WrfDYcfHDKYbkcX99Y+JxPqjbGAaem8/n49s/xmvr + l02ybGvlcw5hbxNPrN+OS/4BphqTmTmtUFgRwIILpLD/5LUJNpOpuHBkeF+nYlr3+VpkxEID6FJP + jnCB6jFvM0mfAH+pfLJbxwXg+8hFatGvhj+KhlPwcsJ+I7FGd4dk6A9J52yE4xr+sDfEjzlG/3T4 + nY/2TT812o9zxryN3RJR1RziZzhOENfBnzvn3cpt49u/FNAtun5TtkCNy00EVr/8yL7ZrLsDtw9v + E33Jq4ULJqff+gBApktPRSS3KaTGImRjn9+smFCNuMTb1+vZqXyyq7H0VU3qvPj5Cndy8XL/svtu + V+jea7213JcuOaMT3rUSeNpLkvEwnqHW40dVOH07mX8jHvphcXJ/fHyqf4aF/wfwwenK7Nbn333l + f2RzY3/rCLS3ZWv8iIgIHOhWdp98+Gbk6p/ioxC/xGm7tH6JLZXHJ6zxCctrnp6suul38fuqPvPC + 4Sv+afKB6clIpVGeegTGgnXqsdoQufKVR9+DJ67b8Psd3+Y1i8WqVC9q6idyeVKJZ4rek4mtCy5Z + 8HFSpShm3PiEVZ7AGJ+d9YTVzSf/PlmZ/eXz77783g2TaFw4pQjENTgl8AjaPAKrBx59G35Z4G3T + 2eQ94xPW4ts6SzbFZUDWRxG9Hc7jobD7Ot9eokg18qXa2Yrf+Qgzu6np7a29XcZ+MW9je7a2HaJ/ + 34lu8ukj+1/9n0ll7J5BBMaCdQbB20h194FHr8P/VH0bPgpxM56QVoRTEpcnpvEJa9PixTjh8ocs + T1BDQffihQrRFDsVk4HiCI4oLsEfsqF3vgUem4/zMwUfW9HCf3ir1x2Yz1f/4vD+yx5s+MbBGUdg + LFhnHMJNCA789xt3zY//yXQ6uwUPAedFkvNyQE0lDNLexljztzNl3efrWDDiCIce7470qR7zBk36 + NoxW+UIv/kDdEU7U4hbfdjb8USycgpccPI3EWr7qZr2MQ39IOmcjHNfwh73Mi/6WdhpiH2R+5+OK + zaemYmbxm78AS59fX9312aPvuuQHCTJ2tzECythtJBypBiLw5ceu3L3afQIrvz+bTi/WJfVr5EUl + tKxIsLmsRcMRNg+CFrdYTIpeXNcFPtkRTvqqHnVefoQ7UdSSf/Qj+eXdBdErQrrkqVhBwVgVF0jg + 6VeSLI69YlHs9PjjSeu07BRSc0gbXOChH/IY3afxj2rdMb1gz+3Pvn3vM1l97G9/BDz7tp94ZByI + wIFHL95tRWsy/QQ+hHNluaS5SOEGlGKzUGScM823RcYvfeaDivHldkr8XhRZRKi/6BeLSPD75Q47 + Q0Wn7BegUpxM38ahPySDNEvHbYudzBv9zI8585fN5q11kyfR+6vn1n/2c5PfnL6oyfH15Y5Am8kv + t7WRv0Rg99efeMvsxPEP4dL/Fr5fHwuluKSixMsdJ2XzmGhxi8WkFKV4dlngE6Fw0lcVqfP0ye0W + PudhsaIf7nnMx0ay7BWhhScWYM0M92kSeNpLUvYMMdB6/Bs+GUF1SzuZ3nmTv0+A4CsnT87uOvzu + Sx/I0LH/ykTA0/GVMTZaGYgAfjPbeV99Yt98Mv8Q7vz7cCAX9YtDHbt+Kj5tkfFLH8Uj45Lpyqfj + r+PE70WRRYQ8i0WRRQQqph9FIswMFR2zVoqS6eXxBkWHvEGaZS4msN/Yy7zob2kn80Y/+OfdM/jL + k3tOzlbvPrxv77ex2Q2qZiiO8uWMgJ3l2HZKBA4+ft7up2fvXem6D+Et434Ugt2bFRO6vUExKXpx + XYlTcalFKMbjExbjZUUKDWIdJfAb+Fly97OvufSfJ784XefC+PJTj8BYsH7qR7CBA/f+4JLz1ifv + xyPJh/FJ+rcDZXdKLRWf8Qlr8e1j80RlgcN3ebLzJ6fyRBhFChDgvg22uyd7ZveMf4DuubbDRFyB + HebW6E4Tgft+dPn568fxa266fd1sdj3elOCfKPOjo1x8uzY+YfnbRASyFCv0c/HCb0l4ZDad3Y8P + Th2c7971b4ffceFTTdzHwY6LwFiwdtyRbO3Q+fc9+dru+ORd+Dfa9uE22q91fq1pWe3Kb/c4wfn2 + mEsxS09q0nPbTtTiFotiwx9PLk7R/JkS+NIfXDdFoxST0B+SztkIx700O92T3by7fzJbOXhsNv/m + kesve7LhHAc7PgJtJu94d0cHhyKw51+fvGblJJ68Jp0Vr+uBubwUmygPC8VJRy+cipFXO5qweXVc + WNFBt85jHRMBqx3hm9deETqV4kW/oBeS/MYz1Hr88Xavm89/BP2D80l3cNKt3P/sb+x9bEh9nFue + CHhWLo/Do6dbRAB/Uvyqb/z4WvwPItdPZngCm3TvxCftL81aLAKYMGmtjjlUFWIxykVq5z9hwfGn + 8Ynzb+Fzbvcfn3YHD++77Hu+o1GcJREYC9ZZcpAbbgMF7Px7/+/nVqbHr5lMZ9fgf8y+Bk8g7OPh + 60345P0q/8ddK156hCJVFDM+oGGmFjVPGce/0k9YcPEE/Hwcny44hBJ6CNs7BPcOrc92HzryzvN/ + CEc3eAzbMELjwhJFYCxYS3RY2+4qfvXzRcefeuNkPkUBQyHDNxLiGpSna3DrX4Nixtuv4vXKPWF1 + MDvt5j/sJrNDkIcwftgK1ImTq4eeX7nwsfFfktn2TFgawrFgLc1RvcKOfqfbdeHzz188Pd5dMpkd + 3zudrO6dTE/unUxWICeXwJu9+EN/jPGNMYqa+jEnd9cg1vBEt4aiszaxD2HOpmvzebcG3TX8Mru1 + ldnkGfwt3RrK4Rp41ubT2drhC161NvmV6XFRjK9jBGoE/h//xb5CiJqhJQAAAABJRU5ErkJggg== + installModes: + - supported: true + type: OwnNamespace + - supported: false + type: SingleNamespace + - supported: false + type: MultiNamespace + - supported: false + type: AllNamespaces + install: + strategy: deployment + spec: + deployments: + - name: clickhouse-operator + spec: + replicas: 1 + selector: + matchLabels: + app: clickhouse-operator + template: + metadata: + labels: + app: clickhouse-operator + spec: + containers: + - env: + - name: OPERATOR_POD_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: OPERATOR_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OPERATOR_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OPERATOR_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: OPERATOR_POD_SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + - name: OPERATOR_CONTAINER_CPU_REQUEST + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: requests.cpu + - name: OPERATOR_CONTAINER_CPU_LIMIT + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: limits.cpu + - name: OPERATOR_CONTAINER_MEM_REQUEST + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: requests.memory + - name: OPERATOR_CONTAINER_MEM_LIMIT + valueFrom: + resourceFieldRef: + containerName: clickhouse-operator + resource: limits.memory + - name: WATCH_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + image: docker.io/altinity/clickhouse-operator:0.24.1 + imagePullPolicy: Always + name: clickhouse-operator + - image: docker.io/altinity/metrics-exporter:0.24.1 + imagePullPolicy: Always + name: metrics-exporter + serviceAccountName: clickhouse-operator + permissions: + - serviceAccountName: clickhouse-operator + rules: + # + # Core API group + # + - apiGroups: + - "" + resources: + - configmaps + - services + - persistentvolumeclaims + - secrets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - apiGroups: + - "" + resources: + - persistentvolumes + verbs: + - get + - list + - patch + - update + - watch + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - patch + - update + - watch + - delete + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list + # + # apps.* resources + # + - apiGroups: + - apps + resources: + - statefulsets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + - apiGroups: + - apps + resources: + - replicasets + verbs: + - get + - patch + - update + - delete + # The operator deployment personally, identified by name + - apiGroups: + - apps + resources: + - deployments + resourceNames: + - clickhouse-operator + verbs: + - get + - patch + - update + - delete + # + # policy.* resources + # + - apiGroups: + - policy + resources: + - poddisruptionbudgets + verbs: + - get + - list + - patch + - update + - watch + - create + - delete + # + # apiextensions + # + - apiGroups: + - apiextensions.k8s.io + resources: + - customresourcedefinitions + verbs: + - get + - list + # clickhouse - related resources + - apiGroups: + - clickhouse.altinity.com + # + # The operator's specific Custom Resources + # + + resources: + - clickhouseinstallations + verbs: + - get + - list + - watch + - patch + - update + - delete + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallationtemplates + - clickhouseoperatorconfigurations + verbs: + - get + - list + - watch + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/finalizers + - clickhouseinstallationtemplates/finalizers + - clickhouseoperatorconfigurations/finalizers + verbs: + - update + - apiGroups: + - clickhouse.altinity.com + resources: + - clickhouseinstallations/status + - clickhouseinstallationtemplates/status + - clickhouseoperatorconfigurations/status + verbs: + - get + - update + - patch + - create + - delete + # clickhouse-keeper - related resources + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations + verbs: + - get + - list + - watch + - patch + - update + - delete + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations/finalizers + verbs: + - update + - apiGroups: + - clickhouse-keeper.altinity.com + resources: + - clickhousekeeperinstallations/status + verbs: + - get + - update + - patch + - create + - delete diff --git a/deploy/operatorhub/0.24.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.24.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml new file mode 100644 index 000000000..5abf359fa --- /dev/null +++ b/deploy/operatorhub/0.24.1/clickhouseinstallations.clickhouse.altinity.com.crd.yaml @@ -0,0 +1,1274 @@ +# Template Parameters: +# +# KIND=ClickHouseInstallation +# SINGULAR=clickhouseinstallation +# PLURAL=clickhouseinstallations +# SHORT=chi +# OPERATOR_VERSION=0.24.1 +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhouseinstallations.clickhouse.altinity.com + labels: + clickhouse.altinity.com/chop: 0.24.1 +spec: + group: clickhouse.altinity.com + scope: Namespaced + names: + kind: ClickHouseInstallation + singular: clickhouseinstallation + plural: clickhouseinstallations + shortNames: + - chi + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: version + type: string + description: Operator version + priority: 1 # show in wide view + jsonPath: .status.chop-version + - name: clusters + type: integer + description: Clusters count + jsonPath: .status.clusters + - name: shards + type: integer + description: Shards count + priority: 1 # show in wide view + jsonPath: .status.shards + - name: hosts + type: integer + description: Hosts count + jsonPath: .status.hosts + - name: taskID + type: string + description: TaskID + priority: 1 # show in wide view + jsonPath: .status.taskID + - name: status + type: string + description: Resource status + jsonPath: .status.status + - name: hosts-unchanged + type: integer + description: Unchanged hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsUnchanged + - name: hosts-updated + type: integer + description: Updated hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsUpdated + - name: hosts-added + type: integer + description: Added hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsAdded + - name: hosts-completed + type: integer + description: Completed hosts count + jsonPath: .status.hostsCompleted + - name: hosts-deleted + type: integer + description: Hosts deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDeleted + - name: hosts-delete + type: integer + description: Hosts to be deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDelete + - name: endpoint + type: string + description: Client access endpoint + priority: 1 # show in wide view + jsonPath: .status.endpoint + - name: age + type: date + description: Age of the resource + # Displayed in all priorities + jsonPath: .metadata.creationTimestamp + subresources: + status: {} + schema: + openAPIV3Schema: + description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more clusters" + type: object + required: + - spec + properties: + apiVersion: + description: | + APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: | + Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + status: + type: object + description: | + Status contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other + properties: + chop-version: + type: string + description: "Operator version" + chop-commit: + type: string + description: "Operator git commit SHA" + chop-date: + type: string + description: "Operator build date" + chop-ip: + type: string + description: "IP address of the operator's pod which managed this resource" + clusters: + type: integer + minimum: 0 + description: "Clusters count" + shards: + type: integer + minimum: 0 + description: "Shards count" + replicas: + type: integer + minimum: 0 + description: "Replicas count" + hosts: + type: integer + minimum: 0 + description: "Hosts count" + status: + type: string + description: "Status" + taskID: + type: string + description: "Current task id" + taskIDsStarted: + type: array + description: "Started task ids" + nullable: true + items: + type: string + taskIDsCompleted: + type: array + description: "Completed task ids" + nullable: true + items: + type: string + action: + type: string + description: "Action" + actions: + type: array + description: "Actions" + nullable: true + items: + type: string + error: + type: string + description: "Last error" + errors: + type: array + description: "Errors" + nullable: true + items: + type: string + hostsUnchanged: + type: integer + minimum: 0 + description: "Unchanged Hosts count" + hostsUpdated: + type: integer + minimum: 0 + description: "Updated Hosts count" + hostsAdded: + type: integer + minimum: 0 + description: "Added Hosts count" + hostsCompleted: + type: integer + minimum: 0 + description: "Completed Hosts count" + hostsDeleted: + type: integer + minimum: 0 + description: "Deleted Hosts count" + hostsDelete: + type: integer + minimum: 0 + description: "About to delete Hosts count" + pods: + type: array + description: "Pods" + nullable: true + items: + type: string + pod-ips: + type: array + description: "Pod IPs" + nullable: true + items: + type: string + fqdns: + type: array + description: "Pods FQDNs" + nullable: true + items: + type: string + endpoint: + type: string + description: "Endpoint" + generation: + type: integer + minimum: 0 + description: "Generation" + normalized: + type: object + description: "Normalized resource requested" + x-kubernetes-preserve-unknown-fields: true + normalizedCompleted: + type: object + description: "Normalized resource completed" + x-kubernetes-preserve-unknown-fields: true + hostsWithTablesCreated: + type: array + description: "List of hosts with tables created by the operator" + nullable: true + items: + type: string + usedTemplates: + type: array + description: "List of templates used to build this CHI" + nullable: true + x-kubernetes-preserve-unknown-fields: true + items: + type: object + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + # x-kubernetes-preserve-unknown-fields: true + description: | + Specification of the desired behavior of one or more ClickHouse clusters + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md + properties: + taskID: + type: string + description: | + Allows to define custom taskID for CHI update and watch status of this update execution. + Displayed in all .status.taskID* fields. + By default (if not filled) every update of CHI manifest will generate random taskID + stop: &TypeStringBool + type: string + description: | + Allows to stop all ClickHouse clusters defined in a CHI. + Works as the following: + - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact. + - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s. + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + restart: + type: string + description: | + In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. + This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. + enum: + - "" + - "RollingUpdate" + troubleshoot: + !!merge <<: *TypeStringBool + description: | + Allows to troubleshoot Pods during CrashLoopBack state. + This may happen when wrong configuration applied, in this case `clickhouse-server` wouldn't start. + Command within ClickHouse container is modified with `sleep` in order to avoid quick restarts + and give time to troubleshoot via CLI. + Liveness and Readiness probes are disabled as well. + namespaceDomainPattern: + type: string + description: | + Custom domain pattern which will be used for DNS names of `Service` or `Pod`. + Typical use scenario - custom cluster domain in Kubernetes cluster + Example: %s.svc.my.test + templating: + type: object + # nullable: true + description: | + Optional, applicable inside ClickHouseInstallationTemplate only. + Defines current ClickHouseInstallationTemplate application options to target ClickHouseInstallation(s)." + properties: + policy: + type: string + description: | + When defined as `auto` inside ClickhouseInstallationTemplate, this ClickhouseInstallationTemplate + will be auto-added into ClickHouseInstallation, selectable by `chiSelector`. + Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. + enum: + - "" + - "auto" + - "manual" + chiSelector: + type: object + description: "Optional, defines selector for ClickHouseInstallation(s) to be templated with ClickhouseInstallationTemplate" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + reconciling: + type: object + description: "Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" + # nullable: true + properties: + policy: + type: string + description: | + DISCUSSED TO BE DEPRECATED + Syntax sugar + Overrides all three 'reconcile.host.wait.{exclude, queries, include}' values from the operator's config + Possible values: + - wait - should wait to exclude host, complete queries and include host back into the cluster + - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + enum: + - "" + - "wait" + - "nowait" + configMapPropagationTimeout: + type: integer + description: | + Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod` + More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically + minimum: 0 + maximum: 3600 + cleanup: + type: object + description: "Optional, defines behavior for cleanup Kubernetes resources during reconcile cycle" + # nullable: true + properties: + unknownObjects: + type: object + description: | + Describes what clickhouse-operator should do with found Kubernetes resources which should be managed by clickhouse-operator, + but do not have `ownerReference` to any currently managed `ClickHouseInstallation` resource. + Default behavior is `Delete`" + # nullable: true + properties: + statefulSet: &TypeObjectsCleanup + type: string + description: "Behavior policy for unknown StatefulSet, `Delete` by default" + enum: + # List ObjectsCleanupXXX constants from model + - "" + - "Retain" + - "Delete" + pvc: + type: string + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown PVC, `Delete` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown ConfigMap, `Delete` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown Service, `Delete` by default" + reconcileFailedObjects: + type: object + description: | + Describes what clickhouse-operator should do with Kubernetes resources which are failed during reconcile. + Default behavior is `Retain`" + # nullable: true + properties: + statefulSet: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed StatefulSet, `Retain` by default" + pvc: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed PVC, `Retain` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed ConfigMap, `Retain` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed Service, `Retain` by default" + defaults: + type: object + description: | + define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults + # nullable: true + properties: + replicasUseFQDN: + !!merge <<: *TypeStringBool + description: | + define should replicas be specified by FQDN in ``. + In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup + "yes" by default + distributedDDL: + type: object + description: | + allows change `` settings + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl + # nullable: true + properties: + profile: + type: string + description: "Settings from this profile will be used to execute DDL queries" + storageManagement: + type: object + description: default storage management options + properties: + provisioner: &TypePVCProvisioner + type: string + description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + enum: + - "" + - "StatefulSet" + - "Operator" + reclaimPolicy: &TypePVCReclaimPolicy + type: string + description: | + defines behavior of `PVC` deletion. + `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet + enum: + - "" + - "Retain" + - "Delete" + templates: &TypeTemplateNames + type: object + description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + serviceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" + clusterServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" + volumeClaimTemplate: + type: string + description: "optional, alias for dataVolumeClaimTemplate, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + configuration: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" + # nullable: true + properties: + zookeeper: &TypeZookeeperConfig + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + `clickhouse-operator` itself doesn't manage Zookeeper, please install Zookeeper separatelly look examples on https://github.com/Altinity/clickhouse-operator/tree/master/deploy/zookeeper/ + currently, zookeeper (or clickhouse-keeper replacement) used for *ReplicatedMergeTree table engines and for `distributed_ddl` + More details: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings_zookeeper + # nullable: true + properties: + nodes: + type: array + description: "describe every available zookeeper cluster node for interaction" + # nullable: true + items: + type: object + #required: + # - host + properties: + host: + type: string + description: "dns name or ip address for Zookeeper node" + port: + type: integer + description: "TCP port which used to connect to Zookeeper node" + minimum: 0 + maximum: 65535 + secure: + !!merge <<: *TypeStringBool + description: "if a secure connection to Zookeeper is required" + session_timeout_ms: + type: integer + description: "session timeout during connect to Zookeeper" + operation_timeout_ms: + type: integer + description: "one operation timeout during Zookeeper transactions" + root: + type: string + description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" + identity: + type: string + description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" + users: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure password hashed, authorization restrictions, database level security row filters etc. + More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers + + any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets + secret value will pass in `pod.spec.containers.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/users.d/chop-generated-users.xml + it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle + + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + + any key with prefix `k8s_secret_` shall has value with format namespace/secret/key or secret/key + in this case value from secret will write directly into XML tag during render *-usersd ConfigMap + + any key with prefix `k8s_secret_env` shall has value with format namespace/secret/key or secret/key + in this case value from secret will write into environment variable and write to XML tag via from_env=XXX + + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + # nullable: true + x-kubernetes-preserve-unknown-fields: true + profiles: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of settings profile + More details: https://clickhouse.tech/docs/en/operations/settings/settings-profiles/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationprofiles + # nullable: true + x-kubernetes-preserve-unknown-fields: true + quotas: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of resource quotas + More details: https://clickhouse.tech/docs/en/operations/quotas/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationquotas + # nullable: true + x-kubernetes-preserve-unknown-fields: true + settings: &TypeSettings + type: object + description: | + allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings + + any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + + secret value will pass in `pod.spec.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/config.d/chop-generated-settings.xml + it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: &TypeFiles + type: object + description: | + allows define content of any setting file inside each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + every key in this object is the file name + every value in this object is the file content + you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html + each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored + More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml + + any key could contains `valueFrom` with `secretKeyRef` which allow pass values from kubernetes secrets + secrets will mounted into pod as separate volume in /etc/clickhouse-server/secrets.d/ + and will automatically update when update secret + it useful for pass SSL certificates from cert-manager or similar tool + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + # nullable: true + x-kubernetes-preserve-unknown-fields: true + clusters: + type: array + description: | + describes clusters layout and allows change settings on cluster-level, shard-level and replica-level + every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` + all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` + Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + If `cluster` contains zookeeper settings (could be inherited from top `chi` level), when you can create *ReplicatedMergeTree tables + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "cluster name, used to identify set of servers and wide used during generate names of related Kubernetes resources" + minLength: 1 + # See namePartClusterMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + zookeeper: + !!merge <<: *TypeZookeeperConfig + description: | + optional, allows configure .. section in each `Pod` only in current ClickHouse cluster, during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.zookeeper` settings + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster + override top-level `chi.spec.configuration.templates` + schemaPolicy: + type: object + description: | + describes how schema is propagated within replicas and shards + properties: + replica: + type: string + description: "how schema is propagated within a replica" + enum: + # List SchemaPolicyReplicaXXX constants from model + - "" + - "None" + - "All" + shard: + type: string + description: "how schema is propagated between shards" + enum: + # List SchemaPolicyShardXXX constants from model + - "" + - "None" + - "All" + - "DistributedTablesOnly" + insecure: + !!merge <<: *TypeStringBool + description: optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: optional, open secure ports for cluster + secret: + type: object + description: "optional, shared secret value to secure cluster communications" + properties: + auto: + !!merge <<: *TypeStringBool + description: "Auto-generate shared secret value to secure cluster communications" + value: + description: "Cluster shared secret value in plain text" + type: string + valueFrom: + description: "Cluster shared secret source" + type: object + properties: + secretKeyRef: + description: | + Selects a key of a secret in the clickhouse installation namespace. + Should not be used if value is not empty. + type: object + properties: + name: + description: | + Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - name + - key + pdbMaxUnavailable: + type: integer + description: | + Pod eviction is allowed if at most "pdbMaxUnavailable" pods are unavailable after the eviction, + i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions + by specifying 0. This is a mutually exclusive setting with "minAvailable". + minimum: 0 + maximum: 65535 + layout: + type: object + description: | + describe current cluster layout, how much shards in cluster, how much replica in shard + allows override settings on each shard and replica separatelly + # nullable: true + properties: + shardsCount: + type: integer + description: | + how much shards for current ClickHouse cluster will run in Kubernetes, + each shard contains shared-nothing part of data and contains set of replicas, + cluster contains 1 shard by default" + replicasCount: + type: integer + description: | + how much replicas in each shards for current cluster will run in Kubernetes, + each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + every shard contains 1 replica by default" + shards: + type: array + description: | + optional, allows override top-level `chi.spec.configuration`, cluster-level + `chi.spec.configuration.clusters` settings for each shard separately, + use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + definitionType: + type: string + description: "DEPRECATED - to be removed soon" + weight: + type: integer + description: | + optional, 1 by default, allows setup shard setting which will use during insert into tables with `Distributed` engine, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + internalReplication: + !!merge <<: *TypeStringBool + description: | + optional, `true` by default when `chi.spec.configuration.clusters[].layout.ReplicaCount` > 1 and 0 otherwise + allows setup setting which will use during insert into tables with `Distributed` engine for insert only in one live replica and other replicas will download inserted data during replication, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` and cluster-level `chi.spec.configuration.clusters.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected shard + override top-level `chi.spec.configuration.templates` and cluster-level `chi.spec.configuration.clusters.templates` + replicasCount: + type: integer + description: | + optional, how much replicas in selected shard for selected ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + shard contains 1 replica by default + override cluster-level `chi.spec.configuration.clusters.layout.replicasCount` + minimum: 1 + replicas: + type: array + description: | + optional, allows override behavior for selected replicas from cluster-level `chi.spec.configuration.clusters` and shard-level `chi.spec.configuration.clusters.layout.shards` + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected replica, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected replica, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected replica, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and shard-level `chi.spec.configuration.clusters.layout.shards.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files`, cluster-level `chi.spec.configuration.clusters.files` and shard-level `chi.spec.configuration.clusters.layout.shards.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` and shard-level `chi.spec.configuration.clusters.layout.shards.templates` + replicas: + type: array + description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` + shardsCount: + type: integer + description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" + minimum: 1 + shards: + type: array + description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected shard, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected shard, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected shard, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` + templates: + type: object + description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" + # nullable: true + properties: + hostTemplates: + type: array + description: "hostTemplate will use during apply to generate `clickhose-server` config files" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" + type: string + portDistribution: + type: array + description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" + enum: + # List PortDistributionXXX constants + - "" + - "Unspecified" + - "ClusterScopeIndex" + spec: + # Host + type: object + properties: + name: + type: string + description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `tcp_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=tcp]` + More info: https://clickhouse.tech/docs/en/interfaces/tcp/ + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=http]` + More info: https://clickhouse.tech/docs/en/interfaces/http/ + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `interserver_http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=interserver]` + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#interserver-http-port + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + templates: + !!merge <<: *TypeTemplateNames + description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" + podTemplates: + type: array + description: | + podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone + More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" + generateName: + type: string + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" + zone: + type: object + description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + #required: + # - values + properties: + key: + type: string + description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" + values: + type: array + description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" + # nullable: true + items: + type: string + distribution: + type: string + description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + enum: + - "" + - "Unspecified" + - "OnePerHost" + podDistribution: + type: array + description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "you can define multiple affinity policy types" + enum: + # List PodDistributionXXX constants + - "" + - "Unspecified" + - "ClickHouseAntiAffinity" + - "ShardAntiAffinity" + - "ReplicaAntiAffinity" + - "AnotherNamespaceAntiAffinity" + - "AnotherClickHouseInstallationAntiAffinity" + - "AnotherClusterAntiAffinity" + - "MaxNumberPerNode" + - "NamespaceAffinity" + - "ClickHouseInstallationAffinity" + - "ClusterAffinity" + - "ShardAffinity" + - "ReplicaAffinity" + - "PreviousTailAffinity" + - "CircularReplication" + scope: + type: string + description: "scope for apply each podDistribution" + enum: + # list PodDistributionScopeXXX constants + - "" + - "Unspecified" + - "Shard" + - "Replica" + - "Cluster" + - "ClickHouseInstallation" + - "Namespace" + number: + type: integer + description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" + minimum: 0 + maximum: 65535 + topologyKey: + type: string + description: | + use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, + more info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" + metadata: + type: object + description: | + allows pass standard object's metadata from template to Pod + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify PodSpec + type: object + description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + volumeClaimTemplates: + type: array + description: | + allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, + cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` + provisioner: *TypePVCProvisioner + reclaimPolicy: *TypePVCReclaimPolicy + metadata: + type: object + description: | + allows to pass standard object's metadata from template to PVC + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + allows define all aspects of `PVC` resource + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims + # nullable: true + x-kubernetes-preserve-unknown-fields: true + serviceTemplates: + type: array + description: | + allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + chi-level `chi.spec.defaults.templates.serviceTemplate` + cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` + generateName: + type: string + description: | + allows define format for generated `Service` name, + look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates + for details about available template variables" + metadata: + # TODO specify ObjectMeta + type: object + description: | + allows pass standard object's metadata from template to Service + Could be use for define specificly for Cloud Provider metadata which impact to behavior of service + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify ServiceSpec + type: object + description: | + describe behavior of generated Service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + useTemplates: + type: array + description: | + list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `CHI` + manifest during render Kubernetes resources to create related ClickHouse clusters" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "name of `ClickHouseInstallationTemplate` (chit) resource" + namespace: + type: string + description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clichouse-operator`" + useType: + type: string + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + enum: + # List useTypeXXX constants from model + - "" + - "merge" diff --git a/deploy/operatorhub/0.24.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.24.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml new file mode 100644 index 000000000..ead66666a --- /dev/null +++ b/deploy/operatorhub/0.24.1/clickhouseinstallationtemplates.clickhouse.altinity.com.crd.yaml @@ -0,0 +1,1274 @@ +# Template Parameters: +# +# KIND=ClickHouseInstallationTemplate +# SINGULAR=clickhouseinstallationtemplate +# PLURAL=clickhouseinstallationtemplates +# SHORT=chit +# OPERATOR_VERSION=0.24.1 +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhouseinstallationtemplates.clickhouse.altinity.com + labels: + clickhouse.altinity.com/chop: 0.24.1 +spec: + group: clickhouse.altinity.com + scope: Namespaced + names: + kind: ClickHouseInstallationTemplate + singular: clickhouseinstallationtemplate + plural: clickhouseinstallationtemplates + shortNames: + - chit + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: version + type: string + description: Operator version + priority: 1 # show in wide view + jsonPath: .status.chop-version + - name: clusters + type: integer + description: Clusters count + jsonPath: .status.clusters + - name: shards + type: integer + description: Shards count + priority: 1 # show in wide view + jsonPath: .status.shards + - name: hosts + type: integer + description: Hosts count + jsonPath: .status.hosts + - name: taskID + type: string + description: TaskID + priority: 1 # show in wide view + jsonPath: .status.taskID + - name: status + type: string + description: Resource status + jsonPath: .status.status + - name: hosts-unchanged + type: integer + description: Unchanged hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsUnchanged + - name: hosts-updated + type: integer + description: Updated hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsUpdated + - name: hosts-added + type: integer + description: Added hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsAdded + - name: hosts-completed + type: integer + description: Completed hosts count + jsonPath: .status.hostsCompleted + - name: hosts-deleted + type: integer + description: Hosts deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDeleted + - name: hosts-delete + type: integer + description: Hosts to be deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDelete + - name: endpoint + type: string + description: Client access endpoint + priority: 1 # show in wide view + jsonPath: .status.endpoint + - name: age + type: date + description: Age of the resource + # Displayed in all priorities + jsonPath: .metadata.creationTimestamp + subresources: + status: {} + schema: + openAPIV3Schema: + description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more clusters" + type: object + required: + - spec + properties: + apiVersion: + description: | + APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: | + Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + status: + type: object + description: | + Status contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other + properties: + chop-version: + type: string + description: "Operator version" + chop-commit: + type: string + description: "Operator git commit SHA" + chop-date: + type: string + description: "Operator build date" + chop-ip: + type: string + description: "IP address of the operator's pod which managed this resource" + clusters: + type: integer + minimum: 0 + description: "Clusters count" + shards: + type: integer + minimum: 0 + description: "Shards count" + replicas: + type: integer + minimum: 0 + description: "Replicas count" + hosts: + type: integer + minimum: 0 + description: "Hosts count" + status: + type: string + description: "Status" + taskID: + type: string + description: "Current task id" + taskIDsStarted: + type: array + description: "Started task ids" + nullable: true + items: + type: string + taskIDsCompleted: + type: array + description: "Completed task ids" + nullable: true + items: + type: string + action: + type: string + description: "Action" + actions: + type: array + description: "Actions" + nullable: true + items: + type: string + error: + type: string + description: "Last error" + errors: + type: array + description: "Errors" + nullable: true + items: + type: string + hostsUnchanged: + type: integer + minimum: 0 + description: "Unchanged Hosts count" + hostsUpdated: + type: integer + minimum: 0 + description: "Updated Hosts count" + hostsAdded: + type: integer + minimum: 0 + description: "Added Hosts count" + hostsCompleted: + type: integer + minimum: 0 + description: "Completed Hosts count" + hostsDeleted: + type: integer + minimum: 0 + description: "Deleted Hosts count" + hostsDelete: + type: integer + minimum: 0 + description: "About to delete Hosts count" + pods: + type: array + description: "Pods" + nullable: true + items: + type: string + pod-ips: + type: array + description: "Pod IPs" + nullable: true + items: + type: string + fqdns: + type: array + description: "Pods FQDNs" + nullable: true + items: + type: string + endpoint: + type: string + description: "Endpoint" + generation: + type: integer + minimum: 0 + description: "Generation" + normalized: + type: object + description: "Normalized resource requested" + x-kubernetes-preserve-unknown-fields: true + normalizedCompleted: + type: object + description: "Normalized resource completed" + x-kubernetes-preserve-unknown-fields: true + hostsWithTablesCreated: + type: array + description: "List of hosts with tables created by the operator" + nullable: true + items: + type: string + usedTemplates: + type: array + description: "List of templates used to build this CHI" + nullable: true + x-kubernetes-preserve-unknown-fields: true + items: + type: object + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + # x-kubernetes-preserve-unknown-fields: true + description: | + Specification of the desired behavior of one or more ClickHouse clusters + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md + properties: + taskID: + type: string + description: | + Allows to define custom taskID for CHI update and watch status of this update execution. + Displayed in all .status.taskID* fields. + By default (if not filled) every update of CHI manifest will generate random taskID + stop: &TypeStringBool + type: string + description: | + Allows to stop all ClickHouse clusters defined in a CHI. + Works as the following: + - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact. + - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s. + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + restart: + type: string + description: | + In case 'RollingUpdate' specified, the operator will always restart ClickHouse pods during reconcile. + This options is used in rare cases when force restart is required and is typically removed after the use in order to avoid unneeded restarts. + enum: + - "" + - "RollingUpdate" + troubleshoot: + !!merge <<: *TypeStringBool + description: | + Allows to troubleshoot Pods during CrashLoopBack state. + This may happen when wrong configuration applied, in this case `clickhouse-server` wouldn't start. + Command within ClickHouse container is modified with `sleep` in order to avoid quick restarts + and give time to troubleshoot via CLI. + Liveness and Readiness probes are disabled as well. + namespaceDomainPattern: + type: string + description: | + Custom domain pattern which will be used for DNS names of `Service` or `Pod`. + Typical use scenario - custom cluster domain in Kubernetes cluster + Example: %s.svc.my.test + templating: + type: object + # nullable: true + description: | + Optional, applicable inside ClickHouseInstallationTemplate only. + Defines current ClickHouseInstallationTemplate application options to target ClickHouseInstallation(s)." + properties: + policy: + type: string + description: | + When defined as `auto` inside ClickhouseInstallationTemplate, this ClickhouseInstallationTemplate + will be auto-added into ClickHouseInstallation, selectable by `chiSelector`. + Default value is `manual`, meaning ClickHouseInstallation should request this ClickhouseInstallationTemplate explicitly. + enum: + - "" + - "auto" + - "manual" + chiSelector: + type: object + description: "Optional, defines selector for ClickHouseInstallation(s) to be templated with ClickhouseInstallationTemplate" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + reconciling: + type: object + description: "Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" + # nullable: true + properties: + policy: + type: string + description: | + DISCUSSED TO BE DEPRECATED + Syntax sugar + Overrides all three 'reconcile.host.wait.{exclude, queries, include}' values from the operator's config + Possible values: + - wait - should wait to exclude host, complete queries and include host back into the cluster + - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + enum: + - "" + - "wait" + - "nowait" + configMapPropagationTimeout: + type: integer + description: | + Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod` + More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically + minimum: 0 + maximum: 3600 + cleanup: + type: object + description: "Optional, defines behavior for cleanup Kubernetes resources during reconcile cycle" + # nullable: true + properties: + unknownObjects: + type: object + description: | + Describes what clickhouse-operator should do with found Kubernetes resources which should be managed by clickhouse-operator, + but do not have `ownerReference` to any currently managed `ClickHouseInstallation` resource. + Default behavior is `Delete`" + # nullable: true + properties: + statefulSet: &TypeObjectsCleanup + type: string + description: "Behavior policy for unknown StatefulSet, `Delete` by default" + enum: + # List ObjectsCleanupXXX constants from model + - "" + - "Retain" + - "Delete" + pvc: + type: string + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown PVC, `Delete` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown ConfigMap, `Delete` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown Service, `Delete` by default" + reconcileFailedObjects: + type: object + description: | + Describes what clickhouse-operator should do with Kubernetes resources which are failed during reconcile. + Default behavior is `Retain`" + # nullable: true + properties: + statefulSet: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed StatefulSet, `Retain` by default" + pvc: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed PVC, `Retain` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed ConfigMap, `Retain` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed Service, `Retain` by default" + defaults: + type: object + description: | + define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults + # nullable: true + properties: + replicasUseFQDN: + !!merge <<: *TypeStringBool + description: | + define should replicas be specified by FQDN in ``. + In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup + "yes" by default + distributedDDL: + type: object + description: | + allows change `` settings + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl + # nullable: true + properties: + profile: + type: string + description: "Settings from this profile will be used to execute DDL queries" + storageManagement: + type: object + description: default storage management options + properties: + provisioner: &TypePVCProvisioner + type: string + description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + enum: + - "" + - "StatefulSet" + - "Operator" + reclaimPolicy: &TypePVCReclaimPolicy + type: string + description: | + defines behavior of `PVC` deletion. + `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet + enum: + - "" + - "Retain" + - "Delete" + templates: &TypeTemplateNames + type: object + description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + serviceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" + clusterServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" + volumeClaimTemplate: + type: string + description: "optional, alias for dataVolumeClaimTemplate, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + configuration: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" + # nullable: true + properties: + zookeeper: &TypeZookeeperConfig + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + `clickhouse-operator` itself doesn't manage Zookeeper, please install Zookeeper separatelly look examples on https://github.com/Altinity/clickhouse-operator/tree/master/deploy/zookeeper/ + currently, zookeeper (or clickhouse-keeper replacement) used for *ReplicatedMergeTree table engines and for `distributed_ddl` + More details: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings_zookeeper + # nullable: true + properties: + nodes: + type: array + description: "describe every available zookeeper cluster node for interaction" + # nullable: true + items: + type: object + #required: + # - host + properties: + host: + type: string + description: "dns name or ip address for Zookeeper node" + port: + type: integer + description: "TCP port which used to connect to Zookeeper node" + minimum: 0 + maximum: 65535 + secure: + !!merge <<: *TypeStringBool + description: "if a secure connection to Zookeeper is required" + session_timeout_ms: + type: integer + description: "session timeout during connect to Zookeeper" + operation_timeout_ms: + type: integer + description: "one operation timeout during Zookeeper transactions" + root: + type: string + description: "optional root znode path inside zookeeper to store ClickHouse related data (replication queue or distributed DDL)" + identity: + type: string + description: "optional access credentials string with `user:password` format used when use digest authorization in Zookeeper" + users: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure password hashed, authorization restrictions, database level security row filters etc. + More details: https://clickhouse.tech/docs/en/operations/settings/settings-users/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationusers + + any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets + secret value will pass in `pod.spec.containers.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/users.d/chop-generated-users.xml + it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle + + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + + any key with prefix `k8s_secret_` shall has value with format namespace/secret/key or secret/key + in this case value from secret will write directly into XML tag during render *-usersd ConfigMap + + any key with prefix `k8s_secret_env` shall has value with format namespace/secret/key or secret/key + in this case value from secret will write into environment variable and write to XML tag via from_env=XXX + + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + # nullable: true + x-kubernetes-preserve-unknown-fields: true + profiles: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of settings profile + More details: https://clickhouse.tech/docs/en/operations/settings/settings-profiles/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationprofiles + # nullable: true + x-kubernetes-preserve-unknown-fields: true + quotas: + type: object + description: | + allows configure .. section in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/users.d/` + you can configure any aspect of resource quotas + More details: https://clickhouse.tech/docs/en/operations/quotas/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationquotas + # nullable: true + x-kubernetes-preserve-unknown-fields: true + settings: &TypeSettings + type: object + description: | + allows configure `clickhouse-server` settings inside ... tag in each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + Your yaml code will convert to XML, see examples https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specconfigurationsettings + + any key could contains `valueFrom` with `secretKeyRef` which allow pass password from kubernetes secrets + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + + secret value will pass in `pod.spec.evn`, and generate with from_env=XXX in XML in /etc/clickhouse-server/config.d/chop-generated-settings.xml + it not allow automatically updates when updates `secret`, change spec.taskID for manually trigger reconcile cycle + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: &TypeFiles + type: object + description: | + allows define content of any setting file inside each `Pod` during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + every key in this object is the file name + every value in this object is the file content + you can use `!!binary |` and base64 for binary files, see details here https://yaml.org/type/binary.html + each key could contains prefix like USERS, COMMON, HOST or config.d, users.d, cond.d, wrong prefixes will ignored, subfolders also will ignored + More details: https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-05-files-nested.yaml + + any key could contains `valueFrom` with `secretKeyRef` which allow pass values from kubernetes secrets + secrets will mounted into pod as separate volume in /etc/clickhouse-server/secrets.d/ + and will automatically update when update secret + it useful for pass SSL certificates from cert-manager or similar tool + look into https://github.com/Altinity/clickhouse-operator/blob/master/docs/chi-examples/05-settings-01-overview.yaml for examples + # nullable: true + x-kubernetes-preserve-unknown-fields: true + clusters: + type: array + description: | + describes clusters layout and allows change settings on cluster-level, shard-level and replica-level + every cluster is a set of StatefulSet, one StatefulSet contains only one Pod with `clickhouse-server` + all Pods will rendered in part of ClickHouse configs, mounted from ConfigMap as `/etc/clickhouse-server/config.d/chop-generated-remote_servers.xml` + Clusters will use for Distributed table engine, more details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + If `cluster` contains zookeeper settings (could be inherited from top `chi` level), when you can create *ReplicatedMergeTree tables + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "cluster name, used to identify set of servers and wide used during generate names of related Kubernetes resources" + minLength: 1 + # See namePartClusterMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + zookeeper: + !!merge <<: *TypeZookeeperConfig + description: | + optional, allows configure .. section in each `Pod` only in current ClickHouse cluster, during generate `ConfigMap` which will mounted in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.zookeeper` settings + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster + override top-level `chi.spec.configuration.templates` + schemaPolicy: + type: object + description: | + describes how schema is propagated within replicas and shards + properties: + replica: + type: string + description: "how schema is propagated within a replica" + enum: + # List SchemaPolicyReplicaXXX constants from model + - "" + - "None" + - "All" + shard: + type: string + description: "how schema is propagated between shards" + enum: + # List SchemaPolicyShardXXX constants from model + - "" + - "None" + - "All" + - "DistributedTablesOnly" + insecure: + !!merge <<: *TypeStringBool + description: optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: optional, open secure ports for cluster + secret: + type: object + description: "optional, shared secret value to secure cluster communications" + properties: + auto: + !!merge <<: *TypeStringBool + description: "Auto-generate shared secret value to secure cluster communications" + value: + description: "Cluster shared secret value in plain text" + type: string + valueFrom: + description: "Cluster shared secret source" + type: object + properties: + secretKeyRef: + description: | + Selects a key of a secret in the clickhouse installation namespace. + Should not be used if value is not empty. + type: object + properties: + name: + description: | + Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + key: + description: The key of the secret to select from. Must be a valid secret key. + type: string + optional: + description: Specify whether the Secret or its key must be defined + type: boolean + required: + - name + - key + pdbMaxUnavailable: + type: integer + description: | + Pod eviction is allowed if at most "pdbMaxUnavailable" pods are unavailable after the eviction, + i.e. even in absence of the evicted pod. For example, one can prevent all voluntary evictions + by specifying 0. This is a mutually exclusive setting with "minAvailable". + minimum: 0 + maximum: 65535 + layout: + type: object + description: | + describe current cluster layout, how much shards in cluster, how much replica in shard + allows override settings on each shard and replica separatelly + # nullable: true + properties: + shardsCount: + type: integer + description: | + how much shards for current ClickHouse cluster will run in Kubernetes, + each shard contains shared-nothing part of data and contains set of replicas, + cluster contains 1 shard by default" + replicasCount: + type: integer + description: | + how much replicas in each shards for current cluster will run in Kubernetes, + each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + every shard contains 1 replica by default" + shards: + type: array + description: | + optional, allows override top-level `chi.spec.configuration`, cluster-level + `chi.spec.configuration.clusters` settings for each shard separately, + use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + definitionType: + type: string + description: "DEPRECATED - to be removed soon" + weight: + type: integer + description: | + optional, 1 by default, allows setup shard setting which will use during insert into tables with `Distributed` engine, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + internalReplication: + !!merge <<: *TypeStringBool + description: | + optional, `true` by default when `chi.spec.configuration.clusters[].layout.ReplicaCount` > 1 and 0 otherwise + allows setup setting which will use during insert into tables with `Distributed` engine for insert only in one live replica and other replicas will download inserted data during replication, + will apply in inside ConfigMap which will mount in /etc/clickhouse-server/config.d/chop-generated-remote_servers.xml + More details: https://clickhouse.tech/docs/en/engines/table-engines/special/distributed/ + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` and cluster-level `chi.spec.configuration.clusters.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected shard + override top-level `chi.spec.configuration.templates` and cluster-level `chi.spec.configuration.clusters.templates` + replicasCount: + type: integer + description: | + optional, how much replicas in selected shard for selected ClickHouse cluster will run in Kubernetes, each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + shard contains 1 replica by default + override cluster-level `chi.spec.configuration.clusters.layout.replicasCount` + minimum: 1 + replicas: + type: array + description: | + optional, allows override behavior for selected replicas from cluster-level `chi.spec.configuration.clusters` and shard-level `chi.spec.configuration.clusters.layout.shards` + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected replica, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected replica, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected replica, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and shard-level `chi.spec.configuration.clusters.layout.shards.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files`, cluster-level `chi.spec.configuration.clusters.files` and shard-level `chi.spec.configuration.clusters.layout.shards.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` and shard-level `chi.spec.configuration.clusters.layout.shards.templates` + replicas: + type: array + description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` + shardsCount: + type: integer + description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" + minimum: 1 + shards: + type: array + description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `tcp` for selected shard, override `chi.spec.templates.hostTemplates.spec.tcpPort` + allows connect to `clickhouse-server` via TCP Native protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `http` for selected shard, override `chi.spec.templates.hostTemplates.spec.httpPort` + allows connect to `clickhouse-server` via HTTP protocol via kubernetes `Service` + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `Pod.spec.containers.ports` with name `interserver` for selected shard, override `chi.spec.templates.hostTemplates.spec.interserverHTTPPort` + allows connect between replicas inside same shard during fetch replicated data parts HTTP protocol + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` + templates: + type: object + description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" + # nullable: true + properties: + hostTemplates: + type: array + description: "hostTemplate will use during apply to generate `clickhose-server` config files" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" + type: string + portDistribution: + type: array + description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" + enum: + # List PortDistributionXXX constants + - "" + - "Unspecified" + - "ClusterScopeIndex" + spec: + # Host + type: object + properties: + name: + type: string + description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + insecure: + !!merge <<: *TypeStringBool + description: | + optional, open insecure ports for cluster, defaults to "yes" + secure: + !!merge <<: *TypeStringBool + description: | + optional, open secure ports + tcpPort: + type: integer + description: | + optional, setup `tcp_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=tcp]` + More info: https://clickhouse.tech/docs/en/interfaces/tcp/ + minimum: 1 + maximum: 65535 + tlsPort: + type: integer + minimum: 1 + maximum: 65535 + httpPort: + type: integer + description: | + optional, setup `http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=http]` + More info: https://clickhouse.tech/docs/en/interfaces/http/ + minimum: 1 + maximum: 65535 + httpsPort: + type: integer + minimum: 1 + maximum: 65535 + interserverHTTPPort: + type: integer + description: | + optional, setup `interserver_http_port` inside `clickhouse-server` settings for each Pod where current template will apply + if specified, should have equal value with `chi.spec.templates.podTemplates.spec.containers.ports[name=interserver]` + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#interserver-http-port + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + templates: + !!merge <<: *TypeTemplateNames + description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" + podTemplates: + type: array + description: | + podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone + More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" + generateName: + type: string + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" + zone: + type: object + description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + #required: + # - values + properties: + key: + type: string + description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" + values: + type: array + description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" + # nullable: true + items: + type: string + distribution: + type: string + description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + enum: + - "" + - "Unspecified" + - "OnePerHost" + podDistribution: + type: array + description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "you can define multiple affinity policy types" + enum: + # List PodDistributionXXX constants + - "" + - "Unspecified" + - "ClickHouseAntiAffinity" + - "ShardAntiAffinity" + - "ReplicaAntiAffinity" + - "AnotherNamespaceAntiAffinity" + - "AnotherClickHouseInstallationAntiAffinity" + - "AnotherClusterAntiAffinity" + - "MaxNumberPerNode" + - "NamespaceAffinity" + - "ClickHouseInstallationAffinity" + - "ClusterAffinity" + - "ShardAffinity" + - "ReplicaAffinity" + - "PreviousTailAffinity" + - "CircularReplication" + scope: + type: string + description: "scope for apply each podDistribution" + enum: + # list PodDistributionScopeXXX constants + - "" + - "Unspecified" + - "Shard" + - "Replica" + - "Cluster" + - "ClickHouseInstallation" + - "Namespace" + number: + type: integer + description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" + minimum: 0 + maximum: 65535 + topologyKey: + type: string + description: | + use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, + more info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" + metadata: + type: object + description: | + allows pass standard object's metadata from template to Pod + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify PodSpec + type: object + description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + volumeClaimTemplates: + type: array + description: | + allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, + cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` + provisioner: *TypePVCProvisioner + reclaimPolicy: *TypePVCReclaimPolicy + metadata: + type: object + description: | + allows to pass standard object's metadata from template to PVC + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + allows define all aspects of `PVC` resource + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims + # nullable: true + x-kubernetes-preserve-unknown-fields: true + serviceTemplates: + type: array + description: | + allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + chi-level `chi.spec.defaults.templates.serviceTemplate` + cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` + generateName: + type: string + description: | + allows define format for generated `Service` name, + look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates + for details about available template variables" + metadata: + # TODO specify ObjectMeta + type: object + description: | + allows pass standard object's metadata from template to Service + Could be use for define specificly for Cloud Provider metadata which impact to behavior of service + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify ServiceSpec + type: object + description: | + describe behavior of generated Service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true + useTemplates: + type: array + description: | + list of `ClickHouseInstallationTemplate` (chit) resource names which will merge with current `CHI` + manifest during render Kubernetes resources to create related ClickHouse clusters" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "name of `ClickHouseInstallationTemplate` (chit) resource" + namespace: + type: string + description: "Kubernetes namespace where need search `chit` resource, depending on `watchNamespaces` settings in `clichouse-operator`" + useType: + type: string + description: "optional, current strategy is only merge, and current `chi` settings have more priority than merged template `chit`" + enum: + # List useTypeXXX constants from model + - "" + - "merge" diff --git a/deploy/operatorhub/0.24.1/clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml b/deploy/operatorhub/0.24.1/clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml new file mode 100644 index 000000000..1662c4051 --- /dev/null +++ b/deploy/operatorhub/0.24.1/clickhousekeeperinstallations.clickhouse-keeper.altinity.com.crd.yaml @@ -0,0 +1,836 @@ +# Template Parameters: +# +# OPERATOR_VERSION=0.24.1 +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhousekeeperinstallations.clickhouse-keeper.altinity.com + labels: + clickhouse-keeper.altinity.com/chop: 0.24.1 +spec: + group: clickhouse-keeper.altinity.com + scope: Namespaced + names: + kind: ClickHouseKeeperInstallation + singular: clickhousekeeperinstallation + plural: clickhousekeeperinstallations + shortNames: + - chk + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: version + type: string + description: Operator version + priority: 1 # show in wide view + jsonPath: .status.chop-version + - name: clusters + type: integer + description: Clusters count + jsonPath: .status.clusters + - name: shards + type: integer + description: Shards count + priority: 1 # show in wide view + jsonPath: .status.shards + - name: hosts + type: integer + description: Hosts count + jsonPath: .status.hosts + - name: taskID + type: string + description: TaskID + priority: 1 # show in wide view + jsonPath: .status.taskID + - name: status + type: string + description: Resource status + jsonPath: .status.status + - name: hosts-unchanged + type: integer + description: Unchanged hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsUnchanged + - name: hosts-updated + type: integer + description: Updated hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsUpdated + - name: hosts-added + type: integer + description: Added hosts count + priority: 1 # show in wide view + jsonPath: .status.hostsAdded + - name: hosts-completed + type: integer + description: Completed hosts count + jsonPath: .status.hostsCompleted + - name: hosts-deleted + type: integer + description: Hosts deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDeleted + - name: hosts-delete + type: integer + description: Hosts to be deleted count + priority: 1 # show in wide view + jsonPath: .status.hostsDelete + - name: endpoint + type: string + description: Client access endpoint + priority: 1 # show in wide view + jsonPath: .status.endpoint + - name: age + type: date + description: Age of the resource + # Displayed in all priorities + jsonPath: .metadata.creationTimestamp + subresources: + status: {} + schema: + openAPIV3Schema: + description: "define a set of Kubernetes resources (StatefulSet, PVC, Service, ConfigMap) which describe behavior one or more clusters" + type: object + required: + - spec + properties: + apiVersion: + description: | + APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: | + Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + status: + type: object + description: | + Status contains many fields like a normalized configuration, clickhouse-operator version, current action and all applied action list, current taskID and all applied taskIDs and other + properties: + chop-version: + type: string + description: "Operator version" + chop-commit: + type: string + description: "Operator git commit SHA" + chop-date: + type: string + description: "Operator build date" + chop-ip: + type: string + description: "IP address of the operator's pod which managed this resource" + clusters: + type: integer + minimum: 0 + description: "Clusters count" + shards: + type: integer + minimum: 0 + description: "Shards count" + replicas: + type: integer + minimum: 0 + description: "Replicas count" + hosts: + type: integer + minimum: 0 + description: "Hosts count" + status: + type: string + description: "Status" + taskID: + type: string + description: "Current task id" + taskIDsStarted: + type: array + description: "Started task ids" + nullable: true + items: + type: string + taskIDsCompleted: + type: array + description: "Completed task ids" + nullable: true + items: + type: string + action: + type: string + description: "Action" + actions: + type: array + description: "Actions" + nullable: true + items: + type: string + error: + type: string + description: "Last error" + errors: + type: array + description: "Errors" + nullable: true + items: + type: string + hostsUnchanged: + type: integer + minimum: 0 + description: "Unchanged Hosts count" + hostsUpdated: + type: integer + minimum: 0 + description: "Updated Hosts count" + hostsAdded: + type: integer + minimum: 0 + description: "Added Hosts count" + hostsCompleted: + type: integer + minimum: 0 + description: "Completed Hosts count" + hostsDeleted: + type: integer + minimum: 0 + description: "Deleted Hosts count" + hostsDelete: + type: integer + minimum: 0 + description: "About to delete Hosts count" + pods: + type: array + description: "Pods" + nullable: true + items: + type: string + pod-ips: + type: array + description: "Pod IPs" + nullable: true + items: + type: string + fqdns: + type: array + description: "Pods FQDNs" + nullable: true + items: + type: string + endpoint: + type: string + description: "Endpoint" + generation: + type: integer + minimum: 0 + description: "Generation" + normalized: + type: object + description: "Normalized resource requested" + x-kubernetes-preserve-unknown-fields: true + normalizedCompleted: + type: object + description: "Normalized resource completed" + x-kubernetes-preserve-unknown-fields: true + hostsWithTablesCreated: + type: array + description: "List of hosts with tables created by the operator" + nullable: true + items: + type: string + usedTemplates: + type: array + description: "List of templates used to build this CHI" + nullable: true + x-kubernetes-preserve-unknown-fields: true + items: + type: object + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + # x-kubernetes-preserve-unknown-fields: true + description: | + Specification of the desired behavior of one or more ClickHouse clusters + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md + properties: + taskID: + type: string + description: | + Allows to define custom taskID for CHI update and watch status of this update execution. + Displayed in all .status.taskID* fields. + By default (if not filled) every update of CHI manifest will generate random taskID + stop: &TypeStringBool + type: string + description: | + Allows to stop all ClickHouse clusters defined in a CHI. + Works as the following: + - When `stop` is `1` operator sets `Replicas: 0` in each StatefulSet. Thie leads to having all `Pods` and `Service` deleted. All PVCs are kept intact. + - When `stop` is `0` operator sets `Replicas: 1` and `Pod`s and `Service`s will created again and all retained PVCs will be attached to `Pod`s. + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + namespaceDomainPattern: + type: string + description: | + Custom domain pattern which will be used for DNS names of `Service` or `Pod`. + Typical use scenario - custom cluster domain in Kubernetes cluster + Example: %s.svc.my.test + reconciling: + type: object + description: "Optional, allows tuning reconciling cycle for ClickhouseInstallation from clickhouse-operator side" + # nullable: true + properties: + policy: + type: string + description: | + DISCUSSED TO BE DEPRECATED + Syntax sugar + Overrides all three 'reconcile.host.wait.{exclude, queries, include}' values from the operator's config + Possible values: + - wait - should wait to exclude host, complete queries and include host back into the cluster + - nowait - should NOT wait to exclude host, complete queries and include host back into the cluster + enum: + - "" + - "wait" + - "nowait" + configMapPropagationTimeout: + type: integer + description: | + Timeout in seconds for `clickhouse-operator` to wait for modified `ConfigMap` to propagate into the `Pod` + More details: https://kubernetes.io/docs/concepts/configuration/configmap/#mounted-configmaps-are-updated-automatically + minimum: 0 + maximum: 3600 + cleanup: + type: object + description: "Optional, defines behavior for cleanup Kubernetes resources during reconcile cycle" + # nullable: true + properties: + unknownObjects: + type: object + description: | + Describes what clickhouse-operator should do with found Kubernetes resources which should be managed by clickhouse-operator, + but do not have `ownerReference` to any currently managed `ClickHouseInstallation` resource. + Default behavior is `Delete`" + # nullable: true + properties: + statefulSet: &TypeObjectsCleanup + type: string + description: "Behavior policy for unknown StatefulSet, `Delete` by default" + enum: + # List ObjectsCleanupXXX constants from model + - "" + - "Retain" + - "Delete" + pvc: + type: string + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown PVC, `Delete` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown ConfigMap, `Delete` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for unknown Service, `Delete` by default" + reconcileFailedObjects: + type: object + description: | + Describes what clickhouse-operator should do with Kubernetes resources which are failed during reconcile. + Default behavior is `Retain`" + # nullable: true + properties: + statefulSet: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed StatefulSet, `Retain` by default" + pvc: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed PVC, `Retain` by default" + configMap: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed ConfigMap, `Retain` by default" + service: + !!merge <<: *TypeObjectsCleanup + description: "Behavior policy for failed Service, `Retain` by default" + defaults: + type: object + description: | + define default behavior for whole ClickHouseInstallation, some behavior can be re-define on cluster, shard and replica level + More info: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#specdefaults + # nullable: true + properties: + replicasUseFQDN: + !!merge <<: *TypeStringBool + description: | + define should replicas be specified by FQDN in ``. + In case of "no" will use short hostname and clickhouse-server will use kubernetes default suffixes for DNS lookup + "yes" by default + distributedDDL: + type: object + description: | + allows change `` settings + More info: https://clickhouse.tech/docs/en/operations/server-configuration-parameters/settings/#server-settings-distributed_ddl + # nullable: true + properties: + profile: + type: string + description: "Settings from this profile will be used to execute DDL queries" + storageManagement: + type: object + description: default storage management options + properties: + provisioner: &TypePVCProvisioner + type: string + description: "defines `PVC` provisioner - be it StatefulSet or the Operator" + enum: + - "" + - "StatefulSet" + - "Operator" + reclaimPolicy: &TypePVCReclaimPolicy + type: string + description: | + defines behavior of `PVC` deletion. + `Delete` by default, if `Retain` specified then `PVC` will be kept when deleting StatefulSet + enum: + - "" + - "Retain" + - "Delete" + templates: &TypeTemplateNames + type: object + description: "optional, configuration of the templates names which will use for generate Kubernetes resources according to one or more ClickHouse clusters described in current ClickHouseInstallation (chi) resource" + # nullable: true + properties: + hostTemplate: + type: string + description: "optional, template name from chi.spec.templates.hostTemplates, which will apply to configure every `clickhouse-server` instance during render ConfigMap resources which will mount into `Pod`" + podTemplate: + type: string + description: "optional, template name from chi.spec.templates.podTemplates, allows customization each `Pod` resource during render and reconcile each StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + dataVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + logVolumeClaimTemplate: + type: string + description: "optional, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse log directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + serviceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for one `Service` resource which will created by `clickhouse-operator` which cover all clusters in whole `chi` resource" + clusterServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each clickhouse cluster described in `chi.spec.configuration.clusters`" + shardServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each shard inside clickhouse cluster described in `chi.spec.configuration.clusters`" + replicaServiceTemplate: + type: string + description: "optional, template name from chi.spec.templates.serviceTemplates, allows customization for each `Service` resource which will created by `clickhouse-operator` which cover each replica inside each shard inside each clickhouse cluster described in `chi.spec.configuration.clusters`" + volumeClaimTemplate: + type: string + description: "optional, alias for dataVolumeClaimTemplate, template name from chi.spec.templates.volumeClaimTemplates, allows customization each `PVC` which will mount for clickhouse data directory in each `Pod` during render and reconcile every StatefulSet.spec resource described in `chi.spec.configuration.clusters`" + configuration: + type: object + description: "allows configure multiple aspects and behavior for `clickhouse-server` instance and also allows describe multiple `clickhouse-server` clusters inside one `chi` resource" + # nullable: true + properties: + settings: &TypeSettings + type: object + description: | + allows configure multiple aspects and behavior for `clickhouse-keeper` instance + # nullable: true + x-kubernetes-preserve-unknown-fields: true + files: &TypeFiles + type: object + description: | + allows define content of any setting + # nullable: true + x-kubernetes-preserve-unknown-fields: true + clusters: + type: array + description: | + describes clusters layout and allows change settings on cluster-level and replica-level + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "cluster name, used to identify set of servers and wide used during generate names of related Kubernetes resources" + minLength: 1 + # See namePartClusterMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` only in one cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` + override top-level `chi.spec.configuration.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` on current cluster during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected cluster + override top-level `chi.spec.configuration.templates` + layout: + type: object + description: | + describe current cluster layout, how much shards in cluster, how much replica in shard + allows override settings on each shard and replica separatelly + # nullable: true + properties: + replicasCount: + type: integer + description: | + how much replicas in each shards for current cluster will run in Kubernetes, + each replica is a separate `StatefulSet` which contains only one `Pod` with `clickhouse-server` instance, + every shard contains 1 replica by default" + replicas: + type: array + description: "optional, allows override top-level `chi.spec.configuration` and cluster-level `chi.spec.configuration.clusters` configuration for each replica and each shard relates to selected replica, use it only if you fully understand what you do" + # nullable: true + items: + type: object + properties: + name: + type: string + description: "optional, by default replica name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartShardMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and will ignore if shard-level `chi.spec.configuration.clusters.layout.shards` present + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates` + shardsCount: + type: integer + description: "optional, count of shards related to current replica, you can override each shard behavior on low-level `chi.spec.configuration.clusters.layout.replicas.shards`" + minimum: 1 + shards: + type: array + description: "optional, list of shards related to current replica, will ignore if `chi.spec.configuration.clusters.layout.shards` presents" + # nullable: true + items: + # Host + type: object + properties: + name: + type: string + description: "optional, by default shard name is generated, but you can override it and setup custom name" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + zkPort: + type: integer + minimum: 1 + maximum: 65535 + raftPort: + type: integer + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + override top-level `chi.spec.configuration.settings`, cluster-level `chi.spec.configuration.clusters.settings` and replica-level `chi.spec.configuration.clusters.layout.replicas.settings` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` only in one shard related to current replica during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + override top-level `chi.spec.configuration.files` and cluster-level `chi.spec.configuration.clusters.files`, will ignore if `chi.spec.configuration.clusters.layout.shards` presents + templates: + !!merge <<: *TypeTemplateNames + description: | + optional, configuration of the templates names which will use for generate Kubernetes resources according to selected replica + override top-level `chi.spec.configuration.templates`, cluster-level `chi.spec.configuration.clusters.templates`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates` + templates: + type: object + description: "allows define templates which will use for render Kubernetes resources like StatefulSet, ConfigMap, Service, PVC, by default, clickhouse-operator have own templates, but you can override it" + # nullable: true + properties: + hostTemplates: + type: array + description: "hostTemplate will use during apply to generate `clickhose-server` config files" + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.hostTemplate`, cluster-level `chi.spec.configuration.clusters.templates.hostTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.hostTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.hostTemplate`" + type: string + portDistribution: + type: array + description: "define how will distribute numeric values of named ports in `Pod.spec.containers.ports` and clickhouse-server configs" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "type of distribution, when `Unspecified` (default value) then all listen ports on clickhouse-server configuration in all Pods will have the same value, when `ClusterScopeIndex` then ports will increment to offset from base value depends on shard and replica index inside cluster with combination of `chi.spec.templates.podTemlates.spec.HostNetwork` it allows setup ClickHouse cluster inside Kubernetes and provide access via external network bypass Kubernetes internal network" + enum: + # List PortDistributionXXX constants + - "" + - "Unspecified" + - "ClusterScopeIndex" + spec: + # Host + type: object + properties: + name: + type: string + description: "by default, hostname will generate, but this allows define custom name for each `clickhuse-server`" + minLength: 1 + # See namePartReplicaMaxLen const + maxLength: 15 + pattern: "^[a-zA-Z0-9-]{0,15}$" + zkPort: + type: integer + minimum: 1 + maximum: 65535 + raftPort: + type: integer + minimum: 1 + maximum: 65535 + settings: + !!merge <<: *TypeSettings + description: | + optional, allows configure `clickhouse-server` settings inside ... tag in each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/conf.d/` + More details: https://clickhouse.tech/docs/en/operations/settings/settings/ + files: + !!merge <<: *TypeFiles + description: | + optional, allows define content of any setting file inside each `Pod` where this template will apply during generate `ConfigMap` which will mount in `/etc/clickhouse-server/config.d/` or `/etc/clickhouse-server/conf.d/` or `/etc/clickhouse-server/users.d/` + templates: + !!merge <<: *TypeTemplateNames + description: "be careful, this part of CRD allows override template inside template, don't use it if you don't understand what you do" + podTemplates: + type: array + description: | + podTemplate will use during render `Pod` inside `StatefulSet.spec` and allows define rendered `Pod.spec`, pod scheduling distribution and pod zone + More information: https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatespodtemplates + # nullable: true + items: + type: object + #required: + # - name + properties: + name: + type: string + description: "template name, could use to link inside top-level `chi.spec.defaults.templates.podTemplate`, cluster-level `chi.spec.configuration.clusters.templates.podTemplate`, shard-level `chi.spec.configuration.clusters.layout.shards.temlates.podTemplate`, replica-level `chi.spec.configuration.clusters.layout.replicas.templates.podTemplate`" + generateName: + type: string + description: "allows define format for generated `Pod` name, look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates for details about available template variables" + zone: + type: object + description: "allows define custom zone name and will separate ClickHouse `Pods` between nodes, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + #required: + # - values + properties: + key: + type: string + description: "optional, if defined, allows select kubernetes nodes by label with `name` equal `key`" + values: + type: array + description: "optional, if defined, allows select kubernetes nodes by label with `value` in `values`" + # nullable: true + items: + type: string + distribution: + type: string + description: "DEPRECATED, shortcut for `chi.spec.templates.podTemplates.spec.affinity.podAntiAffinity`" + enum: + - "" + - "Unspecified" + - "OnePerHost" + podDistribution: + type: array + description: "define ClickHouse Pod distribution policy between Kubernetes Nodes inside Shard, Replica, Namespace, CHI, another ClickHouse cluster" + # nullable: true + items: + type: object + #required: + # - type + properties: + type: + type: string + description: "you can define multiple affinity policy types" + enum: + # List PodDistributionXXX constants + - "" + - "Unspecified" + - "ClickHouseAntiAffinity" + - "ShardAntiAffinity" + - "ReplicaAntiAffinity" + - "AnotherNamespaceAntiAffinity" + - "AnotherClickHouseInstallationAntiAffinity" + - "AnotherClusterAntiAffinity" + - "MaxNumberPerNode" + - "NamespaceAffinity" + - "ClickHouseInstallationAffinity" + - "ClusterAffinity" + - "ShardAffinity" + - "ReplicaAffinity" + - "PreviousTailAffinity" + - "CircularReplication" + scope: + type: string + description: "scope for apply each podDistribution" + enum: + # list PodDistributionScopeXXX constants + - "" + - "Unspecified" + - "Shard" + - "Replica" + - "Cluster" + - "ClickHouseInstallation" + - "Namespace" + number: + type: integer + description: "define, how much ClickHouse Pods could be inside selected scope with selected distribution type" + minimum: 0 + maximum: 65535 + topologyKey: + type: string + description: | + use for inter-pod affinity look to `pod.spec.affinity.podAntiAffinity.preferredDuringSchedulingIgnoredDuringExecution.podAffinityTerm.topologyKey`, + more info: https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node/#inter-pod-affinity-and-anti-affinity" + metadata: + type: object + description: | + allows pass standard object's metadata from template to Pod + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify PodSpec + type: object + description: "allows define whole Pod.spec inside StaefulSet.spec, look to https://kubernetes.io/docs/concepts/workloads/pods/#pod-templates for details" + # nullable: true + x-kubernetes-preserve-unknown-fields: true + volumeClaimTemplates: + type: array + description: | + allows define template for rendering `PVC` kubernetes resource, which would use inside `Pod` for mount clickhouse `data`, clickhouse `logs` or something else + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + top-level `chi.spec.defaults.templates.dataVolumeClaimTemplate` or `chi.spec.defaults.templates.logVolumeClaimTemplate`, + cluster-level `chi.spec.configuration.clusters.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.templates.logVolumeClaimTemplate`, + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.shards.temlates.logVolumeClaimTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.dataVolumeClaimTemplate` or `chi.spec.configuration.clusters.layout.replicas.templates.logVolumeClaimTemplate` + provisioner: *TypePVCProvisioner + reclaimPolicy: *TypePVCReclaimPolicy + metadata: + type: object + description: | + allows to pass standard object's metadata from template to PVC + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + allows define all aspects of `PVC` resource + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes/#persistentvolumeclaims + # nullable: true + x-kubernetes-preserve-unknown-fields: true + serviceTemplates: + type: array + description: | + allows define template for rendering `Service` which would get endpoint from Pods which scoped chi-wide, cluster-wide, shard-wide, replica-wide level + # nullable: true + items: + type: object + #required: + # - name + # - spec + properties: + name: + type: string + description: | + template name, could use to link inside + chi-level `chi.spec.defaults.templates.serviceTemplate` + cluster-level `chi.spec.configuration.clusters.templates.clusterServiceTemplate` + shard-level `chi.spec.configuration.clusters.layout.shards.temlates.shardServiceTemplate` + replica-level `chi.spec.configuration.clusters.layout.replicas.templates.replicaServiceTemplate` or `chi.spec.configuration.clusters.layout.shards.replicas.replicaServiceTemplate` + generateName: + type: string + description: | + allows define format for generated `Service` name, + look to https://github.com/Altinity/clickhouse-operator/blob/master/docs/custom_resource_explained.md#spectemplatesservicetemplates + for details about available template variables" + metadata: + # TODO specify ObjectMeta + type: object + description: | + allows pass standard object's metadata from template to Service + Could be use for define specificly for Cloud Provider metadata which impact to behavior of service + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata + # nullable: true + x-kubernetes-preserve-unknown-fields: true + spec: + # TODO specify ServiceSpec + type: object + description: | + describe behavior of generated Service + More info: https://kubernetes.io/docs/concepts/services-networking/service/ + # nullable: true + x-kubernetes-preserve-unknown-fields: true diff --git a/deploy/operatorhub/0.24.1/clickhouseoperatorconfigurations.clickhouse.altinity.com.crd.yaml b/deploy/operatorhub/0.24.1/clickhouseoperatorconfigurations.clickhouse.altinity.com.crd.yaml new file mode 100644 index 000000000..057ee8b2e --- /dev/null +++ b/deploy/operatorhub/0.24.1/clickhouseoperatorconfigurations.clickhouse.altinity.com.crd.yaml @@ -0,0 +1,449 @@ +# Template Parameters: +# +# NONE +# +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: clickhouseoperatorconfigurations.clickhouse.altinity.com + labels: + clickhouse.altinity.com/chop: 0.24.1 +spec: + group: clickhouse.altinity.com + scope: Namespaced + names: + kind: ClickHouseOperatorConfiguration + singular: clickhouseoperatorconfiguration + plural: clickhouseoperatorconfigurations + shortNames: + - chopconf + versions: + - name: v1 + served: true + storage: true + additionalPrinterColumns: + - name: namespaces + type: string + description: Watch namespaces + jsonPath: .status + - name: age + type: date + description: Age of the resource + # Displayed in all priorities + jsonPath: .metadata.creationTimestamp + schema: + openAPIV3Schema: + type: object + description: "allows customize `clickhouse-operator` settings, need restart clickhouse-operator pod after adding, more details https://github.com/Altinity/clickhouse-operator/blob/master/docs/operator_configuration.md" + x-kubernetes-preserve-unknown-fields: true + properties: + status: + type: object + x-kubernetes-preserve-unknown-fields: true + spec: + type: object + description: | + Allows to define settings of the clickhouse-operator. + More info: https://github.com/Altinity/clickhouse-operator/blob/master/config/config.yaml + Check into etc-clickhouse-operator* ConfigMaps if you need more control + x-kubernetes-preserve-unknown-fields: true + properties: + watch: + type: object + description: "Parameters for watch kubernetes resources which used by clickhouse-operator deployment" + properties: + namespaces: + type: array + description: "List of namespaces where clickhouse-operator watches for events." + items: + type: string + clickhouse: + type: object + description: "Clickhouse related parameters used by clickhouse-operator" + properties: + configuration: + type: object + properties: + file: + type: object + properties: + path: + type: object + description: | + Each 'path' can be either absolute or relative. + In case path is absolute - it is used as is. + In case path is relative - it is relative to the folder where configuration file you are reading right now is located. + properties: + common: + type: string + description: | + Path to the folder where ClickHouse configuration files common for all instances within a CHI are located. + Default value - config.d + host: + type: string + description: | + Path to the folder where ClickHouse configuration files unique for each instance (host) within a CHI are located. + Default value - conf.d + user: + type: string + description: | + Path to the folder where ClickHouse configuration files with users settings are located. + Files are common for all instances within a CHI. + Default value - users.d + user: + type: object + description: "Default parameters for any user which will create" + properties: + default: + type: object + properties: + profile: + type: string + description: "ClickHouse server configuration `...` for any " + quota: + type: string + description: "ClickHouse server configuration `...` for any " + networksIP: + type: array + description: "ClickHouse server configuration `...` for any " + items: + type: string + password: + type: string + description: "ClickHouse server configuration `...` for any " + network: + type: object + description: "Default network parameters for any user which will create" + properties: + hostRegexpTemplate: + type: string + description: "ClickHouse server configuration `...` for any " + configurationRestartPolicy: + type: object + description: "Configuration restart policy describes what configuration changes require ClickHouse restart" + properties: + rules: + type: array + description: "Array of set of rules per specified ClickHouse versions" + items: + type: object + properties: + version: + type: string + description: "ClickHouse version expression" + rules: + type: array + description: "Set of configuration rules for specified ClickHouse version" + items: + type: object + description: "setting: value pairs for configuration restart policy" + access: + type: object + description: "parameters which use for connect to clickhouse from clickhouse-operator deployment" + properties: + scheme: + type: string + description: "The scheme to user for connecting to ClickHouse. Possible values: http, https, auto" + username: + type: string + description: "ClickHouse username to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName" + password: + type: string + description: "ClickHouse password to be used by operator to connect to ClickHouse instances, deprecated, use chCredentialsSecretName" + rootCA: + type: string + description: "Root certificate authority that clients use when verifying server certificates. Used for https connection to ClickHouse" + secret: + type: object + properties: + namespace: + type: string + description: "Location of k8s Secret with username and password to be used by operator to connect to ClickHouse instances" + name: + type: string + description: "Name of k8s Secret with username and password to be used by operator to connect to ClickHouse instances" + port: + type: integer + minimum: 1 + maximum: 65535 + description: "Port to be used by operator to connect to ClickHouse instances" + timeouts: + type: object + description: "Timeouts used to limit connection and queries from the operator to ClickHouse instances, In seconds" + properties: + connect: + type: integer + minimum: 1 + maximum: 10 + description: "Timout to setup connection from the operator to ClickHouse instances. In seconds." + query: + type: integer + minimum: 1 + maximum: 600 + description: "Timout to perform SQL query from the operator to ClickHouse instances. In seconds." + metrics: + type: object + description: "parameters which use for connect to fetch metrics from clickhouse by clickhouse-operator" + properties: + timeouts: + type: object + description: | + Timeouts used to limit connection and queries from the metrics exporter to ClickHouse instances + Specified in seconds. + properties: + collect: + type: integer + minimum: 1 + maximum: 600 + description: | + Timeout used to limit metrics collection request. In seconds. + Upon reaching this timeout metrics collection is aborted and no more metrics are collected in this cycle. + All collected metrics are returned. + template: + type: object + description: "Parameters which are used if you want to generate ClickHouseInstallationTemplate custom resources from files which are stored inside clickhouse-operator deployment" + properties: + chi: + type: object + properties: + policy: + type: string + description: | + CHI template updates handling policy + Possible policy values: + - ReadOnStart. Accept CHIT updates on the operators start only. + - ApplyOnNextReconcile. Accept CHIT updates at all time. Apply news CHITs on next regular reconcile of the CHI + enum: + - "" + - "ReadOnStart" + - "ApplyOnNextReconcile" + path: + type: string + description: "Path to folder where ClickHouseInstallationTemplate .yaml manifests are located." + reconcile: + type: object + description: "allow tuning reconciling process" + properties: + runtime: + type: object + description: "runtime parameters for clickhouse-operator process which are used during reconcile cycle" + properties: + reconcileCHIsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile CHIs in parallel, 10 by default" + reconcileShardsThreadsNumber: + type: integer + minimum: 1 + maximum: 65535 + description: "How many goroutines will be used to reconcile shards of a cluster in parallel, 1 by default" + reconcileShardsMaxConcurrencyPercent: + type: integer + minimum: 0 + maximum: 100 + description: "The maximum percentage of cluster shards that may be reconciled in parallel, 50 percent by default." + statefulSet: + type: object + description: "Allow change default behavior for reconciling StatefulSet which generated by clickhouse-operator" + properties: + create: + type: object + description: "Behavior during create StatefulSet" + properties: + onFailure: + type: string + description: | + What to do in case created StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds + Possible options: + 1. abort - do nothing, just break the process and wait for admin. + 2. delete - delete newly created problematic StatefulSet. + 3. ignore (default) - ignore error, pretend nothing happened and move on to the next StatefulSet. + update: + type: object + description: "Behavior during update StatefulSet" + properties: + timeout: + type: integer + description: "How many seconds to wait for created/updated StatefulSet to be Ready" + pollInterval: + type: integer + description: "How many seconds to wait between checks for created/updated StatefulSet status" + onFailure: + type: string + description: | + What to do in case updated StatefulSet is not in Ready after `statefulSetUpdateTimeout` seconds + Possible options: + 1. abort - do nothing, just break the process and wait for admin. + 2. rollback (default) - delete Pod and rollback StatefulSet to previous Generation. Pod would be recreated by StatefulSet based on rollback-ed configuration. + 3. ignore - ignore error, pretend nothing happened and move on to the next StatefulSet. + host: + type: object + description: | + Whether the operator during reconcile procedure should wait for a ClickHouse host: + - to be excluded from a ClickHouse cluster + - to complete all running queries + - to be included into a ClickHouse cluster + respectfully before moving forward + properties: + wait: + type: object + properties: + exclude: &TypeStringBool + type: string + description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be excluded from a ClickHouse cluster" + enum: + # List StringBoolXXX constants from model + - "" + - "0" + - "1" + - "False" + - "false" + - "True" + - "true" + - "No" + - "no" + - "Yes" + - "yes" + - "Off" + - "off" + - "On" + - "on" + - "Disable" + - "disable" + - "Enable" + - "enable" + - "Disabled" + - "disabled" + - "Enabled" + - "enabled" + queries: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to complete all running queries" + include: + !!merge <<: *TypeStringBool + description: "Whether the operator during reconcile procedure should wait for a ClickHouse host to be included into a ClickHouse cluster" + annotation: + type: object + description: "defines which metadata.annotations items will include or exclude during render StatefulSet, Pod, PVC resources" + properties: + include: + type: array + description: | + When propagating labels from the chi's `metadata.annotations` section to child objects' `metadata.annotations`, + include annotations with names from the following list + items: + type: string + exclude: + type: array + description: | + When propagating labels from the chi's `metadata.annotations` section to child objects' `metadata.annotations`, + exclude annotations with names from the following list + items: + type: string + label: + type: object + description: "defines which metadata.labels will include or exclude during render StatefulSet, Pod, PVC resources" + properties: + include: + type: array + description: | + When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`, + include labels from the following list + items: + type: string + exclude: + type: array + items: + type: string + description: | + When propagating labels from the chi's `metadata.labels` section to child objects' `metadata.labels`, + exclude labels from the following list + appendScope: + !!merge <<: *TypeStringBool + description: | + Whether to append *Scope* labels to StatefulSet and Pod + - "LabelShardScopeIndex" + - "LabelReplicaScopeIndex" + - "LabelCHIScopeIndex" + - "LabelCHIScopeCycleSize" + - "LabelCHIScopeCycleIndex" + - "LabelCHIScopeCycleOffset" + - "LabelClusterScopeIndex" + - "LabelClusterScopeCycleSize" + - "LabelClusterScopeCycleIndex" + - "LabelClusterScopeCycleOffset" + metrics: + type: object + description: "defines metrics exporter options" + properties: + labels: + type: object + description: "defines metric labels options" + properties: + exclude: + type: array + description: | + When adding labels to a metric exclude labels with names from the following list + items: + type: string + status: + type: object + description: "defines status options" + properties: + fields: + type: object + description: "defines status fields options" + properties: + action: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'action'" + actions: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'actions'" + error: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'error'" + errors: + !!merge <<: *TypeStringBool + description: "Whether the operator should fill status field 'errors'" + statefulSet: + type: object + description: "define StatefulSet-specific parameters" + properties: + revisionHistoryLimit: + type: integer + description: "revisionHistoryLimit is the maximum number of revisions that will be\nmaintained in the StatefulSet's revision history. \nLook details in `statefulset.spec.revisionHistoryLimit`\n" + pod: + type: object + description: "define pod specific parameters" + properties: + terminationGracePeriod: + type: integer + description: "Optional duration in seconds the pod needs to terminate gracefully. \nLook details in `pod.spec.terminationGracePeriodSeconds`\n" + logger: + type: object + description: "allow setup clickhouse-operator logger behavior" + properties: + logtostderr: + type: string + description: "boolean, allows logs to stderr" + alsologtostderr: + type: string + description: "boolean allows logs to stderr and files both" + v: + type: string + description: "verbosity level of clickhouse-operator log, default - 1 max - 9" + stderrthreshold: + type: string + vmodule: + type: string + description: | + Comma-separated list of filename=N, where filename (can be a pattern) must have no .go ext, and N is a V level. + Ex.: file*=2 sets the 'V' to 2 in all files with names like file*. + log_backtrace_at: + type: string + description: | + It can be set to a file and line number with a logging line. + Ex.: file.go:123 + Each time when this line is being executed, a stack trace will be written to the Info log. diff --git a/deploy/zookeeper/zookeeper-manually/advanced/05-stateful-set-persistent-volume.yaml b/deploy/zookeeper/zookeeper-manually/advanced/05-stateful-set-persistent-volume.yaml index a9231c1b5..1a8848de1 100644 --- a/deploy/zookeeper/zookeeper-manually/advanced/05-stateful-set-persistent-volume.yaml +++ b/deploy/zookeeper/zookeeper-manually/advanced/05-stateful-set-persistent-volume.yaml @@ -42,7 +42,7 @@ spec: containers: - name: kubernetes-zookeeper imagePullPolicy: IfNotPresent - image: "docker.io/zookeeper:3.8.3" + image: "docker.io/zookeeper:3.8.4" resources: requests: memory: "1Gi" diff --git a/deploy/zookeeper/zookeeper-manually/advanced/05-stateful-set-volume-emptyDir.yaml b/deploy/zookeeper/zookeeper-manually/advanced/05-stateful-set-volume-emptyDir.yaml index 31351b2be..c1149bcd0 100644 --- a/deploy/zookeeper/zookeeper-manually/advanced/05-stateful-set-volume-emptyDir.yaml +++ b/deploy/zookeeper/zookeeper-manually/advanced/05-stateful-set-volume-emptyDir.yaml @@ -42,7 +42,7 @@ spec: containers: - name: kubernetes-zookeeper imagePullPolicy: IfNotPresent - image: "docker.io/zookeeper:3.8.3" + image: "docker.io/zookeeper:3.8.4" resources: requests: memory: "1Gi" diff --git a/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc-secure.yaml b/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc-secure.yaml index 693ecfc70..f2385fcbb 100644 --- a/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc-secure.yaml +++ b/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc-secure.yaml @@ -592,7 +592,7 @@ spec: - name: zookeeper command: - /conf/zookeeperStart.sh - image: docker.io/zookeeper:3.8.3 + image: docker.io/zookeeper:3.8.4 imagePullPolicy: Always lifecycle: preStop: diff --git a/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc.yaml b/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc.yaml index 5914ff190..9a40392eb 100644 --- a/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc.yaml +++ b/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only-scaleout-pvc.yaml @@ -545,7 +545,7 @@ spec: - name: zookeeper command: - /conf/zookeeperStart.sh - image: docker.io/zookeeper:3.8.3 + image: docker.io/zookeeper:3.8.4 imagePullPolicy: Always lifecycle: preStop: diff --git a/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only.yaml b/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only.yaml index 980534dfc..0aa1e7918 100644 --- a/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only.yaml +++ b/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only.yaml @@ -82,7 +82,7 @@ spec: containers: - name: kubernetes-zookeeper imagePullPolicy: IfNotPresent - image: "docker.io/zookeeper:3.8.3" + image: "docker.io/zookeeper:3.8.4" ports: - containerPort: 2181 name: client diff --git a/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node-for-test-probes.yaml b/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node-for-test-probes.yaml index 34fa8f1ee..634c490ce 100644 --- a/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node-for-test-probes.yaml +++ b/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node-for-test-probes.yaml @@ -93,7 +93,7 @@ spec: containers: - name: kubernetes-zookeeper imagePullPolicy: IfNotPresent - image: "docker.io/zookeeper:3.8.3" + image: "docker.io/zookeeper:3.8.4" resources: requests: memory: "512M" diff --git a/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node.yaml b/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node.yaml index 82f735c4d..c24f27789 100644 --- a/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node.yaml +++ b/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node.yaml @@ -94,7 +94,7 @@ spec: containers: - name: kubernetes-zookeeper imagePullPolicy: IfNotPresent - image: "docker.io/zookeeper:3.8.3" + image: "docker.io/zookeeper:3.8.4" resources: requests: memory: "512M" diff --git a/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only-scaleout-pvc.yaml b/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only-scaleout-pvc.yaml index cbc0edc1a..9044829dc 100644 --- a/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only-scaleout-pvc.yaml +++ b/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only-scaleout-pvc.yaml @@ -545,7 +545,7 @@ spec: - name: zookeeper command: - /conf/zookeeperStart.sh - image: docker.io/zookeeper:3.8.3 + image: docker.io/zookeeper:3.8.4 imagePullPolicy: Always lifecycle: preStop: diff --git a/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only.yaml b/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only.yaml index 83b8b3fd5..68bf22703 100644 --- a/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only.yaml +++ b/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only.yaml @@ -82,7 +82,7 @@ spec: containers: - name: kubernetes-zookeeper imagePullPolicy: IfNotPresent - image: "docker.io/zookeeper:3.8.3" + image: "docker.io/zookeeper:3.8.4" ports: - containerPort: 2181 name: client diff --git a/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-3-nodes-for-test-probes.yaml b/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-3-nodes-for-test-probes.yaml index 72169eb38..8d043a0bc 100644 --- a/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-3-nodes-for-test-probes.yaml +++ b/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-3-nodes-for-test-probes.yaml @@ -93,7 +93,7 @@ spec: containers: - name: kubernetes-zookeeper imagePullPolicy: IfNotPresent - image: "docker.io/zookeeper:3.8.3" + image: "docker.io/zookeeper:3.8.4" resources: requests: memory: "512M" diff --git a/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-3-nodes.yaml b/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-3-nodes.yaml index a26e9b02f..2ab53183f 100644 --- a/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-3-nodes.yaml +++ b/deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-3-nodes.yaml @@ -94,7 +94,7 @@ spec: containers: - name: kubernetes-zookeeper imagePullPolicy: IfNotPresent - image: "docker.io/zookeeper:3.8.3" + image: "docker.io/zookeeper:3.8.4" resources: requests: memory: "512M" diff --git a/deploy/zookeeper/zookeeper-manually/quick-start-volume-emptyDir/zookeeper-1-node.yaml b/deploy/zookeeper/zookeeper-manually/quick-start-volume-emptyDir/zookeeper-1-node.yaml index 0bf66db35..247af99b7 100644 --- a/deploy/zookeeper/zookeeper-manually/quick-start-volume-emptyDir/zookeeper-1-node.yaml +++ b/deploy/zookeeper/zookeeper-manually/quick-start-volume-emptyDir/zookeeper-1-node.yaml @@ -94,7 +94,7 @@ spec: containers: - name: kubernetes-zookeeper imagePullPolicy: IfNotPresent - image: "docker.io/zookeeper:3.8.3" + image: "docker.io/zookeeper:3.8.4" resources: requests: memory: "512M" diff --git a/deploy/zookeeper/zookeeper-manually/quick-start-volume-emptyDir/zookeeper-3-nodes.yaml b/deploy/zookeeper/zookeeper-manually/quick-start-volume-emptyDir/zookeeper-3-nodes.yaml index e7aada9d5..e4dece68c 100644 --- a/deploy/zookeeper/zookeeper-manually/quick-start-volume-emptyDir/zookeeper-3-nodes.yaml +++ b/deploy/zookeeper/zookeeper-manually/quick-start-volume-emptyDir/zookeeper-3-nodes.yaml @@ -94,7 +94,7 @@ spec: containers: - name: kubernetes-zookeeper imagePullPolicy: IfNotPresent - image: "docker.io/zookeeper:3.8.3" + image: "docker.io/zookeeper:3.8.4" resources: requests: memory: "512M" diff --git a/deploy/zookeeper/zookeeper-with-zookeeper-operator/zookeeper-operator-1-node-with-custom-probes.yaml b/deploy/zookeeper/zookeeper-with-zookeeper-operator/zookeeper-operator-1-node-with-custom-probes.yaml index fc4834295..3a8d95a3c 100644 --- a/deploy/zookeeper/zookeeper-with-zookeeper-operator/zookeeper-operator-1-node-with-custom-probes.yaml +++ b/deploy/zookeeper/zookeeper-with-zookeeper-operator/zookeeper-operator-1-node-with-custom-probes.yaml @@ -453,7 +453,7 @@ spec: fieldRef: apiVersion: v1 fieldPath: metadata.annotations['sidecar.istio.io/status'] - image: docker.io/zookeeper:3.8.3 + image: docker.io/zookeeper:3.8.4 imagePullPolicy: Always lifecycle: preStop: diff --git a/deploy/zookeeper/zookeeper-with-zookeeper-operator/zookeeper-operator-3-nodes-with-custom-probes.yaml b/deploy/zookeeper/zookeeper-with-zookeeper-operator/zookeeper-operator-3-nodes-with-custom-probes.yaml index 4a6fdab58..b2268a8f2 100644 --- a/deploy/zookeeper/zookeeper-with-zookeeper-operator/zookeeper-operator-3-nodes-with-custom-probes.yaml +++ b/deploy/zookeeper/zookeeper-with-zookeeper-operator/zookeeper-operator-3-nodes-with-custom-probes.yaml @@ -453,7 +453,7 @@ spec: fieldRef: apiVersion: v1 fieldPath: metadata.annotations['sidecar.istio.io/status'] - image: docker.io/zookeeper:3.8.3 + image: docker.io/zookeeper:3.8.4 imagePullPolicy: Always lifecycle: preStop: diff --git a/dev/generate_helm_chart.sh b/dev/generate_helm_chart.sh index 2dd20e40b..7caef9137 100755 --- a/dev/generate_helm_chart.sh +++ b/dev/generate_helm_chart.sh @@ -4,7 +4,7 @@ function usage() { cat << EOT Script splits clickhouse-operator-install-bundle.yaml to separate files and adjusts them to conform the helm standards NOTE script requires some pre-installed tools: - - yq ( https://mikefarah.gitbook.io/yq/ ) > v4.14.x. Do not use brew install yq in MacOS，Version is lower than it. + - yq ( https://mikefarah.gitbook.io/yq/ ) > v4.14.x - jq ( https://github.com/stedolan/jq ) - helm-docs ( https://github.com/norwoodj/helm-docs ) - perl ( https://learn.perl.org/installing/ ) diff --git a/dev/start_new_release_branch.sh b/dev/start_new_release_branch.sh index 8ea53f39d..e9701eb83 100755 --- a/dev/start_new_release_branch.sh +++ b/dev/start_new_release_branch.sh @@ -34,7 +34,7 @@ NEW_RELEASE_MINOR=$(increment_version "${CUR_RELEASE}" 1) NEW_RELEASE_PATCH=$(increment_version "${CUR_RELEASE}" 2) echo "Starting new release." echo "Current release: ${CUR_RELEASE}" -echo "What would you like to start. Possible options:" +echo "What would you like to start? Possible options:" echo " 1 - new MAJOR version: ${NEW_RELEASE_MAJOR}" echo " 2 - new MINOR version: ${NEW_RELEASE_MINOR}" echo " 3 - new PATCH version: ${NEW_RELEASE_PATCH}" diff --git a/dockerfile/metrics-exporter/Dockerfile b/dockerfile/metrics-exporter/Dockerfile index e277e4612..597e250f9 100644 --- a/dockerfile/metrics-exporter/Dockerfile +++ b/dockerfile/metrics-exporter/Dockerfile @@ -2,7 +2,7 @@ # ===== Builder ===== # =================== -FROM --platform=${BUILDPLATFORM} golang:1.21 AS builder +FROM --platform=${BUILDPLATFORM} golang:1.23 AS builder ARG TARGETOS ARG TARGETARCH @@ -34,7 +34,7 @@ RUN METRICS_EXPORTER_BIN=/tmp/metrics-exporter bash -xe ./dev/go_build_metrics_e # =================== # == Delve builder == # =================== -FROM --platform=${BUILDPLATFORM} golang:1.19 AS delve-builder +FROM --platform=${BUILDPLATFORM} golang:1.23 AS delve-builder RUN CGO_ENABLED=0 GO111MODULE=on GOOS="${TARGETOS}" GOARCH="${TARGETARCH}" \ go install -ldflags "-s -w -extldflags '-static'" github.com/go-delve/delve/cmd/dlv@latest && \ rm -rf /root/.cache/go-build/ /go/pkg/mod/ diff --git a/dockerfile/operator/Dockerfile b/dockerfile/operator/Dockerfile index 142af615c..9be1d8211 100644 --- a/dockerfile/operator/Dockerfile +++ b/dockerfile/operator/Dockerfile @@ -2,7 +2,7 @@ # ===== Builder ===== # =================== -FROM --platform=${BUILDPLATFORM} golang:1.21 AS builder +FROM --platform=${BUILDPLATFORM} golang:1.23 AS builder ARG TARGETOS ARG TARGETARCH @@ -34,7 +34,7 @@ RUN OPERATOR_BIN=/tmp/clickhouse-operator bash -xe ./dev/go_build_operator.sh # =================== # == Delve builder == # =================== -FROM --platform=${BUILDPLATFORM} golang:1.19 AS delve-builder +FROM --platform=${BUILDPLATFORM} golang:1.23 AS delve-builder RUN CGO_ENABLED=0 GO111MODULE=on GOOS="${TARGETOS}" GOARCH="${TARGETARCH}" \ go install -ldflags "-s -w -extldflags '-static'" github.com/go-delve/delve/cmd/dlv@latest && \ rm -rf /root/.cache/go-build/ /go/pkg/mod/ diff --git a/docs/chi-examples/70-chop-config.yaml b/docs/chi-examples/70-chop-config.yaml index b8c2bcd8e..323773bee 100644 --- a/docs/chi-examples/70-chop-config.yaml +++ b/docs/chi-examples/70-chop-config.yaml @@ -66,7 +66,6 @@ spec: # for: # 1. Metrics requests # 2. Schema maintenance - # 3. DROP DNS CACHE # User with such credentials can be specified in additional ClickHouse .xml config files, # located in `chUsersConfigsPath` folder username: "clickhouse_operator" diff --git a/docs/chi_update_clickhouse_version.md b/docs/chi_update_clickhouse_version.md index 2d1caa427..b9617c2f9 100644 --- a/docs/chi_update_clickhouse_version.md +++ b/docs/chi_update_clickhouse_version.md @@ -51,7 +51,7 @@ configmap/chi-06791a-deploy-confd-28a0-0-0 1 9s configmap/chi-06791a-deploy-confd-28a0-0-1 1 9s configmap/chi-06791a-deploy-confd-28a0-0-2 1 9s ``` -We expect all Pods to run ClickHouse version `19.1.10` as specified in [initial manifest][initial-manifest]. +We expect all Pods to run ClickHouse version `23.3.0` as specified in [initial manifest][initial-manifest]. Let's explore all Pods in order to check available ClickHouse version. Navigate directly inside each Pod @@ -59,29 +59,29 @@ Navigate directly inside each Pod kubectl -n dev exec -it chi-06791a-28a0-0-0-0 -- clickhouse-client ``` ```text -ClickHouse client version 19.1.10. +ClickHouse client version 23.3.0. Connecting to localhost:9000. -Connected to ClickHouse server version 19.1.10 revision 54413. +Connected to ClickHouse server version 23.3.0 revision 54413. ``` Repeat for all Pods ```bash kubectl -n dev exec -it chi-06791a-28a0-0-1-0 -- clickhouse-client ``` ```text -ClickHouse client version 19.1.10. +ClickHouse client version 23.3.0. Connecting to localhost:9000. -Connected to ClickHouse server version 19.1.10 revision 54413. +Connected to ClickHouse server version 23.3.0 revision 54413. ``` And the last Pod ```bash kubectl -n dev exec -it chi-06791a-28a0-0-2-0 -- clickhouse-client ``` ```text -ClickHouse client version 19.1.10. +ClickHouse client version 23.3.0. Connecting to localhost:9000. -Connected to ClickHouse server version 19.1.10 revision 54413. +Connected to ClickHouse server version 23.3.0 revision 54413. ``` -All is fine, all Pods are running `19.1.10` +All is fine, all Pods are running `23.3.0` We'll make Version Update in two steps: 1. Update one ClickHouse instance (one Pod) @@ -93,40 +93,40 @@ Now let's update version of only one instance of ClickHouse. Let it be the last We can do by explicitly specifying `templates` with different ClickHouse version: ```yaml templates: - podTemplate: clickhouse:19.3.7 + podTemplate: clickhouse:23.8.0 ``` Manifest file with one ClickHouse instance update is [08-clickhouse-version-update-02-apply-update-one.yaml][08-clickhouse-version-update-02-apply-update-one.yaml]: ```bash kubectl -n dev apply -f 08-clickhouse-version-update-02-apply-update-one.yaml ``` -And let's check what ClickHouse versions are running over the whole cluster. We expect the last instance to run specific version `19.3.7`. Check the first Pod: +And let's check what ClickHouse versions are running over the whole cluster. We expect the last instance to run specific version `23.8.0`. Check the first Pod: ```bash kubectl -n dev exec -it chi-06791a-28a0-0-0-0 -- clickhouse-client ``` ```text -ClickHouse client version 19.1.10. +ClickHouse client version 23.3.0. Connecting to localhost:9000. -Connected to ClickHouse server version 19.1.10 revision 54413. +Connected to ClickHouse server version 23.3.0 revision 54413. ``` The second Pod: ```bash kubectl -n dev exec -it chi-06791a-28a0-0-1-0 -- clickhouse-client ``` ```text -ClickHouse client version 19.1.10. +ClickHouse client version 23.3.0. Connecting to localhost:9000. -Connected to ClickHouse server version 19.1.10 revision 54413. +Connected to ClickHouse server version 23.3.0 revision 54413. ``` And the most interesting part - the last one: ```bash kubectl -n dev exec -it chi-06791a-28a0-0-2-0 -- clickhouse-client ``` ```text -ClickHouse client version 19.3.7. +ClickHouse client version 23.8.0. Connecting to localhost:9000 as user default. -Connected to ClickHouse server version 19.3.7 revision 54415. +Connected to ClickHouse server version 23.8.0 revision 54415. ``` -As we can see - it runs different, explicitly specified version `19.3.7`. +As we can see - it runs different, explicitly specified version `23.8.0`. All seems to be good. Let's update the whole cluster now. ## Update the whole cluster @@ -136,32 +136,32 @@ Manifest file with all ClickHouse instance updated is [08-clickhouse-version-upd ```bash kubectl -n dev apply -f 08-clickhouse-version-update-03-apply-update-all.yaml ``` -And let's check the results - we expect all Pods to have ClickHouse `19.3.7` running. The first Pod +And let's check the results - we expect all Pods to have ClickHouse `23.8.0` running. The first Pod ```bash kubectl -n dev exec -it chi-06791a-28a0-0-0-0 -- clickhouse-client ``` ```text -ClickHouse client version 19.3.7. +ClickHouse client version 23.8.0. Connecting to localhost:9000 as user default. -Connected to ClickHouse server version 19.3.7 revision 54415.``` +Connected to ClickHouse server version 23.8.0 revision 54415.``` ``` The second Pod ```bash kubectl -n dev exec -it chi-06791a-28a0-0-1-0 -- clickhouse-client ``` ```text -ClickHouse client version 19.3.7. +ClickHouse client version 23.8.0. Connecting to localhost:9000 as user default. -Connected to ClickHouse server version 19.3.7 revision 54415. +Connected to ClickHouse server version 23.8.0 revision 54415. ``` And the last Pod ```bash kubectl -n dev exec -it chi-06791a-28a0-0-2-0 -- clickhouse-client ``` ```text -ClickHouse client version 19.3.7. +ClickHouse client version 23.8.0. Connecting to localhost:9000 as user default. -Connected to ClickHouse server version 19.3.7 revision 54415. +Connected to ClickHouse server version 23.8.0 revision 54415. ``` All looks fine. diff --git a/docs/chk-examples/02-extended-1-node.yaml b/docs/chk-examples/02-extended-1-node.yaml index 28e481bb9..8a0b5bfae 100644 --- a/docs/chk-examples/02-extended-1-node.yaml +++ b/docs/chk-examples/02-extended-1-node.yaml @@ -25,6 +25,7 @@ spec: templates: # Templates are specified as default for all clusters podTemplate: default + dataVolumeClaimTemplate: default templates: podTemplates: @@ -33,7 +34,7 @@ spec: containers: - name: clickhouse-keeper imagePullPolicy: IfNotPresent - image: "clickhouse/clickhouse-keeper:24.3.5.46" + image: "clickhouse/clickhouse-keeper:latest" resources: requests: memory: "256M" diff --git a/docs/chk-examples/02-extended-3-nodes.yaml b/docs/chk-examples/02-extended-3-nodes.yaml index da3cd3f68..e38389ade 100644 --- a/docs/chk-examples/02-extended-3-nodes.yaml +++ b/docs/chk-examples/02-extended-3-nodes.yaml @@ -25,6 +25,7 @@ spec: templates: # Templates are specified as default for all clusters podTemplate: default + dataVolumeClaimTemplate: default templates: podTemplates: @@ -44,7 +45,7 @@ spec: containers: - name: clickhouse-keeper imagePullPolicy: IfNotPresent - image: "clickhouse/clickhouse-keeper:24.3.5.46" + image: "clickhouse/clickhouse-keeper:latest" resources: requests: memory: "256M" diff --git a/docs/chk-examples/04-private-image-secret.yaml b/docs/chk-examples/04-private-image-secret.yaml new file mode 100644 index 000000000..5b8eeb890 --- /dev/null +++ b/docs/chk-examples/04-private-image-secret.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Secret +metadata: + name: my-registry-secret +type: kubernetes.io/dockerconfigjson +stringData: + .dockerconfigjson: | + { + "auths": { + "https://index.docker.io/v1/": { + "username": "your-login", + "password": "your-password" + } + } + } + +--- +apiVersion: clickhouse-keeper.altinity.com/v1 +kind: ClickHouseKeeperInstallation +metadata: + name: custom-image +spec: + defaults: + templates: + podTemplate: private-image + configuration: + clusters: + - name: keeper + + templates: + podTemplates: + - name: private-image + spec: + imagePullSecrets: + - name: image-pull-secret + containers: + - name: clickhouse + image: registry/repo/clickhouse-keeper:latest \ No newline at end of file diff --git a/docs/chk-examples/clickhouse-keeper-1-node-for-test-only.yaml b/docs/chk-examples/clickhouse-keeper-1-node-for-test-only.yaml index 6c1672289..818f0672d 100644 --- a/docs/chk-examples/clickhouse-keeper-1-node-for-test-only.yaml +++ b/docs/chk-examples/clickhouse-keeper-1-node-for-test-only.yaml @@ -1,46 +1,27 @@ ---- -# Fake Service to drop-in replacement in tests -apiVersion: v1 -kind: Service -metadata: - # DNS would be like zookeeper.namespace.svc - name: zookeeper - labels: - app: zookeeper -spec: - ports: - - port: 2181 - name: client - - port: 7000 - name: prometheus - selector: - clickhouse-keeper.altinity.com/chk: clickhouse-keeper - clickhouse-keeper.altinity.com/ready: "yes" ---- apiVersion: "clickhouse-keeper.altinity.com/v1" kind: "ClickHouseKeeperInstallation" metadata: name: clickhouse-keeper - labels: - app: clickhouse-keeper spec: defaults: templates: + podTemplate: clickhouse-keeper volumeClaimTemplate: data-volume - podTemplate: latest-with-volume-mounts configuration: clusters: - - name: "test-only" + - name: test layout: replicasCount: 1 + settings: + keeper_server/tcp_port: "2181" templates: podTemplates: - - name: latest-with-volume-mounts + - name: clickhouse-keeper spec: containers: - name: clickhouse-keeper imagePullPolicy: Always - image: "clickhouse/clickhouse-keeper:latest-alpine" + image: "clickhouse/clickhouse-keeper:24.8" volumeClaimTemplates: - name: data-volume spec: @@ -48,4 +29,22 @@ spec: - ReadWriteOnce resources: requests: - storage: 1Gi + storage: 100Mi +--- +# Fake Service to drop-in replacement in tests +apiVersion: v1 +kind: Service +metadata: + name: zookeeper + labels: + clickhouse-keeper.altinity.com/app: chop + clickhouse-keeper.altinity.com/chk: clickhouse-keeper +spec: + ports: + - port: 2181 + name: client + - port: 7000 + name: prometheus + selector: + clickhouse-keeper.altinity.com/chk: clickhouse-keeper + clickhouse-keeper.altinity.com/ready: "yes" diff --git a/docs/chk-examples/clickhouse-keeper-3-node-for-test-only-version-24.yaml b/docs/chk-examples/clickhouse-keeper-3-node-for-test-only-version-24.yaml deleted file mode 100644 index 8c0d8c905..000000000 --- a/docs/chk-examples/clickhouse-keeper-3-node-for-test-only-version-24.yaml +++ /dev/null @@ -1,60 +0,0 @@ ---- -# Fake Service to drop-in replacement in tests -apiVersion: v1 -kind: Service -metadata: - # DNS would be like zookeeper.namespace.svc - name: zookeeper - labels: - app: zookeeper -spec: - ports: - - port: 2181 - name: client - - port: 7000 - name: prometheus - selector: - clickhouse-keeper.altinity.com/chk: clickhouse-keeper - clickhouse-keeper.altinity.com/ready: "yes" ---- -apiVersion: "clickhouse-keeper.altinity.com/v1" -kind: "ClickHouseKeeperInstallation" -metadata: - name: clickhouse-keeper -spec: - defaults: - templates: - podTemplate: default - volumeClaimTemplate: default - templates: - podTemplates: - - name: default - spec: - containers: - - name: clickhouse-keeper - imagePullPolicy: IfNotPresent - # IMPORTANT !!! - # clickhouse-keeper:24.3.5.46 version IS CHECKED IN TESTS and can be changed with TESTS only! - # DO NOT CHANGE THE VERSION ! - image: "clickhouse/clickhouse-keeper:24.3.5.46" - volumeClaimTemplates: - - name: default - spec: - accessModes: - - ReadWriteOnce - resources: - requests: - storage: 1Gi - configuration: - clusters: - - name: "test-only" - layout: - replicasCount: 3 - settings: - logger/level: "trace" - prometheus/endpoint: "/metrics" - prometheus/port: "7000" - prometheus/metrics: "true" - prometheus/events: "true" - prometheus/asynchronous_metrics: "true" - prometheus/status_info: "false" diff --git a/docs/chk-examples/clickhouse-keeper-3-node-for-test-only.yaml b/docs/chk-examples/clickhouse-keeper-3-node-for-test-only.yaml index 67beb494c..3bbee607d 100644 --- a/docs/chk-examples/clickhouse-keeper-3-node-for-test-only.yaml +++ b/docs/chk-examples/clickhouse-keeper-3-node-for-test-only.yaml @@ -1,46 +1,27 @@ ---- -# Fake Service to drop-in replacement in tests -apiVersion: v1 -kind: Service -metadata: - # DNS would be like zookeeper.namespace.svc - name: zookeeper - labels: - app: zookeeper -spec: - ports: - - port: 2181 - name: client - - port: 7000 - name: prometheus - selector: - clickhouse-keeper.altinity.com/chk: clickhouse-keeper - clickhouse-keeper.altinity.com/ready: "yes" ---- apiVersion: "clickhouse-keeper.altinity.com/v1" kind: "ClickHouseKeeperInstallation" metadata: name: clickhouse-keeper - labels: - app: clickhouse-keeper spec: defaults: templates: + podTemplate: clickhouse-keeper volumeClaimTemplate: data-volume - podTemplate: latest-with-volume-mounts configuration: clusters: - - name: "test-only" + - name: test layout: replicasCount: 3 + settings: + keeper_server/tcp_port: "2181" templates: podTemplates: - - name: latest-with-volume-mounts + - name: clickhouse-keeper spec: containers: - name: clickhouse-keeper imagePullPolicy: Always - image: "clickhouse/clickhouse-keeper:latest-alpine" + image: "clickhouse/clickhouse-keeper:24.8" volumeClaimTemplates: - name: data-volume spec: @@ -48,4 +29,22 @@ spec: - ReadWriteOnce resources: requests: - storage: 1Gi + storage: 100Mi +--- +# Fake Service to drop-in replacement in tests +apiVersion: v1 +kind: Service +metadata: + name: zookeeper + labels: + clickhouse-keeper.altinity.com/app: chop + clickhouse-keeper.altinity.com/chk: clickhouse-keeper +spec: + ports: + - port: 2181 + name: client + - port: 7000 + name: prometheus + selector: + clickhouse-keeper.altinity.com/chk: clickhouse-keeper + clickhouse-keeper.altinity.com/ready: "yes" \ No newline at end of file diff --git a/docs/custom_resource_explained.md b/docs/custom_resource_explained.md index 60944e9c8..648dbf079 100644 --- a/docs/custom_resource_explained.md +++ b/docs/custom_resource_explained.md @@ -440,42 +440,49 @@ Another example with selectively described replicas. Note - `replicasCount` spec templates: serviceTemplates: - name: chi-service-template - # generateName understands different sets of macroses, + # generateName understands different sets of macros, # depending on the level of the object, for which Service is being created: - # - # For CHI-level Service: - # 1. {chi} - ClickHouseInstallation name - # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature) + # For CHI/CHK-level Service: + # {chi} - ClickHouseInstallation name + # {chk} - ClickHouseKeeperInstallation name + # {chiID} - short hashed ClickHouseInstallation name + # {chkID} - short hashed ClickHouseKeeperInstallation name # # For Cluster-level Service: - # 1. {chi} - ClickHouseInstallation name - # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature) - # 3. {cluster} - cluster name - # 4. {clusterID} - short hashed cluster name (BEWARE, this is an experimental feature) - # 5. {clusterIndex} - 0-based index of the cluster in the CHI (BEWARE, this is an experimental feature) + # {chi} - ClickHouseInstallation name + # {chk} - ClickHouseKeeperInstallation name + # {chiID} - short hashed ClickHouseInstallation name + # {chkID} - short hashed ClickHouseKeeperInstallation name + # {cluster} - cluster name + # {clusterID} - short hashed cluster name (BEWARE, this is an experimental feature) + # {clusterIndex} - 0-based index of the cluster in the CHI (BEWARE, this is an experimental feature) # # For Shard-level Service: - # 1. {chi} - ClickHouseInstallation name - # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature) - # 3. {cluster} - cluster name - # 4. {clusterID} - short hashed cluster name (BEWARE, this is an experimental feature) - # 5. {clusterIndex} - 0-based index of the cluster in the CHI (BEWARE, this is an experimental feature) - # 6. {shard} - shard name - # 7. {shardID} - short hashed shard name (BEWARE, this is an experimental feature) - # 8. {shardIndex} - 0-based index of the shard in the cluster (BEWARE, this is an experimental feature) + # {chi} - ClickHouseInstallation name + # {chk} - ClickHouseKeeperInstallation name + # {chiID} - short hashed ClickHouseInstallation name + # {chkID} - short hashed ClickHouseKeeperInstallation name + # {cluster} - cluster name + # {clusterID} - short hashed cluster name (BEWARE, this is an experimental feature) + # {clusterIndex} - 0-based index of the cluster in the CHI + # {shard} - shard name + # {shardID} - short hashed shard name (BEWARE, this is an experimental feature) + # {shardIndex} - 0-based index of the shard in the cluster # # For Replica-level Service: - # 1. {chi} - ClickHouseInstallation name - # 2. {chiID} - short hashed ClickHouseInstallation name (BEWARE, this is an experimental feature) - # 3. {cluster} - cluster name - # 4. {clusterID} - short hashed cluster name (BEWARE, this is an experimental feature) - # 5. {clusterIndex} - 0-based index of the cluster in the CHI (BEWARE, this is an experimental feature) - # 6. {shard} - shard name - # 7. {shardID} - short hashed shard name (BEWARE, this is an experimental feature) - # 8. {shardIndex} - 0-based index of the shard in the cluster (BEWARE, this is an experimental feature) - # 9. {replica} - replica name - # 10. {replicaID} - short hashed replica name (BEWARE, this is an experimental feature) - # 11. {replicaIndex} - 0-based index of the replica in the shard (BEWARE, this is an experimental feature) + # {chi} - ClickHouseInstallation name + # {chk} - ClickHouseKeeperInstallation name + # {chiID} - short hashed ClickHouseInstallation name + # {chkID} - short hashed ClickHouseKeeperInstallation name + # {cluster} - cluster name + # {clusterID} - short hashed cluster name + # {clusterIndex} - 0-based index of the cluster in the CHI + # {shard} - shard name + # {shardID} - short hashed shard name (BEWARE, this is an experimental feature) + # {shardIndex} - 0-based index of the shard in the cluster + # {replica} - replica name + # {replicaID} - short hashed replica name + # {replicaIndex} - 0-based index of the replica in the shard generateName: "service-{chi}" # type ObjectMeta struct from k8s.io/meta/v1 metadata: diff --git a/go.mod b/go.mod index 5a3e18f3a..e6d1c223e 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/altinity/clickhouse-operator -go 1.21 +go 1.23 replace ( github.com/emicklei/go-restful/v3 => github.com/emicklei/go-restful/v3 v3.10.0 diff --git a/pkg/announcer/announcer.go b/pkg/announcer/announcer.go index 70b50c7bb..52697e711 100644 --- a/pkg/announcer/announcer.go +++ b/pkg/announcer/announcer.go @@ -44,14 +44,6 @@ type Announcer struct { meta string } -// announcer which would be used in top-level functions, can be called as a 'default announcer' -var announcer Announcer - -// init creates default announcer -func init() { - announcer = New() -} - // skip specifies file name which to be skipped from address const skip = "announcer.go" @@ -69,11 +61,6 @@ func (a Announcer) Silence() Announcer { return b } -// Silence produces silent announcer -func Silence() Announcer { - return announcer.Silence() -} - // V is inspired by log.V() func (a Announcer) V(level log.Level) Announcer { b := a @@ -81,11 +68,6 @@ func (a Announcer) V(level log.Level) Announcer { return b } -// V is inspired by log.V() -func V(level log.Level) Announcer { - return announcer.V(level) -} - // F adds function name func (a Announcer) F() Announcer { b := a @@ -93,11 +75,6 @@ func (a Announcer) F() Announcer { return b } -// F adds function name -func F() Announcer { - return announcer.F() -} - // L adds line number func (a Announcer) L() Announcer { b := a @@ -105,11 +82,6 @@ func (a Announcer) L() Announcer { return b } -// L adds line number -func L() Announcer { - return announcer.L() -} - // FL adds filename func (a Announcer) FL() Announcer { b := a @@ -117,11 +89,6 @@ func (a Announcer) FL() Announcer { return b } -// FL adds filename -func FL() Announcer { - return announcer.FL() -} - // A adds full code address as 'file:line:function' func (a Announcer) A() Announcer { b := a @@ -129,11 +96,6 @@ func (a Announcer) A() Announcer { return b } -// A adds full code address as 'file:line:function' -func A() Announcer { - return announcer.A() -} - // S adds 'start of the function' tag, which includes: // file, line, function and start prefix func (a Announcer) S() Announcer { @@ -143,12 +105,6 @@ func (a Announcer) S() Announcer { return b } -// S adds 'start of the function' tag, which includes: -// file, line, function and start prefix -func S() Announcer { - return announcer.S() -} - // E adds 'end of the function' tag, which includes: // file, line, function and start prefix func (a Announcer) E() Announcer { @@ -158,12 +114,6 @@ func (a Announcer) E() Announcer { return b } -// E adds 'end of the function' tag, which includes: -// file, line, function and start prefix -func E() Announcer { - return announcer.E() -} - // M adds object meta as 'namespace/name' func (a Announcer) M(m ...interface{}) Announcer { if len(m) == 0 { @@ -199,21 +149,11 @@ func (a Announcer) M(m ...interface{}) Announcer { return b } -// M adds object meta as 'namespace/name' -func M(m ...interface{}) Announcer { - return announcer.M(m...) -} - // P triggers log to print line func (a Announcer) P() { a.Info("") } -// P triggers log to print line -func P() { - announcer.P() -} - // Info is inspired by log.Infof() func (a Announcer) Info(format string, args ...interface{}) { // Produce classic log line @@ -237,11 +177,6 @@ func (a Announcer) Info(format string, args ...interface{}) { } } -// Info is inspired by log.Infof() -func Info(format string, args ...interface{}) { - announcer.Info(format, args...) -} - // Warning is inspired by log.Warningf() func (a Announcer) Warning(format string, args ...interface{}) { // Produce classic log line @@ -257,11 +192,6 @@ func (a Announcer) Warning(format string, args ...interface{}) { } } -// Warning is inspired by log.Warningf() -func Warning(format string, args ...interface{}) { - announcer.Warning(format, args...) -} - // Error is inspired by log.Errorf() func (a Announcer) Error(format string, args ...interface{}) { // Produce classic log line @@ -277,11 +207,6 @@ func (a Announcer) Error(format string, args ...interface{}) { } } -// Error is inspired by log.Errorf() -func Error(format string, args ...interface{}) { - announcer.Error(format, args...) -} - // Fatal is inspired by log.Fatalf() func (a Announcer) Fatal(format string, args ...interface{}) { format = a.prependFormat(format) @@ -293,11 +218,6 @@ func (a Announcer) Fatal(format string, args ...interface{}) { } } -// Fatal is inspired by log.Fatalf() -func Fatal(format string, args ...interface{}) { - announcer.Fatal(format, args...) -} - // prependFormat func (a Announcer) prependFormat(format string) string { // Result format is expected to be 'file:line:function:prefix:meta:_start_format_' @@ -351,7 +271,7 @@ func (a Announcer) tryToFindNamespaceNameEverywhere(m interface{}) (string, bool return "", false } -// findInObjectMeta +// findNamespaceName func (a Announcer) findNamespaceName(m interface{}) (string, bool) { if m == nil { return "", false diff --git a/pkg/announcer/export.go b/pkg/announcer/export.go new file mode 100644 index 000000000..22f1a8f35 --- /dev/null +++ b/pkg/announcer/export.go @@ -0,0 +1,99 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package announcer + +import ( + log "github.com/golang/glog" +) + +// announcer which would be used in top-level functions, can be called as a 'default announcer' +var announcer Announcer + +// init creates default announcer +func init() { + announcer = New() +} + +// Silence produces silent announcer +func Silence() Announcer { + return announcer.Silence() +} + +// V is inspired by log.V() +func V(level log.Level) Announcer { + return announcer.V(level) +} + +// F adds function name +func F() Announcer { + return announcer.F() +} + +// L adds line number +func L() Announcer { + return announcer.L() +} + +// FL adds filename +func FL() Announcer { + return announcer.FL() +} + +// A adds full code address as 'file:line:function' +func A() Announcer { + return announcer.A() +} + +// S adds 'start of the function' tag, which includes: +// file, line, function and start prefix +func S() Announcer { + return announcer.S() +} + +// E adds 'end of the function' tag, which includes: +// file, line, function and start prefix +func E() Announcer { + return announcer.E() +} + +// M adds object meta as 'namespace/name' +func M(m ...interface{}) Announcer { + return announcer.M(m...) +} + +// P triggers log to print line +func P() { + announcer.P() +} + +// Info is inspired by log.Infof() +func Info(format string, args ...interface{}) { + announcer.Info(format, args...) +} + +// Warning is inspired by log.Warningf() +func Warning(format string, args ...interface{}) { + announcer.Warning(format, args...) +} + +// Error is inspired by log.Errorf() +func Error(format string, args ...interface{}) { + announcer.Error(format, args...) +} + +// Fatal is inspired by log.Fatalf() +func Fatal(format string, args ...interface{}) { + announcer.Fatal(format, args...) +} diff --git a/pkg/apis/clickhouse-keeper.altinity.com/v1/type_chk.go b/pkg/apis/clickhouse-keeper.altinity.com/v1/type_chk.go index d52406022..1a7dbfef1 100644 --- a/pkg/apis/clickhouse-keeper.altinity.com/v1/type_chk.go +++ b/pkg/apis/clickhouse-keeper.altinity.com/v1/type_chk.go @@ -27,10 +27,6 @@ import ( "github.com/altinity/clickhouse-operator/pkg/util" ) -func (cr *ClickHouseKeeperInstallation) IsNonZero() bool { - return cr != nil -} - func (cr *ClickHouseKeeperInstallation) GetSpec() apiChi.ICRSpec { return &cr.Spec } @@ -227,7 +223,9 @@ func (cr *ClickHouseKeeperInstallation) MergeFrom(from *ClickHouseKeeperInstalla //cr.ensureRuntime().attributes = from.ensureRuntime().attributes cr.EnsureStatus().CopyFrom(from.Status, types.CopyStatusOptions{ - InheritableFields: true, + CopyStatusFieldGroup: types.CopyStatusFieldGroup{ + FieldGroupInheritable: true, + }, }) } @@ -653,3 +651,11 @@ func (cr *ClickHouseKeeperInstallation) WalkTillError( return nil } + +func (cr *ClickHouseKeeperInstallation) IsZero() bool { + return cr == nil +} + +func (cr *ClickHouseKeeperInstallation) IsNonZero() bool { + return cr != nil +} diff --git a/pkg/apis/clickhouse-keeper.altinity.com/v1/type_cluster.go b/pkg/apis/clickhouse-keeper.altinity.com/v1/type_cluster.go index 45f25c1c6..ec4a084e9 100644 --- a/pkg/apis/clickhouse-keeper.altinity.com/v1/type_cluster.go +++ b/pkg/apis/clickhouse-keeper.altinity.com/v1/type_cluster.go @@ -189,6 +189,10 @@ func (cluster *Cluster) GetServiceTemplate() (*apiChi.ServiceTemplate, bool) { return nil, false } +func (cluster *Cluster) GetAncestor() apiChi.ICluster { + return (*Cluster)(nil) +} + // GetShard gets shard with specified index func (cluster *Cluster) GetShard(shard int) *ChkShard { return cluster.Layout.Shards[shard] @@ -330,6 +334,14 @@ func (cluster *Cluster) HostsCount() int { return count } +func (cluster *Cluster) IsZero() bool { + return cluster == nil +} + +func (cluster *Cluster) IsNonZero() bool { + return cluster != nil +} + // ChkClusterLayout defines layout section of .spec.configuration.clusters type ChkClusterLayout struct { ShardsCount int `json:"shardsCount,omitempty" yaml:"shardsCount,omitempty"` diff --git a/pkg/apis/clickhouse-keeper.altinity.com/v1/type_configuration.go b/pkg/apis/clickhouse-keeper.altinity.com/v1/type_configuration.go index 1636f2452..374196e7f 100644 --- a/pkg/apis/clickhouse-keeper.altinity.com/v1/type_configuration.go +++ b/pkg/apis/clickhouse-keeper.altinity.com/v1/type_configuration.go @@ -30,6 +30,10 @@ func NewConfiguration() *Configuration { return new(Configuration) } +func (c *Configuration) GetUsers() *apiChi.Settings { + return nil +} + func (c *Configuration) GetProfiles() *apiChi.Settings { return nil } @@ -42,31 +46,14 @@ func (c *Configuration) GetSettings() *apiChi.Settings { if c == nil { return nil } - return c.Settings } func (c *Configuration) GetFiles() *apiChi.Settings { - return c.Files -} - -func (c *Configuration) GetClusters() []*Cluster { if c == nil { return nil } - - return c.Clusters -} - -func (c *Configuration) GetCluster(i int) *Cluster { - clusters := c.GetClusters() - if clusters == nil { - return nil - } - if i >= len(clusters) { - return nil - } - return clusters[i] + return c.Files } // MergeFrom merges from specified source diff --git a/pkg/apis/clickhouse-keeper.altinity.com/v1/type_shard.go b/pkg/apis/clickhouse-keeper.altinity.com/v1/type_shard.go index 89ed9e15f..050ceca5c 100644 --- a/pkg/apis/clickhouse-keeper.altinity.com/v1/type_shard.go +++ b/pkg/apis/clickhouse-keeper.altinity.com/v1/type_shard.go @@ -175,6 +175,10 @@ func (shard *ChkShard) GetCHK() *ClickHouseKeeperInstallation { return shard.Runtime.CHK } +func (shard *ChkShard) GetAncestor() apiChi.IShard { + return (*ChkShard)(nil) +} + // GetCluster gets cluster of the shard func (shard *ChkShard) GetCluster() *Cluster { return shard.Runtime.CHK.GetSpecT().Configuration.Clusters[shard.Runtime.Address.ClusterIndex] @@ -239,6 +243,14 @@ func (shard *ChkShard) GetTemplates() *apiChi.TemplatesList { return shard.Templates } +func (shard *ChkShard) IsZero() bool { + return shard == nil +} + +func (shard *ChkShard) IsNonZero() bool { + return shard != nil +} + // ChiShardAddress defines address of a shard within ClickHouseInstallation type ChkShardAddress struct { Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"` diff --git a/pkg/apis/clickhouse-keeper.altinity.com/v1/type_spec.go b/pkg/apis/clickhouse-keeper.altinity.com/v1/type_spec.go index 9a097628b..faa798fe5 100644 --- a/pkg/apis/clickhouse-keeper.altinity.com/v1/type_spec.go +++ b/pkg/apis/clickhouse-keeper.altinity.com/v1/type_spec.go @@ -31,27 +31,45 @@ type ChkSpec struct { // HasTaskID checks whether task id is specified func (spec *ChkSpec) HasTaskID() bool { + if spec == nil { + return false + } return len(spec.TaskID.Value()) > 0 } // GetTaskID gets task id as a string func (spec *ChkSpec) GetTaskID() string { + if spec == nil { + return "" + } return spec.TaskID.Value() } func (spec *ChkSpec) GetNamespaceDomainPattern() *types.String { + if spec == nil { + return (*types.String)(nil) + } return spec.NamespaceDomainPattern } func (spec *ChkSpec) GetDefaults() *apiChi.Defaults { + if spec == nil { + return (*apiChi.Defaults)(nil) + } return spec.Defaults } func (spec *ChkSpec) GetConfiguration() apiChi.IConfiguration { + if spec == nil { + return (*Configuration)(nil) + } return spec.Configuration } func (spec *ChkSpec) GetTemplates() *apiChi.Templates { + if spec == nil { + return (*apiChi.Templates)(nil) + } return spec.Templates } @@ -61,6 +79,10 @@ func (spec *ChkSpec) MergeFrom(from *ChkSpec, _type apiChi.MergeType) { return } + if spec == nil { + spec = &ChkSpec{} + } + switch _type { case apiChi.MergeTypeFillEmptyValues: if !spec.HasTaskID() { diff --git a/pkg/apis/clickhouse-keeper.altinity.com/v1/type_status.go b/pkg/apis/clickhouse-keeper.altinity.com/v1/type_status.go index 400440c9a..16356e13d 100644 --- a/pkg/apis/clickhouse-keeper.altinity.com/v1/type_status.go +++ b/pkg/apis/clickhouse-keeper.altinity.com/v1/type_status.go @@ -18,7 +18,7 @@ import ( "sort" "sync" - apiChi "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" + chi "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" "github.com/altinity/clickhouse-operator/pkg/apis/common/types" "github.com/altinity/clickhouse-operator/pkg/util" "github.com/altinity/clickhouse-operator/pkg/version" @@ -75,7 +75,7 @@ type Status struct { NormalizedCR *ClickHouseKeeperInstallation `json:"normalized,omitempty" yaml:"normalized,omitempty"` NormalizedCRCompleted *ClickHouseKeeperInstallation `json:"normalizedCompleted,omitempty" yaml:"normalizedCompleted,omitempty"` HostsWithTablesCreated []string `json:"hostsWithTablesCreated,omitempty" yaml:"hostsWithTablesCreated,omitempty"` - UsedTemplates []*apiChi.TemplateRef `json:"usedTemplates,omitempty" yaml:"usedTemplates,omitempty"` + UsedTemplates []*chi.TemplateRef `json:"usedTemplates,omitempty" yaml:"usedTemplates,omitempty"` mu sync.RWMutex `json:"-" yaml:"-"` } @@ -135,6 +135,16 @@ func (s *Status) SetError(err string) { }) } +// PushError sets and pushes error into status +func (s *Status) PushError(error string) { + doWithWriteLock(s, func(s *Status) { + s.Errors = append([]string{error}, s.Errors...) + if len(s.Errors) > maxErrors { + s.Errors = s.Errors[:maxErrors] + } + }) +} + // SetAndPushError sets and pushes error into status func (s *Status) SetAndPushError(err string) { doWithWriteLock(s, func(s *Status) { @@ -166,11 +176,13 @@ func (s *Status) SyncHostTablesCreated() { }) } -// PushUsedTemplate pushes used template to the list of used templates -func (s *Status) PushUsedTemplate(templateRef *apiChi.TemplateRef) { - doWithWriteLock(s, func(s *Status) { - s.UsedTemplates = append(s.UsedTemplates, templateRef) - }) +// PushUsedTemplate pushes used templates to the list of used templates +func (s *Status) PushUsedTemplate(templateRefs ...*chi.TemplateRef) { + if len(templateRefs) > 0 { + doWithWriteLock(s, func(s *Status) { + s.UsedTemplates = append(s.UsedTemplates, templateRefs...) + }) + } } // GetUsedTemplatesCount gets used templates count @@ -187,16 +199,6 @@ func (s *Status) SetAction(action string) { }) } -// HasNormalizedCRCompleted is a checker -func (s *Status) HasNormalizedCRCompleted() bool { - return s.GetNormalizedCRCompleted() != nil -} - -// HasNormalizedCR is a checker -func (s *Status) HasNormalizedCR() bool { - return s.GetNormalizedCR() != nil -} - // PushAction pushes action into status func (s *Status) PushAction(action string) { doWithWriteLock(s, func(s *Status) { @@ -205,14 +207,14 @@ func (s *Status) PushAction(action string) { }) } -// PushError sets and pushes error into status -func (s *Status) PushError(error string) { - doWithWriteLock(s, func(s *Status) { - s.Errors = append([]string{error}, s.Errors...) - if len(s.Errors) > maxErrors { - s.Errors = s.Errors[:maxErrors] - } - }) +// HasNormalizedCRCompleted is a checker +func (s *Status) HasNormalizedCRCompleted() bool { + return s.GetNormalizedCRCompleted() != nil +} + +// HasNormalizedCR is a checker +func (s *Status) HasNormalizedCR() bool { + return s.GetNormalizedCR() != nil } // SetPodIPs sets pod IPs @@ -322,6 +324,98 @@ func (s *Status) DeleteStart() { }) } +func prepareOptions(opts types.CopyStatusOptions) types.CopyStatusOptions { + if opts.FieldGroupInheritable { + opts.Copy.TaskIDsStarted = true + opts.Copy.TaskIDsCompleted = true + opts.Copy.Actions = true + opts.Copy.Errors = true + opts.Copy.HostsWithTablesCreated = true + opts.Copy.UsedTemplates = true + } + + if opts.FieldGroupActions { + opts.Copy.Action = true + opts.Merge.Actions = true + opts.Copy.HostsWithTablesCreated = true + opts.Copy.UsedTemplates = true + } + + if opts.FieldGroupErrors { + opts.Copy.Error = true + opts.Merge.Errors = true + } + + if opts.FieldGroupMain { + opts.Copy.CHOpVersion = true + opts.Copy.CHOpCommit = true + opts.Copy.CHOpDate = true + opts.Copy.CHOpIP = true + opts.Copy.ClustersCount = true + opts.Copy.ShardsCount = true + opts.Copy.ReplicasCount = true + opts.Copy.HostsCount = true + opts.Copy.Status = true + opts.Copy.TaskID = true + opts.Copy.TaskIDsStarted = true + opts.Copy.TaskIDsCompleted = true + opts.Copy.Action = true + opts.Merge.Actions = true + opts.Copy.Error = true + opts.Copy.Errors = true + opts.Copy.HostsUpdatedCount = true + opts.Copy.HostsAddedCount = true + opts.Copy.HostsUnchangedCount = true + opts.Copy.HostsCompletedCount = true + opts.Copy.HostsDeletedCount = true + opts.Copy.HostsDeleteCount = true + opts.Copy.Pods = true + opts.Copy.PodIPs = true + opts.Copy.FQDNs = true + opts.Copy.Endpoint = true + opts.Copy.NormalizedCR = true + opts.Copy.UsedTemplates = true + } + + if opts.FieldGroupNormalized { + opts.Copy.NormalizedCR = true + } + + if opts.FieldGroupWholeStatus { + opts.Copy.CHOpVersion = true + opts.Copy.CHOpCommit = true + opts.Copy.CHOpDate = true + opts.Copy.CHOpIP = true + opts.Copy.ClustersCount = true + opts.Copy.ShardsCount = true + opts.Copy.ReplicasCount = true + opts.Copy.HostsCount = true + opts.Copy.Status = true + opts.Copy.TaskID = true + opts.Copy.TaskIDsStarted = true + opts.Copy.TaskIDsCompleted = true + opts.Copy.Action = true + opts.Merge.Actions = true + opts.Copy.Error = true + opts.Copy.Errors = true + opts.Copy.HostsUpdatedCount = true + opts.Copy.HostsAddedCount = true + opts.Copy.HostsUnchangedCount = true + opts.Copy.HostsCompletedCount = true + opts.Copy.HostsDeletedCount = true + opts.Copy.HostsDeleteCount = true + opts.Copy.Pods = true + opts.Copy.PodIPs = true + opts.Copy.FQDNs = true + opts.Copy.Endpoint = true + opts.Copy.NormalizedCR = true + opts.Copy.NormalizedCRCompleted = true + opts.Copy.UsedTemplates = true + } + + return opts +} + // CopyFrom copies the state of a given Status f into the receiver Status of the call. func (s *Status) CopyFrom(f *Status, opts types.CopyStatusOptions) { doWithWriteLock(s, func(s *Status) { @@ -330,97 +424,109 @@ func (s *Status) CopyFrom(f *Status, opts types.CopyStatusOptions) { return } - if opts.InheritableFields { - s.TaskIDsStarted = from.TaskIDsStarted - s.TaskIDsCompleted = from.TaskIDsCompleted - s.Actions = from.Actions - s.Errors = from.Errors - s.HostsWithTablesCreated = from.HostsWithTablesCreated - } + opts = prepareOptions(opts) - if opts.Actions { - s.Action = from.Action - mergeActionsNoSync(s, from) - s.HostsWithTablesCreated = nil - if len(from.HostsWithTablesCreated) > 0 { - s.HostsWithTablesCreated = append(s.HostsWithTablesCreated, from.HostsWithTablesCreated...) - } - s.UsedTemplates = nil - if len(from.UsedTemplates) > 0 { - s.UsedTemplates = append(s.UsedTemplates, from.UsedTemplates...) - } - } - - if opts.Errors { - s.Error = from.Error - s.Errors = util.MergeStringArrays(s.Errors, from.Errors) - sort.Sort(sort.Reverse(sort.StringSlice(s.Errors))) - } - - if opts.MainFields { + // Copy fields + if opts.Copy.CHOpVersion { s.CHOpVersion = from.CHOpVersion + } + if opts.Copy.CHOpCommit { s.CHOpCommit = from.CHOpCommit + } + if opts.Copy.CHOpDate { s.CHOpDate = from.CHOpDate + } + if opts.Copy.CHOpIP { s.CHOpIP = from.CHOpIP + } + if opts.Copy.ClustersCount { s.ClustersCount = from.ClustersCount + } + if opts.Copy.ShardsCount { s.ShardsCount = from.ShardsCount + } + if opts.Copy.ReplicasCount { s.ReplicasCount = from.ReplicasCount + } + if opts.Copy.HostsCount { s.HostsCount = from.HostsCount + } + if opts.Copy.Status { s.Status = from.Status + } + if opts.Copy.TaskID { s.TaskID = from.TaskID + } + if opts.Copy.TaskIDsStarted { s.TaskIDsStarted = from.TaskIDsStarted + } + if opts.Copy.TaskIDsCompleted { s.TaskIDsCompleted = from.TaskIDsCompleted + } + if opts.Copy.Action { s.Action = from.Action + } + if opts.Merge.Actions { mergeActionsNoSync(s, from) + } + if opts.Copy.Error { s.Error = from.Error + } + if opts.Copy.Errors { s.Errors = from.Errors - s.HostsUpdatedCount = from.HostsUpdatedCount - s.HostsAddedCount = from.HostsAddedCount - s.HostsUnchangedCount = from.HostsUnchangedCount - s.HostsCompletedCount = from.HostsCompletedCount - s.HostsDeletedCount = from.HostsDeletedCount - s.HostsDeleteCount = from.HostsDeleteCount - s.Pods = from.Pods - s.PodIPs = from.PodIPs - s.FQDNs = from.FQDNs - s.Endpoint = from.Endpoint - s.NormalizedCR = from.NormalizedCR } - - if opts.Normalized { - s.NormalizedCR = from.NormalizedCR + if opts.Merge.Errors { + s.Errors = util.MergeStringArrays(s.Errors, from.Errors) + sort.Sort(sort.Reverse(sort.StringSlice(s.Errors))) } - - if opts.WholeStatus { - s.CHOpVersion = from.CHOpVersion - s.CHOpCommit = from.CHOpCommit - s.CHOpDate = from.CHOpDate - s.CHOpIP = from.CHOpIP - s.ClustersCount = from.ClustersCount - s.ShardsCount = from.ShardsCount - s.ReplicasCount = from.ReplicasCount - s.HostsCount = from.HostsCount - s.Status = from.Status - s.TaskID = from.TaskID - s.TaskIDsStarted = from.TaskIDsStarted - s.TaskIDsCompleted = from.TaskIDsCompleted - s.Action = from.Action - mergeActionsNoSync(s, from) - s.Error = from.Error - s.Errors = from.Errors + if opts.Copy.HostsUpdatedCount { s.HostsUpdatedCount = from.HostsUpdatedCount + } + if opts.Copy.HostsAddedCount { s.HostsAddedCount = from.HostsAddedCount + } + if opts.Copy.HostsUnchangedCount { s.HostsUnchangedCount = from.HostsUnchangedCount + } + if opts.Copy.HostsCompletedCount { s.HostsCompletedCount = from.HostsCompletedCount + } + if opts.Copy.HostsDeletedCount { s.HostsDeletedCount = from.HostsDeletedCount + } + if opts.Copy.HostsDeleteCount { s.HostsDeleteCount = from.HostsDeleteCount + } + if opts.Copy.Pods { s.Pods = from.Pods + } + if opts.Copy.PodIPs { s.PodIPs = from.PodIPs + } + if opts.Copy.FQDNs { s.FQDNs = from.FQDNs + } + if opts.Copy.Endpoint { s.Endpoint = from.Endpoint + } + if opts.Copy.NormalizedCR { s.NormalizedCR = from.NormalizedCR + } + if opts.Copy.NormalizedCRCompleted { s.NormalizedCRCompleted = from.NormalizedCRCompleted } + if opts.Copy.HostsWithTablesCreated { + s.HostsWithTablesCreated = nil + if len(from.HostsWithTablesCreated) > 0 { + s.HostsWithTablesCreated = append(s.HostsWithTablesCreated, from.HostsWithTablesCreated...) + } + } + if opts.Copy.UsedTemplates { + if len(from.UsedTemplates) > len(s.UsedTemplates) { + s.UsedTemplates = nil + s.UsedTemplates = append(s.UsedTemplates, from.UsedTemplates...) + } + } }) }) } diff --git a/pkg/apis/clickhouse.altinity.com/v1/interface.go b/pkg/apis/clickhouse.altinity.com/v1/interface.go index 263ac3a75..09b6ad399 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/interface.go +++ b/pkg/apis/clickhouse.altinity.com/v1/interface.go @@ -23,6 +23,7 @@ type ICustomResource interface { meta.Object IsNonZero() bool + IsZero() bool GetSpecA() any GetSpec() ICRSpec @@ -67,6 +68,7 @@ type ICRSpec interface { } type IConfiguration interface { + GetUsers() *Settings GetProfiles() *Settings GetQuotas() *Settings GetSettings() *Settings @@ -100,6 +102,9 @@ type IStatus interface { } type ICluster interface { + IsNonZero() bool + IsZero() bool + GetName() string GetZookeeper() *ZookeeperConfig GetSchemaPolicy() *SchemaPolicy @@ -120,6 +125,7 @@ type ICluster interface { GetRuntime() IClusterRuntime GetServiceTemplate() (*ServiceTemplate, bool) + GetAncestor() ICluster } type IClusterRuntime interface { @@ -143,6 +149,9 @@ type IClusterAddress interface { } type IShard interface { + IsNonZero() bool + IsZero() bool + GetName() string GetRuntime() IShardRuntime GetServiceTemplate() (*ServiceTemplate, bool) @@ -163,6 +172,7 @@ type IShard interface { FirstHost() *Host HostsCount() int + GetAncestor() IShard } type IShardRuntime interface { diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_chi.go b/pkg/apis/clickhouse.altinity.com/v1/type_chi.go index 71727284d..f152842aa 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_chi.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_chi.go @@ -26,10 +26,6 @@ import ( "github.com/altinity/clickhouse-operator/pkg/util" ) -func (cr *ClickHouseInstallation) IsNonZero() bool { - return cr != nil -} - func (cr *ClickHouseInstallation) GetSpec() ICRSpec { return &cr.Spec } @@ -226,7 +222,9 @@ func (cr *ClickHouseInstallation) MergeFrom(from *ClickHouseInstallation, _type cr.ensureRuntime().attributes = from.ensureRuntime().attributes cr.EnsureStatus().CopyFrom(from.Status, types.CopyStatusOptions{ - InheritableFields: true, + CopyStatusFieldGroup: types.CopyStatusFieldGroup{ + FieldGroupInheritable: true, + }, }) } @@ -681,3 +679,11 @@ func (cr *ClickHouseInstallation) WalkTillError( return nil } + +func (cr *ClickHouseInstallation) IsZero() bool { + return cr == nil +} + +func (cr *ClickHouseInstallation) IsNonZero() bool { + return cr != nil +} diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_cluster.go b/pkg/apis/clickhouse.altinity.com/v1/type_cluster.go index f9f10a90f..f38686234 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_cluster.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_cluster.go @@ -224,6 +224,10 @@ func (cluster *Cluster) GetCHI() *ClickHouseInstallation { return cluster.Runtime.CHI } +func (cluster *Cluster) GetAncestor() ICluster { + return cluster.GetCHI().GetAncestor().FindCluster(cluster.GetName()) +} + // GetShard gets shard with specified index func (cluster *Cluster) GetShard(shard int) *ChiShard { return cluster.Layout.Shards[shard] @@ -365,6 +369,14 @@ func (cluster *Cluster) HostsCount() int { return count } +func (cluster *Cluster) IsZero() bool { + return cluster == nil +} + +func (cluster *Cluster) IsNonZero() bool { + return cluster != nil +} + // ChiClusterLayout defines layout section of .spec.configuration.clusters type ChiClusterLayout struct { ShardsCount int `json:"shardsCount,omitempty" yaml:"shardsCount,omitempty"` diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chi.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chi.go index 245c27c42..006b961d7 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chi.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chi.go @@ -50,8 +50,7 @@ type Configuration struct { Quotas *Settings `json:"quotas,omitempty" yaml:"quotas,omitempty"` Settings *Settings `json:"settings,omitempty" yaml:"settings,omitempty"` Files *Settings `json:"files,omitempty" yaml:"files,omitempty"` - // TODO refactor into map[string]ChiCluster - Clusters []*Cluster `json:"clusters,omitempty" yaml:"clusters,omitempty"` + Clusters []*Cluster `json:"clusters,omitempty" yaml:"clusters,omitempty"` } // NewConfiguration creates new Configuration objects @@ -59,19 +58,38 @@ func NewConfiguration() *Configuration { return new(Configuration) } +func (c *Configuration) GetUsers() *Settings { + if c == nil { + return nil + } + return c.Users +} + func (c *Configuration) GetProfiles() *Settings { + if c == nil { + return nil + } return c.Profiles } func (c *Configuration) GetQuotas() *Settings { + if c == nil { + return nil + } return c.Quotas } func (c *Configuration) GetSettings() *Settings { + if c == nil { + return nil + } return c.Settings } func (c *Configuration) GetFiles() *Settings { + if c == nil { + return nil + } return c.Files } diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go index a4f631e09..302c5fb86 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_configuration_chop.go @@ -210,6 +210,24 @@ type OperatorConfigFileRuntime struct { UsersConfigFiles map[string]string `json:"-" yaml:"-"` } +type IOperatorConfigFilesPathsGetter interface { + GetCommonConfigFiles() map[string]string + GetHostConfigFiles() map[string]string + GetUsersConfigFiles() map[string]string +} + +func (r OperatorConfigFileRuntime) GetCommonConfigFiles() map[string]string { + return r.CommonConfigFiles +} + +func (r OperatorConfigFileRuntime) GetHostConfigFiles() map[string]string { + return r.HostConfigFiles +} + +func (r OperatorConfigFileRuntime) GetUsersConfigFiles() map[string]string { + return r.UsersConfigFiles +} + // OperatorConfigUser specifies User section type OperatorConfigUser struct { Default OperatorConfigDefault `json:"default" yaml:"default"` @@ -396,6 +414,21 @@ type OperatorConfigLabel struct { } `json:"runtime" yaml:"runtime"` } +type OperatorConfigMetrics struct { + Labels struct { + Exclude []string `json:"exclude" yaml:"exclude"` + } `json:"labels" yaml:"labels"` +} + +type OperatorConfigStatus struct { + Fields struct { + Action *types.StringBool `json:"action,omitempty" yaml:"action,omitempty"` + Actions *types.StringBool `json:"actions,omitempty" yaml:"actions,omitempty"` + Error *types.StringBool `json:"error,omitempty" yaml:"error,omitempty"` + Errors *types.StringBool `json:"errors,omitempty" yaml:"errors,omitempty"` + } `json:"fields" yaml:"fields"` +} + type ConfigCRSource struct { Namespace string Name string @@ -411,6 +444,8 @@ type OperatorConfig struct { Reconcile OperatorConfigReconcile `json:"reconcile" yaml:"reconcile"` Annotation OperatorConfigAnnotation `json:"annotation" yaml:"annotation"` Label OperatorConfigLabel `json:"label" yaml:"label"` + Metrics OperatorConfigMetrics `json:"metrics" yaml:"metrics"` + Status OperatorConfigStatus `json:"status" yaml:"status"` StatefulSet struct { // Revision history limit RevisionHistoryLimit int `json:"revisionHistoryLimit" yaml:"revisionHistoryLimit"` diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_defaults.go b/pkg/apis/clickhouse.altinity.com/v1/type_defaults.go index 8da292d7f..96053df13 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_defaults.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_defaults.go @@ -29,6 +29,13 @@ func NewDefaults() *Defaults { return new(Defaults) } +func (d *Defaults) GetDistributedDDL() *DistributedDDL { + if d == nil { + return nil + } + return d.DistributedDDL +} + // MergeFrom merges from specified object func (defaults *Defaults) MergeFrom(from *Defaults, _type MergeType) *Defaults { if from == nil { diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_host.go b/pkg/apis/clickhouse.altinity.com/v1/type_host.go index 2af2add9e..2bb97a28e 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_host.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_host.go @@ -74,21 +74,47 @@ type HostRuntime struct { } func (r *HostRuntime) GetAddress() IHostAddress { + if r == nil { + return (*HostAddress)(nil) + } return &r.Address } func (r *HostRuntime) SetCR(cr ICustomResource) { + if r == nil { + return + } r.cr = cr } func (r *HostRuntime) GetCR() ICustomResource { - return r.cr.(ICustomResource) + if r == nil { + return nil + } + return r.cr } func (host *Host) GetRuntime() IHostRuntime { + if host == nil { + return (*HostRuntime)(nil) + } return &host.Runtime } +func (host *Host) GetTemplates() *TemplatesList { + if host == nil { + return nil + } + return host.Templates +} + +func (host *Host) SetTemplates(tl *TemplatesList) { + if host == nil { + return + } + host.Templates = tl +} + // GetReconcileAttributes is an ensurer getter func (host *Host) GetReconcileAttributes() *HostReconcileAttributes { if host == nil { @@ -102,6 +128,9 @@ func (host *Host) GetReconcileAttributes() *HostReconcileAttributes { // InheritSettingsFrom inherits settings from specified shard and replica func (host *Host) InheritSettingsFrom(shard IShard, replica IReplica) { + if host == nil { + return + } if (shard != nil) && shard.HasSettings() { host.Settings = host.Settings.MergeFrom(shard.GetSettings()) } @@ -113,6 +142,9 @@ func (host *Host) InheritSettingsFrom(shard IShard, replica IReplica) { // InheritFilesFrom inherits files from specified shard and replica func (host *Host) InheritFilesFrom(shard IShard, replica IReplica) { + if host == nil { + return + } if (shard != nil) && shard.HasFiles() { host.Files = host.Files.MergeFrom(shard.GetFiles()) } @@ -124,27 +156,30 @@ func (host *Host) InheritFilesFrom(shard IShard, replica IReplica) { // InheritTemplatesFrom inherits templates from specified shard, replica or template func (host *Host) InheritTemplatesFrom(sources ...any) { + if host == nil { + return + } for _, source := range sources { switch typed := source.(type) { case IShard: shard := typed if shard.HasTemplates() { - host.Templates = host.Templates.MergeFrom(shard.GetTemplates(), MergeTypeFillEmptyValues) + host.SetTemplates(host.GetTemplates().MergeFrom(shard.GetTemplates(), MergeTypeFillEmptyValues)) } case IReplica: replica := typed if replica.HasTemplates() { - host.Templates = host.Templates.MergeFrom(replica.GetTemplates(), MergeTypeFillEmptyValues) + host.SetTemplates(host.GetTemplates().MergeFrom(replica.GetTemplates(), MergeTypeFillEmptyValues)) } case *HostTemplate: template := typed if template != nil { - host.Templates = host.Templates.MergeFrom(template.Spec.Templates, MergeTypeFillEmptyValues) + host.SetTemplates(host.GetTemplates().MergeFrom(template.Spec.GetTemplates(), MergeTypeFillEmptyValues)) } } } - host.Templates.HandleDeprecatedFields() + host.GetTemplates().HandleDeprecatedFields() } // MergeFrom merges from specified host @@ -178,34 +213,34 @@ func (host *Host) MergeFrom(from *Host) { host.RaftPort.MergeFrom(from.RaftPort) } - host.Templates = host.Templates.MergeFrom(from.Templates, MergeTypeFillEmptyValues) - host.Templates.HandleDeprecatedFields() + host.SetTemplates(host.GetTemplates().MergeFrom(from.GetTemplates(), MergeTypeFillEmptyValues)) + host.GetTemplates().HandleDeprecatedFields() } // GetHostTemplate gets host template func (host *Host) GetHostTemplate() (*HostTemplate, bool) { - if !host.Templates.HasHostTemplate() { + if !host.GetTemplates().HasHostTemplate() { return nil, false } - name := host.Templates.GetHostTemplate() + name := host.GetTemplates().GetHostTemplate() return host.GetCR().GetHostTemplate(name) } // GetPodTemplate gets pod template func (host *Host) GetPodTemplate() (*PodTemplate, bool) { - if !host.Templates.HasPodTemplate() { + if !host.GetTemplates().HasPodTemplate() { return nil, false } - name := host.Templates.GetPodTemplate() + name := host.GetTemplates().GetPodTemplate() return host.GetCR().GetPodTemplate(name) } // GetServiceTemplate gets service template func (host *Host) GetServiceTemplate() (*ServiceTemplate, bool) { - if !host.Templates.HasReplicaServiceTemplate() { + if !host.GetTemplates().HasReplicaServiceTemplate() { return nil, false } - name := host.Templates.GetReplicaServiceTemplate() + name := host.GetTemplates().GetReplicaServiceTemplate() return host.GetCR().GetServiceTemplate(name) } @@ -227,6 +262,9 @@ func (host *Host) GetStatefulSetReplicasNum(shutdown bool) *int32 { // GetSettings gets settings func (host *Host) GetSettings() *Settings { + if host == nil { + return nil + } return host.Settings } @@ -557,3 +595,22 @@ func (host *Host) SetHasData(hasData bool) { } host.Runtime.hasData = hasData } + +func (host *Host) IsZero() bool { + return host == nil +} + +func (host *Host) IsNonZero() bool { + return host != nil +} + +func (host *Host) ShouldIncludeIntoCluster() bool { + switch { + case host.IsStopped(): + return false + case host.GetCluster().HostsCount() < 2: + return false + default: + return true + } +} diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_host_reconcile_attributes.go b/pkg/apis/clickhouse.altinity.com/v1/type_host_reconcile_attributes.go index 012c7b6b9..2a04dfc49 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_host_reconcile_attributes.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_host_reconcile_attributes.go @@ -310,3 +310,7 @@ func (s *HostReconcileAttributesCounters) GetExclude() int { func (s *HostReconcileAttributesCounters) AddOnly() bool { return s.GetAdd() > 0 && s.GetFound() == 0 && s.GetModify() == 0 && s.GetRemove() == 0 } + +func (s *HostReconcileAttributesCounters) String() string { + return fmt.Sprintf("a: %d f: %d m: %d r: %d", s.GetAdd(), s.GetFound(), s.GetModify(), s.GetRemove()) +} diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_shard.go b/pkg/apis/clickhouse.altinity.com/v1/type_shard.go index 22c07ae79..950a5d456 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_shard.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_shard.go @@ -174,6 +174,10 @@ func (shard *ChiShard) GetCHI() *ClickHouseInstallation { return shard.Runtime.CHI } +func (shard *ChiShard) GetAncestor() IShard { + return shard.GetCHI().GetAncestor().FindShard(shard.GetCluster().GetName(), shard.GetName()) +} + // GetCluster gets cluster of the shard func (shard *ChiShard) GetCluster() *Cluster { return shard.Runtime.CHI.GetSpecT().Configuration.Clusters[shard.Runtime.Address.ClusterIndex] @@ -238,6 +242,14 @@ func (shard *ChiShard) GetTemplates() *TemplatesList { return shard.Templates } +func (shard *ChiShard) IsZero() bool { + return shard == nil +} + +func (shard *ChiShard) IsNonZero() bool { + return shard != nil +} + // ChiShardAddress defines address of a shard within ClickHouseInstallation type ChiShardAddress struct { Namespace string `json:"namespace,omitempty" yaml:"namespace,omitempty"` diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_spec.go b/pkg/apis/clickhouse.altinity.com/v1/type_spec.go index 11b50e4a5..1464d5da5 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_spec.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_spec.go @@ -35,43 +35,73 @@ type ChiSpec struct { // HasTaskID checks whether task id is specified func (spec *ChiSpec) HasTaskID() bool { + if spec == nil { + return false + } return len(spec.TaskID.Value()) > 0 } // GetTaskID gets task id as a string func (spec *ChiSpec) GetTaskID() string { + if spec == nil { + return "" + } return spec.TaskID.Value() } func (spec *ChiSpec) GetStop() *types.StringBool { + if spec == nil { + return (*types.StringBool)(nil) + } return spec.Stop } func (spec *ChiSpec) GetRestart() *types.String { + if spec == nil { + return (*types.String)(nil) + } return spec.Restart } func (spec *ChiSpec) GetTroubleshoot() *types.StringBool { + if spec == nil { + return (*types.StringBool)(nil) + } return spec.Troubleshoot } func (spec *ChiSpec) GetNamespaceDomainPattern() *types.String { + if spec == nil { + return (*types.String)(nil) + } return spec.NamespaceDomainPattern } func (spec *ChiSpec) GetTemplating() *ChiTemplating { + if spec == nil { + return (*ChiTemplating)(nil) + } return spec.Templating } func (spec *ChiSpec) GetDefaults() *Defaults { + if spec == nil { + return (*Defaults)(nil) + } return spec.Defaults } func (spec *ChiSpec) GetConfiguration() IConfiguration { + if spec == nil { + return (*Configuration)(nil) + } return spec.Configuration } func (spec *ChiSpec) GetTemplates() *Templates { + if spec == nil { + return (*Templates)(nil) + } return spec.Templates } @@ -81,6 +111,10 @@ func (spec *ChiSpec) MergeFrom(from *ChiSpec, _type MergeType) { return } + if spec == nil { + spec = &ChiSpec{} + } + switch _type { case MergeTypeFillEmptyValues: if !spec.HasTaskID() { diff --git a/pkg/apis/clickhouse.altinity.com/v1/type_status.go b/pkg/apis/clickhouse.altinity.com/v1/type_status.go index bb5be021d..3f6709682 100644 --- a/pkg/apis/clickhouse.altinity.com/v1/type_status.go +++ b/pkg/apis/clickhouse.altinity.com/v1/type_status.go @@ -134,6 +134,16 @@ func (s *Status) SetError(err string) { }) } +// PushError sets and pushes error into status +func (s *Status) PushError(error string) { + doWithWriteLock(s, func(s *Status) { + s.Errors = append([]string{error}, s.Errors...) + if len(s.Errors) > maxErrors { + s.Errors = s.Errors[:maxErrors] + } + }) +} + // SetAndPushError sets and pushes error into status func (s *Status) SetAndPushError(err string) { doWithWriteLock(s, func(s *Status) { @@ -165,11 +175,13 @@ func (s *Status) SyncHostTablesCreated() { }) } -// PushUsedTemplate pushes used template to the list of used templates -func (s *Status) PushUsedTemplate(templateRef *TemplateRef) { - doWithWriteLock(s, func(s *Status) { - s.UsedTemplates = append(s.UsedTemplates, templateRef) - }) +// PushUsedTemplate pushes used templates to the list of used templates +func (s *Status) PushUsedTemplate(templateRefs ...*TemplateRef) { + if len(templateRefs) > 0 { + doWithWriteLock(s, func(s *Status) { + s.UsedTemplates = append(s.UsedTemplates, templateRefs...) + }) + } } // GetUsedTemplatesCount gets used templates count @@ -186,16 +198,6 @@ func (s *Status) SetAction(action string) { }) } -// HasNormalizedCRCompleted is a checker -func (s *Status) HasNormalizedCRCompleted() bool { - return s.GetNormalizedCRCompleted() != nil -} - -// HasNormalizedCR is a checker -func (s *Status) HasNormalizedCR() bool { - return s.GetNormalizedCR() != nil -} - // PushAction pushes action into status func (s *Status) PushAction(action string) { doWithWriteLock(s, func(s *Status) { @@ -204,14 +206,14 @@ func (s *Status) PushAction(action string) { }) } -// PushError sets and pushes error into status -func (s *Status) PushError(error string) { - doWithWriteLock(s, func(s *Status) { - s.Errors = append([]string{error}, s.Errors...) - if len(s.Errors) > maxErrors { - s.Errors = s.Errors[:maxErrors] - } - }) +// HasNormalizedCRCompleted is a checker +func (s *Status) HasNormalizedCRCompleted() bool { + return s.GetNormalizedCRCompleted() != nil +} + +// HasNormalizedCR is a checker +func (s *Status) HasNormalizedCR() bool { + return s.GetNormalizedCR() != nil } // SetPodIPs sets pod IPs @@ -321,6 +323,98 @@ func (s *Status) DeleteStart() { }) } +func prepareOptions(opts types.CopyStatusOptions) types.CopyStatusOptions { + if opts.FieldGroupInheritable { + opts.Copy.TaskIDsStarted = true + opts.Copy.TaskIDsCompleted = true + opts.Copy.Actions = true + opts.Copy.Errors = true + opts.Copy.HostsWithTablesCreated = true + opts.Copy.UsedTemplates = true + } + + if opts.FieldGroupActions { + opts.Copy.Action = true + opts.Merge.Actions = true + opts.Copy.HostsWithTablesCreated = true + opts.Copy.UsedTemplates = true + } + + if opts.FieldGroupErrors { + opts.Copy.Error = true + opts.Merge.Errors = true + } + + if opts.FieldGroupMain { + opts.Copy.CHOpVersion = true + opts.Copy.CHOpCommit = true + opts.Copy.CHOpDate = true + opts.Copy.CHOpIP = true + opts.Copy.ClustersCount = true + opts.Copy.ShardsCount = true + opts.Copy.ReplicasCount = true + opts.Copy.HostsCount = true + opts.Copy.Status = true + opts.Copy.TaskID = true + opts.Copy.TaskIDsStarted = true + opts.Copy.TaskIDsCompleted = true + opts.Copy.Action = true + opts.Merge.Actions = true + opts.Copy.Error = true + opts.Copy.Errors = true + opts.Copy.HostsUpdatedCount = true + opts.Copy.HostsAddedCount = true + opts.Copy.HostsUnchangedCount = true + opts.Copy.HostsCompletedCount = true + opts.Copy.HostsDeletedCount = true + opts.Copy.HostsDeleteCount = true + opts.Copy.Pods = true + opts.Copy.PodIPs = true + opts.Copy.FQDNs = true + opts.Copy.Endpoint = true + opts.Copy.NormalizedCR = true + opts.Copy.UsedTemplates = true + } + + if opts.FieldGroupNormalized { + opts.Copy.NormalizedCR = true + } + + if opts.FieldGroupWholeStatus { + opts.Copy.CHOpVersion = true + opts.Copy.CHOpCommit = true + opts.Copy.CHOpDate = true + opts.Copy.CHOpIP = true + opts.Copy.ClustersCount = true + opts.Copy.ShardsCount = true + opts.Copy.ReplicasCount = true + opts.Copy.HostsCount = true + opts.Copy.Status = true + opts.Copy.TaskID = true + opts.Copy.TaskIDsStarted = true + opts.Copy.TaskIDsCompleted = true + opts.Copy.Action = true + opts.Merge.Actions = true + opts.Copy.Error = true + opts.Copy.Errors = true + opts.Copy.HostsUpdatedCount = true + opts.Copy.HostsAddedCount = true + opts.Copy.HostsUnchangedCount = true + opts.Copy.HostsCompletedCount = true + opts.Copy.HostsDeletedCount = true + opts.Copy.HostsDeleteCount = true + opts.Copy.Pods = true + opts.Copy.PodIPs = true + opts.Copy.FQDNs = true + opts.Copy.Endpoint = true + opts.Copy.NormalizedCR = true + opts.Copy.NormalizedCRCompleted = true + opts.Copy.UsedTemplates = true + } + + return opts +} + // CopyFrom copies the state of a given Status f into the receiver Status of the call. func (s *Status) CopyFrom(f *Status, opts types.CopyStatusOptions) { doWithWriteLock(s, func(s *Status) { @@ -329,97 +423,109 @@ func (s *Status) CopyFrom(f *Status, opts types.CopyStatusOptions) { return } - if opts.InheritableFields { - s.TaskIDsStarted = from.TaskIDsStarted - s.TaskIDsCompleted = from.TaskIDsCompleted - s.Actions = from.Actions - s.Errors = from.Errors - s.HostsWithTablesCreated = from.HostsWithTablesCreated - } + opts = prepareOptions(opts) - if opts.Actions { - s.Action = from.Action - mergeActionsNoSync(s, from) - s.HostsWithTablesCreated = nil - if len(from.HostsWithTablesCreated) > 0 { - s.HostsWithTablesCreated = append(s.HostsWithTablesCreated, from.HostsWithTablesCreated...) - } - s.UsedTemplates = nil - if len(from.UsedTemplates) > 0 { - s.UsedTemplates = append(s.UsedTemplates, from.UsedTemplates...) - } - } - - if opts.Errors { - s.Error = from.Error - s.Errors = util.MergeStringArrays(s.Errors, from.Errors) - sort.Sort(sort.Reverse(sort.StringSlice(s.Errors))) - } - - if opts.MainFields { + // Copy fields + if opts.Copy.CHOpVersion { s.CHOpVersion = from.CHOpVersion + } + if opts.Copy.CHOpCommit { s.CHOpCommit = from.CHOpCommit + } + if opts.Copy.CHOpDate { s.CHOpDate = from.CHOpDate + } + if opts.Copy.CHOpIP { s.CHOpIP = from.CHOpIP + } + if opts.Copy.ClustersCount { s.ClustersCount = from.ClustersCount + } + if opts.Copy.ShardsCount { s.ShardsCount = from.ShardsCount + } + if opts.Copy.ReplicasCount { s.ReplicasCount = from.ReplicasCount + } + if opts.Copy.HostsCount { s.HostsCount = from.HostsCount + } + if opts.Copy.Status { s.Status = from.Status + } + if opts.Copy.TaskID { s.TaskID = from.TaskID + } + if opts.Copy.TaskIDsStarted { s.TaskIDsStarted = from.TaskIDsStarted + } + if opts.Copy.TaskIDsCompleted { s.TaskIDsCompleted = from.TaskIDsCompleted + } + if opts.Copy.Action { s.Action = from.Action + } + if opts.Merge.Actions { mergeActionsNoSync(s, from) + } + if opts.Copy.Error { s.Error = from.Error + } + if opts.Copy.Errors { s.Errors = from.Errors - s.HostsUpdatedCount = from.HostsUpdatedCount - s.HostsAddedCount = from.HostsAddedCount - s.HostsUnchangedCount = from.HostsUnchangedCount - s.HostsCompletedCount = from.HostsCompletedCount - s.HostsDeletedCount = from.HostsDeletedCount - s.HostsDeleteCount = from.HostsDeleteCount - s.Pods = from.Pods - s.PodIPs = from.PodIPs - s.FQDNs = from.FQDNs - s.Endpoint = from.Endpoint - s.NormalizedCR = from.NormalizedCR } - - if opts.Normalized { - s.NormalizedCR = from.NormalizedCR + if opts.Merge.Errors { + s.Errors = util.MergeStringArrays(s.Errors, from.Errors) + sort.Sort(sort.Reverse(sort.StringSlice(s.Errors))) } - - if opts.WholeStatus { - s.CHOpVersion = from.CHOpVersion - s.CHOpCommit = from.CHOpCommit - s.CHOpDate = from.CHOpDate - s.CHOpIP = from.CHOpIP - s.ClustersCount = from.ClustersCount - s.ShardsCount = from.ShardsCount - s.ReplicasCount = from.ReplicasCount - s.HostsCount = from.HostsCount - s.Status = from.Status - s.TaskID = from.TaskID - s.TaskIDsStarted = from.TaskIDsStarted - s.TaskIDsCompleted = from.TaskIDsCompleted - s.Action = from.Action - mergeActionsNoSync(s, from) - s.Error = from.Error - s.Errors = from.Errors + if opts.Copy.HostsUpdatedCount { s.HostsUpdatedCount = from.HostsUpdatedCount + } + if opts.Copy.HostsAddedCount { s.HostsAddedCount = from.HostsAddedCount + } + if opts.Copy.HostsUnchangedCount { s.HostsUnchangedCount = from.HostsUnchangedCount + } + if opts.Copy.HostsCompletedCount { s.HostsCompletedCount = from.HostsCompletedCount + } + if opts.Copy.HostsDeletedCount { s.HostsDeletedCount = from.HostsDeletedCount + } + if opts.Copy.HostsDeleteCount { s.HostsDeleteCount = from.HostsDeleteCount + } + if opts.Copy.Pods { s.Pods = from.Pods + } + if opts.Copy.PodIPs { s.PodIPs = from.PodIPs + } + if opts.Copy.FQDNs { s.FQDNs = from.FQDNs + } + if opts.Copy.Endpoint { s.Endpoint = from.Endpoint + } + if opts.Copy.NormalizedCR { s.NormalizedCR = from.NormalizedCR + } + if opts.Copy.NormalizedCRCompleted { s.NormalizedCRCompleted = from.NormalizedCRCompleted } + if opts.Copy.HostsWithTablesCreated { + s.HostsWithTablesCreated = nil + if len(from.HostsWithTablesCreated) > 0 { + s.HostsWithTablesCreated = append(s.HostsWithTablesCreated, from.HostsWithTablesCreated...) + } + } + if opts.Copy.UsedTemplates { + if len(from.UsedTemplates) > len(s.UsedTemplates) { + s.UsedTemplates = nil + s.UsedTemplates = append(s.UsedTemplates, from.UsedTemplates...) + } + } }) }) } diff --git a/pkg/apis/common/types/status_options.go b/pkg/apis/common/types/status_options.go index cbab9cbbb..697a523cd 100644 --- a/pkg/apis/common/types/status_options.go +++ b/pkg/apis/common/types/status_options.go @@ -16,12 +16,22 @@ package types // CopyStatusOptions specifies what parts to copy in status type CopyStatusOptions struct { - Actions bool - Errors bool - Normalized bool - MainFields bool - WholeStatus bool - InheritableFields bool + CopyStatusFieldGroup + CopyStatusField +} + +type CopyStatusFieldGroup struct { + FieldGroupActions bool + FieldGroupErrors bool + FieldGroupNormalized bool + FieldGroupMain bool + FieldGroupWholeStatus bool + FieldGroupInheritable bool +} + +type CopyStatusField struct { + Copy Status + Merge Status } // UpdateStatusOptions defines how to update CHI status @@ -29,3 +39,37 @@ type UpdateStatusOptions struct { CopyStatusOptions TolerateAbsence bool } + +type Status struct { + CHOpVersion bool + CHOpCommit bool + CHOpDate bool + CHOpIP bool + ClustersCount bool + ShardsCount bool + ReplicasCount bool + HostsCount bool + Status bool + TaskID bool + TaskIDsStarted bool + TaskIDsCompleted bool + Action bool + Actions bool + Error bool + Errors bool + HostsUpdatedCount bool + HostsAddedCount bool + HostsUnchangedCount bool + HostsFailedCount bool + HostsCompletedCount bool + HostsDeletedCount bool + HostsDeleteCount bool + Pods bool + PodIPs bool + FQDNs bool + Endpoint bool + NormalizedCR bool + NormalizedCRCompleted bool + HostsWithTablesCreated bool + UsedTemplates bool +} diff --git a/pkg/apis/metrics/watched_chi.go b/pkg/apis/metrics/watched_resource.go similarity index 64% rename from pkg/apis/metrics/watched_chi.go rename to pkg/apis/metrics/watched_resource.go index 40ab02fbd..22110450c 100644 --- a/pkg/apis/metrics/watched_chi.go +++ b/pkg/apis/metrics/watched_resource.go @@ -20,8 +20,8 @@ import ( api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" ) -// WatchedCHI specifies watched ClickHouseInstallation -type WatchedCHI struct { +// WatchedCR specifies watched ClickHouseInstallation +type WatchedCR struct { Namespace string `json:"namespace"` Name string `json:"name"` Labels map[string]string `json:"labels"` @@ -45,87 +45,87 @@ type WatchedHost struct { HTTPSPort int32 `json:"httpsPort,omitempty" yaml:"httpsPort,omitempty"` } -// NewWatchedCHI creates new watched CHI -func NewWatchedCHI(cr api.ICustomResource) *WatchedCHI { - chi := &WatchedCHI{} - chi.readFrom(cr) - return chi +// NewWatchedCR creates new watched CR +func NewWatchedCR(src api.ICustomResource) *WatchedCR { + cr := &WatchedCR{} + cr.readFrom(src) + return cr } -func (chi *WatchedCHI) readFrom(cr api.ICustomResource) { - if chi == nil { +func (cr *WatchedCR) readFrom(src api.ICustomResource) { + if cr == nil { return } - chi.Namespace = cr.GetNamespace() - chi.Name = cr.GetName() - chi.Labels = cr.GetLabels() - chi.Annotations = cr.GetAnnotations() + cr.Namespace = src.GetNamespace() + cr.Name = src.GetName() + cr.Labels = src.GetLabels() + cr.Annotations = src.GetAnnotations() - cr.WalkClusters(func(cl api.ICluster) error { + src.WalkClusters(func(cl api.ICluster) error { cluster := &WatchedCluster{} cluster.readFrom(cl) - chi.Clusters = append(chi.Clusters, cluster) + cr.Clusters = append(cr.Clusters, cluster) return nil }) } -func (chi *WatchedCHI) IsValid() bool { - return !chi.empty() +func (cr *WatchedCR) IsValid() bool { + return !cr.empty() } -func (chi *WatchedCHI) empty() bool { - return (len(chi.Namespace) == 0) && (len(chi.Name) == 0) && (len(chi.Clusters) == 0) +func (cr *WatchedCR) empty() bool { + return (len(cr.Namespace) == 0) && (len(cr.Name) == 0) && (len(cr.Clusters) == 0) } -func (chi *WatchedCHI) IndexKey() string { - return chi.Namespace + ":" + chi.Name +func (cr *WatchedCR) IndexKey() string { + return cr.Namespace + ":" + cr.Name } -func (chi *WatchedCHI) WalkHosts(f func(*WatchedCHI, *WatchedCluster, *WatchedHost)) { - if chi == nil { +func (cr *WatchedCR) WalkHosts(f func(*WatchedCR, *WatchedCluster, *WatchedHost)) { + if cr == nil { return } - for _, cluster := range chi.Clusters { + for _, cluster := range cr.Clusters { for _, host := range cluster.Hosts { - f(chi, cluster, host) + f(cr, cluster, host) } } } -func (chi *WatchedCHI) GetName() string { - if chi == nil { +func (cr *WatchedCR) GetName() string { + if cr == nil { return "" } - return chi.Name + return cr.Name } -func (chi *WatchedCHI) GetNamespace() string { - if chi == nil { +func (cr *WatchedCR) GetNamespace() string { + if cr == nil { return "" } - return chi.Namespace + return cr.Namespace } -func (chi *WatchedCHI) GetLabels() map[string]string { - if chi == nil { +func (cr *WatchedCR) GetLabels() map[string]string { + if cr == nil { return nil } - return chi.Labels + return cr.Labels } -func (chi *WatchedCHI) GetAnnotations() map[string]string { - if chi == nil { +func (cr *WatchedCR) GetAnnotations() map[string]string { + if cr == nil { return nil } - return chi.Annotations + return cr.Annotations } // String is a stringifier -func (chi *WatchedCHI) String() string { - if chi == nil { +func (cr *WatchedCR) String() string { + if cr == nil { return "nil" } - bytes, _ := json.Marshal(chi) + bytes, _ := json.Marshal(cr) return string(bytes) } diff --git a/pkg/controller/chi/cmd_queue/type_cmd_queue.go b/pkg/controller/chi/cmd_queue/type_cmd_queue.go index 69d66836b..135247ead 100644 --- a/pkg/controller/chi/cmd_queue/type_cmd_queue.go +++ b/pkg/controller/chi/cmd_queue/type_cmd_queue.go @@ -15,10 +15,8 @@ package cmd_queue import ( - core "k8s.io/api/core/v1" - meta "k8s.io/apimachinery/pkg/apis/meta/v1" - "github.com/altinity/queue" + core "k8s.io/api/core/v1" api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" ) @@ -44,7 +42,6 @@ const ( priorityReconcileCHIT int = 5 priorityReconcileChopConfig int = 3 priorityReconcileEndpoints int = 15 - priorityDropDNS int = 7 ) // ReconcileCHI specifies reconcile request queue item @@ -197,32 +194,6 @@ func NewReconcileEndpoints(cmd string, old, new *core.Endpoints) *ReconcileEndpo } } -// DropDns specifies drop dns queue item -type DropDns struct { - PriorityQueueItem - Initiator meta.Object -} - -var _ queue.PriorityQueueItem = &DropDns{} - -// Handle returns handle of the queue item -func (r DropDns) Handle() queue.T { - if r.Initiator != nil { - return "DropDNS" + ":" + r.Initiator.GetNamespace() + "/" + r.Initiator.GetName() - } - return "" -} - -// NewDropDns creates new drop dns queue item -func NewDropDns(initiator meta.Object) *DropDns { - return &DropDns{ - PriorityQueueItem: PriorityQueueItem{ - priority: priorityDropDNS, - }, - Initiator: initiator, - } -} - // ReconcilePod specifies pod reconcile type ReconcilePod struct { PriorityQueueItem diff --git a/pkg/controller/chi/controller.go b/pkg/controller/chi/controller.go index a433a14c3..1bfaf0cef 100644 --- a/pkg/controller/chi/controller.go +++ b/pkg/controller/chi/controller.go @@ -358,7 +358,6 @@ func (c *Controller) addEventHandlersEndpoint( log.V(3).M(newEndpoints).Info("endpointsInformer.UpdateFunc") if updated(oldEndpoints, newEndpoints) { c.enqueueObject(cmd_queue.NewReconcileEndpoints(cmd_queue.ReconcileUpdate, oldEndpoints, newEndpoints)) - c.enqueueObject(cmd_queue.NewDropDns(&newEndpoints.ObjectMeta)) } }, DeleteFunc: func(obj interface{}) { @@ -603,8 +602,7 @@ func (c *Controller) enqueueObject(obj queue.PriorityQueueItem) { *cmd_queue.ReconcileCHIT, *cmd_queue.ReconcileChopConfig, *cmd_queue.ReconcileEndpoints, - *cmd_queue.ReconcilePod, - *cmd_queue.DropDns: + *cmd_queue.ReconcilePod: variants := api.DefaultReconcileSystemThreadsNumber index = util.HashIntoIntTopped(handle, variants) enqueue = true @@ -617,12 +615,12 @@ func (c *Controller) enqueueObject(obj queue.PriorityQueueItem) { // updateWatch func (c *Controller) updateWatch(chi *api.ClickHouseInstallation) { - watched := metrics.NewWatchedCHI(chi) + watched := metrics.NewWatchedCR(chi) go c.updateWatchAsync(watched) } // updateWatchAsync -func (c *Controller) updateWatchAsync(chi *metrics.WatchedCHI) { +func (c *Controller) updateWatchAsync(chi *metrics.WatchedCR) { if err := clickhouse.InformMetricsExporterAboutWatchedCHI(chi); err != nil { log.V(1).F().Info("FAIL update watch (%s/%s): %q", chi.Namespace, chi.Name, err) } else { @@ -632,12 +630,12 @@ func (c *Controller) updateWatchAsync(chi *metrics.WatchedCHI) { // deleteWatch func (c *Controller) deleteWatch(chi *api.ClickHouseInstallation) { - watched := metrics.NewWatchedCHI(chi) + watched := metrics.NewWatchedCR(chi) go c.deleteWatchAsync(watched) } // deleteWatchAsync -func (c *Controller) deleteWatchAsync(chi *metrics.WatchedCHI) { +func (c *Controller) deleteWatchAsync(chi *metrics.WatchedCR) { if err := clickhouse.InformMetricsExporterToDeleteWatchedCHI(chi); err != nil { log.V(1).F().Info("FAIL delete watch (%s/%s): %q", chi.Namespace, chi.Name, err) } else { diff --git a/pkg/controller/chi/metrics/interface.go b/pkg/controller/chi/metrics/interface.go new file mode 100644 index 000000000..1fb2d5182 --- /dev/null +++ b/pkg/controller/chi/metrics/interface.go @@ -0,0 +1,22 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +type labelsSource interface { + GetName() string + GetNamespace() string + GetLabels() map[string]string + GetAnnotations() map[string]string +} diff --git a/pkg/controller/chi/metrics/labels.go b/pkg/controller/chi/metrics/labels.go new file mode 100644 index 000000000..aed84f963 --- /dev/null +++ b/pkg/controller/chi/metrics/labels.go @@ -0,0 +1,53 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package metrics + +import ( + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" + + "github.com/altinity/clickhouse-operator/pkg/chop" + "github.com/altinity/clickhouse-operator/pkg/metrics/operator" + "github.com/altinity/clickhouse-operator/pkg/util" +) + +func labels(src labelsSource) metric.MeasurementOption { + return metric.WithAttributes(prepareLabels(src)...) +} + +func prepareLabels(cr labelsSource) []attribute.KeyValue { + // Prepare base set of labels + labels := getBaseLabels(cr) + // Append particular metric labels + // not yet... + // Filter out metrics to be skipped + labels = util.CopyMapFilter( + labels, + nil, + chop.Config().Metrics.Labels.Exclude, + ) + return convert(labels) +} + +func getBaseLabels(cr labelsSource) map[string]string { + return operator.GetLabelsFromSource(cr) +} + +func convert(labels map[string]string) (attributes []attribute.KeyValue) { + for name, value := range labels { + attributes = append(attributes, attribute.String(name, value)) + } + return attributes +} diff --git a/pkg/controller/chi/metrics/metrics.go b/pkg/controller/chi/metrics/metrics.go index 671f3ce6c..c3e221128 100644 --- a/pkg/controller/chi/metrics/metrics.go +++ b/pkg/controller/chi/metrics/metrics.go @@ -17,7 +17,6 @@ package metrics import ( "context" - "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" "github.com/altinity/clickhouse-operator/pkg/metrics/operator" @@ -31,7 +30,7 @@ type Metrics struct { // In ideal world number of completed reconciles should be equal to CHIReconcilesStarted CHIReconcilesCompleted metric.Int64Counter // CHIReconcilesAborted is a number (counter) of explicitly aborted CHI reconciles. - // This counter does not includes reconciles that we not completed due to external rasons, such as operator restart + // This counter does not includes reconciles that we not completed due to external reasons, such as operator restart CHIReconcilesAborted metric.Int64Counter // CHIReconcilesTimings is a histogram of durations of successfully completed CHI reconciles CHIReconcilesTimings metric.Float64Histogram @@ -53,8 +52,6 @@ type Metrics struct { PodDeleteEvents metric.Int64Counter } -var m *Metrics - func createMetrics() *Metrics { // The unit u should be defined using the appropriate [UCUM](https://ucum.org) case-sensitive code. CHIReconcilesStarted, _ := operator.Meter().Int64Counter( @@ -138,6 +135,8 @@ func createMetrics() *Metrics { } } +var m *Metrics + func ensureMetrics() *Metrics { if m == nil { m = createMetrics() @@ -145,66 +144,48 @@ func ensureMetrics() *Metrics { return m } -type BaseInfoGetter interface { - GetName() string - GetNamespace() string - GetLabels() map[string]string - GetAnnotations() map[string]string -} - -func prepareLabels(cr BaseInfoGetter) (attributes []attribute.KeyValue) { - labels, values := operator.GetMandatoryLabelsAndValues(cr) - for i := range labels { - label := labels[i] - value := values[i] - attributes = append(attributes, attribute.String(label, value)) - } - - return attributes -} - -// metricsCHIInitZeroValues initializes all metrics for CHI to zero values if not already present with appropriate labels +// CHIInitZeroValues initializes all metrics for CHI to zero values if not already present with appropriate labels // // This is due to `rate` prometheus function limitation where it expects the metric to be 0-initialized with all possible labels // and doesn't default to 0 if the metric is not present. -func CHIInitZeroValues(ctx context.Context, chi BaseInfoGetter) { - ensureMetrics().CHIReconcilesStarted.Add(ctx, 0, metric.WithAttributes(prepareLabels(chi)...)) - ensureMetrics().CHIReconcilesCompleted.Add(ctx, 0, metric.WithAttributes(prepareLabels(chi)...)) - ensureMetrics().CHIReconcilesAborted.Add(ctx, 0, metric.WithAttributes(prepareLabels(chi)...)) +func CHIInitZeroValues(ctx context.Context, src labelsSource) { + ensureMetrics().CHIReconcilesStarted.Add(ctx, 0, labels(src)) + ensureMetrics().CHIReconcilesCompleted.Add(ctx, 0, labels(src)) + ensureMetrics().CHIReconcilesAborted.Add(ctx, 0, labels(src)) - ensureMetrics().HostReconcilesStarted.Add(ctx, 0, metric.WithAttributes(prepareLabels(chi)...)) - ensureMetrics().HostReconcilesCompleted.Add(ctx, 0, metric.WithAttributes(prepareLabels(chi)...)) - ensureMetrics().HostReconcilesRestarts.Add(ctx, 0, metric.WithAttributes(prepareLabels(chi)...)) - ensureMetrics().HostReconcilesErrors.Add(ctx, 0, metric.WithAttributes(prepareLabels(chi)...)) + ensureMetrics().HostReconcilesStarted.Add(ctx, 0, labels(src)) + ensureMetrics().HostReconcilesCompleted.Add(ctx, 0, labels(src)) + ensureMetrics().HostReconcilesRestarts.Add(ctx, 0, labels(src)) + ensureMetrics().HostReconcilesErrors.Add(ctx, 0, labels(src)) } -func CHIReconcilesStarted(ctx context.Context, chi BaseInfoGetter) { - ensureMetrics().CHIReconcilesStarted.Add(ctx, 1, metric.WithAttributes(prepareLabels(chi)...)) +func CHIReconcilesStarted(ctx context.Context, src labelsSource) { + ensureMetrics().CHIReconcilesStarted.Add(ctx, 1, labels(src)) } -func CHIReconcilesCompleted(ctx context.Context, chi BaseInfoGetter) { - ensureMetrics().CHIReconcilesCompleted.Add(ctx, 1, metric.WithAttributes(prepareLabels(chi)...)) +func CHIReconcilesCompleted(ctx context.Context, src labelsSource) { + ensureMetrics().CHIReconcilesCompleted.Add(ctx, 1, labels(src)) } -func CHIReconcilesAborted(ctx context.Context, chi BaseInfoGetter) { - ensureMetrics().CHIReconcilesAborted.Add(ctx, 1, metric.WithAttributes(prepareLabels(chi)...)) +func CHIReconcilesAborted(ctx context.Context, src labelsSource) { + ensureMetrics().CHIReconcilesAborted.Add(ctx, 1, labels(src)) } -func CHIReconcilesTimings(ctx context.Context, chi BaseInfoGetter, seconds float64) { - ensureMetrics().CHIReconcilesTimings.Record(ctx, seconds, metric.WithAttributes(prepareLabels(chi)...)) +func CHIReconcilesTimings(ctx context.Context, src labelsSource, seconds float64) { + ensureMetrics().CHIReconcilesTimings.Record(ctx, seconds, labels(src)) } -func HostReconcilesStarted(ctx context.Context, chi BaseInfoGetter) { - ensureMetrics().HostReconcilesStarted.Add(ctx, 1, metric.WithAttributes(prepareLabels(chi)...)) +func HostReconcilesStarted(ctx context.Context, src labelsSource) { + ensureMetrics().HostReconcilesStarted.Add(ctx, 1, labels(src)) } -func HostReconcilesCompleted(ctx context.Context, chi BaseInfoGetter) { - ensureMetrics().HostReconcilesCompleted.Add(ctx, 1, metric.WithAttributes(prepareLabels(chi)...)) +func HostReconcilesCompleted(ctx context.Context, src labelsSource) { + ensureMetrics().HostReconcilesCompleted.Add(ctx, 1, labels(src)) } -func HostReconcilesRestart(ctx context.Context, chi BaseInfoGetter) { - ensureMetrics().HostReconcilesRestarts.Add(ctx, 1, metric.WithAttributes(prepareLabels(chi)...)) +func HostReconcilesRestart(ctx context.Context, src labelsSource) { + ensureMetrics().HostReconcilesRestarts.Add(ctx, 1, labels(src)) } -func HostReconcilesErrors(ctx context.Context, chi BaseInfoGetter) { - ensureMetrics().HostReconcilesErrors.Add(ctx, 1, metric.WithAttributes(prepareLabels(chi)...)) +func HostReconcilesErrors(ctx context.Context, src labelsSource) { + ensureMetrics().HostReconcilesErrors.Add(ctx, 1, labels(src)) } -func HostReconcilesTimings(ctx context.Context, chi BaseInfoGetter, seconds float64) { - ensureMetrics().HostReconcilesTimings.Record(ctx, seconds, metric.WithAttributes(prepareLabels(chi)...)) +func HostReconcilesTimings(ctx context.Context, src labelsSource, seconds float64) { + ensureMetrics().HostReconcilesTimings.Record(ctx, seconds, labels(src)) } func PodAdd(ctx context.Context) { diff --git a/pkg/controller/chi/worker-boilerplate.go b/pkg/controller/chi/worker-boilerplate.go index 929a46479..44ff11f57 100644 --- a/pkg/controller/chi/worker-boilerplate.go +++ b/pkg/controller/chi/worker-boilerplate.go @@ -23,7 +23,6 @@ import ( log "github.com/altinity/clickhouse-operator/pkg/announcer" "github.com/altinity/clickhouse-operator/pkg/controller/chi/cmd_queue" "github.com/altinity/clickhouse-operator/pkg/controller/chi/metrics" - normalizerCommon "github.com/altinity/clickhouse-operator/pkg/model/common/normalizer" "github.com/altinity/clickhouse-operator/pkg/util" ) @@ -150,16 +149,6 @@ func (w *worker) processReconcilePod(ctx context.Context, cmd *cmd_queue.Reconci return nil } -func (w *worker) processDropDns(ctx context.Context, cmd *cmd_queue.DropDns) error { - if chi, err := w.createCRFromObjectMeta(cmd.Initiator, false, normalizerCommon.NewOptions()); err == nil { - w.a.V(2).M(cmd.Initiator).Info("flushing DNS for CHI %s", chi.Name) - _ = w.ensureClusterSchemer(chi.FirstHost()).CHIDropDnsCache(ctx, chi) - } else { - w.a.M(cmd.Initiator).F().Error("unable to find CHI by %v err: %v", cmd.Initiator.GetLabels(), err) - } - return nil -} - // processItem processes one work item according to its type func (w *worker) processItem(ctx context.Context, item interface{}) error { if util.IsContextDone(ctx) { @@ -181,8 +170,6 @@ func (w *worker) processItem(ctx context.Context, item interface{}) error { return w.processReconcileEndpoints(ctx, cmd) case *cmd_queue.ReconcilePod: return w.processReconcilePod(ctx, cmd) - case *cmd_queue.DropDns: - return w.processDropDns(ctx, cmd) } // Unknown item type, don't know what to do with it diff --git a/pkg/controller/chi/worker-chi-reconciler.go b/pkg/controller/chi/worker-chi-reconciler.go index bb9850518..fea240869 100644 --- a/pkg/controller/chi/worker-chi-reconciler.go +++ b/pkg/controller/chi/worker-chi-reconciler.go @@ -27,6 +27,7 @@ import ( "github.com/altinity/clickhouse-operator/pkg/chop" "github.com/altinity/clickhouse-operator/pkg/controller/chi/metrics" "github.com/altinity/clickhouse-operator/pkg/controller/common" + a "github.com/altinity/clickhouse-operator/pkg/controller/common/announcer" "github.com/altinity/clickhouse-operator/pkg/controller/common/statefulset" "github.com/altinity/clickhouse-operator/pkg/controller/common/storage" "github.com/altinity/clickhouse-operator/pkg/interfaces" @@ -96,15 +97,15 @@ func (w *worker) reconcileCR(ctx context.Context, old, new *api.ClickHouseInstal return nil } - w.newTask(new) + w.newTask(new, old) w.markReconcileStart(ctx, new, actionPlan) w.excludeStoppedCHIFromMonitoring(new) w.walkHosts(ctx, new, actionPlan) if err := w.reconcile(ctx, new); err != nil { // Something went wrong - w.a.WithEvent(new, common.EventActionReconcile, common.EventReasonReconcileFailed). - WithStatusError(new). + w.a.WithEvent(new, a.EventActionReconcile, a.EventReasonReconcileFailed). + WithError(new). M(new).F(). Error("FAILED to reconcile CR %s, err: %v", util.NamespaceNameString(new), err) w.markReconcileCompletedUnsuccessfully(ctx, new, err) @@ -148,10 +149,12 @@ func (w *worker) reconcile(ctx context.Context, cr *api.ClickHouseInstallation) }) if counters.AddOnly() { - w.a.V(1).M(cr).Info("Enabling full fan-out mode. CHI: %s", util.NamespaceNameString(cr)) + w.a.V(1).M(cr).Info("Enabling full fan-out mode. CR: %s", util.NamespaceNameString(cr)) ctx = context.WithValue(ctx, common.ReconcileShardsAndHostsOptionsCtxKey, &common.ReconcileShardsAndHostsOptions{ FullFanOut: true, }) + } else { + w.a.V(1).M(cr).Info("Unable to use full fan-out mode. Counters: %s. CR: %s", counters, util.NamespaceNameString(cr)) } return cr.WalkTillError( @@ -206,7 +209,8 @@ func (w *worker) reconcileCRServiceFinal(ctx context.Context, cr api.ICustomReso // Create entry point for the whole CHI if service := w.task.Creator().CreateService(interfaces.ServiceCR); service != nil { - if err := w.reconcileService(ctx, cr, service); err != nil { + prevService := w.task.CreatorPrev().CreateService(interfaces.ServiceCR) + if err := w.reconcileService(ctx, cr, service, prevService); err != nil { // Service not reconciled w.task.RegistryFailed().RegisterService(service.GetObjectMeta()) return err @@ -229,8 +233,17 @@ func (w *worker) reconcileCRAuxObjectsFinal(ctx context.Context, cr *api.ClickHo // CR ConfigMaps with update cr.GetRuntime().LockCommonConfig() - err = w.reconcileConfigMapCommon(ctx, cr, nil) + err = w.reconcileConfigMapCommon(ctx, cr) cr.GetRuntime().UnlockCommonConfig() + + // Wait for all hosts to be included into cluster + cr.WalkHosts(func(host *api.Host) error { + if host.ShouldIncludeIntoCluster() { + _ = w.waitHostInCluster(ctx, host) + } + return nil + }) + return err } @@ -238,17 +251,22 @@ func (w *worker) reconcileCRAuxObjectsFinal(ctx context.Context, cr *api.ClickHo func (w *worker) reconcileConfigMapCommon( ctx context.Context, cr api.ICustomResource, - options *config.FilesGeneratorOptions, + options ...*config.FilesGeneratorOptions, ) error { if util.IsContextDone(ctx) { log.V(2).Info("task is done") return nil } - // ConfigMap common for all resources in CHI + var opts *config.FilesGeneratorOptions + if len(options) > 0 { + opts = options[0] + } + + // ConfigMap common for all resources in CR // contains several sections, mapped as separated chopConfig files, // such as remote servers, zookeeper setup, etc - configMapCommon := w.task.Creator().CreateConfigMap(interfaces.ConfigMapCommon, options) + configMapCommon := w.task.Creator().CreateConfigMap(interfaces.ConfigMapCommon, opts) err := w.reconcileConfigMap(ctx, cr, configMapCommon) if err == nil { w.task.RegistryReconciled().RegisterConfigMap(configMapCommon.GetObjectMeta()) @@ -336,9 +354,9 @@ func (w *worker) reconcileHostStatefulSet(ctx context.Context, host *api.Host, o } host.GetCR().IEnsureStatus().HostFailed() - w.a.WithEvent(host.GetCR(), common.EventActionReconcile, common.EventReasonReconcileFailed). - WithStatusAction(host.GetCR()). - WithStatusError(host.GetCR()). + w.a.WithEvent(host.GetCR(), a.EventActionReconcile, a.EventReasonReconcileFailed). + WithAction(host.GetCR()). + WithError(host.GetCR()). M(host).F(). Error("FAILED to reconcile StatefulSet for host: %s", host.GetName()) } @@ -369,7 +387,8 @@ func (w *worker) reconcileHostService(ctx context.Context, host *api.Host) error // This is not a problem, service may be omitted return nil } - err := w.reconcileService(ctx, host.GetCR(), service) + prevService := w.task.CreatorPrev().CreateService(interfaces.ServiceHost, host.GetAncestor()) + err := w.reconcileService(ctx, host.GetCR(), service, prevService) if err == nil { w.a.V(1).M(host).F().Info("DONE Reconcile service of the host: %s", host.GetName()) w.task.RegistryReconciled().RegisterService(service.GetObjectMeta()) @@ -392,7 +411,8 @@ func (w *worker) reconcileCluster(ctx context.Context, cluster *api.Cluster) err // Add Cluster Service if service := w.task.Creator().CreateService(interfaces.ServiceCluster, cluster); service != nil { - if err := w.reconcileService(ctx, cluster.GetRuntime().GetCR(), service); err == nil { + prevService := w.task.CreatorPrev().CreateService(interfaces.ServiceCluster, cluster.GetAncestor()) + if err := w.reconcileService(ctx, cluster.GetRuntime().GetCR(), service, prevService); err == nil { w.task.RegistryReconciled().RegisterService(service.GetObjectMeta()) } else { w.task.RegistryFailed().RegisterService(service.GetObjectMeta()) @@ -479,20 +499,77 @@ func (w *worker) reconcileShardsAndHosts(ctx context.Context, shards []*api.ChiS return err } - // Since shard with 0 index is already done, we'll proceed with the 1-st + // Since shard with 0 index is already done, we'll proceed concurrently starting with the 1-st startShard = 1 } // Process shards using specified concurrency level while maintaining specified max concurrency percentage. // Loop over shards. workersNum := w.getReconcileShardsWorkersNum(shards, opts) - w.a.V(1).Info("Starting rest of shards on workers: %d", workersNum) - for startShardIndex := startShard; startShardIndex < len(shards); startShardIndex += workersNum { - endShardIndex := startShardIndex + workersNum - if endShardIndex > len(shards) { - endShardIndex = len(shards) + w.a.V(1).Info("Starting rest of shards on workers. Workers num: %d", workersNum) + if err := w.runConcurrently(ctx, workersNum, startShard, shards[startShard:]); err != nil { + w.a.V(1).Info("Finished with ERROR rest of shards on workers: %d, err: %v", workersNum, err) + return err + } + w.a.V(1).Info("Finished successfully rest of shards on workers: %d", workersNum) + return nil +} + +func (w *worker) runConcurrently(ctx context.Context, workersNum int, startShardIndex int, shards []*api.ChiShard) error { + if len(shards) == 0 { + return nil + } + + type shardReconcile struct { + shard *api.ChiShard + index int + } + + ch := make(chan *shardReconcile) + wg := sync.WaitGroup{} + + // Launch tasks feeder + wg.Add(1) + go func() { + defer wg.Done() + defer close(ch) + for i, shard := range shards { + ch <- &shardReconcile{ + shard, + startShardIndex + i, + } } + }() + + // Launch workers + var err error + var errLock sync.Mutex + for i := 0; i < workersNum; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for rq := range ch { + w.a.V(1).Info("Starting shard index: %d on worker", rq.index) + if e := w.reconcileShardWithHosts(ctx, rq.shard); e != nil { + errLock.Lock() + err = e + errLock.Unlock() + } + } + }() + } + + w.a.V(1).Info("Starting to wait shards from index: %d on workers.", startShardIndex) + wg.Wait() + w.a.V(1).Info("Finished to wait shards from index: %d on workers.", startShardIndex) + return err +} + +func (w *worker) runConcurrentlyInBatches(ctx context.Context, workersNum int, start int, shards []*api.ChiShard) error { + for startShardIndex := 0; startShardIndex < len(shards); startShardIndex += workersNum { + endShardIndex := util.IncTopped(startShardIndex, workersNum, len(shards)) concurrentlyProcessedShards := shards[startShardIndex:endShardIndex] + w.a.V(1).Info("Starting shards from index: %d on workers. Shards indexes [%d:%d)", start+startShardIndex, start+startShardIndex, start+endShardIndex) // Processing error protected with mutex var err error @@ -503,17 +580,21 @@ func (w *worker) reconcileShardsAndHosts(ctx context.Context, shards []*api.ChiS // Launch shard concurrent processing for j := range concurrentlyProcessedShards { shard := concurrentlyProcessedShards[j] + w.a.V(1).Info("Starting shard on worker. Shard index: %d", start+startShardIndex+j) go func() { defer wg.Done() + w.a.V(1).Info("Starting shard on goroutine. Shard index: %d", start+startShardIndex+j) if e := w.reconcileShardWithHosts(ctx, shard); e != nil { errLock.Lock() err = e errLock.Unlock() - return } + w.a.V(1).Info("Finished shard on goroutine. Shard index: %d", start+startShardIndex+j) }() } + w.a.V(1).Info("Starting to wait shards from index: %d on workers. Shards indexes [%d:%d)", start+startShardIndex, start+startShardIndex, start+endShardIndex) wg.Wait() + w.a.V(1).Info("Finished to wait shards from index: %d on workers. Shards indexes [%d:%d)", start+startShardIndex, start+startShardIndex, start+endShardIndex) if err != nil { w.a.V(1).Warning("Skipping rest of shards due to an error: %v", err) return err @@ -553,7 +634,8 @@ func (w *worker) reconcileShardService(ctx context.Context, shard api.IShard) er // This is not a problem, ServiceShard may be omitted return nil } - err := w.reconcileService(ctx, shard.GetRuntime().GetCR(), service) + prevService := w.task.CreatorPrev().CreateService(interfaces.ServiceShard, shard.GetAncestor()) + err := w.reconcileService(ctx, shard.GetRuntime().GetCR(), service, prevService) if err == nil { w.task.RegistryReconciled().RegisterService(service.GetObjectMeta()) } else { @@ -572,6 +654,17 @@ func (w *worker) reconcileHost(ctx context.Context, host *api.Host) error { w.a.V(2).M(host).S().P() defer w.a.V(2).M(host).E().P() + //si := host.GetRuntime().GetAddress().GetShardIndex() + //ri := host.GetRuntime().GetAddress().GetReplicaIndex() + ////sleep := util.DecBottomed(si, 1, 0)*(si % 3)*20 + //sleep := (2 - si)*90 + //if ri > 0 { + // sleep = 0 + //} + //w.a.V(1).Info("Host [%d/%d]. Going to sleep %d sec", si, ri, sleep) + //time.Sleep((time.Duration)(sleep)*time.Second) + //w.a.V(1).Info("Host [%d/%d]. Done to sleep %d sec", si, ri) + metrics.HostReconcilesStarted(ctx, host.GetCR()) startTime := time.Now() @@ -604,14 +697,16 @@ func (w *worker) reconcileHost(ctx context.Context, host *api.Host) error { hostsCount = host.GetCR().GetStatus().GetHostsCount() } w.a.V(1). - WithEvent(host.GetCR(), common.EventActionProgress, common.EventReasonProgressHostsCompleted). - WithStatusAction(host.GetCR()). + WithEvent(host.GetCR(), a.EventActionProgress, a.EventReasonProgressHostsCompleted). + WithAction(host.GetCR()). M(host).F(). - Info("[now: %s] %s: %d of %d", now, common.EventReasonProgressHostsCompleted, hostsCompleted, hostsCount) + Info("[now: %s] %s: %d of %d", now, a.EventReasonProgressHostsCompleted, hostsCompleted, hostsCount) _ = w.c.updateCRObjectStatus(ctx, host.GetCR(), types.UpdateStatusOptions{ CopyStatusOptions: types.CopyStatusOptions{ - MainFields: true, + CopyStatusFieldGroup: types.CopyStatusFieldGroup{ + FieldGroupMain: true, + }, }, }) @@ -626,14 +721,14 @@ func (w *worker) reconcileHostPrepare(ctx context.Context, host *api.Host) error // Check whether ClickHouse is running and accessible and what version is available if version, err := w.getHostClickHouseVersion(ctx, host, versionOptions{skipNew: true, skipStoppedAncestor: true}); err == nil { w.a.V(1). - WithEvent(host.GetCR(), common.EventActionReconcile, common.EventReasonReconcileStarted). - WithStatusAction(host.GetCR()). + WithEvent(host.GetCR(), a.EventActionReconcile, a.EventReasonReconcileStarted). + WithAction(host.GetCR()). M(host).F(). Info("Reconcile Host start. Host: %s ClickHouse version running: %s", host.GetName(), version) } else { w.a.V(1). - WithEvent(host.GetCR(), common.EventActionReconcile, common.EventReasonReconcileStarted). - WithStatusAction(host.GetCR()). + WithEvent(host.GetCR(), a.EventActionReconcile, a.EventReasonReconcileStarted). + WithAction(host.GetCR()). M(host).F(). Warning("Reconcile Host start. Host: %s Failed to get ClickHouse version: %s", host.GetName(), version) } @@ -735,14 +830,14 @@ func (w *worker) reconcileHostBootstrap(ctx context.Context, host *api.Host) err // Sometimes service needs some time to start after creation|modification before being accessible for usage if version, err := w.pollHostForClickHouseVersion(ctx, host); err == nil { w.a.V(1). - WithEvent(host.GetCR(), common.EventActionReconcile, common.EventReasonReconcileCompleted). - WithStatusAction(host.GetCR()). + WithEvent(host.GetCR(), a.EventActionReconcile, a.EventReasonReconcileCompleted). + WithAction(host.GetCR()). M(host).F(). Info("Reconcile Host completed. Host: %s ClickHouse version running: %s", host.GetName(), version) } else { w.a.V(1). - WithEvent(host.GetCR(), common.EventActionReconcile, common.EventReasonReconcileCompleted). - WithStatusAction(host.GetCR()). + WithEvent(host.GetCR(), a.EventActionReconcile, a.EventReasonReconcileCompleted). + WithAction(host.GetCR()). M(host).F(). Warning("Reconcile Host completed. Host: %s Failed to get ClickHouse version: %s", host.GetName(), version) } diff --git a/pkg/controller/chi/worker-config-map.go b/pkg/controller/chi/worker-config-map.go index 1817a7ec2..d91b134b1 100644 --- a/pkg/controller/chi/worker-config-map.go +++ b/pkg/controller/chi/worker-config-map.go @@ -23,7 +23,7 @@ import ( log "github.com/altinity/clickhouse-operator/pkg/announcer" api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" - "github.com/altinity/clickhouse-operator/pkg/controller/common" + a "github.com/altinity/clickhouse-operator/pkg/controller/common/announcer" "github.com/altinity/clickhouse-operator/pkg/util" ) @@ -55,9 +55,9 @@ func (w *worker) reconcileConfigMap( } if err != nil { - w.a.WithEvent(cr, common.EventActionReconcile, common.EventReasonReconcileFailed). - WithStatusAction(cr). - WithStatusError(cr). + w.a.WithEvent(cr, a.EventActionReconcile, a.EventReasonReconcileFailed). + WithAction(cr). + WithError(cr). M(cr).F(). Error("FAILED to reconcile ConfigMap: %s CHI: %s ", configMap.GetName(), cr.GetName()) } @@ -75,17 +75,17 @@ func (w *worker) updateConfigMap(ctx context.Context, cr api.ICustomResource, co updatedConfigMap, err := w.c.updateConfigMap(ctx, configMap) if err == nil { w.a.V(1). - WithEvent(cr, common.EventActionUpdate, common.EventReasonUpdateCompleted). - WithStatusAction(cr). + WithEvent(cr, a.EventActionUpdate, a.EventReasonUpdateCompleted). + WithAction(cr). M(cr).F(). Info("Update ConfigMap %s/%s", configMap.Namespace, configMap.Name) if updatedConfigMap.ResourceVersion != configMap.ResourceVersion { w.task.SetCmUpdate(time.Now()) } } else { - w.a.WithEvent(cr, common.EventActionUpdate, common.EventReasonUpdateFailed). - WithStatusAction(cr). - WithStatusError(cr). + w.a.WithEvent(cr, a.EventActionUpdate, a.EventReasonUpdateFailed). + WithAction(cr). + WithError(cr). M(cr).F(). Error("Update ConfigMap %s/%s failed with error %v", configMap.Namespace, configMap.Name, err) } @@ -103,14 +103,14 @@ func (w *worker) createConfigMap(ctx context.Context, cr api.ICustomResource, co err := w.c.createConfigMap(ctx, configMap) if err == nil { w.a.V(1). - WithEvent(cr, common.EventActionCreate, common.EventReasonCreateCompleted). - WithStatusAction(cr). + WithEvent(cr, a.EventActionCreate, a.EventReasonCreateCompleted). + WithAction(cr). M(cr).F(). Info("Create ConfigMap %s", util.NamespaceNameString(configMap)) } else { - w.a.WithEvent(cr, common.EventActionCreate, common.EventReasonCreateFailed). - WithStatusAction(cr). - WithStatusError(cr). + w.a.WithEvent(cr, a.EventActionCreate, a.EventReasonCreateFailed). + WithAction(cr). + WithError(cr). M(cr).F(). Error("Create ConfigMap %s failed with error %v", util.NamespaceNameString(configMap), err) } diff --git a/pkg/controller/chi/worker-deleter.go b/pkg/controller/chi/worker-deleter.go index 003c3104e..4555a9b29 100644 --- a/pkg/controller/chi/worker-deleter.go +++ b/pkg/controller/chi/worker-deleter.go @@ -25,8 +25,7 @@ import ( api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" "github.com/altinity/clickhouse-operator/pkg/apis/common/types" "github.com/altinity/clickhouse-operator/pkg/controller" - "github.com/altinity/clickhouse-operator/pkg/controller/chi/cmd_queue" - "github.com/altinity/clickhouse-operator/pkg/controller/common" + a "github.com/altinity/clickhouse-operator/pkg/controller/common/announcer" "github.com/altinity/clickhouse-operator/pkg/controller/common/storage" "github.com/altinity/clickhouse-operator/pkg/model" chiLabeler "github.com/altinity/clickhouse-operator/pkg/model/chi/tags/labeler" @@ -42,8 +41,8 @@ func (w *worker) clean(ctx context.Context, cr api.ICustomResource) { } w.a.V(1). - WithEvent(cr, common.EventActionReconcile, common.EventReasonReconcileInProgress). - WithStatusAction(cr). + WithEvent(cr, a.EventActionReconcile, a.EventReasonReconcileInProgress). + WithAction(cr). M(cr).F(). Info("remove items scheduled for deletion") @@ -56,7 +55,6 @@ func (w *worker) clean(ctx context.Context, cr api.ICustomResource) { objs.Subtract(need) w.a.V(1).M(cr).F().Info("Non-reconciled objects:\n%s", objs) if w.purge(ctx, cr, objs, w.task.RegistryFailed()) > 0 { - w.c.enqueueObject(cmd_queue.NewDropDns(cr)) util.WaitContextDoneOrTimeout(ctx, 1*time.Minute) } @@ -274,8 +272,8 @@ func (w *worker) deleteCHIProtocol(ctx context.Context, chi *api.ClickHouseInsta var err error chi, err = w.normalizer.CreateTemplated(chi, normalizer.NewOptions()) if err != nil { - w.a.WithEvent(chi, common.EventActionDelete, common.EventReasonDeleteFailed). - WithStatusError(chi). + w.a.WithEvent(chi, a.EventActionDelete, a.EventReasonDeleteFailed). + WithError(chi). M(chi).F(). Error("Delete CHI failed - unable to normalize: %q", err) return err @@ -283,8 +281,8 @@ func (w *worker) deleteCHIProtocol(ctx context.Context, chi *api.ClickHouseInsta // Announce delete procedure w.a.V(1). - WithEvent(chi, common.EventActionDelete, common.EventReasonDeleteStarted). - WithStatusAction(chi). + WithEvent(chi, a.EventActionDelete, a.EventReasonDeleteStarted). + WithAction(chi). M(chi).F(). Info("Delete CHI started") @@ -292,7 +290,9 @@ func (w *worker) deleteCHIProtocol(ctx context.Context, chi *api.ClickHouseInsta if err := w.c.updateCRObjectStatus(ctx, chi, types.UpdateStatusOptions{ TolerateAbsence: true, CopyStatusOptions: types.CopyStatusOptions{ - MainFields: true, + CopyStatusFieldGroup: types.CopyStatusFieldGroup{ + FieldGroupMain: true, + }, }, }); err != nil { w.a.V(1).M(chi).F().Error("UNABLE to write normalized CHI. err: %q", err) @@ -326,8 +326,8 @@ func (w *worker) deleteCHIProtocol(ctx context.Context, chi *api.ClickHouseInsta _ = w.c.deleteConfigMapsCHI(ctx, chi) w.a.V(1). - WithEvent(chi, common.EventActionDelete, common.EventReasonDeleteCompleted). - WithStatusAction(chi). + WithEvent(chi, a.EventActionDelete, a.EventReasonDeleteCompleted). + WithAction(chi). M(chi).F(). Info("Delete CHI completed") @@ -413,13 +413,13 @@ func (w *worker) dropReplica(ctx context.Context, hostToDrop *api.Host, opts ... if err == nil { w.a.V(1). - WithEvent(hostToRunOn.GetCR(), common.EventActionDelete, common.EventReasonDeleteCompleted). - WithStatusAction(hostToRunOn.GetCR()). + WithEvent(hostToRunOn.GetCR(), a.EventActionDelete, a.EventReasonDeleteCompleted). + WithAction(hostToRunOn.GetCR()). M(hostToRunOn).F(). Info("Drop replica host: %s in cluster: %s", hostToDrop.GetName(), hostToDrop.Runtime.Address.ClusterName) } else { - w.a.WithEvent(hostToRunOn.GetCR(), common.EventActionDelete, common.EventReasonDeleteFailed). - WithStatusError(hostToRunOn.GetCR()). + w.a.WithEvent(hostToRunOn.GetCR(), a.EventActionDelete, a.EventReasonDeleteFailed). + WithError(hostToRunOn.GetCR()). M(hostToRunOn).F(). Error("FAILED to drop replica on host: %s with error: %v", hostToDrop.GetName(), err) } @@ -441,14 +441,14 @@ func (w *worker) deleteTables(ctx context.Context, host *api.Host) error { if err == nil { w.a.V(1). - WithEvent(host.GetCR(), common.EventActionDelete, common.EventReasonDeleteCompleted). - WithStatusAction(host.GetCR()). + WithEvent(host.GetCR(), a.EventActionDelete, a.EventReasonDeleteCompleted). + WithAction(host.GetCR()). M(host).F(). Info("Deleted tables on host: %s replica: %d to shard: %d in cluster: %s", host.GetName(), host.Runtime.Address.ReplicaIndex, host.Runtime.Address.ShardIndex, host.Runtime.Address.ClusterName) } else { - w.a.WithEvent(host.GetCR(), common.EventActionDelete, common.EventReasonDeleteFailed). - WithStatusError(host.GetCR()). + w.a.WithEvent(host.GetCR(), a.EventActionDelete, a.EventReasonDeleteFailed). + WithError(host.GetCR()). M(host).F(). Error("FAILED to delete tables on host: %s with error: %v", host.GetName(), err) } @@ -468,15 +468,15 @@ func (w *worker) deleteHost(ctx context.Context, chi *api.ClickHouseInstallation defer w.a.V(2).M(host).E().Info(host.Runtime.Address.HostName) w.a.V(1). - WithEvent(host.GetCR(), common.EventActionDelete, common.EventReasonDeleteStarted). - WithStatusAction(host.GetCR()). + WithEvent(host.GetCR(), a.EventActionDelete, a.EventReasonDeleteStarted). + WithAction(host.GetCR()). M(host).F(). Info("Delete host: %s/%s - started", host.Runtime.Address.ClusterName, host.GetName()) var err error if host.Runtime.CurStatefulSet, err = w.c.kube.STS().Get(ctx, host); err != nil { - w.a.WithEvent(host.GetCR(), common.EventActionDelete, common.EventReasonDeleteCompleted). - WithStatusAction(host.GetCR()). + w.a.WithEvent(host.GetCR(), a.EventActionDelete, a.EventReasonDeleteCompleted). + WithAction(host.GetCR()). M(host).F(). Info("Delete host: %s/%s - completed StatefulSet not found - already deleted? err: %v", host.Runtime.Address.ClusterName, host.GetName(), err) @@ -498,19 +498,21 @@ func (w *worker) deleteHost(ctx context.Context, chi *api.ClickHouseInstallation _ = w.c.updateCRObjectStatus(ctx, chi, types.UpdateStatusOptions{ TolerateAbsence: true, CopyStatusOptions: types.CopyStatusOptions{ - MainFields: true, + CopyStatusFieldGroup: types.CopyStatusFieldGroup{ + FieldGroupMain: true, + }, }, }) if err == nil { w.a.V(1). - WithEvent(host.GetCR(), common.EventActionDelete, common.EventReasonDeleteCompleted). - WithStatusAction(host.GetCR()). + WithEvent(host.GetCR(), a.EventActionDelete, a.EventReasonDeleteCompleted). + WithAction(host.GetCR()). M(host).F(). Info("Delete host: %s/%s - completed", host.Runtime.Address.ClusterName, host.GetName()) } else { - w.a.WithEvent(host.GetCR(), common.EventActionDelete, common.EventReasonDeleteFailed). - WithStatusError(host.GetCR()). + w.a.WithEvent(host.GetCR(), a.EventActionDelete, a.EventReasonDeleteFailed). + WithError(host.GetCR()). M(host).F(). Error("FAILED Delete host: %s/%s - completed", host.Runtime.Address.ClusterName, host.GetName()) } @@ -530,8 +532,8 @@ func (w *worker) deleteShard(ctx context.Context, chi *api.ClickHouseInstallatio defer w.a.V(2).M(shard).E().P() w.a.V(1). - WithEvent(shard.Runtime.CHI, common.EventActionDelete, common.EventReasonDeleteStarted). - WithStatusAction(shard.Runtime.CHI). + WithEvent(shard.Runtime.CHI, a.EventActionDelete, a.EventReasonDeleteStarted). + WithAction(shard.Runtime.CHI). M(shard).F(). Info("Delete shard: %s/%s - started", shard.Runtime.Address.Namespace, shard.Name) @@ -544,8 +546,8 @@ func (w *worker) deleteShard(ctx context.Context, chi *api.ClickHouseInstallatio }) w.a.V(1). - WithEvent(shard.Runtime.CHI, common.EventActionDelete, common.EventReasonDeleteCompleted). - WithStatusAction(shard.Runtime.CHI). + WithEvent(shard.Runtime.CHI, a.EventActionDelete, a.EventReasonDeleteCompleted). + WithAction(shard.Runtime.CHI). M(shard).F(). Info("Delete shard: %s/%s - completed", shard.Runtime.Address.Namespace, shard.Name) @@ -564,8 +566,8 @@ func (w *worker) deleteCluster(ctx context.Context, chi *api.ClickHouseInstallat defer w.a.V(2).M(cluster).E().P() w.a.V(1). - WithEvent(cluster.Runtime.CHI, common.EventActionDelete, common.EventReasonDeleteStarted). - WithStatusAction(cluster.Runtime.CHI). + WithEvent(cluster.Runtime.CHI, a.EventActionDelete, a.EventReasonDeleteStarted). + WithAction(cluster.Runtime.CHI). M(cluster).F(). Info("Delete cluster: %s/%s - started", cluster.Runtime.Address.Namespace, cluster.Name) @@ -584,8 +586,8 @@ func (w *worker) deleteCluster(ctx context.Context, chi *api.ClickHouseInstallat }) w.a.V(1). - WithEvent(cluster.Runtime.CHI, common.EventActionDelete, common.EventReasonDeleteCompleted). - WithStatusAction(cluster.Runtime.CHI). + WithEvent(cluster.Runtime.CHI, a.EventActionDelete, a.EventReasonDeleteCompleted). + WithAction(cluster.Runtime.CHI). M(cluster).F(). Info("Delete cluster: %s/%s - completed", cluster.Runtime.Address.Namespace, cluster.Name) diff --git a/pkg/controller/chi/worker-migrator.go b/pkg/controller/chi/worker-migrator.go index f563b324d..4d3a5c014 100644 --- a/pkg/controller/chi/worker-migrator.go +++ b/pkg/controller/chi/worker-migrator.go @@ -20,7 +20,7 @@ import ( log "github.com/altinity/clickhouse-operator/pkg/announcer" api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" "github.com/altinity/clickhouse-operator/pkg/chop" - "github.com/altinity/clickhouse-operator/pkg/controller/common" + a "github.com/altinity/clickhouse-operator/pkg/controller/common/announcer" "github.com/altinity/clickhouse-operator/pkg/interfaces" "github.com/altinity/clickhouse-operator/pkg/model/chi/schemer" "github.com/altinity/clickhouse-operator/pkg/model/clickhouse" @@ -89,8 +89,8 @@ func (w *worker) migrateTables(ctx context.Context, host *api.Host, opts ...*mig } w.a.V(1). - WithEvent(host.GetCR(), common.EventActionCreate, common.EventReasonCreateStarted). - WithStatusAction(host.GetCR()). + WithEvent(host.GetCR(), a.EventActionCreate, a.EventReasonCreateStarted). + WithAction(host.GetCR()). M(host).F(). Info( "Adding tables on shard/host:%d/%d cluster:%s", @@ -99,16 +99,16 @@ func (w *worker) migrateTables(ctx context.Context, host *api.Host, opts ...*mig err := w.ensureClusterSchemer(host).HostCreateTables(ctx, host) if err == nil { w.a.V(1). - WithEvent(host.GetCR(), common.EventActionCreate, common.EventReasonCreateCompleted). - WithStatusAction(host.GetCR()). + WithEvent(host.GetCR(), a.EventActionCreate, a.EventReasonCreateCompleted). + WithAction(host.GetCR()). M(host).F(). Info("Tables added successfully on shard/host:%d/%d cluster:%s", host.Runtime.Address.ShardIndex, host.Runtime.Address.ReplicaIndex, host.Runtime.Address.ClusterName) host.GetCR().IEnsureStatus().PushHostTablesCreated(w.c.namer.Name(interfaces.NameFQDN, host)) } else { w.a.V(1). - WithEvent(host.GetCR(), common.EventActionCreate, common.EventReasonCreateFailed). - WithStatusAction(host.GetCR()). + WithEvent(host.GetCR(), a.EventActionCreate, a.EventReasonCreateFailed). + WithAction(host.GetCR()). M(host).F(). Error("ERROR add tables added successfully on shard/host:%d/%d cluster:%s err:%v", host.Runtime.Address.ShardIndex, host.Runtime.Address.ReplicaIndex, host.Runtime.Address.ClusterName, err) diff --git a/pkg/controller/chi/worker-secret.go b/pkg/controller/chi/worker-secret.go index 0ba50de27..f38f1aaa9 100644 --- a/pkg/controller/chi/worker-secret.go +++ b/pkg/controller/chi/worker-secret.go @@ -21,7 +21,7 @@ import ( log "github.com/altinity/clickhouse-operator/pkg/announcer" api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" - "github.com/altinity/clickhouse-operator/pkg/controller/common" + a "github.com/altinity/clickhouse-operator/pkg/controller/common/announcer" "github.com/altinity/clickhouse-operator/pkg/util" ) @@ -45,9 +45,9 @@ func (w *worker) reconcileSecret(ctx context.Context, cr api.ICustomResource, se _ = w.c.deleteSecretIfExists(ctx, secret.Namespace, secret.Name) err := w.createSecret(ctx, cr, secret) if err != nil { - w.a.WithEvent(cr, common.EventActionReconcile, common.EventReasonReconcileFailed). - WithStatusAction(cr). - WithStatusError(cr). + w.a.WithEvent(cr, a.EventActionReconcile, a.EventReasonReconcileFailed). + WithAction(cr). + WithError(cr). M(cr).F(). Error("FAILED to reconcile Secret: %s CHI: %s ", secret.Name, cr.GetName()) } @@ -65,14 +65,14 @@ func (w *worker) createSecret(ctx context.Context, cr api.ICustomResource, secre err := w.c.createSecret(ctx, secret) if err == nil { w.a.V(1). - WithEvent(cr, common.EventActionCreate, common.EventReasonCreateCompleted). - WithStatusAction(cr). + WithEvent(cr, a.EventActionCreate, a.EventReasonCreateCompleted). + WithAction(cr). M(cr).F(). Info("Create Secret %s/%s", secret.Namespace, secret.Name) } else { - w.a.WithEvent(cr, common.EventActionCreate, common.EventReasonCreateFailed). - WithStatusAction(cr). - WithStatusError(cr). + w.a.WithEvent(cr, a.EventActionCreate, a.EventReasonCreateFailed). + WithAction(cr). + WithError(cr). M(cr).F(). Error("Create Secret %s/%s failed with error %v", secret.Namespace, secret.Name, err) } diff --git a/pkg/controller/chi/worker-service.go b/pkg/controller/chi/worker-service.go index 89655ec92..3c269c5a6 100644 --- a/pkg/controller/chi/worker-service.go +++ b/pkg/controller/chi/worker-service.go @@ -22,13 +22,13 @@ import ( apiErrors "k8s.io/apimachinery/pkg/api/errors" log "github.com/altinity/clickhouse-operator/pkg/announcer" - api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" - "github.com/altinity/clickhouse-operator/pkg/controller/common" + chi "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" + a "github.com/altinity/clickhouse-operator/pkg/controller/common/announcer" "github.com/altinity/clickhouse-operator/pkg/util" ) // reconcileService reconciles core.Service -func (w *worker) reconcileService(ctx context.Context, cr api.ICustomResource, service *core.Service) error { +func (w *worker) reconcileService(ctx context.Context, cr chi.ICustomResource, service, prevService *core.Service) error { if util.IsContextDone(ctx) { log.V(2).Info("task is done") return nil @@ -43,7 +43,7 @@ func (w *worker) reconcileService(ctx context.Context, cr api.ICustomResource, s if curService != nil { // We have the Service - try to update it w.a.V(1).M(cr).F().Info("Service found: %s. Will try to update", util.NamespaceNameString(service)) - err = w.updateService(ctx, cr, curService, service) + err = w.updateService(ctx, cr, curService, service, prevService) } if err != nil { @@ -52,9 +52,9 @@ func (w *worker) reconcileService(ctx context.Context, cr api.ICustomResource, s w.a.V(1).M(cr).F().Info("Service: %s not found. err: %v", util.NamespaceNameString(service), err) } else { // The Service is either not found or not updated. Try to recreate it - w.a.WithEvent(cr, common.EventActionUpdate, common.EventReasonUpdateFailed). - WithStatusAction(cr). - WithStatusError(cr). + w.a.WithEvent(cr, a.EventActionUpdate, a.EventReasonUpdateFailed). + WithAction(cr). + WithError(cr). M(cr).F(). Error("Update Service: %s failed with error: %v", util.NamespaceNameString(service), err) } @@ -66,9 +66,9 @@ func (w *worker) reconcileService(ctx context.Context, cr api.ICustomResource, s if err == nil { w.a.V(1).M(cr).F().Info("Service reconcile successful: %s", util.NamespaceNameString(service)) } else { - w.a.WithEvent(cr, common.EventActionReconcile, common.EventReasonReconcileFailed). - WithStatusAction(cr). - WithStatusError(cr). + w.a.WithEvent(cr, a.EventActionReconcile, a.EventReasonReconcileFailed). + WithAction(cr). + WithError(cr). M(cr).F(). Error("FAILED to reconcile Service: %s CHI: %s ", util.NamespaceNameString(service), cr.GetName()) } @@ -79,9 +79,10 @@ func (w *worker) reconcileService(ctx context.Context, cr api.ICustomResource, s // updateService func (w *worker) updateService( ctx context.Context, - cr api.ICustomResource, + cr chi.ICustomResource, curService *core.Service, targetService *core.Service, + prevService *core.Service, ) error { if util.IsContextDone(ctx) { log.V(2).Info("task is done") @@ -166,9 +167,9 @@ func (w *worker) updateService( // // Migrate labels, annotations and finalizers to the new service // - newService.GetObjectMeta().SetLabels(util.MergeStringMapsPreserve(newService.GetObjectMeta().GetLabels(), curService.GetObjectMeta().GetLabels())) - newService.GetObjectMeta().SetAnnotations(util.MergeStringMapsPreserve(newService.GetObjectMeta().GetAnnotations(), curService.GetObjectMeta().GetAnnotations())) - newService.GetObjectMeta().SetFinalizers(util.MergeStringArrays(newService.GetObjectMeta().GetFinalizers(), curService.GetObjectMeta().GetFinalizers())) + newService.SetLabels(w.prepareLabels(curService, newService, ensureService(prevService))) + newService.SetAnnotations(w.prepareAnnotations(curService, newService, ensureService(prevService))) + newService.SetFinalizers(w.prepareFinalizers(curService, newService, ensureService(prevService))) // // And only now we are ready to actually update the service with new version of the service @@ -177,8 +178,8 @@ func (w *worker) updateService( err := w.c.updateService(ctx, newService) if err == nil { w.a.V(1). - WithEvent(cr, common.EventActionUpdate, common.EventReasonUpdateCompleted). - WithStatusAction(cr). + WithEvent(cr, a.EventActionUpdate, a.EventReasonUpdateCompleted). + WithAction(cr). M(cr).F(). Info("Update Service success: %s", util.NamespaceNameString(newService)) } else { @@ -188,8 +189,27 @@ func (w *worker) updateService( return err } +func ensureService(svc *core.Service) *core.Service { + if svc == nil { + return &core.Service{} + } + return svc +} + +func (w *worker) prepareLabels(curService, newService, oldService *core.Service) map[string]string { + return util.MapMigrate(curService.GetLabels(), newService.GetLabels(), oldService.GetLabels()) +} + +func (w *worker) prepareAnnotations(curService, newService, oldService *core.Service) map[string]string { + return util.MapMigrate(curService.GetAnnotations(), newService.GetAnnotations(), oldService.GetAnnotations()) +} + +func (w *worker) prepareFinalizers(curService, newService, oldService *core.Service) []string { + return util.MergeStringArrays(newService.GetFinalizers(), curService.GetFinalizers()) +} + // createService -func (w *worker) createService(ctx context.Context, cr api.ICustomResource, service *core.Service) error { +func (w *worker) createService(ctx context.Context, cr chi.ICustomResource, service *core.Service) error { if util.IsContextDone(ctx) { log.V(2).Info("task is done") return nil @@ -198,14 +218,14 @@ func (w *worker) createService(ctx context.Context, cr api.ICustomResource, serv err := w.c.createService(ctx, service) if err == nil { w.a.V(1). - WithEvent(cr, common.EventActionCreate, common.EventReasonCreateCompleted). - WithStatusAction(cr). + WithEvent(cr, a.EventActionCreate, a.EventReasonCreateCompleted). + WithAction(cr). M(cr).F(). Info("OK Create Service: %s", util.NamespaceNameString(service)) } else { - w.a.WithEvent(cr, common.EventActionCreate, common.EventReasonCreateFailed). - WithStatusAction(cr). - WithStatusError(cr). + w.a.WithEvent(cr, a.EventActionCreate, a.EventReasonCreateFailed). + WithAction(cr). + WithError(cr). M(cr).F(). Error("FAILED Create Service: %s err: %v", util.NamespaceNameString(service), err) } diff --git a/pkg/controller/chi/worker.go b/pkg/controller/chi/worker.go index 91b15a7c1..438daa81b 100644 --- a/pkg/controller/chi/worker.go +++ b/pkg/controller/chi/worker.go @@ -28,6 +28,7 @@ import ( "github.com/altinity/clickhouse-operator/pkg/apis/deployment" "github.com/altinity/clickhouse-operator/pkg/chop" "github.com/altinity/clickhouse-operator/pkg/controller/common" + a "github.com/altinity/clickhouse-operator/pkg/controller/common/announcer" "github.com/altinity/clickhouse-operator/pkg/controller/common/poller/domain" "github.com/altinity/clickhouse-operator/pkg/controller/common/statefulset" "github.com/altinity/clickhouse-operator/pkg/controller/common/storage" @@ -55,7 +56,7 @@ const FinalizerName = "finalizer.clickhouseinstallation.altinity.com" // worker represents worker thread which runs reconcile tasks type worker struct { c *Controller - a common.Announcer + a a.Announcer //queue workqueue.RateLimitingInterface queue queue.PriorityQueue @@ -78,8 +79,8 @@ func (c *Controller) newWorker(q queue.PriorityQueue, sys bool) *worker { generateName := "chop-chi-" component := componentName - announcer := common.NewAnnouncer( - common.NewEventEmitter(c.kube.Event(), kind, generateName, component), + announcer := a.NewAnnouncer( + a.NewEventEmitter(c.kube.Event(), kind, generateName, component), c.kube.CR(), ) @@ -105,40 +106,44 @@ func (c *Controller) newWorker(q queue.PriorityQueue, sys bool) *worker { func configGeneratorOptions(cr *api.ClickHouseInstallation) *config.GeneratorOptions { return &config.GeneratorOptions{ - Users: cr.GetSpecT().Configuration.Users, - Profiles: cr.GetSpecT().Configuration.Profiles, - Quotas: cr.GetSpecT().Configuration.Quotas, - Settings: cr.GetSpecT().Configuration.Settings, - Files: cr.GetSpecT().Configuration.Files, - DistributedDDL: cr.GetSpecT().Defaults.DistributedDDL, + Users: cr.GetSpecT().GetConfiguration().GetUsers(), + Profiles: cr.GetSpecT().GetConfiguration().GetProfiles(), + Quotas: cr.GetSpecT().GetConfiguration().GetQuotas(), + Settings: cr.GetSpecT().GetConfiguration().GetSettings(), + Files: cr.GetSpecT().GetConfiguration().GetFiles(), + DistributedDDL: cr.GetSpecT().GetDefaults().GetDistributedDDL(), } } -func (w *worker) newTask(cr *api.ClickHouseInstallation) { - w.task = common.NewTask( - commonCreator.NewCreator( - cr, - managers.NewConfigFilesGenerator(managers.FilesGeneratorTypeClickHouse, cr, configGeneratorOptions(cr)), - managers.NewContainerManager(managers.ContainerManagerTypeClickHouse), - managers.NewTagManager(managers.TagManagerTypeClickHouse, cr), - managers.NewProbeManager(managers.ProbeManagerTypeClickHouse), - managers.NewServiceManager(managers.ServiceManagerTypeClickHouse), - managers.NewVolumeManager(managers.VolumeManagerTypeClickHouse), - managers.NewConfigMapManager(managers.ConfigMapManagerTypeClickHouse), - managers.NewNameManager(managers.NameManagerTypeClickHouse), - managers.NewOwnerReferencesManager(managers.OwnerReferencesManagerTypeClickHouse), - namer.New(), - commonMacro.New(macro.List), - labeler.New(cr), - ), +func (w *worker) buildCreator(cr *api.ClickHouseInstallation) *commonCreator.Creator { + if cr == nil { + cr = &api.ClickHouseInstallation{} + } + return commonCreator.NewCreator( + cr, + managers.NewConfigFilesGenerator(managers.FilesGeneratorTypeClickHouse, cr, configGeneratorOptions(cr)), + managers.NewContainerManager(managers.ContainerManagerTypeClickHouse), + managers.NewTagManager(managers.TagManagerTypeClickHouse, cr), + managers.NewProbeManager(managers.ProbeManagerTypeClickHouse), + managers.NewServiceManager(managers.ServiceManagerTypeClickHouse), + managers.NewVolumeManager(managers.VolumeManagerTypeClickHouse), + managers.NewConfigMapManager(managers.ConfigMapManagerTypeClickHouse), + managers.NewNameManager(managers.NameManagerTypeClickHouse), + managers.NewOwnerReferencesManager(managers.OwnerReferencesManagerTypeClickHouse), + namer.New(), + commonMacro.New(macro.List), + labeler.New(cr), ) +} +func (w *worker) newTask(new, old *api.ClickHouseInstallation) { + w.task = common.NewTask(w.buildCreator(new), w.buildCreator(old)) w.stsReconciler = statefulset.NewReconciler( w.a, w.task, domain.NewHostStatefulSetPoller(domain.NewStatefulSetPoller(w.c.kube), w.c.kube, w.c.ctrlLabeler), w.c.namer, - labeler.New(cr), + labeler.New(new), storage.NewStorageReconciler(w.task, w.c.namer, w.c.kube.Storage()), w.c.kube, w.c, @@ -178,24 +183,26 @@ func (w *worker) shouldForceRestartHost(host *api.Host) bool { return true } - podIsCrushed := false + if host.Runtime.Version.IsUnknown() && w.isPodCrushed(host) { + w.a.V(1).M(host).F().Info("Host with unknown version and in CrashLoopBackOff should be restarted. It most likely is unable to start due to bad config. Host: %s", host.GetName()) + return true + } + + w.a.V(1).M(host).F().Info("Host restart is not required. Host: %s", host.GetName()) + return false +} + +func (w *worker) isPodCrushed(host *api.Host) bool { // pod.Status.ContainerStatuses[0].State.Waiting.Reason if pod, err := w.c.kube.Pod().Get(host); err == nil { if len(pod.Status.ContainerStatuses) > 0 { if pod.Status.ContainerStatuses[0].State.Waiting != nil { if pod.Status.ContainerStatuses[0].State.Waiting.Reason == "CrashLoopBackOff" { - podIsCrushed = true + return true } } } } - - if host.Runtime.Version.IsUnknown() && podIsCrushed { - w.a.V(1).M(host).F().Info("Host with unknown version and in CrashLoopBackOff should be restarted. It most likely is unable to start due to bad config. Host: %s", host.GetName()) - return true - } - - w.a.V(1).M(host).F().Info("Host restart is not required. Host: %s", host.GetName()) return false } @@ -203,8 +210,8 @@ func (w *worker) shouldForceRestartHost(host *api.Host) bool { func (w *worker) normalize(c *api.ClickHouseInstallation) *api.ClickHouseInstallation { chi, err := w.normalizer.CreateTemplated(c, commonNormalizer.NewOptions()) if err != nil { - w.a.WithEvent(chi, common.EventActionReconcile, common.EventReasonReconcileFailed). - WithStatusError(chi). + w.a.WithEvent(chi, a.EventActionReconcile, a.EventReasonReconcileFailed). + WithError(chi). M(chi).F(). Error("FAILED to normalize CR 1: %v", err) } @@ -216,8 +223,8 @@ func (w *worker) normalize(c *api.ClickHouseInstallation) *api.ClickHouseInstall chi, err = w.normalizer.CreateTemplated(c, opts) if err != nil { - w.a.WithEvent(chi, common.EventActionReconcile, common.EventReasonReconcileFailed). - WithStatusError(chi). + w.a.WithEvent(chi, a.EventActionReconcile, a.EventReasonReconcileFailed). + WithError(chi). M(chi).F(). Error("FAILED to normalize CHI 2: %v", err) } @@ -267,12 +274,14 @@ func (w *worker) updateEndpoints(ctx context.Context, old, new *core.Endpoints) w.a.V(1).M(chi).Info("Update users IPS-1") // TODO unify with finalize reconcile - w.newTask(chi) + w.newTask(chi, chi.GetAncestorT()) w.reconcileConfigMapCommonUsers(ctx, chi) w.c.updateCRObjectStatus(ctx, chi, types.UpdateStatusOptions{ TolerateAbsence: true, CopyStatusOptions: types.CopyStatusOptions{ - Normalized: true, + CopyStatusFieldGroup: types.CopyStatusFieldGroup{ + FieldGroupNormalized: true, + }, }, }) } else { @@ -441,8 +450,8 @@ func (w *worker) excludeStoppedCHIFromMonitoring(chi *api.ClickHouseInstallation } w.a.V(1). - WithEvent(chi, common.EventActionReconcile, common.EventReasonReconcileInProgress). - WithStatusAction(chi). + WithEvent(chi, a.EventActionReconcile, a.EventReasonReconcileInProgress). + WithAction(chi). M(chi).F(). Info("exclude CHI from monitoring") w.c.deleteWatch(chi) @@ -456,8 +465,8 @@ func (w *worker) addCHIToMonitoring(chi *api.ClickHouseInstallation) { } w.a.V(1). - WithEvent(chi, common.EventActionReconcile, common.EventReasonReconcileInProgress). - WithStatusAction(chi). + WithEvent(chi, a.EventActionReconcile, a.EventReasonReconcileInProgress). + WithAction(chi). M(chi).F(). Info("add CHI to monitoring") w.c.updateWatch(chi) @@ -473,14 +482,16 @@ func (w *worker) markReconcileStart(ctx context.Context, cr *api.ClickHouseInsta cr.EnsureStatus().ReconcileStart(ap.GetRemovedHostsNum()) _ = w.c.updateCRObjectStatus(ctx, cr, types.UpdateStatusOptions{ CopyStatusOptions: types.CopyStatusOptions{ - MainFields: true, + CopyStatusFieldGroup: types.CopyStatusFieldGroup{ + FieldGroupMain: true, + }, }, }) w.a.V(1). - WithEvent(cr, common.EventActionReconcile, common.EventReasonReconcileStarted). - WithStatusAction(cr). - WithStatusActions(cr). + WithEvent(cr, a.EventActionReconcile, a.EventReasonReconcileStarted). + WithAction(cr). + WithActions(cr). M(cr).F(). Info("reconcile started, task id: %s", cr.GetSpecT().GetTaskID()) w.a.V(2).M(cr).F().Info("action plan\n%s\n", ap.String()) @@ -507,11 +518,13 @@ func (w *worker) finalizeReconcileAndMarkCompleted(ctx context.Context, _chi *ap chi.SetTarget(nil) chi.EnsureStatus().ReconcileComplete() // TODO unify with update endpoints - w.newTask(chi) + w.newTask(chi, chi.GetAncestorT()) w.reconcileConfigMapCommonUsers(ctx, chi) w.c.updateCRObjectStatus(ctx, chi, types.UpdateStatusOptions{ CopyStatusOptions: types.CopyStatusOptions{ - WholeStatus: true, + CopyStatusFieldGroup: types.CopyStatusFieldGroup{ + FieldGroupWholeStatus: true, + }, }, }) } else { @@ -522,9 +535,9 @@ func (w *worker) finalizeReconcileAndMarkCompleted(ctx context.Context, _chi *ap } w.a.V(1). - WithEvent(_chi, common.EventActionReconcile, common.EventReasonReconcileCompleted). - WithStatusAction(_chi). - WithStatusActions(_chi). + WithEvent(_chi, a.EventActionReconcile, a.EventReasonReconcileCompleted). + WithAction(_chi). + WithActions(_chi). M(_chi).F(). Info("reconcile completed successfully, task id: %s", _chi.GetSpecT().GetTaskID()) } @@ -543,14 +556,16 @@ func (w *worker) markReconcileCompletedUnsuccessfully(ctx context.Context, chi * } w.c.updateCRObjectStatus(ctx, chi, types.UpdateStatusOptions{ CopyStatusOptions: types.CopyStatusOptions{ - MainFields: true, + CopyStatusFieldGroup: types.CopyStatusFieldGroup{ + FieldGroupMain: true, + }, }, }) w.a.V(1). - WithEvent(chi, common.EventActionReconcile, common.EventReasonReconcileFailed). - WithStatusAction(chi). - WithStatusActions(chi). + WithEvent(chi, a.EventActionReconcile, a.EventReasonReconcileFailed). + WithAction(chi). + WithActions(chi). M(chi).F(). Warning("reconcile completed UNSUCCESSFULLY, task id: %s", chi.GetSpecT().GetTaskID()) } @@ -631,19 +646,27 @@ func (w *worker) walkHosts(ctx context.Context, chi *api.ClickHouseInstallation, chi.WalkHosts(func(host *api.Host) error { w.a.V(3).M(chi).Info("Walking over CR hosts. Host: %s", host.GetName()) + _, err := w.c.kube.STS().Get(ctx, host) switch { case host.GetReconcileAttributes().IsAdd(): - w.a.V(3).M(chi).Info("Walking over CR hosts. Host: is already added Host: %s", host.GetName()) + w.a.V(3).M(chi).Info("Walking over CR hosts. Host: is already listed as ADD. Host: %s", host.GetName()) return nil case host.GetReconcileAttributes().IsModify(): - w.a.V(3).M(chi).Info("Walking over CR hosts. Host: is already modified Host: %s", host.GetName()) + w.a.V(3).M(chi).Info("Walking over CR hosts. Host: is already listed as MODIFIED. Host: %s", host.GetName()) return nil - default: - w.a.V(3).M(chi).Info("Walking over CR hosts. Host: is not clear yet (not detected as added or modified) Host: %s", host.GetName()) - w.a.V(1).M(chi).Info("Add host as FOUND via host. Host: %s", host.GetName()) + case host.HasAncestor(): + w.a.V(1).M(chi).Info("Add host as FOUND via host because host has ancestor. Host: %s", host.GetName()) + host.GetReconcileAttributes().SetFound() + return nil + case err == nil: + w.a.V(1).M(chi).Info("Add host as FOUND via host because has found sts. Host: %s", host.GetName()) host.GetReconcileAttributes().SetFound() + return nil + default: + w.a.V(1).M(chi).Info("Add host as ADD via host. Host: %s", host.GetName()) + host.GetReconcileAttributes().SetAdd() + return nil } - return nil }) // Log hosts statuses diff --git a/pkg/controller/chk/worker-chk-reconciler.go b/pkg/controller/chk/worker-chk-reconciler.go index eea2aecf7..f438471d8 100644 --- a/pkg/controller/chk/worker-chk-reconciler.go +++ b/pkg/controller/chk/worker-chk-reconciler.go @@ -27,6 +27,7 @@ import ( "github.com/altinity/clickhouse-operator/pkg/controller/chi/metrics" "github.com/altinity/clickhouse-operator/pkg/controller/chk/kube" "github.com/altinity/clickhouse-operator/pkg/controller/common" + a "github.com/altinity/clickhouse-operator/pkg/controller/common/announcer" "github.com/altinity/clickhouse-operator/pkg/controller/common/statefulset" "github.com/altinity/clickhouse-operator/pkg/controller/common/storage" "github.com/altinity/clickhouse-operator/pkg/interfaces" @@ -86,14 +87,14 @@ func (w *worker) reconcileCR(ctx context.Context, old, new *apiChk.ClickHouseKee return nil } - w.newTask(new) + w.newTask(new, old) w.markReconcileStart(ctx, new, actionPlan) w.walkHosts(ctx, new, actionPlan) if err := w.reconcile(ctx, new); err != nil { // Something went wrong - w.a.WithEvent(new, common.EventActionReconcile, common.EventReasonReconcileFailed). - WithStatusError(new). + w.a.WithEvent(new, a.EventActionReconcile, a.EventReasonReconcileFailed). + WithError(new). M(new).F(). Error("FAILED to reconcile CR %s, err: %v", util.NamespaceNameString(new), err) w.markReconcileCompletedUnsuccessfully(ctx, new, err) @@ -131,7 +132,7 @@ func (w *worker) reconcile(ctx context.Context, cr *apiChk.ClickHouseKeeperInsta }) if counters.AddOnly() { - w.a.V(1).M(cr).Info("Enabling full fan-out mode. CHI: %s", util.NamespaceNameString(cr)) + w.a.V(1).M(cr).Info("Enabling full fan-out mode. CR: %s", util.NamespaceNameString(cr)) ctx = context.WithValue(ctx, common.ReconcileShardsAndHostsOptionsCtxKey, &common.ReconcileShardsAndHostsOptions{ FullFanOut: true, }) @@ -168,6 +169,21 @@ func (w *worker) reconcileCRAuxObjectsPreliminary(ctx context.Context, cr *apiCh w.a.F().Error("failed to reconcile config map users. err: %v", err) } + return w.reconcileCRAuxObjectsPreliminaryDomain(ctx, cr) +} + +func (w *worker) reconcileCRAuxObjectsPreliminaryDomain(ctx context.Context, cr *apiChk.ClickHouseKeeperInstallation) error { + switch { + case cr.HostsCount() < cr.GetAncestor().HostsCount(): + // Downscale + time.Sleep(120 * time.Second) + case cr.HostsCount() > cr.GetAncestor().HostsCount(): + // Upscale + time.Sleep(30 * time.Second) + default: + // Same size + time.Sleep(10 * time.Second) + } return nil } @@ -189,7 +205,8 @@ func (w *worker) reconcileCRServiceFinal(ctx context.Context, cr api.ICustomReso // Create entry point for the whole CHI if service := w.task.Creator().CreateService(interfaces.ServiceCR); service != nil { - if err := w.reconcileService(ctx, cr, service); err != nil { + prevService := w.task.CreatorPrev().CreateService(interfaces.ServiceCR) + if err := w.reconcileService(ctx, cr, service, prevService); err != nil { // Service not reconciled w.task.RegistryFailed().RegisterService(service.GetObjectMeta()) return err @@ -212,7 +229,7 @@ func (w *worker) reconcileCRAuxObjectsFinal(ctx context.Context, cr *apiChk.Clic // CR ConfigMaps with update cr.GetRuntime().LockCommonConfig() - err = w.reconcileConfigMapCommon(ctx, cr, nil) + err = w.reconcileConfigMapCommon(ctx, cr) cr.GetRuntime().UnlockCommonConfig() return err } @@ -221,17 +238,22 @@ func (w *worker) reconcileCRAuxObjectsFinal(ctx context.Context, cr *apiChk.Clic func (w *worker) reconcileConfigMapCommon( ctx context.Context, cr api.ICustomResource, - options *config.FilesGeneratorOptions, + options ...*config.FilesGeneratorOptions, ) error { if util.IsContextDone(ctx) { log.V(2).Info("task is done") return nil } - // ConfigMap common for all resources in CHI + var opts *config.FilesGeneratorOptions + if len(options) > 0 { + opts = options[0] + } + + // ConfigMap common for all resources in CR // contains several sections, mapped as separated chopConfig files, // such as remote servers, zookeeper setup, etc - configMapCommon := w.task.Creator().CreateConfigMap(interfaces.ConfigMapCommon, options) + configMapCommon := w.task.Creator().CreateConfigMap(interfaces.ConfigMapCommon, opts) err := w.reconcileConfigMap(ctx, cr, configMapCommon) if err == nil { w.task.RegistryReconciled().RegisterConfigMap(configMapCommon.GetObjectMeta()) @@ -319,9 +341,9 @@ func (w *worker) reconcileHostStatefulSet(ctx context.Context, host *api.Host, o } host.GetCR().IEnsureStatus().HostFailed() - w.a.WithEvent(host.GetCR(), common.EventActionReconcile, common.EventReasonReconcileFailed). - WithStatusAction(host.GetCR()). - WithStatusError(host.GetCR()). + w.a.WithEvent(host.GetCR(), a.EventActionReconcile, a.EventReasonReconcileFailed). + WithAction(host.GetCR()). + WithError(host.GetCR()). M(host).F(). Error("FAILED to reconcile StatefulSet for host: %s", host.GetName()) } @@ -344,7 +366,8 @@ func (w *worker) reconcileHostService(ctx context.Context, host *api.Host) error // This is not a problem, service may be omitted return nil } - err := w.reconcileService(ctx, host.GetCR(), service) + prevService := w.task.CreatorPrev().CreateService(interfaces.ServiceHost, host.GetAncestor()) + err := w.reconcileService(ctx, host.GetCR(), service, prevService) if err == nil { w.a.V(1).M(host).F().Info("DONE Reconcile service of the host: %s", host.GetName()) w.task.RegistryReconciled().RegisterService(service.GetObjectMeta()) @@ -367,7 +390,8 @@ func (w *worker) reconcileCluster(ctx context.Context, cluster *apiChk.Cluster) // Add Cluster Service if service := w.task.Creator().CreateService(interfaces.ServiceCluster, cluster); service != nil { - if err := w.reconcileService(ctx, cluster.GetRuntime().GetCR(), service); err == nil { + prevService := w.task.CreatorPrev().CreateService(interfaces.ServiceCluster, cluster.GetAncestor()) + if err := w.reconcileService(ctx, cluster.GetRuntime().GetCR(), service, prevService); err == nil { w.task.RegistryReconciled().RegisterService(service.GetObjectMeta()) } else { w.task.RegistryFailed().RegisterService(service.GetObjectMeta()) @@ -539,14 +563,16 @@ func (w *worker) reconcileHost(ctx context.Context, host *api.Host) error { hostsCount = host.GetCR().GetStatus().GetHostsCount() } w.a.V(1). - WithEvent(host.GetCR(), common.EventActionProgress, common.EventReasonProgressHostsCompleted). - WithStatusAction(host.GetCR()). + WithEvent(host.GetCR(), a.EventActionProgress, a.EventReasonProgressHostsCompleted). + WithAction(host.GetCR()). M(host).F(). - Info("[now: %s] %s: %d of %d", now, common.EventReasonProgressHostsCompleted, hostsCompleted, hostsCount) + Info("[now: %s] %s: %d of %d", now, a.EventReasonProgressHostsCompleted, hostsCompleted, hostsCount) _ = w.c.updateCRObjectStatus(ctx, host.GetCR(), types.UpdateStatusOptions{ CopyStatusOptions: types.CopyStatusOptions{ - MainFields: true, + CopyStatusFieldGroup: types.CopyStatusFieldGroup{ + FieldGroupMain: true, + }, }, }) return nil @@ -569,9 +595,9 @@ func (w *worker) reconcileHostMain(ctx context.Context, host *api.Host) error { reconcileStatefulSetOpts *statefulset.ReconcileOptions ) - if host.IsFirst() || host.IsLast() { - reconcileStatefulSetOpts = reconcileStatefulSetOpts.SetDoNotWait() - } + //if !host.IsLast() { + // reconcileStatefulSetOpts = reconcileStatefulSetOpts.SetDoNotWait() + //} if err := w.reconcileConfigMapHost(ctx, host); err != nil { w.a.V(1). @@ -599,6 +625,8 @@ func (w *worker) reconcileHostMain(ctx context.Context, host *api.Host) error { Info("Data loss detected for host: %s. Will do force migrate", host.GetName()) } + _ = w.reconcileHostService(ctx, host) + if err := w.reconcileHostStatefulSet(ctx, host, reconcileStatefulSetOpts); err != nil { w.a.V(1). M(host).F(). @@ -612,8 +640,23 @@ func (w *worker) reconcileHostMain(ctx context.Context, host *api.Host) error { storage.NewStoragePVC(kube.NewPVC(w.c.Client)), ).ReconcilePVCs(ctx, host, api.DesiredStatefulSet) - _ = w.reconcileHostService(ctx, host) + // _ = w.reconcileHostService(ctx, host) + + return w.reconcileHostMainDomain(ctx, host) +} +func (w *worker) reconcileHostMainDomain(ctx context.Context, host *api.Host) error { + // Should we wait for host to startup + wait := false + + if host.GetReconcileAttributes().IsAdd() { + wait = true + } + + // Wait for host to startup + if wait { + time.Sleep(7 * time.Second) + } return nil } diff --git a/pkg/controller/chk/worker-config-map.go b/pkg/controller/chk/worker-config-map.go index 5a3970296..515d8f313 100644 --- a/pkg/controller/chk/worker-config-map.go +++ b/pkg/controller/chk/worker-config-map.go @@ -23,7 +23,7 @@ import ( log "github.com/altinity/clickhouse-operator/pkg/announcer" apiChi "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" - "github.com/altinity/clickhouse-operator/pkg/controller/common" + a "github.com/altinity/clickhouse-operator/pkg/controller/common/announcer" "github.com/altinity/clickhouse-operator/pkg/util" ) @@ -55,9 +55,9 @@ func (w *worker) reconcileConfigMap( } if err != nil { - w.a.WithEvent(cr, common.EventActionReconcile, common.EventReasonReconcileFailed). - WithStatusAction(cr). - WithStatusError(cr). + w.a.WithEvent(cr, a.EventActionReconcile, a.EventReasonReconcileFailed). + WithAction(cr). + WithError(cr). M(cr).F(). Error("FAILED to reconcile ConfigMap: %s CHI: %s ", configMap.GetName(), cr.GetName()) } @@ -75,17 +75,17 @@ func (w *worker) updateConfigMap(ctx context.Context, cr apiChi.ICustomResource, updatedConfigMap, err := w.c.updateConfigMap(ctx, configMap) if err == nil { w.a.V(1). - WithEvent(cr, common.EventActionUpdate, common.EventReasonUpdateCompleted). - WithStatusAction(cr). + WithEvent(cr, a.EventActionUpdate, a.EventReasonUpdateCompleted). + WithAction(cr). M(cr).F(). Info("Update ConfigMap %s/%s", configMap.Namespace, configMap.Name) if updatedConfigMap.ResourceVersion != configMap.ResourceVersion { w.task.SetCmUpdate(time.Now()) } } else { - w.a.WithEvent(cr, common.EventActionUpdate, common.EventReasonUpdateFailed). - WithStatusAction(cr). - WithStatusError(cr). + w.a.WithEvent(cr, a.EventActionUpdate, a.EventReasonUpdateFailed). + WithAction(cr). + WithError(cr). M(cr).F(). Error("Update ConfigMap %s/%s failed with error %v", configMap.Namespace, configMap.Name, err) } @@ -103,14 +103,14 @@ func (w *worker) createConfigMap(ctx context.Context, cr apiChi.ICustomResource, err := w.c.createConfigMap(ctx, configMap) if err == nil { w.a.V(1). - WithEvent(cr, common.EventActionCreate, common.EventReasonCreateCompleted). - WithStatusAction(cr). + WithEvent(cr, a.EventActionCreate, a.EventReasonCreateCompleted). + WithAction(cr). M(cr).F(). Info("Create ConfigMap %s", util.NamespaceNameString(configMap)) } else { - w.a.WithEvent(cr, common.EventActionCreate, common.EventReasonCreateFailed). - WithStatusAction(cr). - WithStatusError(cr). + w.a.WithEvent(cr, a.EventActionCreate, a.EventReasonCreateFailed). + WithAction(cr). + WithError(cr). M(cr).F(). Error("Create ConfigMap %s failed with error %v", util.NamespaceNameString(configMap), err) } diff --git a/pkg/controller/chk/worker-deleter.go b/pkg/controller/chk/worker-deleter.go index db6810b9e..19e77c1b5 100644 --- a/pkg/controller/chk/worker-deleter.go +++ b/pkg/controller/chk/worker-deleter.go @@ -22,7 +22,7 @@ import ( log "github.com/altinity/clickhouse-operator/pkg/announcer" api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" - "github.com/altinity/clickhouse-operator/pkg/controller/common" + a "github.com/altinity/clickhouse-operator/pkg/controller/common/announcer" "github.com/altinity/clickhouse-operator/pkg/model" chkLabeler "github.com/altinity/clickhouse-operator/pkg/model/chk/tags/labeler" "github.com/altinity/clickhouse-operator/pkg/util" @@ -35,8 +35,8 @@ func (w *worker) clean(ctx context.Context, cr api.ICustomResource) { } w.a.V(1). - WithEvent(cr, common.EventActionReconcile, common.EventReasonReconcileInProgress). - WithStatusAction(cr). + WithEvent(cr, a.EventActionReconcile, a.EventReasonReconcileInProgress). + WithAction(cr). M(cr).F(). Info("remove items scheduled for deletion") @@ -49,7 +49,6 @@ func (w *worker) clean(ctx context.Context, cr api.ICustomResource) { objs.Subtract(need) w.a.V(1).M(cr).F().Info("Non-reconciled objects:\n%s", objs) if w.purge(ctx, cr, objs, w.task.RegistryFailed()) > 0 { - //w.c.enqueueObject(cmd_queue.NewDropDns(chk)) util.WaitContextDoneOrTimeout(ctx, 1*time.Minute) } diff --git a/pkg/controller/chk/worker-service.go b/pkg/controller/chk/worker-service.go index 7b6c7ab9b..6647499d4 100644 --- a/pkg/controller/chk/worker-service.go +++ b/pkg/controller/chk/worker-service.go @@ -20,54 +20,15 @@ import ( core "k8s.io/api/core/v1" apiErrors "k8s.io/apimachinery/pkg/api/errors" - "sigs.k8s.io/controller-runtime/pkg/client" log "github.com/altinity/clickhouse-operator/pkg/announcer" - apiChk "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse-keeper.altinity.com/v1" - apiChi "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" - "github.com/altinity/clickhouse-operator/pkg/controller/common" - "github.com/altinity/clickhouse-operator/pkg/interfaces" + chi "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" + a "github.com/altinity/clickhouse-operator/pkg/controller/common/announcer" "github.com/altinity/clickhouse-operator/pkg/util" ) -func (w *worker) reconcileClientService(chk *apiChk.ClickHouseKeeperInstallation) error { - return w.c.reconcile( - chk, - &core.Service{}, - w.task.Creator().CreateService(interfaces.ServiceCR, chk), - "Client Service", - reconcileUpdaterService, - ) -} - -func (w *worker) reconcileHeadlessService(chk *apiChk.ClickHouseKeeperInstallation) error { - return w.c.reconcile( - chk, - &core.Service{}, - w.task.Creator().CreateService(interfaces.ServiceHost, chk), - "Headless Service", - reconcileUpdaterService, - ) -} - -func reconcileUpdaterService(_cur, _new client.Object) error { - cur, ok1 := _cur.(*core.Service) - new, ok2 := _new.(*core.Service) - if !ok1 || !ok2 { - return fmt.Errorf("unable to cast") - } - return updateService(cur, new) -} - -func updateService(cur, new *core.Service) error { - cur.Spec.Ports = new.Spec.Ports - cur.Spec.Type = new.Spec.Type - cur.SetAnnotations(new.GetAnnotations()) - return nil -} - // reconcileService reconciles core.Service -func (w *worker) reconcileService(ctx context.Context, cr apiChi.ICustomResource, service *core.Service) error { +func (w *worker) reconcileService(ctx context.Context, cr chi.ICustomResource, service, prevService *core.Service) error { if util.IsContextDone(ctx) { log.V(2).Info("task is done") return nil @@ -82,7 +43,7 @@ func (w *worker) reconcileService(ctx context.Context, cr apiChi.ICustomResource if curService != nil { // We have the Service - try to update it w.a.V(1).M(cr).F().Info("Service found: %s. Will try to update", util.NamespaceNameString(service)) - err = w.updateService(ctx, cr, curService, service) + err = w.updateService(ctx, cr, curService, service, prevService) } if err != nil { @@ -91,9 +52,9 @@ func (w *worker) reconcileService(ctx context.Context, cr apiChi.ICustomResource w.a.V(1).M(cr).F().Info("Service: %s not found. err: %v", util.NamespaceNameString(service), err) } else { // The Service is either not found or not updated. Try to recreate it - w.a.WithEvent(cr, common.EventActionUpdate, common.EventReasonUpdateFailed). - WithStatusAction(cr). - WithStatusError(cr). + w.a.WithEvent(cr, a.EventActionUpdate, a.EventReasonUpdateFailed). + WithAction(cr). + WithError(cr). M(cr).F(). Error("Update Service: %s failed with error: %v", util.NamespaceNameString(service), err) } @@ -105,9 +66,9 @@ func (w *worker) reconcileService(ctx context.Context, cr apiChi.ICustomResource if err == nil { w.a.V(1).M(cr).F().Info("Service reconcile successful: %s", util.NamespaceNameString(service)) } else { - w.a.WithEvent(cr, common.EventActionReconcile, common.EventReasonReconcileFailed). - WithStatusAction(cr). - WithStatusError(cr). + w.a.WithEvent(cr, a.EventActionReconcile, a.EventReasonReconcileFailed). + WithAction(cr). + WithError(cr). M(cr).F(). Error("FAILED to reconcile Service: %s CHI: %s ", util.NamespaceNameString(service), cr.GetName()) } @@ -118,9 +79,10 @@ func (w *worker) reconcileService(ctx context.Context, cr apiChi.ICustomResource // updateService func (w *worker) updateService( ctx context.Context, - cr apiChi.ICustomResource, + cr chi.ICustomResource, curService *core.Service, targetService *core.Service, + prevService *core.Service, ) error { if util.IsContextDone(ctx) { log.V(2).Info("task is done") @@ -205,9 +167,9 @@ func (w *worker) updateService( // // Migrate labels, annotations and finalizers to the new service // - newService.GetObjectMeta().SetLabels(util.MergeStringMapsPreserve(newService.GetObjectMeta().GetLabels(), curService.GetObjectMeta().GetLabels())) - newService.GetObjectMeta().SetAnnotations(util.MergeStringMapsPreserve(newService.GetObjectMeta().GetAnnotations(), curService.GetObjectMeta().GetAnnotations())) - newService.GetObjectMeta().SetFinalizers(util.MergeStringArrays(newService.GetObjectMeta().GetFinalizers(), curService.GetObjectMeta().GetFinalizers())) + newService.SetLabels(w.prepareLabels(curService, newService, ensureService(prevService))) + newService.SetAnnotations(w.prepareAnnotations(curService, newService, ensureService(prevService))) + newService.SetFinalizers(w.prepareFinalizers(curService, newService, ensureService(prevService))) // // And only now we are ready to actually update the service with new version of the service @@ -216,8 +178,8 @@ func (w *worker) updateService( err := w.c.updateService(ctx, newService) if err == nil { w.a.V(1). - WithEvent(cr, common.EventActionUpdate, common.EventReasonUpdateCompleted). - WithStatusAction(cr). + WithEvent(cr, a.EventActionUpdate, a.EventReasonUpdateCompleted). + WithAction(cr). M(cr).F(). Info("Update Service success: %s", util.NamespaceNameString(newService)) } else { @@ -227,8 +189,27 @@ func (w *worker) updateService( return err } +func ensureService(svc *core.Service) *core.Service { + if svc == nil { + return &core.Service{} + } + return svc +} + +func (w *worker) prepareLabels(curService, newService, oldService *core.Service) map[string]string { + return util.MapMigrate(curService.GetLabels(), newService.GetLabels(), oldService.GetLabels()) +} + +func (w *worker) prepareAnnotations(curService, newService, oldService *core.Service) map[string]string { + return util.MapMigrate(curService.GetAnnotations(), newService.GetAnnotations(), oldService.GetAnnotations()) +} + +func (w *worker) prepareFinalizers(curService, newService, oldService *core.Service) []string { + return util.MergeStringArrays(newService.GetFinalizers(), curService.GetFinalizers()) +} + // createService -func (w *worker) createService(ctx context.Context, cr apiChi.ICustomResource, service *core.Service) error { +func (w *worker) createService(ctx context.Context, cr chi.ICustomResource, service *core.Service) error { if util.IsContextDone(ctx) { log.V(2).Info("task is done") return nil @@ -237,14 +218,14 @@ func (w *worker) createService(ctx context.Context, cr apiChi.ICustomResource, s err := w.c.createService(ctx, service) if err == nil { w.a.V(1). - WithEvent(cr, common.EventActionCreate, common.EventReasonCreateCompleted). - WithStatusAction(cr). + WithEvent(cr, a.EventActionCreate, a.EventReasonCreateCompleted). + WithAction(cr). M(cr).F(). Info("OK Create Service: %s", util.NamespaceNameString(service)) } else { - w.a.WithEvent(cr, common.EventActionCreate, common.EventReasonCreateFailed). - WithStatusAction(cr). - WithStatusError(cr). + w.a.WithEvent(cr, a.EventActionCreate, a.EventReasonCreateFailed). + WithAction(cr). + WithError(cr). M(cr).F(). Error("FAILED Create Service: %s err: %v", util.NamespaceNameString(service), err) } diff --git a/pkg/controller/chk/worker.go b/pkg/controller/chk/worker.go index 869ce7ffa..2e8c7e032 100644 --- a/pkg/controller/chk/worker.go +++ b/pkg/controller/chk/worker.go @@ -26,6 +26,7 @@ import ( api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" "github.com/altinity/clickhouse-operator/pkg/apis/common/types" "github.com/altinity/clickhouse-operator/pkg/controller/common" + a "github.com/altinity/clickhouse-operator/pkg/controller/common/announcer" "github.com/altinity/clickhouse-operator/pkg/controller/common/poller/domain" "github.com/altinity/clickhouse-operator/pkg/controller/common/statefulset" "github.com/altinity/clickhouse-operator/pkg/controller/common/storage" @@ -48,7 +49,7 @@ import ( // worker represents worker thread which runs reconcile tasks type worker struct { c *Controller - a common.Announcer + a a.Announcer normalizer *normalizer.Normalizer task *common.Task @@ -64,7 +65,7 @@ func (c *Controller) newWorker() *worker { //generateName := "chop-chk-" //component := componentName - announcer := common.NewAnnouncer( + announcer := a.NewAnnouncer( //common.NewEventEmitter(c.kube.Event(), kind, generateName, component), nil, c.kube.CR(), @@ -82,37 +83,41 @@ func (c *Controller) newWorker() *worker { func configGeneratorOptions(cr *apiChk.ClickHouseKeeperInstallation) *config.GeneratorOptions { return &config.GeneratorOptions{ - Settings: cr.GetSpecT().Configuration.Settings, - Files: cr.GetSpecT().Configuration.Files, + Settings: cr.GetSpecT().GetConfiguration().GetSettings(), + Files: cr.GetSpecT().GetConfiguration().GetFiles(), } } -func (w *worker) newTask(cr *apiChk.ClickHouseKeeperInstallation) { - w.task = common.NewTask( - commonCreator.NewCreator( - cr, - managers.NewConfigFilesGenerator(managers.FilesGeneratorTypeKeeper, cr, configGeneratorOptions(cr)), - managers.NewContainerManager(managers.ContainerManagerTypeKeeper), - managers.NewTagManager(managers.TagManagerTypeKeeper, cr), - managers.NewProbeManager(managers.ProbeManagerTypeKeeper), - managers.NewServiceManager(managers.ServiceManagerTypeKeeper), - managers.NewVolumeManager(managers.VolumeManagerTypeKeeper), - managers.NewConfigMapManager(managers.ConfigMapManagerTypeKeeper), - managers.NewNameManager(managers.NameManagerTypeKeeper), - managers.NewOwnerReferencesManager(managers.OwnerReferencesManagerTypeKeeper), - namer.New(), - commonMacro.New(macro.List), - labeler.New(cr), - ), +func (w *worker) buildCreator(cr *apiChk.ClickHouseKeeperInstallation) *commonCreator.Creator { + if cr == nil { + cr = &apiChk.ClickHouseKeeperInstallation{} + } + return commonCreator.NewCreator( + cr, + managers.NewConfigFilesGenerator(managers.FilesGeneratorTypeKeeper, cr, configGeneratorOptions(cr)), + managers.NewContainerManager(managers.ContainerManagerTypeKeeper), + managers.NewTagManager(managers.TagManagerTypeKeeper, cr), + managers.NewProbeManager(managers.ProbeManagerTypeKeeper), + managers.NewServiceManager(managers.ServiceManagerTypeKeeper), + managers.NewVolumeManager(managers.VolumeManagerTypeKeeper), + managers.NewConfigMapManager(managers.ConfigMapManagerTypeKeeper), + managers.NewNameManager(managers.NameManagerTypeKeeper), + managers.NewOwnerReferencesManager(managers.OwnerReferencesManagerTypeKeeper), + namer.New(), + commonMacro.New(macro.List), + labeler.New(cr), ) +} +func (w *worker) newTask(new, old *apiChk.ClickHouseKeeperInstallation) { + w.task = common.NewTask(w.buildCreator(new), w.buildCreator(old)) w.stsReconciler = statefulset.NewReconciler( w.a, w.task, //poller.NewHostStatefulSetPoller(poller.NewStatefulSetPoller(w.c.kube), w.c.kube, w.c.labeler), domain.NewHostStatefulSetPoller(domain.NewStatefulSetPoller(w.c.kube), w.c.kube, nil), w.c.namer, - labeler.New(cr), + labeler.New(new), storage.NewStorageReconciler(w.task, w.c.namer, w.c.kube.Storage()), w.c.kube, statefulset.NewDefaultFallback(), @@ -169,8 +174,8 @@ func (w *worker) shouldForceRestartHost(host *api.Host) bool { func (w *worker) normalize(c *apiChk.ClickHouseKeeperInstallation) *apiChk.ClickHouseKeeperInstallation { chk, err := normalizer.New().CreateTemplated(c, commonNormalizer.NewOptions()) if err != nil { - w.a.WithEvent(chk, common.EventActionReconcile, common.EventReasonReconcileFailed). - WithStatusError(chk). + w.a.WithEvent(chk, a.EventActionReconcile, a.EventReasonReconcileFailed). + WithError(chk). M(chk).F(). Error("FAILED to normalize CR 1: %v", err) } @@ -207,14 +212,16 @@ func (w *worker) markReconcileStart(ctx context.Context, cr *apiChk.ClickHouseKe cr.EnsureStatus().ReconcileStart(ap.GetRemovedHostsNum()) _ = w.c.updateCRObjectStatus(ctx, cr, types.UpdateStatusOptions{ CopyStatusOptions: types.CopyStatusOptions{ - MainFields: true, + CopyStatusFieldGroup: types.CopyStatusFieldGroup{ + FieldGroupMain: true, + }, }, }) w.a.V(1). - WithEvent(cr, common.EventActionReconcile, common.EventReasonReconcileStarted). - WithStatusAction(cr). - WithStatusActions(cr). + WithEvent(cr, a.EventActionReconcile, a.EventReasonReconcileStarted). + WithAction(cr). + WithActions(cr). M(cr).F(). Info("reconcile started, task id: %s", cr.GetSpecT().GetTaskID()) w.a.V(2).M(cr).F().Info("action plan\n%s\n", ap.String()) @@ -241,11 +248,13 @@ func (w *worker) finalizeReconcileAndMarkCompleted(ctx context.Context, _chk *ap chi.SetTarget(nil) chi.EnsureStatus().ReconcileComplete() // TODO unify with update endpoints - w.newTask(chi) + w.newTask(chi, chi.GetAncestorT()) //w.reconcileConfigMapCommonUsers(ctx, chi) w.c.updateCRObjectStatus(ctx, chi, types.UpdateStatusOptions{ CopyStatusOptions: types.CopyStatusOptions{ - WholeStatus: true, + CopyStatusFieldGroup: types.CopyStatusFieldGroup{ + FieldGroupWholeStatus: true, + }, }, }) } else { @@ -256,9 +265,9 @@ func (w *worker) finalizeReconcileAndMarkCompleted(ctx context.Context, _chk *ap } w.a.V(1). - WithEvent(_chk, common.EventActionReconcile, common.EventReasonReconcileCompleted). - WithStatusAction(_chk). - WithStatusActions(_chk). + WithEvent(_chk, a.EventActionReconcile, a.EventReasonReconcileCompleted). + WithAction(_chk). + WithActions(_chk). M(_chk).F(). Info("reconcile completed successfully, task id: %s", _chk.GetSpecT().GetTaskID()) } @@ -277,14 +286,16 @@ func (w *worker) markReconcileCompletedUnsuccessfully(ctx context.Context, chk * } w.c.updateCRObjectStatus(ctx, chk, types.UpdateStatusOptions{ CopyStatusOptions: types.CopyStatusOptions{ - MainFields: true, + CopyStatusFieldGroup: types.CopyStatusFieldGroup{ + FieldGroupMain: true, + }, }, }) w.a.V(1). - WithEvent(chk, common.EventActionReconcile, common.EventReasonReconcileFailed). - WithStatusAction(chk). - WithStatusActions(chk). + WithEvent(chk, a.EventActionReconcile, a.EventReasonReconcileFailed). + WithAction(chk). + WithActions(chk). M(chk).F(). Warning("reconcile completed UNSUCCESSFULLY, task id: %s", chk.GetSpecT().GetTaskID()) } diff --git a/pkg/controller/common/announcer.go b/pkg/controller/common/announcer/announcer.go similarity index 62% rename from pkg/controller/common/announcer.go rename to pkg/controller/common/announcer/announcer.go index 714e47bfc..e18861400 100644 --- a/pkg/controller/common/announcer.go +++ b/pkg/controller/common/announcer/announcer.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package common +package announcer import ( "context" @@ -24,6 +24,7 @@ import ( a "github.com/altinity/clickhouse-operator/pkg/announcer" api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" "github.com/altinity/clickhouse-operator/pkg/apis/common/types" + "github.com/altinity/clickhouse-operator/pkg/chop" "github.com/altinity/clickhouse-operator/pkg/interfaces" ) @@ -44,15 +45,15 @@ type Announcer struct { // event reason specifies k8s event reason eventReason string - // writeStatusAction specifies whether to produce action into `ClickHouseInstallation.Status.Action` of chi, + // writeAction specifies whether to produce action into `ClickHouseInstallation.Status.Action` of chi, // therefore requires chi to be specified - writeStatusAction bool - // writeStatusAction specifies whether to produce action into `ClickHouseInstallation.Status.Actions` of chi, + writeAction bool + // writeActions specifies whether to produce action into `ClickHouseInstallation.Status.Actions` of chi, // therefore requires chi to be specified - writeStatusActions bool - // writeStatusAction specifies whether to produce action into `ClickHouseInstallation.Status.Error` of chi, + writeActions bool + // writeError specifies whether to produce action into `ClickHouseInstallation.Status.Error` of chi, // therefore requires chi to be specified - writeStatusError bool + writeError bool } // NewAnnouncer creates new announcer @@ -138,13 +139,7 @@ func (a Announcer) Info(format string, args ...interface{}) { a.Announcer.Info(format, args...) // Produce k8s event - if a.writeEvent && a.capable() { - if len(args) > 0 { - a.eventEmitter.EventInfo(a.cr, a.eventAction, a.eventReason, fmt.Sprintf(format, args...)) - } else { - a.eventEmitter.EventInfo(a.cr, a.eventAction, a.eventReason, fmt.Sprint(format)) - } - } + a.emitEvent("info", format, args...) // Produce chi status record a.writeStatus(format, args...) @@ -156,13 +151,7 @@ func (a Announcer) Warning(format string, args ...interface{}) { a.Announcer.Warning(format, args...) // Produce k8s event - if a.writeEvent && a.capable() { - if len(args) > 0 { - a.eventEmitter.EventWarning(a.cr, a.eventAction, a.eventReason, fmt.Sprintf(format, args...)) - } else { - a.eventEmitter.EventWarning(a.cr, a.eventAction, a.eventReason, fmt.Sprint(format)) - } - } + a.emitEvent("warning", format, args...) // Produce chi status record a.writeStatus(format, args...) @@ -174,13 +163,7 @@ func (a Announcer) Error(format string, args ...interface{}) { a.Announcer.Error(format, args...) // Produce k8s event - if a.writeEvent && a.capable() { - if len(args) > 0 { - a.eventEmitter.EventError(a.cr, a.eventAction, a.eventReason, fmt.Sprintf(format, args...)) - } else { - a.eventEmitter.EventError(a.cr, a.eventAction, a.eventReason, fmt.Sprint(format)) - } - } + a.emitEvent("error", format, args...) // Produce chi status record a.writeStatus(format, args...) @@ -189,13 +172,7 @@ func (a Announcer) Error(format string, args ...interface{}) { // Fatal is inspired by log.Fatalf() func (a Announcer) Fatal(format string, args ...interface{}) { // Produce k8s event - if a.writeEvent && a.capable() { - if len(args) > 0 { - a.eventEmitter.EventError(a.cr, a.eventAction, a.eventReason, fmt.Sprintf(format, args...)) - } else { - a.eventEmitter.EventError(a.cr, a.eventAction, a.eventReason, fmt.Sprint(format)) - } - } + a.emitEvent("error", format, args...) // Produce chi status record a.writeStatus(format, args...) @@ -221,41 +198,41 @@ func (a Announcer) WithEvent(cr api.ICustomResource, action string, reason strin return b } -// WithStatusAction is used in chained calls in order to produce action into `ClickHouseInstallation.Status.Action` -func (a Announcer) WithStatusAction(cr api.ICustomResource) Announcer { +// WithAction is used in chained calls in order to produce action into `ClickHouseInstallation.Status.Action` +func (a Announcer) WithAction(cr api.ICustomResource) Announcer { b := a if cr == nil { b.cr = nil - b.writeStatusAction = false + b.writeAction = false } else { b.cr = cr - b.writeStatusAction = true + b.writeAction = true } return b } -// WithStatusActions is used in chained calls in order to produce action in ClickHouseInstallation.Status.Actions -func (a Announcer) WithStatusActions(cr api.ICustomResource) Announcer { +// WithActions is used in chained calls in order to produce action in ClickHouseInstallation.Status.Actions +func (a Announcer) WithActions(cr api.ICustomResource) Announcer { b := a if cr == nil { b.cr = nil - b.writeStatusActions = false + b.writeActions = false } else { b.cr = cr - b.writeStatusActions = true + b.writeActions = true } return b } -// WithStatusError is used in chained calls in order to produce error in ClickHouseInstallation.Status.Error -func (a Announcer) WithStatusError(cr api.ICustomResource) Announcer { +// WithError is used in chained calls in order to produce error in ClickHouseInstallation.Status.Error +func (a Announcer) WithError(cr api.ICustomResource) Announcer { b := a if cr == nil { b.cr = nil - b.writeStatusError = false + b.writeError = false } else { b.cr = cr - b.writeStatusError = true + b.writeError = true } return b } @@ -265,6 +242,20 @@ func (a Announcer) capable() bool { return (a.eventEmitter != nil) && (a.cr != nil) } +func (a Announcer) emitEvent(level string, format string, args ...interface{}) { + if !a.capable() { + return + } + if !a.writeEvent { + return + } + if len(args) > 0 { + a.eventEmitter.Event(level, a.cr, a.eventAction, a.eventReason, fmt.Sprintf(format, args...)) + } else { + a.eventEmitter.Event(level, a.cr, a.eventAction, a.eventReason, fmt.Sprint(format)) + } +} + // writeStatus is internal function which writes ClickHouseInstallation.Status func (a Announcer) writeStatus(format string, args ...interface{}) { if !a.capable() { @@ -275,42 +266,68 @@ func (a Announcer) writeStatus(format string, args ...interface{}) { prefix := now.Format(time.RFC3339Nano) + " " shouldUpdateStatus := false - if a.writeStatusAction { - shouldUpdateStatus = true - if len(args) > 0 { - a.cr.IEnsureStatus().SetAction(fmt.Sprintf(format, args...)) - } else { - a.cr.IEnsureStatus().SetAction(fmt.Sprint(format)) - } - } - if a.writeStatusActions { - shouldUpdateStatus = true - if len(args) > 0 { - a.cr.IEnsureStatus().PushAction(prefix + fmt.Sprintf(format, args...)) - } else { - a.cr.IEnsureStatus().PushAction(prefix + fmt.Sprint(format)) - } - } - if a.writeStatusError { - shouldUpdateStatus = true - if len(args) > 0 { - // PR review question: should we prefix the string in the SetError call? If so, we can SetAndPushError. - a.cr.IEnsureStatus().SetError(fmt.Sprintf(format, args...)) - a.cr.IEnsureStatus().PushError(prefix + fmt.Sprintf(format, args...)) - } else { - a.cr.IEnsureStatus().SetError(fmt.Sprint(format)) - a.cr.IEnsureStatus().PushError(prefix + fmt.Sprint(format)) - } - } + shouldUpdateStatus = shouldUpdateStatus || a._writeActions(prefix, format, args...) + shouldUpdateStatus = shouldUpdateStatus || a._writeErrors(prefix, format, args...) // Propagate status updates into object if shouldUpdateStatus { _ = a.statusUpdater.StatusUpdate(context.Background(), a.cr, types.UpdateStatusOptions{ TolerateAbsence: true, CopyStatusOptions: types.CopyStatusOptions{ - Actions: true, - Errors: true, + CopyStatusFieldGroup: types.CopyStatusFieldGroup{ + FieldGroupActions: true, + FieldGroupErrors: true, + }, }, }) } } + +func (a Announcer) _writeActions(prefix, format string, args ...interface{}) (shouldUpdateStatus bool) { + if a.writeAction { + if chop.Config().Status.Fields.Action.IsTrue() { + shouldUpdateStatus = true + if len(args) > 0 { + a.cr.IEnsureStatus().SetAction(fmt.Sprintf(format, args...)) + } else { + a.cr.IEnsureStatus().SetAction(fmt.Sprint(format)) + } + } + } + if a.writeActions { + if chop.Config().Status.Fields.Actions.IsTrue() { + shouldUpdateStatus = true + if len(args) > 0 { + a.cr.IEnsureStatus().PushAction(prefix + fmt.Sprintf(format, args...)) + } else { + a.cr.IEnsureStatus().PushAction(prefix + fmt.Sprint(format)) + } + } + } + return +} + +func (a Announcer) _writeErrors(prefix, format string, args ...interface{}) (shouldUpdateStatus bool) { + if a.writeError { + if chop.Config().Status.Fields.Error.IsTrue() { + shouldUpdateStatus = true + if len(args) > 0 { + // PR review question: should we prefix the string in the SetError call? If so, we can SetAndPushError. + a.cr.IEnsureStatus().SetError(fmt.Sprintf(format, args...)) + } else { + a.cr.IEnsureStatus().SetError(fmt.Sprint(format)) + } + } + } + if a.writeError { + if chop.Config().Status.Fields.Errors.IsTrue() { + shouldUpdateStatus = true + if len(args) > 0 { + a.cr.IEnsureStatus().PushError(prefix + fmt.Sprintf(format, args...)) + } else { + a.cr.IEnsureStatus().PushError(prefix + fmt.Sprint(format)) + } + } + } + return +} diff --git a/pkg/controller/common/event-emitter.go b/pkg/controller/common/announcer/event-emitter.go similarity index 93% rename from pkg/controller/common/event-emitter.go rename to pkg/controller/common/announcer/event-emitter.go index e44b7b841..c96e3e8f8 100644 --- a/pkg/controller/common/event-emitter.go +++ b/pkg/controller/common/announcer/event-emitter.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package common +package announcer import ( "time" @@ -83,6 +83,17 @@ func NewEventEmitter( } } +func (c *EventEmitter) Event(level string, obj meta.Object, action string, reason string, message string) { + switch level { + case "info": + c.EventInfo(obj, action, reason, message) + case "warning": + c.EventWarning(obj, action, reason, message) + case "error": + c.EventError(obj, action, reason, message) + } +} + // EventInfo emits event Info func (c *EventEmitter) EventInfo(obj meta.Object, action string, reason string, message string) { c.emitEvent(obj, eventTypeInfo, action, reason, message) diff --git a/pkg/controller/common/statefulset/statefulset-reconciler.go b/pkg/controller/common/statefulset/statefulset-reconciler.go index 9eed289bb..67234b998 100644 --- a/pkg/controller/common/statefulset/statefulset-reconciler.go +++ b/pkg/controller/common/statefulset/statefulset-reconciler.go @@ -25,6 +25,7 @@ import ( api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" "github.com/altinity/clickhouse-operator/pkg/apis/common/types" "github.com/altinity/clickhouse-operator/pkg/controller/common" + a "github.com/altinity/clickhouse-operator/pkg/controller/common/announcer" "github.com/altinity/clickhouse-operator/pkg/controller/common/storage" "github.com/altinity/clickhouse-operator/pkg/interfaces" "github.com/altinity/clickhouse-operator/pkg/model/k8s" @@ -32,7 +33,7 @@ import ( ) type Reconciler struct { - a common.Announcer + a a.Announcer task *common.Task hostSTSPoller IHostStatefulSetPoller @@ -47,7 +48,7 @@ type Reconciler struct { } func NewReconciler( - a common.Announcer, + a a.Announcer, task *common.Task, hostSTSPoller IHostStatefulSetPoller, namer interfaces.INameManager, @@ -100,25 +101,25 @@ func (r *Reconciler) getStatefulSetStatus(host *api.Host) api.ObjectStatus { host.GetCR().IEnsureStatus().GetHostsAddedCount(), ) - curStatefulSet, err := r.sts.Get(context.TODO(), new) + cur, err := r.sts.Get(context.TODO(), new) switch { - case curStatefulSet != nil: + case cur != nil: r.a.V(1).M(new).Info("Have StatefulSet available, try to perform label-based comparison for sts: %s", util.NamespaceNameString(new)) - return common.GetObjectStatusFromMetas(r.labeler, curStatefulSet, new) + return common.GetObjectStatusFromMetas(r.labeler, cur, new) case apiErrors.IsNotFound(err): // StatefulSet is not found at the moment. // However, it may be just deleted r.a.V(1).M(new).Info("No cur StatefulSet available and the reason is - not found. Either new one or a deleted sts: %s", util.NamespaceNameString(new)) if host.HasAncestor() { - r.a.V(1).M(new).Warning("No cur StatefulSet available but host has an ancestor. Found deleted StatefulSet. for: %s", util.NamespaceNameString(new)) + r.a.V(1).M(new).Warning("No cur StatefulSet available but host has an ancestor. Found deleted sts. for: %s", util.NamespaceNameString(new)) return api.ObjectStatusModified } - r.a.V(1).M(new).Info("No cur StatefulSet available and it is not found and is a new one. New one for: %s", util.NamespaceNameString(new)) + r.a.V(1).M(new).Info("No cur StatefulSet available and it is not found and is a new one. New sts: %s", util.NamespaceNameString(new)) return api.ObjectStatusNew default: - r.a.V(1).M(new).Warning("Have no StatefulSet available, nor it is not found for: %s err: %v", util.NamespaceNameString(new), err) + r.a.V(1).M(new).Warning("Have no StatefulSet available, nor it is not found. sts: %s err: %v", util.NamespaceNameString(new), err) return api.ObjectStatusUnknown } } @@ -146,7 +147,9 @@ func (r *Reconciler) ReconcileStatefulSet( host.GetCR().IEnsureStatus().HostUnchanged() _ = r.cr.StatusUpdate(ctx, host.GetCR(), types.UpdateStatusOptions{ CopyStatusOptions: types.CopyStatusOptions{ - MainFields: true, + CopyStatusFieldGroup: types.CopyStatusFieldGroup{ + FieldGroupMain: true, + }, }, }) } @@ -215,8 +218,8 @@ func (r *Reconciler) updateStatefulSet(ctx context.Context, host *api.Host, regi name := newStatefulSet.Name r.a.V(1). - WithEvent(host.GetCR(), common.EventActionCreate, common.EventReasonCreateStarted). - WithStatusAction(host.GetCR()). + WithEvent(host.GetCR(), a.EventActionCreate, a.EventReasonCreateStarted). + WithAction(host.GetCR()). M(host).F(). Info("Update StatefulSet(%s) - started", util.NamespaceNameString(newStatefulSet)) @@ -236,13 +239,15 @@ func (r *Reconciler) updateStatefulSet(ctx context.Context, host *api.Host, regi host.GetCR().IEnsureStatus().HostUpdated() _ = r.cr.StatusUpdate(ctx, host.GetCR(), types.UpdateStatusOptions{ CopyStatusOptions: types.CopyStatusOptions{ - MainFields: true, + CopyStatusFieldGroup: types.CopyStatusFieldGroup{ + FieldGroupMain: true, + }, }, }) } r.a.V(1). - WithEvent(host.GetCR(), common.EventActionUpdate, common.EventReasonUpdateCompleted). - WithStatusAction(host.GetCR()). + WithEvent(host.GetCR(), a.EventActionUpdate, a.EventReasonUpdateCompleted). + WithAction(host.GetCR()). M(host).F(). Info("Update StatefulSet(%s/%s) - completed", namespace, name) return nil @@ -253,8 +258,8 @@ func (r *Reconciler) updateStatefulSet(ctx context.Context, host *api.Host, regi r.a.V(1).M(host).Info("Update StatefulSet(%s/%s) - got ignore. Ignore", namespace, name) return nil case common.ErrCRUDRecreate: - r.a.WithEvent(host.GetCR(), common.EventActionUpdate, common.EventReasonUpdateInProgress). - WithStatusAction(host.GetCR()). + r.a.WithEvent(host.GetCR(), a.EventActionUpdate, a.EventReasonUpdateInProgress). + WithAction(host.GetCR()). M(host).F(). Info("Update StatefulSet(%s/%s) switch from Update to Recreate", namespace, name) common.DumpStatefulSetDiff(host, curStatefulSet, newStatefulSet) @@ -281,8 +286,8 @@ func (r *Reconciler) createStatefulSet(ctx context.Context, host *api.Host, regi defer r.a.V(2).M(host).E().Info(util.NamespaceNameString(statefulSet.GetObjectMeta())) r.a.V(1). - WithEvent(host.GetCR(), common.EventActionCreate, common.EventReasonCreateStarted). - WithStatusAction(host.GetCR()). + WithEvent(host.GetCR(), a.EventActionCreate, a.EventReasonCreateStarted). + WithAction(host.GetCR()). M(host).F(). Info("Create StatefulSet %s - started", util.NamespaceNameString(statefulSet)) @@ -292,7 +297,9 @@ func (r *Reconciler) createStatefulSet(ctx context.Context, host *api.Host, regi host.GetCR().IEnsureStatus().HostAdded() _ = r.cr.StatusUpdate(ctx, host.GetCR(), types.UpdateStatusOptions{ CopyStatusOptions: types.CopyStatusOptions{ - MainFields: true, + CopyStatusFieldGroup: types.CopyStatusFieldGroup{ + FieldGroupMain: true, + }, }, }) } @@ -300,21 +307,21 @@ func (r *Reconciler) createStatefulSet(ctx context.Context, host *api.Host, regi switch action { case nil: r.a.V(1). - WithEvent(host.GetCR(), common.EventActionCreate, common.EventReasonCreateCompleted). - WithStatusAction(host.GetCR()). + WithEvent(host.GetCR(), a.EventActionCreate, a.EventReasonCreateCompleted). + WithAction(host.GetCR()). M(host).F(). Info("Create StatefulSet: %s - completed", util.NamespaceNameString(statefulSet)) return nil case common.ErrCRUDAbort: - r.a.WithEvent(host.GetCR(), common.EventActionCreate, common.EventReasonCreateFailed). - WithStatusAction(host.GetCR()). - WithStatusError(host.GetCR()). + r.a.WithEvent(host.GetCR(), a.EventActionCreate, a.EventReasonCreateFailed). + WithAction(host.GetCR()). + WithError(host.GetCR()). M(host).F(). Error("Create StatefulSet: %s - failed with error: %v", util.NamespaceNameString(statefulSet), action) return action case common.ErrCRUDIgnore: - r.a.WithEvent(host.GetCR(), common.EventActionCreate, common.EventReasonCreateFailed). - WithStatusAction(host.GetCR()). + r.a.WithEvent(host.GetCR(), a.EventActionCreate, a.EventReasonCreateFailed). + WithAction(host.GetCR()). M(host).F(). Warning("Create StatefulSet: %s - error ignored", util.NamespaceNameString(statefulSet)) return nil @@ -334,7 +341,7 @@ func (r *Reconciler) createStatefulSet(ctx context.Context, host *api.Host, regi func (r *Reconciler) waitForConfigMapPropagation(ctx context.Context, host *api.Host) bool { // No need to wait for ConfigMap propagation on stopped host if host.IsStopped() { - r.a.V(1).M(host).F().Info("No need to wait for ConfigMap propagation - on stopped host") + r.a.V(1).M(host).F().Info("No need to wait for ConfigMap propagation - host is stopped") return false } @@ -347,7 +354,7 @@ func (r *Reconciler) waitForConfigMapPropagation(ctx context.Context, host *api. // What timeout is expected to be enough for ConfigMap propagation? // In case timeout is not specified, no need to wait if !host.GetCR().GetReconciling().HasConfigMapPropagationTimeout() { - r.a.V(1).M(host).F().Info("No need to wait for ConfigMap propagation - not applicable") + r.a.V(1).M(host).F().Info("No need to wait for ConfigMap propagation - not applicable due to missing timeout value") return false } @@ -357,18 +364,19 @@ func (r *Reconciler) waitForConfigMapPropagation(ctx context.Context, host *api. // May be there is no need to wait already elapsed := time.Now().Sub(r.task.CmUpdate()) if elapsed >= timeout { - r.a.V(1).M(host).F().Info("No need to wait for ConfigMap propagation - already elapsed. %s/%s", elapsed, timeout) + r.a.V(1).M(host).F().Info("No need to wait for ConfigMap propagation - already elapsed. [elapsed/timeout: %s/%s]", elapsed, timeout) return false } // Looks like we need to wait for Configmap propagation, after all wait := timeout - elapsed - r.a.V(1).M(host).F().Info("Wait for ConfigMap propagation for %s %s/%s", wait, elapsed, timeout) + r.a.V(1).M(host).F().Info("Going to wait for ConfigMap propagation for: %s [elapsed/timeout: %s/%s]", wait, elapsed, timeout) if util.WaitContextDoneOrTimeout(ctx, wait) { log.V(2).Info("task is done") return true } + r.a.V(1).M(host).F().Info("Wait completed for: %s of timeout: %s]", wait, timeout) return false } diff --git a/pkg/controller/common/storage/storage-pvc.go b/pkg/controller/common/storage/storage-pvc.go index e7a59d19f..acdbb8d3b 100644 --- a/pkg/controller/common/storage/storage-pvc.go +++ b/pkg/controller/common/storage/storage-pvc.go @@ -49,7 +49,7 @@ func (c *PVC) UpdateOrCreate(ctx context.Context, pvc *core.PersistentVolumeClai return nil, fmt.Errorf("task is done") } - _, err := c.Get(ctx, pvc.Namespace, pvc.Name) + oldPvc, err := c.Get(ctx, pvc.Namespace, pvc.Name) if err != nil { if apiErrors.IsNotFound(err) { log.V(1).M(pvc).F().Error("PVC not found, need to create %s", util.NamespacedName(pvc)) @@ -62,6 +62,14 @@ func (c *PVC) UpdateOrCreate(ctx context.Context, pvc *core.PersistentVolumeClai // In case of any non-NotFound API error - unable to proceed log.V(1).M(pvc).F().Error("ERROR unable to get PVC(%s) err: %v", util.NamespacedName(pvc), err) return nil, err + } + + oldStorageRequest := oldPvc.Spec.Resources.Requests[core.ResourceStorage] + newStorageRequest := pvc.Spec.Resources.Requests[core.ResourceStorage] + + if oldStorageRequest.Cmp(newStorageRequest) == 1 { + log.V(1).M(pvc).F().Info("PVC storage was increased externally to greater value and value cannot be decreased, using greater value") + pvc.Spec.Resources.Requests[core.ResourceStorage] = oldStorageRequest } pvcUpdated, err := c.Update(ctx, pvc) diff --git a/pkg/controller/common/task.go b/pkg/controller/common/task.go index f8ab9f1e2..8d37ffaa9 100644 --- a/pkg/controller/common/task.go +++ b/pkg/controller/common/task.go @@ -23,7 +23,8 @@ import ( // task represents context of a worker. This also can be called "a reconcile task" type Task struct { - creator interfaces.ICreator + creatorNew interfaces.ICreator + creatorOld interfaces.ICreator registryReconciled *model.Registry registryFailed *model.Registry cmUpdate time.Time @@ -31,9 +32,10 @@ type Task struct { } // NewTask creates new context -func NewTask(creator interfaces.ICreator) *Task { +func NewTask(creatorNew, creatorOld interfaces.ICreator) *Task { return &Task{ - creator: creator, + creatorNew: creatorNew, + creatorOld: creatorOld, registryReconciled: model.NewRegistry(), registryFailed: model.NewRegistry(), cmUpdate: time.Time{}, @@ -42,7 +44,11 @@ func NewTask(creator interfaces.ICreator) *Task { } func (t *Task) Creator() interfaces.ICreator { - return t.creator + return t.creatorNew +} + +func (t *Task) CreatorPrev() interfaces.ICreator { + return t.creatorOld } func (t *Task) RegistryReconciled() *model.Registry { diff --git a/pkg/interfaces/interfaces-main.go b/pkg/interfaces/interfaces-main.go index 13edc5ead..77592b2ee 100644 --- a/pkg/interfaces/interfaces-main.go +++ b/pkg/interfaces/interfaces-main.go @@ -106,6 +106,7 @@ type ICreator interface { } type IEventEmitter interface { + Event(level string, obj meta.Object, action string, reason string, message string) EventInfo(obj meta.Object, action string, reason string, message string) EventWarning(obj meta.Object, action string, reason string, message string) EventError(obj meta.Object, action string, reason string, message string) diff --git a/pkg/metrics/clickhouse/chi_index.go b/pkg/metrics/clickhouse/chi_index.go index f70209b3f..29904d6db 100644 --- a/pkg/metrics/clickhouse/chi_index.go +++ b/pkg/metrics/clickhouse/chi_index.go @@ -16,17 +16,17 @@ package clickhouse import "github.com/altinity/clickhouse-operator/pkg/apis/metrics" -type chInstallationsIndex map[string]*metrics.WatchedCHI +type chInstallationsIndex map[string]*metrics.WatchedCR -func (i chInstallationsIndex) slice() []*metrics.WatchedCHI { - res := make([]*metrics.WatchedCHI, 0) +func (i chInstallationsIndex) slice() []*metrics.WatchedCR { + res := make([]*metrics.WatchedCR, 0) for _, chi := range i { res = append(res, chi) } return res } -func (i chInstallationsIndex) get(key string) (*metrics.WatchedCHI, bool) { +func (i chInstallationsIndex) get(key string) (*metrics.WatchedCR, bool) { if i == nil { return nil, false } @@ -36,7 +36,7 @@ func (i chInstallationsIndex) get(key string) (*metrics.WatchedCHI, bool) { return nil, false } -func (i chInstallationsIndex) set(key string, value *metrics.WatchedCHI) { +func (i chInstallationsIndex) set(key string, value *metrics.WatchedCR) { if i == nil { return } @@ -52,7 +52,7 @@ func (i chInstallationsIndex) remove(key string) { } } -func (i chInstallationsIndex) walk(f func(*metrics.WatchedCHI, *metrics.WatchedCluster, *metrics.WatchedHost)) { +func (i chInstallationsIndex) walk(f func(*metrics.WatchedCR, *metrics.WatchedCluster, *metrics.WatchedHost)) { // Loop over ClickHouseInstallations for _, chi := range i { chi.WalkHosts(f) diff --git a/pkg/metrics/clickhouse/clickhouse_metrics_fetcher.go b/pkg/metrics/clickhouse/clickhouse_metrics_fetcher.go index cbad28ebc..c3c0ab373 100644 --- a/pkg/metrics/clickhouse/clickhouse_metrics_fetcher.go +++ b/pkg/metrics/clickhouse/clickhouse_metrics_fetcher.go @@ -248,21 +248,11 @@ func (f *ClickHouseMetricsFetcher) getClickHouseQueryDetachedParts(ctx context.C ) } -// ScanFunction defines function to scan rows -type ScanFunction func(rows *sql.Rows, data *Table) error - -// Table defines tables of strings -type Table [][]string - -func newTable() Table { - return make(Table, 0) -} - // clickHouseQueryScanRows scan all rows by external scan function func (f *ClickHouseMetricsFetcher) clickHouseQueryScanRows( ctx context.Context, sql string, - scan ScanFunction, + scanner ScanFunction, ) (Table, error) { if util.IsContextDone(ctx) { return nil, ctx.Err() @@ -277,7 +267,7 @@ func (f *ClickHouseMetricsFetcher) clickHouseQueryScanRows( if util.IsContextDone(ctx) { return nil, ctx.Err() } - _ = scan(query.Rows, &data) + _ = scanner(query.Rows, &data) } return data, nil } diff --git a/pkg/metrics/clickhouse/exporter.go b/pkg/metrics/clickhouse/exporter.go index 6a07d36ca..891e3f1d2 100644 --- a/pkg/metrics/clickhouse/exporter.go +++ b/pkg/metrics/clickhouse/exporter.go @@ -56,13 +56,13 @@ var _ prometheus.Collector = &Exporter{} // NewExporter returns a new instance of Exporter type func NewExporter(collectorTimeout time.Duration) *Exporter { return &Exporter{ - chInstallations: make(map[string]*metrics.WatchedCHI), + chInstallations: make(map[string]*metrics.WatchedCR), collectorTimeout: collectorTimeout, } } // getWatchedCHIs -func (e *Exporter) getWatchedCHIs() []*metrics.WatchedCHI { +func (e *Exporter) getWatchedCHIs() []*metrics.WatchedCR { return e.chInstallations.slice() } @@ -94,9 +94,9 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { log.V(1).Infof("Launching host collectors [%s]", time.Now().Sub(start)) var wg = sync.WaitGroup{} - e.chInstallations.walk(func(chi *metrics.WatchedCHI, _ *metrics.WatchedCluster, host *metrics.WatchedHost) { + e.chInstallations.walk(func(chi *metrics.WatchedCR, _ *metrics.WatchedCluster, host *metrics.WatchedHost) { wg.Add(1) - go func(ctx context.Context, chi *metrics.WatchedCHI, host *metrics.WatchedHost, ch chan<- prometheus.Metric) { + go func(ctx context.Context, chi *metrics.WatchedCR, host *metrics.WatchedHost, ch chan<- prometheus.Metric) { defer wg.Done() e.collectHostMetrics(ctx, chi, host, ch) }(ctx, chi, host, ch) @@ -110,7 +110,7 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { } // enqueueToRemoveFromWatched -func (e *Exporter) enqueueToRemoveFromWatched(chi *metrics.WatchedCHI) { +func (e *Exporter) enqueueToRemoveFromWatched(chi *metrics.WatchedCR) { e.toRemoveFromWatched.Store(chi, struct{}{}) } @@ -120,10 +120,10 @@ func (e *Exporter) cleanup() { log.V(2).Info("Starting cleanup") e.toRemoveFromWatched.Range(func(key, value interface{}) bool { switch key.(type) { - case *metrics.WatchedCHI: + case *metrics.WatchedCR: e.toRemoveFromWatched.Delete(key) - e.removeFromWatched(key.(*metrics.WatchedCHI)) - log.V(1).Infof("Removed ClickHouseInstallation (%s/%s) from Exporter", key.(*metrics.WatchedCHI).Name, key.(*metrics.WatchedCHI).Namespace) + e.removeFromWatched(key.(*metrics.WatchedCR)) + log.V(1).Infof("Removed ClickHouseInstallation (%s/%s) from Exporter", key.(*metrics.WatchedCR).Name, key.(*metrics.WatchedCR).Namespace) } return true }) @@ -131,7 +131,7 @@ func (e *Exporter) cleanup() { } // removeFromWatched deletes record from Exporter.chInstallation map identified by chiName key -func (e *Exporter) removeFromWatched(chi *metrics.WatchedCHI) { +func (e *Exporter) removeFromWatched(chi *metrics.WatchedCR) { e.mutex.Lock() defer e.mutex.Unlock() log.V(1).Infof("Remove ClickHouseInstallation (%s/%s)", chi.Namespace, chi.Name) @@ -139,7 +139,7 @@ func (e *Exporter) removeFromWatched(chi *metrics.WatchedCHI) { } // updateWatched updates Exporter.chInstallation map with values from chInstances slice -func (e *Exporter) updateWatched(chi *metrics.WatchedCHI) { +func (e *Exporter) updateWatched(chi *metrics.WatchedCR) { e.mutex.Lock() defer e.mutex.Unlock() log.V(1).Infof("Update ClickHouseInstallation (%s/%s): %s", chi.Namespace, chi.Name, chi) @@ -171,7 +171,7 @@ func (e *Exporter) newHostFetcher(host *metrics.WatchedHost) *ClickHouseMetricsF } // collectHostMetrics collects metrics from one host and writes them into chan -func (e *Exporter) collectHostMetrics(ctx context.Context, chi *metrics.WatchedCHI, host *metrics.WatchedHost, c chan<- prometheus.Metric) { +func (e *Exporter) collectHostMetrics(ctx context.Context, chi *metrics.WatchedCR, host *metrics.WatchedHost, c chan<- prometheus.Metric) { fetcher := e.newHostFetcher(host) writer := NewCHIPrometheusWriter(c, chi, host) @@ -340,8 +340,8 @@ func (e *Exporter) getWatchedCHI(w http.ResponseWriter, r *http.Request) { } // fetchCHI decodes chi from the request -func (e *Exporter) fetchCHI(r *http.Request) (*metrics.WatchedCHI, error) { - chi := &metrics.WatchedCHI{} +func (e *Exporter) fetchCHI(r *http.Request) (*metrics.WatchedCR, error) { + chi := &metrics.WatchedCR{} if err := json.NewDecoder(r.Body).Decode(chi); err == nil { if chi.IsValid() { return chi, nil @@ -405,7 +405,7 @@ func (e *Exporter) DiscoveryWatchedCHIs(kubeClient kube.Interface, chopClient *c }) normalized, _ := normalizer.CreateTemplated(chi, normalizerCommon.NewOptions()) - watchedCHI := metrics.NewWatchedCHI(normalized) + watchedCHI := metrics.NewWatchedCR(normalized) e.updateWatched(watchedCHI) } } diff --git a/pkg/metrics/clickhouse/prometheus_writer.go b/pkg/metrics/clickhouse/prometheus_writer.go index 723d37049..cc7cad5ed 100644 --- a/pkg/metrics/clickhouse/prometheus_writer.go +++ b/pkg/metrics/clickhouse/prometheus_writer.go @@ -16,8 +16,6 @@ package clickhouse import ( "fmt" - "github.com/altinity/clickhouse-operator/pkg/apis/metrics" - "github.com/altinity/clickhouse-operator/pkg/metrics/operator" "strconv" "time" @@ -25,6 +23,9 @@ import ( // log "k8s.io/klog" "github.com/prometheus/client_golang/prometheus" + "github.com/altinity/clickhouse-operator/pkg/apis/metrics" + "github.com/altinity/clickhouse-operator/pkg/chop" + "github.com/altinity/clickhouse-operator/pkg/metrics/operator" "github.com/altinity/clickhouse-operator/pkg/util" ) @@ -41,14 +42,14 @@ const ( // CHIPrometheusWriter specifies writer to prometheus type CHIPrometheusWriter struct { out chan<- prometheus.Metric - chi *metrics.WatchedCHI + chi *metrics.WatchedCR host *metrics.WatchedHost } // NewCHIPrometheusWriter creates new CHI prometheus writer func NewCHIPrometheusWriter( out chan<- prometheus.Metric, - chi *metrics.WatchedCHI, + chi *metrics.WatchedCR, host *metrics.WatchedHost, ) *CHIPrometheusWriter { return &CHIPrometheusWriter{ @@ -77,10 +78,7 @@ func (w *CHIPrometheusWriter) WriteMetrics(data [][]string) { } else { metricType = prometheus.GaugeValue } - w.writeSingleMetricToPrometheus( - name, desc, - metricType, value, - nil, nil) + w.writeSingleMetricToPrometheus(name, desc, metricType, value, nil) } } @@ -92,28 +90,31 @@ func (w *CHIPrometheusWriter) WriteTableSizes(data [][]string) { if len(metric) < 2 { continue } - labelNames := []string{"database", "table", "active"} - labelValues := []string{metric[0], metric[1], metric[2]} + labels := map[string]string{ + "database": metric[0], + "table": metric[1], + "active": metric[2], + } w.writeSingleMetricToPrometheus( "table_partitions", "Number of partitions of the table", prometheus.GaugeValue, metric[3], - labelNames, labelValues) + labels) w.writeSingleMetricToPrometheus( "table_parts", "Number of parts of the table", prometheus.GaugeValue, metric[4], - labelNames, labelValues) + labels) w.writeSingleMetricToPrometheus( "table_parts_bytes", "Table size in bytes", prometheus.GaugeValue, metric[5], - labelNames, labelValues) + labels) w.writeSingleMetricToPrometheus( "table_parts_bytes_uncompressed", "Table size in bytes uncompressed", prometheus.GaugeValue, metric[6], - labelNames, labelValues) + labels) w.writeSingleMetricToPrometheus( "table_parts_rows", "Number of rows in the table", prometheus.GaugeValue, metric[7], - labelNames, labelValues) + labels) } } @@ -149,90 +150,103 @@ func (w *CHIPrometheusWriter) WriteSystemParts(data [][]string) { // WriteSystemReplicas writes system replicas func (w *CHIPrometheusWriter) WriteSystemReplicas(data [][]string) { for _, metric := range data { - labelNames := []string{"database", "table"} - labelValues := []string{metric[0], metric[1]} + labels := map[string]string{ + "database": metric[0], + "table": metric[1], + } w.writeSingleMetricToPrometheus( "system_replicas_is_session_expired", "Number of expired Zookeeper sessions of the table", prometheus.GaugeValue, metric[2], - labelNames, labelValues) + labels) } } // WriteMutations writes mutations func (w *CHIPrometheusWriter) WriteMutations(data [][]string) { for _, metric := range data { - labelNames := []string{"database", "table"} - labelValues := []string{metric[0], metric[1]} + labels := map[string]string{ + "database": metric[0], + "table": metric[1], + } w.writeSingleMetricToPrometheus( "table_mutations", "Number of active mutations for the table", prometheus.GaugeValue, metric[2], - labelNames, labelValues) + labels) w.writeSingleMetricToPrometheus( "table_mutations_parts_to_do", "Number of data parts that need to be mutated for the mutation to finish", prometheus.GaugeValue, metric[3], - labelNames, labelValues) + labels) } } // WriteSystemDisks writes system disks func (w *CHIPrometheusWriter) WriteSystemDisks(data [][]string) { for _, metric := range data { - labelNames := []string{"disk"} - labelValues := []string{metric[0]} + labels := map[string]string{ + "disk": metric[0], + } w.writeSingleMetricToPrometheus( "metric_DiskFreeBytes", "Free disk space available from system.disks", prometheus.GaugeValue, metric[1], - labelNames, labelValues) + labels) w.writeSingleMetricToPrometheus( "metric_DiskTotalBytes", "Total disk space available from system.disks", prometheus.GaugeValue, metric[2], - labelNames, labelValues) + labels) } } // WriteDetachedParts writes detached parts func (w *CHIPrometheusWriter) WriteDetachedParts(data [][]string) { for _, metric := range data { - labelNames := []string{"database", "table", "disk", "reason"} - labelValues := []string{metric[1], metric[2], metric[3], metric[4]} + labels := map[string]string{ + "database": metric[1], + "table": metric[2], + "disk": metric[3], + "reason": metric[4], + } w.writeSingleMetricToPrometheus( "metric_DetachedParts", "Count of currently detached parts from system.detached_parts", prometheus.GaugeValue, metric[0], - labelNames, labelValues) + labels) } } // WriteErrorFetch writes error fetch func (w *CHIPrometheusWriter) WriteErrorFetch(fetchType string) { - labelNames := []string{"fetch_type"} - labelValues := []string{fetchType} + labels := map[string]string{ + "fetch_type": fetchType, + } w.writeSingleMetricToPrometheus( "metric_fetch_errors", "status of fetching metrics from ClickHouse 1 - unsuccessful, 0 - successful", prometheus.GaugeValue, "1", - labelNames, labelValues) + labels) } // WriteOKFetch writes successful fetch func (w *CHIPrometheusWriter) WriteOKFetch(fetchType string) { - labelNames := []string{"fetch_type"} - labelValues := []string{fetchType} + labels := map[string]string{ + "fetch_type": fetchType, + } w.writeSingleMetricToPrometheus( "metric_fetch_errors", "status of fetching metrics from ClickHouse 1 - unsuccessful, 0 - successful", prometheus.GaugeValue, "0", - labelNames, labelValues) + labels) } -func (w *CHIPrometheusWriter) appendHostLabel(labels, values []string) ([]string, []string) { - return append(labels, "hostname"), append(values, w.host.Hostname) +func (w *CHIPrometheusWriter) appendHostLabel(labels map[string]string) map[string]string { + return util.MergeStringMapsOverwrite(labels, map[string]string{ + "hostname": w.host.Hostname, + }) } -func (w *CHIPrometheusWriter) getMandatoryLabelsAndValues() (labelNames []string, labelValues []string) { - // Prepare mandatory set of labels - labelNames, labelValues = operator.GetMandatoryLabelsAndValues(w.chi) +func (w *CHIPrometheusWriter) getBaseSetLabelsAndValues() map[string]string { + // Prepare set of labels from watched CR + labels := operator.GetLabelsFromSource(w.chi) // Append current host label - labelNames, labelValues = w.appendHostLabel(labelNames, labelValues) + labels = w.appendHostLabel(labels) - return labelNames, labelValues + return labels } func (w *CHIPrometheusWriter) writeSingleMetricToPrometheus( @@ -240,16 +254,13 @@ func (w *CHIPrometheusWriter) writeSingleMetricToPrometheus( desc string, metricType prometheus.ValueType, value string, - optionalLabels []string, - optionalLabelValues []string, + metricLabels map[string]string, ) { - // Prepare mandatory set of labels - labelNames, labelValues := w.getMandatoryLabelsAndValues() - // Append optional labels - labelNames = append(labelNames, optionalLabels...) - labelValues = append(labelValues, optionalLabelValues...) - + // Prepare metrics labels + labelNames, labelValues := w.prepareLabels(metricLabels) + // Prepare metrics value floatValue, _ := strconv.ParseFloat(value, 64) + // Prepare metric from value and labels metric, err := prometheus.NewConstMetric( newMetricDescriptor(name, desc, labelNames), metricType, @@ -268,6 +279,19 @@ func (w *CHIPrometheusWriter) writeSingleMetricToPrometheus( } } +func (w *CHIPrometheusWriter) prepareLabels(extraLabels map[string]string) (labelNames []string, labelValues []string) { + // Prepare base set of labels + // Append particular metric labels + labels := util.MergeStringMapsOverwrite(w.getBaseSetLabelsAndValues(), extraLabels) + // Filter out metrics to be skipped + labels = util.CopyMapFilter( + labels, + nil, + chop.Config().Metrics.Labels.Exclude, + ) + return util.MapGetSortedKeysAndValues(labels) +} + // newMetricDescriptor creates a new prometheus.Desc object func newMetricDescriptor(name, help string, labels []string) *prometheus.Desc { return prometheus.NewDesc( diff --git a/pkg/metrics/clickhouse/rest_client.go b/pkg/metrics/clickhouse/rest_client.go index 4c91d22bb..55510d3a7 100644 --- a/pkg/metrics/clickhouse/rest_client.go +++ b/pkg/metrics/clickhouse/rest_client.go @@ -17,11 +17,11 @@ package clickhouse import "github.com/altinity/clickhouse-operator/pkg/apis/metrics" // InformMetricsExporterAboutWatchedCHI informs exporter about new watched CHI -func InformMetricsExporterAboutWatchedCHI(chi *metrics.WatchedCHI) error { +func InformMetricsExporterAboutWatchedCHI(chi *metrics.WatchedCR) error { return makeRESTCall(chi, "POST") } // InformMetricsExporterToDeleteWatchedCHI informs exporter to delete/forget watched CHI -func InformMetricsExporterToDeleteWatchedCHI(chi *metrics.WatchedCHI) error { +func InformMetricsExporterToDeleteWatchedCHI(chi *metrics.WatchedCR) error { return makeRESTCall(chi, "DELETE") } diff --git a/pkg/metrics/clickhouse/rest_machinery.go b/pkg/metrics/clickhouse/rest_machinery.go index 1eb014433..767a9ce7e 100644 --- a/pkg/metrics/clickhouse/rest_machinery.go +++ b/pkg/metrics/clickhouse/rest_machinery.go @@ -23,7 +23,7 @@ import ( "net/http" ) -func makeRESTCall(chi *metrics.WatchedCHI, method string) error { +func makeRESTCall(chi *metrics.WatchedCR, method string) error { url := "http://127.0.0.1:8888/chi" json, err := json.Marshal(chi) diff --git a/pkg/metrics/clickhouse/types.go b/pkg/metrics/clickhouse/types.go new file mode 100644 index 000000000..8935fd13b --- /dev/null +++ b/pkg/metrics/clickhouse/types.go @@ -0,0 +1,29 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package clickhouse + +import ( + "database/sql" +) + +// ScanFunction defines function to scan rows +type ScanFunction func(rows *sql.Rows, data *Table) error + +// Table defines tables of strings +type Table [][]string + +func newTable() Table { + return make(Table, 0) +} diff --git a/pkg/metrics/operator/interface.go b/pkg/metrics/operator/interface.go new file mode 100644 index 000000000..807b1bc1f --- /dev/null +++ b/pkg/metrics/operator/interface.go @@ -0,0 +1,22 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator + +type labelsSource interface { + GetName() string + GetNamespace() string + GetLabels() map[string]string + GetAnnotations() map[string]string +} diff --git a/pkg/metrics/operator/labels.go b/pkg/metrics/operator/labels.go new file mode 100644 index 000000000..28aff43d0 --- /dev/null +++ b/pkg/metrics/operator/labels.go @@ -0,0 +1,49 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package operator + +import ( + "github.com/altinity/clickhouse-operator/pkg/util" +) + +func GetLabelsFromSource(src labelsSource) (labels map[string]string) { + return util.MergeStringMapsOverwrite( + util.MergeStringMapsOverwrite( + util.MergeStringMapsOverwrite(labels, getLabelsFromName(src)), + getLabelsFromLabels(src), + ), + getLabelsFromAnnotations(src), + ) +} + +func getLabelsFromName(chi labelsSource) map[string]string { + return map[string]string{ + "chi": chi.GetName(), + "namespace": chi.GetNamespace(), + } +} + +func getLabelsFromLabels(chi labelsSource) map[string]string { + return chi.GetLabels() +} + +func getLabelsFromAnnotations(chi labelsSource) map[string]string { + // Exclude skipped annotations + return util.CopyMapFilter( + chi.GetAnnotations(), + nil, + util.ListSkippedAnnotations(), + ) +} diff --git a/pkg/metrics/operator/metrics.go b/pkg/metrics/operator/machinery.go similarity index 72% rename from pkg/metrics/operator/metrics.go rename to pkg/metrics/operator/machinery.go index 84e4b4689..f519d6af9 100644 --- a/pkg/metrics/operator/metrics.go +++ b/pkg/metrics/operator/machinery.go @@ -28,7 +28,6 @@ import ( log "github.com/altinity/clickhouse-operator/pkg/announcer" "github.com/altinity/clickhouse-operator/pkg/apis/deployment" "github.com/altinity/clickhouse-operator/pkg/chop" - "github.com/altinity/clickhouse-operator/pkg/util" "github.com/altinity/clickhouse-operator/pkg/version" ) @@ -105,45 +104,3 @@ func serveMetrics(addr, path string) { } fmt.Printf("end serving metrics at: %s%s\n", addr, path) } - -type BaseInfoGetter interface { - GetName() string - GetNamespace() string - GetLabels() map[string]string - GetAnnotations() map[string]string -} - -func getLabelsFromName(chi BaseInfoGetter) (labels []string, values []string) { - return []string{"chi", "namespace"}, []string{chi.GetName(), chi.GetNamespace()} -} - -func getLabelsFromLabels(chi BaseInfoGetter) (labels []string, values []string) { - return util.MapGetSortedKeysAndValues(chi.GetLabels()) -} - -func getLabelsFromAnnotations(chi BaseInfoGetter) (labels []string, values []string) { - return util.MapGetSortedKeysAndValues( - // Exclude skipped annotations - util.CopyMapFilter( - chi.GetAnnotations(), - nil, - util.ListSkippedAnnotations(), - ), - ) -} - -func GetMandatoryLabelsAndValues(cr BaseInfoGetter) (labels []string, values []string) { - labelsFromNames, valuesFromNames := getLabelsFromName(cr) - labels = append(labels, labelsFromNames...) - values = append(values, valuesFromNames...) - - labelsFromLabels, valuesFromLabels := getLabelsFromLabels(cr) - labels = append(labels, labelsFromLabels...) - values = append(values, valuesFromLabels...) - - labelsFromAnnotations, valuesFromAnnotations := getLabelsFromAnnotations(cr) - labels = append(labels, labelsFromAnnotations...) - values = append(values, valuesFromAnnotations...) - - return labels, values -} diff --git a/pkg/model/chi/config/files_generator.go b/pkg/model/chi/config/files_generator.go index 36fc796fb..bc108a70a 100644 --- a/pkg/model/chi/config/files_generator.go +++ b/pkg/model/chi/config/files_generator.go @@ -16,23 +16,41 @@ package config import ( chi "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" - "github.com/altinity/clickhouse-operator/pkg/chop" "github.com/altinity/clickhouse-operator/pkg/interfaces" "github.com/altinity/clickhouse-operator/pkg/util" ) // FilesGenerator specifies configuration generator object type FilesGenerator struct { - configGenerator *Generator - // clickhouse-operator configuration - chopConfig *chi.OperatorConfig + configGeneratorGeneric IConfigGeneratorGeneric + // paths to additional config files + pathsGetter chi.IOperatorConfigFilesPathsGetter + // configFilesGeneratorDomain files generator + configFilesGeneratorDomain IFilesGeneratorDomain +} + +type IConfigGeneratorGeneric interface { + GetGlobalSettings() string + GetSectionFromFiles(section chi.SettingsSection, includeUnspecified bool, host *chi.Host) map[string]string + GetHostSettings(host *chi.Host) string +} + +type IFilesGeneratorDomain interface { + CreateConfigFilesGroupCommon(configSections map[string]string, options *FilesGeneratorOptions) + CreateConfigFilesGroupUsers(configSections map[string]string) + CreateConfigFilesGroupHost(configSections map[string]string, options *FilesGeneratorOptions) } // NewFilesGenerator creates new configuration files generator object -func NewFilesGenerator(cr chi.ICustomResource, namer interfaces.INameManager, opts *GeneratorOptions) *FilesGenerator { +func NewFilesGenerator( + configGeneratorGeneric IConfigGeneratorGeneric, + pathsGetter chi.IOperatorConfigFilesPathsGetter, + configFilesGeneratorDomain IFilesGeneratorDomain, +) *FilesGenerator { return &FilesGenerator{ - configGenerator: newGenerator(cr, namer, opts), - chopConfig: chop.Config(), + configGeneratorGeneric: configGeneratorGeneric, + pathsGetter: pathsGetter, + configFilesGeneratorDomain: configFilesGeneratorDomain, } } @@ -71,16 +89,16 @@ func (c *FilesGenerator) createConfigFilesGroupCommon(options *FilesGeneratorOpt } func (c *FilesGenerator) createConfigFilesGroupCommonDomain(configSections map[string]string, options *FilesGeneratorOptions) { - util.IncludeNonEmpty(configSections, createConfigSectionFilename(configRemoteServers), c.configGenerator.getRemoteServers(options.GetRemoteServersOptions())) + c.configFilesGeneratorDomain.CreateConfigFilesGroupCommon(configSections, options) } func (c *FilesGenerator) createConfigFilesGroupCommonGeneric(configSections map[string]string, options *FilesGeneratorOptions) { // common settings - util.IncludeNonEmpty(configSections, createConfigSectionFilename(configSettings), c.configGenerator.getGlobalSettings()) + util.IncludeNonEmpty(configSections, createConfigSectionFilename(configSettings), c.configGeneratorGeneric.GetGlobalSettings()) // common files - util.MergeStringMapsOverwrite(configSections, c.configGenerator.getSectionFromFiles(chi.SectionCommon, true, nil)) + util.MergeStringMapsOverwrite(configSections, c.configGeneratorGeneric.GetSectionFromFiles(chi.SectionCommon, true, nil)) // Extra user-specified config files - util.MergeStringMapsOverwrite(configSections, c.chopConfig.ClickHouse.Config.File.Runtime.CommonConfigFiles) + util.MergeStringMapsOverwrite(configSections, c.pathsGetter.GetCommonConfigFiles()) } // createConfigFilesGroupUsers creates users config files @@ -95,19 +113,14 @@ func (c *FilesGenerator) createConfigFilesGroupUsers() map[string]string { } func (c *FilesGenerator) createConfigFilesGroupUsersDomain(configSections map[string]string) { - // users - util.IncludeNonEmpty(configSections, createConfigSectionFilename(configUsers), c.configGenerator.getUsers()) - // quotas - util.IncludeNonEmpty(configSections, createConfigSectionFilename(configQuotas), c.configGenerator.getQuotas()) - // profiles - util.IncludeNonEmpty(configSections, createConfigSectionFilename(configProfiles), c.configGenerator.getProfiles()) + c.configFilesGeneratorDomain.CreateConfigFilesGroupUsers(configSections) } func (c *FilesGenerator) createConfigFilesGroupUsersGeneric(configSections map[string]string) { // user files - util.MergeStringMapsOverwrite(configSections, c.configGenerator.getSectionFromFiles(chi.SectionUsers, false, nil)) + util.MergeStringMapsOverwrite(configSections, c.configGeneratorGeneric.GetSectionFromFiles(chi.SectionUsers, false, nil)) // Extra user-specified config files - util.MergeStringMapsOverwrite(configSections, c.chopConfig.ClickHouse.Config.File.Runtime.UsersConfigFiles) + util.MergeStringMapsOverwrite(configSections, c.pathsGetter.GetUsersConfigFiles()) } // createConfigFilesGroupHost creates host config files @@ -122,16 +135,14 @@ func (c *FilesGenerator) createConfigFilesGroupHost(options *FilesGeneratorOptio } func (c *FilesGenerator) createConfigFilesGroupHostDomain(configSections map[string]string, options *FilesGeneratorOptions) { - util.IncludeNonEmpty(configSections, createConfigSectionFilename(configMacros), c.configGenerator.getHostMacros(options.GetHost())) - util.IncludeNonEmpty(configSections, createConfigSectionFilename(configHostnamePorts), c.configGenerator.getHostHostnameAndPorts(options.GetHost())) - util.IncludeNonEmpty(configSections, createConfigSectionFilename(configZookeeper), c.configGenerator.getHostZookeeper(options.GetHost())) + c.configFilesGeneratorDomain.CreateConfigFilesGroupHost(configSections, options) } func (c *FilesGenerator) createConfigFilesGroupHostGeneric(configSections map[string]string, options *FilesGeneratorOptions) { - util.IncludeNonEmpty(configSections, createConfigSectionFilename(configSettings), c.configGenerator.getHostSettings(options.GetHost())) - util.MergeStringMapsOverwrite(configSections, c.configGenerator.getSectionFromFiles(chi.SectionHost, true, options.GetHost())) + util.IncludeNonEmpty(configSections, createConfigSectionFilename(configSettings), c.configGeneratorGeneric.GetHostSettings(options.GetHost())) + util.MergeStringMapsOverwrite(configSections, c.configGeneratorGeneric.GetSectionFromFiles(chi.SectionHost, true, options.GetHost())) // Extra user-specified config files - util.MergeStringMapsOverwrite(configSections, c.chopConfig.ClickHouse.Config.File.Runtime.HostConfigFiles) + util.MergeStringMapsOverwrite(configSections, c.pathsGetter.GetHostConfigFiles()) } // createConfigSectionFilename creates filename of a configuration file. diff --git a/pkg/model/chi/config/files_generator_domain.go b/pkg/model/chi/config/files_generator_domain.go new file mode 100644 index 000000000..1966ef80c --- /dev/null +++ b/pkg/model/chi/config/files_generator_domain.go @@ -0,0 +1,48 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import "github.com/altinity/clickhouse-operator/pkg/util" + +// FilesGenerator specifies configuration generator object +type FilesGeneratorDomain struct { + configGenerator *Generator +} + +// NewFilesGenerator creates new configuration files generator object +func NewFilesGeneratorDomain(configGenerator *Generator) *FilesGeneratorDomain { + return &FilesGeneratorDomain{ + configGenerator: configGenerator, + } +} + +func (c *FilesGeneratorDomain) CreateConfigFilesGroupCommon(configSections map[string]string, options *FilesGeneratorOptions) { + util.IncludeNonEmpty(configSections, createConfigSectionFilename(configRemoteServers), c.configGenerator.getRemoteServers(options.GetRemoteServersOptions())) +} + +func (c *FilesGeneratorDomain) CreateConfigFilesGroupUsers(configSections map[string]string) { + // users + util.IncludeNonEmpty(configSections, createConfigSectionFilename(configUsers), c.configGenerator.getUsers()) + // quotas + util.IncludeNonEmpty(configSections, createConfigSectionFilename(configQuotas), c.configGenerator.getQuotas()) + // profiles + util.IncludeNonEmpty(configSections, createConfigSectionFilename(configProfiles), c.configGenerator.getProfiles()) +} + +func (c *FilesGeneratorDomain) CreateConfigFilesGroupHost(configSections map[string]string, options *FilesGeneratorOptions) { + util.IncludeNonEmpty(configSections, createConfigSectionFilename(configMacros), c.configGenerator.getHostMacros(options.GetHost())) + util.IncludeNonEmpty(configSections, createConfigSectionFilename(configHostnamePorts), c.configGenerator.getHostHostnameAndPorts(options.GetHost())) + util.IncludeNonEmpty(configSections, createConfigSectionFilename(configZookeeper), c.configGenerator.getHostZookeeper(options.GetHost())) +} diff --git a/pkg/model/chi/config/generator.go b/pkg/model/chi/config/generator.go index acd4b7ae3..01cf41172 100644 --- a/pkg/model/chi/config/generator.go +++ b/pkg/model/chi/config/generator.go @@ -17,11 +17,11 @@ package config import ( "bytes" "fmt" - "github.com/altinity/clickhouse-operator/pkg/model/common/config" log "github.com/altinity/clickhouse-operator/pkg/announcer" chi "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" "github.com/altinity/clickhouse-operator/pkg/interfaces" + "github.com/altinity/clickhouse-operator/pkg/model/common/config" "github.com/altinity/clickhouse-operator/pkg/util" ) @@ -50,8 +50,8 @@ type Generator struct { opts *GeneratorOptions } -// newGenerator returns new Generator struct -func newGenerator(cr chi.ICustomResource, namer interfaces.INameManager, opts *GeneratorOptions) *Generator { +// NewGenerator returns new Generator struct +func NewGenerator(cr chi.ICustomResource, namer interfaces.INameManager, opts *GeneratorOptions) *Generator { return &Generator{ cr: cr, namer: namer, @@ -59,20 +59,20 @@ func newGenerator(cr chi.ICustomResource, namer interfaces.INameManager, opts *G } } -// getGlobalSettings creates data for global section of "settings.xml" -func (c *Generator) getGlobalSettings() string { +// GetGlobalSettings creates data for global section of "settings.xml" +func (c *Generator) GetGlobalSettings() string { // No host specified means request to generate common config return c.opts.Settings.ClickHouseConfig() } -// getHostSettings creates data for host section of "settings.xml" -func (c *Generator) getHostSettings(host *chi.Host) string { +// GetHostSettings creates data for host section of "settings.xml" +func (c *Generator) GetHostSettings(host *chi.Host) string { // Generate config for the specified host return host.Settings.ClickHouseConfig() } -// getSectionFromFiles creates data for custom common config files -func (c *Generator) getSectionFromFiles(section chi.SettingsSection, includeUnspecified bool, host *chi.Host) map[string]string { +// GetSectionFromFiles creates data for custom common config files +func (c *Generator) GetSectionFromFiles(section chi.SettingsSection, includeUnspecified bool, host *chi.Host) map[string]string { var files *chi.Settings if host == nil { // We are looking into Common files diff --git a/pkg/model/chi/creator/service.go b/pkg/model/chi/creator/service.go index 2cf7302be..a3e66ca80 100644 --- a/pkg/model/chi/creator/service.go +++ b/pkg/model/chi/creator/service.go @@ -87,6 +87,9 @@ func (m *ServiceManager) SetTagger(tagger interfaces.ITagger) { // createServiceCR creates new core.Service for specified CR func (m *ServiceManager) createServiceCR() *core.Service { + if m.cr.IsZero() { + return nil + } if template, ok := m.cr.GetRootServiceTemplate(); ok { // .templates.ServiceTemplate specified return creator.CreateServiceFromTemplate( @@ -139,6 +142,10 @@ func (m *ServiceManager) createServiceCR() *core.Service { // createServiceCluster creates new core.Service for specified Cluster func (m *ServiceManager) createServiceCluster(cluster chi.ICluster) *core.Service { + if cluster.IsZero() { + return nil + } + serviceName := m.namer.Name(interfaces.NameClusterService, cluster) ownerReferences := m.or.CreateOwnerReferences(m.cr) @@ -162,6 +169,9 @@ func (m *ServiceManager) createServiceCluster(cluster chi.ICluster) *core.Servic // createServiceShard creates new core.Service for specified Shard func (m *ServiceManager) createServiceShard(shard chi.IShard) *core.Service { + if shard.IsZero() { + return nil + } if template, ok := shard.GetServiceTemplate(); ok { // .templates.ServiceTemplate specified return creator.CreateServiceFromTemplate( @@ -182,6 +192,9 @@ func (m *ServiceManager) createServiceShard(shard chi.IShard) *core.Service { // createServiceHost creates new core.Service for specified host func (m *ServiceManager) createServiceHost(host *chi.Host) *core.Service { + if host.IsZero() { + return nil + } if template, ok := host.GetServiceTemplate(); ok { // .templates.ServiceTemplate specified return creator.CreateServiceFromTemplate( diff --git a/pkg/model/chi/normalizer/normalizer.go b/pkg/model/chi/normalizer/normalizer.go index 229537610..4e0ef1dea 100644 --- a/pkg/model/chi/normalizer/normalizer.go +++ b/pkg/model/chi/normalizer/normalizer.go @@ -94,9 +94,8 @@ func (n *Normalizer) buildTargetFromTemplates(subj *chi.ClickHouseInstallation) } func (n *Normalizer) applyCRTemplatesOnTarget(subj crTemplatesNormalizer.TemplateSubject) { - for _, template := range crTemplatesNormalizer.ApplyTemplates(n.req.GetTarget(), subj) { - n.req.GetTarget().EnsureStatus().PushUsedTemplate(template) - } + usedTemplates := crTemplatesNormalizer.ApplyTemplates(n.req.GetTarget(), subj) + n.req.GetTarget().EnsureStatus().PushUsedTemplate(usedTemplates...) } func (n *Normalizer) newSubject() *chi.ClickHouseInstallation { diff --git a/pkg/model/chi/schemer/schemer.go b/pkg/model/chi/schemer/schemer.go index 3902b025a..7f133ccbb 100644 --- a/pkg/model/chi/schemer/schemer.go +++ b/pkg/model/chi/schemer/schemer.go @@ -143,14 +143,6 @@ func (s *ClusterSchemer) IsHostInCluster(ctx context.Context, host *api.Host) bo return inside } -// CHIDropDnsCache runs 'DROP DNS CACHE' over the whole CHI -func (s *ClusterSchemer) CHIDropDnsCache(ctx context.Context, chi *api.ClickHouseInstallation) error { - chi.WalkHosts(func(host *api.Host) error { - return s.ExecHost(ctx, host, []string{s.sqlDropDNSCache()}) - }) - return nil -} - // HostActiveQueriesNum returns how many active queries are on the host func (s *ClusterSchemer) HostActiveQueriesNum(ctx context.Context, host *api.Host) (int, error) { return s.QueryHostInt(ctx, host, s.sqlActiveQueriesNum()) diff --git a/pkg/model/chi/schemer/sql.go b/pkg/model/chi/schemer/sql.go index ff8172525..4387ff872 100644 --- a/pkg/model/chi/schemer/sql.go +++ b/pkg/model/chi/schemer/sql.go @@ -238,10 +238,6 @@ func (s *ClusterSchemer) sqlDropReplica(shard int, replica string) []string { } } -func (s *ClusterSchemer) sqlDropDNSCache() string { - return `SYSTEM DROP DNS CACHE` -} - func (s *ClusterSchemer) sqlActiveQueriesNum() string { return `SELECT count() FROM system.processes` } diff --git a/pkg/model/chi/volume/volume.go b/pkg/model/chi/volume/volume.go index 7f0710ad4..832372e27 100644 --- a/pkg/model/chi/volume/volume.go +++ b/pkg/model/chi/volume/volume.go @@ -82,7 +82,7 @@ func (m *Manager) stsSetupVolumesUserDataWithFixedPaths(statefulSet *apps.Statef // Mount all named (data and log so far) VolumeClaimTemplates into all containers k8s.StatefulSetAppendVolumeMountsInAllContainers( statefulSet, - k8s.CreateVolumeMount(host.Templates.GetDataVolumeClaimTemplate(), config.DirPathDataStorage), - k8s.CreateVolumeMount(host.Templates.GetLogVolumeClaimTemplate(), config.DirPathLogStorage), + k8s.CreateVolumeMount(host.GetTemplates().GetDataVolumeClaimTemplate(), config.DirPathDataStorage), + k8s.CreateVolumeMount(host.GetTemplates().GetLogVolumeClaimTemplate(), config.DirPathLogStorage), ) } diff --git a/pkg/model/chk/config/files_generator.go b/pkg/model/chk/config/files_generator.go index 98a280d49..bc108a70a 100644 --- a/pkg/model/chk/config/files_generator.go +++ b/pkg/model/chk/config/files_generator.go @@ -16,23 +16,41 @@ package config import ( chi "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" - "github.com/altinity/clickhouse-operator/pkg/chop" "github.com/altinity/clickhouse-operator/pkg/interfaces" "github.com/altinity/clickhouse-operator/pkg/util" ) // FilesGenerator specifies configuration generator object type FilesGenerator struct { - configGenerator *Generator - // clickhouse-operator configuration - chopConfig *chi.OperatorConfig + configGeneratorGeneric IConfigGeneratorGeneric + // paths to additional config files + pathsGetter chi.IOperatorConfigFilesPathsGetter + // configFilesGeneratorDomain files generator + configFilesGeneratorDomain IFilesGeneratorDomain +} + +type IConfigGeneratorGeneric interface { + GetGlobalSettings() string + GetSectionFromFiles(section chi.SettingsSection, includeUnspecified bool, host *chi.Host) map[string]string + GetHostSettings(host *chi.Host) string +} + +type IFilesGeneratorDomain interface { + CreateConfigFilesGroupCommon(configSections map[string]string, options *FilesGeneratorOptions) + CreateConfigFilesGroupUsers(configSections map[string]string) + CreateConfigFilesGroupHost(configSections map[string]string, options *FilesGeneratorOptions) } // NewFilesGenerator creates new configuration files generator object -func NewFilesGenerator(cr chi.ICustomResource, namer interfaces.INameManager, opts *GeneratorOptions) *FilesGenerator { +func NewFilesGenerator( + configGeneratorGeneric IConfigGeneratorGeneric, + pathsGetter chi.IOperatorConfigFilesPathsGetter, + configFilesGeneratorDomain IFilesGeneratorDomain, +) *FilesGenerator { return &FilesGenerator{ - configGenerator: newGenerator(cr, namer, opts), - chopConfig: chop.Config(), + configGeneratorGeneric: configGeneratorGeneric, + pathsGetter: pathsGetter, + configFilesGeneratorDomain: configFilesGeneratorDomain, } } @@ -71,16 +89,16 @@ func (c *FilesGenerator) createConfigFilesGroupCommon(options *FilesGeneratorOpt } func (c *FilesGenerator) createConfigFilesGroupCommonDomain(configSections map[string]string, options *FilesGeneratorOptions) { - util.IncludeNonEmpty(configSections, createConfigSectionFilename(configRaft), c.configGenerator.getRaftConfig(options.GetRaftOptions())) + c.configFilesGeneratorDomain.CreateConfigFilesGroupCommon(configSections, options) } func (c *FilesGenerator) createConfigFilesGroupCommonGeneric(configSections map[string]string, options *FilesGeneratorOptions) { // common settings - util.IncludeNonEmpty(configSections, createConfigSectionFilename(configSettings), c.configGenerator.getGlobalSettings()) + util.IncludeNonEmpty(configSections, createConfigSectionFilename(configSettings), c.configGeneratorGeneric.GetGlobalSettings()) // common files - util.MergeStringMapsOverwrite(configSections, c.configGenerator.getSectionFromFiles(chi.SectionCommon, true, nil)) + util.MergeStringMapsOverwrite(configSections, c.configGeneratorGeneric.GetSectionFromFiles(chi.SectionCommon, true, nil)) // Extra user-specified config files - util.MergeStringMapsOverwrite(configSections, c.chopConfig.Keeper.Config.File.Runtime.CommonConfigFiles) + util.MergeStringMapsOverwrite(configSections, c.pathsGetter.GetCommonConfigFiles()) } // createConfigFilesGroupUsers creates users config files @@ -95,13 +113,14 @@ func (c *FilesGenerator) createConfigFilesGroupUsers() map[string]string { } func (c *FilesGenerator) createConfigFilesGroupUsersDomain(configSections map[string]string) { + c.configFilesGeneratorDomain.CreateConfigFilesGroupUsers(configSections) } func (c *FilesGenerator) createConfigFilesGroupUsersGeneric(configSections map[string]string) { // user files - util.MergeStringMapsOverwrite(configSections, c.configGenerator.getSectionFromFiles(chi.SectionUsers, false, nil)) + util.MergeStringMapsOverwrite(configSections, c.configGeneratorGeneric.GetSectionFromFiles(chi.SectionUsers, false, nil)) // Extra user-specified config files - util.MergeStringMapsOverwrite(configSections, c.chopConfig.Keeper.Config.File.Runtime.UsersConfigFiles) + util.MergeStringMapsOverwrite(configSections, c.pathsGetter.GetUsersConfigFiles()) } // createConfigFilesGroupHost creates host config files @@ -116,14 +135,14 @@ func (c *FilesGenerator) createConfigFilesGroupHost(options *FilesGeneratorOptio } func (c *FilesGenerator) createConfigFilesGroupHostDomain(configSections map[string]string, options *FilesGeneratorOptions) { - util.IncludeNonEmpty(configSections, createConfigSectionFilename(configServerId), c.configGenerator.getHostServerId(options.GetHost())) + c.configFilesGeneratorDomain.CreateConfigFilesGroupHost(configSections, options) } func (c *FilesGenerator) createConfigFilesGroupHostGeneric(configSections map[string]string, options *FilesGeneratorOptions) { - util.IncludeNonEmpty(configSections, createConfigSectionFilename(configSettings), c.configGenerator.getHostSettings(options.GetHost())) - util.MergeStringMapsOverwrite(configSections, c.configGenerator.getSectionFromFiles(chi.SectionHost, true, options.GetHost())) + util.IncludeNonEmpty(configSections, createConfigSectionFilename(configSettings), c.configGeneratorGeneric.GetHostSettings(options.GetHost())) + util.MergeStringMapsOverwrite(configSections, c.configGeneratorGeneric.GetSectionFromFiles(chi.SectionHost, true, options.GetHost())) // Extra user-specified config files - util.MergeStringMapsOverwrite(configSections, c.chopConfig.Keeper.Config.File.Runtime.HostConfigFiles) + util.MergeStringMapsOverwrite(configSections, c.pathsGetter.GetHostConfigFiles()) } // createConfigSectionFilename creates filename of a configuration file. diff --git a/pkg/model/chk/config/files_generator_domain.go b/pkg/model/chk/config/files_generator_domain.go new file mode 100644 index 000000000..b5ee397bc --- /dev/null +++ b/pkg/model/chk/config/files_generator_domain.go @@ -0,0 +1,40 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import "github.com/altinity/clickhouse-operator/pkg/util" + +// FilesGenerator specifies configuration generator object +type FilesGeneratorDomain struct { + configGenerator *Generator +} + +// NewFilesGenerator creates new configuration files generator object +func NewFilesGeneratorDomain(configGenerator *Generator) *FilesGeneratorDomain { + return &FilesGeneratorDomain{ + configGenerator: configGenerator, + } +} + +func (c *FilesGeneratorDomain) CreateConfigFilesGroupCommon(configSections map[string]string, options *FilesGeneratorOptions) { + util.IncludeNonEmpty(configSections, createConfigSectionFilename(configRaft), c.configGenerator.getRaftConfig(options.GetRaftOptions())) +} + +func (c *FilesGeneratorDomain) CreateConfigFilesGroupUsers(configSections map[string]string) { +} + +func (c *FilesGeneratorDomain) CreateConfigFilesGroupHost(configSections map[string]string, options *FilesGeneratorOptions) { + util.IncludeNonEmpty(configSections, createConfigSectionFilename(configServerId), c.configGenerator.getHostServerId(options.GetHost())) +} diff --git a/pkg/model/chk/config/generator.go b/pkg/model/chk/config/generator.go index 85f7eee87..ed5c13231 100644 --- a/pkg/model/chk/config/generator.go +++ b/pkg/model/chk/config/generator.go @@ -17,6 +17,7 @@ package config import ( "bytes" "fmt" + log "github.com/altinity/clickhouse-operator/pkg/announcer" chi "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" "github.com/altinity/clickhouse-operator/pkg/interfaces" @@ -33,8 +34,8 @@ type Generator struct { opts *GeneratorOptions } -// newGenerator returns new Generator struct -func newGenerator(cr chi.ICustomResource, namer interfaces.INameManager, opts *GeneratorOptions) *Generator { +// NewGenerator returns new Generator struct +func NewGenerator(cr chi.ICustomResource, namer interfaces.INameManager, opts *GeneratorOptions) *Generator { return &Generator{ cr: cr, namer: namer, @@ -42,20 +43,20 @@ func newGenerator(cr chi.ICustomResource, namer interfaces.INameManager, opts *G } } -// getGlobalSettings creates data for global section of "settings.xml" -func (c *Generator) getGlobalSettings() string { +// GetGlobalSettings creates data for global section of "settings.xml" +func (c *Generator) GetGlobalSettings() string { // No host specified means request to generate common config return c.opts.Settings.ClickHouseConfig() } -// getHostSettings creates data for host section of "settings.xml" -func (c *Generator) getHostSettings(host *chi.Host) string { +// GetHostSettings creates data for host section of "settings.xml" +func (c *Generator) GetHostSettings(host *chi.Host) string { // Generate config for the specified host return host.Settings.ClickHouseConfig() } -// getSectionFromFiles creates data for custom common config files -func (c *Generator) getSectionFromFiles(section chi.SettingsSection, includeUnspecified bool, host *chi.Host) map[string]string { +// GetSectionFromFiles creates data for custom common config files +func (c *Generator) GetSectionFromFiles(section chi.SettingsSection, includeUnspecified bool, host *chi.Host) map[string]string { var files *chi.Settings if host == nil { // We are looking into Common files diff --git a/pkg/model/chk/creator/probe.go b/pkg/model/chk/creator/probe.go index 5fd7040a2..3b63cee5b 100644 --- a/pkg/model/chk/creator/probe.go +++ b/pkg/model/chk/creator/probe.go @@ -36,7 +36,8 @@ func (m *ProbeManager) CreateProbe(what interfaces.ProbeType, host *api.Host) *c case interfaces.ProbeDefaultLiveness: return m.createDefaultLivenessProbe(host) case interfaces.ProbeDefaultReadiness: - return m.createDefaultReadinessProbe(host) + return nil + //return m.createDefaultReadinessProbe(host) } panic("unknown probe type") } @@ -53,9 +54,9 @@ func (m *ProbeManager) createDefaultLivenessProbe(host *api.Host) *core.Probe { }, }, }, - InitialDelaySeconds: 60, - PeriodSeconds: 3, - FailureThreshold: 10, + InitialDelaySeconds: 5, + PeriodSeconds: 5, + FailureThreshold: 12, } } @@ -77,7 +78,8 @@ func (m *ProbeManager) createDefaultReadinessProbe(host *api.Host) *core.Probe { Port: intstr.Parse("9182"), }, }, - InitialDelaySeconds: 10, - PeriodSeconds: 3, + InitialDelaySeconds: 5, + PeriodSeconds: 5, + FailureThreshold: 12, } } diff --git a/pkg/model/chk/creator/service.go b/pkg/model/chk/creator/service.go index 6ab70a0e8..997692b8b 100644 --- a/pkg/model/chk/creator/service.go +++ b/pkg/model/chk/creator/service.go @@ -61,6 +61,12 @@ func (m *ServiceManager) CreateService(what interfaces.ServiceType, params ...an cluster = params[0].(chi.ICluster) return m.createServiceCluster(cluster) } + case interfaces.ServiceShard: + var shard chi.IShard + if len(params) > 0 { + shard = params[0].(chi.IShard) + return m.createServiceShard(shard) + } case interfaces.ServiceHost: var host *chi.Host if len(params) > 0 { @@ -81,6 +87,9 @@ func (m *ServiceManager) SetTagger(tagger interfaces.ITagger) { // createServiceCR creates new core.Service for specified CR func (m *ServiceManager) createServiceCR() *core.Service { + if m.cr.IsZero() { + return nil + } if template, ok := m.cr.GetRootServiceTemplate(); ok { // .templates.ServiceTemplate specified return creator.CreateServiceFromTemplate( @@ -133,6 +142,10 @@ func (m *ServiceManager) createServiceCR() *core.Service { // createServiceCluster creates new core.Service for specified Cluster func (m *ServiceManager) createServiceCluster(cluster chi.ICluster) *core.Service { + if cluster.IsZero() { + return nil + } + serviceName := m.namer.Name(interfaces.NameClusterService, cluster) ownerReferences := m.or.CreateOwnerReferences(m.cr) @@ -154,8 +167,34 @@ func (m *ServiceManager) createServiceCluster(cluster chi.ICluster) *core.Servic return nil } +// createServiceShard creates new core.Service for specified Shard +func (m *ServiceManager) createServiceShard(shard chi.IShard) *core.Service { + if shard.IsZero() { + return nil + } + if template, ok := shard.GetServiceTemplate(); ok { + // .templates.ServiceTemplate specified + return creator.CreateServiceFromTemplate( + template, + shard.GetRuntime().GetAddress().GetNamespace(), + m.namer.Name(interfaces.NameShardService, shard), + m.tagger.Label(interfaces.LabelServiceShard, shard), + m.tagger.Annotate(interfaces.AnnotateServiceShard, shard), + m.tagger.Selector(interfaces.SelectorShardScopeReady, shard), + m.or.CreateOwnerReferences(m.cr), + m.macro.Scope(shard), + m.labeler, + ) + } + // No template specified, no need to create service + return nil +} + // createServiceHost creates new core.Service for specified host func (m *ServiceManager) createServiceHost(host *chi.Host) *core.Service { + if host.IsZero() { + return nil + } if template, ok := host.GetServiceTemplate(); ok { // .templates.ServiceTemplate specified return creator.CreateServiceFromTemplate( diff --git a/pkg/model/chk/volume/volume.go b/pkg/model/chk/volume/volume.go index 23354c77e..73342b8aa 100644 --- a/pkg/model/chk/volume/volume.go +++ b/pkg/model/chk/volume/volume.go @@ -82,7 +82,7 @@ func (m *Manager) stsSetupVolumesUserDataWithFixedPaths(statefulSet *apps.Statef // Mount all named (data and log so far) VolumeClaimTemplates into all containers k8s.StatefulSetAppendVolumeMountsInAllContainers( statefulSet, - k8s.CreateVolumeMount(host.Templates.GetDataVolumeClaimTemplate(), config.DirPathDataStorage), - k8s.CreateVolumeMount(host.Templates.GetLogVolumeClaimTemplate(), config.DirPathLogStorage), + k8s.CreateVolumeMount(host.GetTemplates().GetDataVolumeClaimTemplate(), config.DirPathDataStorage), + k8s.CreateVolumeMount(host.GetTemplates().GetLogVolumeClaimTemplate(), config.DirPathLogStorage), ) } diff --git a/pkg/model/common/creator/stateful-set-application.go b/pkg/model/common/creator/stateful-set-application.go index ef1533114..d3b81207e 100644 --- a/pkg/model/common/creator/stateful-set-application.go +++ b/pkg/model/common/creator/stateful-set-application.go @@ -156,7 +156,7 @@ func (c *Creator) stsAppContainerSetupTroubleshootingMode(statefulSet *apps.Stat // stsSetupLogContainer func (c *Creator) stsSetupLogContainer(statefulSet *apps.StatefulSet, host *api.Host) { // In case we have default LogVolumeClaimTemplate specified - need to append log container to Pod Template - if host.Templates.HasLogVolumeClaimTemplate() { + if host.GetTemplates().HasLogVolumeClaimTemplate() { c.stsEnsureLogContainerSpecified(statefulSet) c.a.V(1).F().Info("add log container for host: %s", host.Runtime.Address.HostName) } diff --git a/pkg/model/managers/files_generator.go b/pkg/model/managers/files_generator.go index 30d685c6b..20871ad39 100644 --- a/pkg/model/managers/files_generator.go +++ b/pkg/model/managers/files_generator.go @@ -16,6 +16,7 @@ package managers import ( api "github.com/altinity/clickhouse-operator/pkg/apis/clickhouse.altinity.com/v1" + "github.com/altinity/clickhouse-operator/pkg/chop" "github.com/altinity/clickhouse-operator/pkg/interfaces" chiConfig "github.com/altinity/clickhouse-operator/pkg/model/chi/config" chkConfig "github.com/altinity/clickhouse-operator/pkg/model/chk/config" @@ -31,9 +32,13 @@ const ( func NewConfigFilesGenerator(what FilesGeneratorType, cr api.ICustomResource, opts any) interfaces.IConfigFilesGenerator { switch what { case FilesGeneratorTypeClickHouse: - return chiConfig.NewFilesGenerator(cr, NewNameManager(NameManagerTypeClickHouse), opts.(*chiConfig.GeneratorOptions)) + gen := chiConfig.NewGenerator(cr, NewNameManager(NameManagerTypeClickHouse), opts.(*chiConfig.GeneratorOptions)) + genDomain := chiConfig.NewFilesGeneratorDomain(gen) + return chiConfig.NewFilesGenerator(gen, chop.Config().ClickHouse.Config.File.Runtime, genDomain) case FilesGeneratorTypeKeeper: - return chkConfig.NewFilesGenerator(cr, NewNameManager(NameManagerTypeKeeper), opts.(*chkConfig.GeneratorOptions)) + gen := chkConfig.NewGenerator(cr, NewNameManager(NameManagerTypeKeeper), opts.(*chkConfig.GeneratorOptions)) + genDomain := chkConfig.NewFilesGeneratorDomain(gen) + return chkConfig.NewFilesGenerator(gen, chop.Config().Keeper.Config.File.Runtime, genDomain) } panic("unknown config files generator type") } diff --git a/pkg/model/zookeeper/connection.go b/pkg/model/zookeeper/connection.go index d49127e08..f309db088 100644 --- a/pkg/model/zookeeper/connection.go +++ b/pkg/model/zookeeper/connection.go @@ -117,7 +117,8 @@ func (c *Connection) retry(ctx context.Context, fn func(*zk.Conn) error) error { for i := 0; i < c.MaxRetriesNum; i++ { if i > 0 { - time.Sleep(1*time.Second + time.Duration(rand.Int63n(int64(1*time.Second)))) + // Progressive delay before each retry + time.Sleep(time.Duration(i)*time.Second + time.Duration(rand.Int63n(int64(1*time.Second)))) } connection, err := c.ensureConnection(ctx) diff --git a/pkg/model/zookeeper/connection_params.go b/pkg/model/zookeeper/connection_params.go index 9d8fd3839..ea2d581a0 100644 --- a/pkg/model/zookeeper/connection_params.go +++ b/pkg/model/zookeeper/connection_params.go @@ -17,11 +17,11 @@ package zookeeper import "time" const ( - maxRetriesNum = 3 - maxConcurrentRequests int64 = 32 + defaultMaxRetriesNum = 25 + defaultMaxConcurrentRequests int64 = 32 - timeoutConnect = 30 * time.Second - timeoutKeepAlive = 30 * time.Second + defaultTimeoutConnect = 30 * time.Second + defaultTimeoutKeepAlive = 30 * time.Second ) type ConnectionParams struct { @@ -42,16 +42,16 @@ func (p *ConnectionParams) Normalize() *ConnectionParams { p = &ConnectionParams{} } if p.MaxRetriesNum == 0 { - p.MaxRetriesNum = maxRetriesNum + p.MaxRetriesNum = defaultMaxRetriesNum } if p.MaxConcurrentRequests == 0 { - p.MaxConcurrentRequests = maxConcurrentRequests + p.MaxConcurrentRequests = defaultMaxConcurrentRequests } if p.TimeoutConnect == 0 { - p.TimeoutConnect = timeoutConnect + p.TimeoutConnect = defaultTimeoutConnect } if p.TimeoutKeepAlive == 0 { - p.TimeoutKeepAlive = timeoutKeepAlive + p.TimeoutKeepAlive = defaultTimeoutKeepAlive } return p } diff --git a/pkg/util/array.go b/pkg/util/array.go index c86963262..ed40850d9 100644 --- a/pkg/util/array.go +++ b/pkg/util/array.go @@ -78,8 +78,8 @@ func RemoveFromArray(needle string, haystack []string) []string { // Unzip makes two 1-value columns (slices) out of one 2-value column (slice) func Unzip(slice [][]string) ([]string, []string) { - col1 := make([]string, len(slice)) - col2 := make([]string, len(slice)) + col1 := make([]string, 0, len(slice)) + col2 := make([]string, 0, len(slice)) for i := 0; i < len(slice); i++ { col1 = append(col1, slice[i][0]) if len(slice[i]) > 1 { diff --git a/pkg/util/int.go b/pkg/util/int.go new file mode 100644 index 000000000..6a0db33af --- /dev/null +++ b/pkg/util/int.go @@ -0,0 +1,31 @@ +// Copyright 2019 Altinity Ltd and/or its affiliates. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package util + +func IncTopped(start, inc, top int) int { + res := start + inc + if res > top { + res = top + } + return res +} + +func DecBottomed(start, dec, bottom int) int { + res := start - dec + if res < bottom { + res = bottom + } + return res +} diff --git a/pkg/util/map.go b/pkg/util/map.go index 2d4d2e1f2..7c92bc15d 100644 --- a/pkg/util/map.go +++ b/pkg/util/map.go @@ -264,3 +264,8 @@ func MapGetSortedKeysAndValues(m map[string]string) (keys []string, values []str } return keys, values } + +func MapMigrate(cur, new, old map[string]string) map[string]string { + removed := MapGetSortedKeys(SubtractStringMaps(CopyMap(old), new)) + return MapDeleteKeys(MergeStringMapsPreserve(new, cur), removed...) +} diff --git a/release b/release index 2094a100c..48b91fd89 100644 --- a/release +++ b/release @@ -1 +1 @@ -0.24.0 +0.24.1 diff --git a/releases b/releases index 7556eb427..e50e7a951 100644 --- a/releases +++ b/releases @@ -1,3 +1,4 @@ +0.24.0 0.23.7 0.23.6 0.23.5 diff --git a/tests/e2e/manifests/chi/test-005-acm.yaml b/tests/e2e/manifests/chi/test-005-acm.yaml index 19a15f4dc..2d925213c 100644 --- a/tests/e2e/manifests/chi/test-005-acm.yaml +++ b/tests/e2e/manifests/chi/test-005-acm.yaml @@ -13,7 +13,7 @@ spec: fsGroup: 101 containers: - name: clickhouse-pod - image: clickhouse/clickhouse-server:24.3 + image: clickhouse/clickhouse-server:24.8.5.115 ports: - name: http containerPort: 8123 @@ -39,7 +39,7 @@ spec: volumeMounts: - mountPath: /var/lib/clickhouse name: default - - image: altinity/clickhouse-backup:2.2.7 + - image: altinity/clickhouse-backup:stable name: clickhouse-backup command: - /bin/bash diff --git a/tests/e2e/manifests/chi/test-010-zkroot.yaml b/tests/e2e/manifests/chi/test-010-zkroot.yaml index 9409f838b..5e0c82ead 100644 --- a/tests/e2e/manifests/chi/test-010-zkroot.yaml +++ b/tests/e2e/manifests/chi/test-010-zkroot.yaml @@ -8,13 +8,13 @@ spec: useTemplates: - name: clickhouse-version defaults: - templates: + templates: logVolumeClaimTemplate: default configuration: zookeeper: # Add Zookeeper nodes: - host: zookeeper - # port: 2181 + port: 2181 root: "/clickhouse/test-010-zkroot" session_timeout_ms: 30000 operation_timeout_ms: 10000 diff --git a/tests/e2e/manifests/chi/test-014-0-replication-2-1.yaml b/tests/e2e/manifests/chi/test-014-0-replication-2-1.yaml new file mode 100644 index 000000000..cadd433ba --- /dev/null +++ b/tests/e2e/manifests/chi/test-014-0-replication-2-1.yaml @@ -0,0 +1,30 @@ +apiVersion: "clickhouse.altinity.com/v1" + +kind: "ClickHouseInstallation" + +metadata: + name: test-014-replication + +spec: + # reconciling: + # policy: wait + useTemplates: + - name: clickhouse-version + - name: persistent-volume + configuration: + zookeeper: + nodes: + - host: zookeeper + port: 2181 + session_timeout_ms: 5000 + operation_timeout_ms: 5000 + root: /test/root/path + clusters: + - name: default + layout: + replicasCount: 2 + shardsCount: 1 + profiles: + default/database_atomic_wait_for_drop_and_detach_synchronously: 1 + default/allow_experimental_live_view: 1 + default/allow_experimental_database_replicated: 1 diff --git a/tests/e2e/manifests/chi/test-050-labels.yaml b/tests/e2e/manifests/chi/test-050-labels.yaml index cd46a1fe1..2ed533ad9 100644 --- a/tests/e2e/manifests/chi/test-050-labels.yaml +++ b/tests/e2e/manifests/chi/test-050-labels.yaml @@ -3,8 +3,11 @@ kind: "ClickHouseInstallation" metadata: name: test-050 labels: - exclude_this_label: test-050 - include_this_label: test-050 + exclude_this_label: test-050-label + include_this_label: test-050-label + annotations: + exclude_this_annotation: test-050-annotation + include_this_annotation: test-050-annotation spec: useTemplates: - name: clickhouse-version diff --git a/tests/e2e/manifests/chi/test-052-keeper-rescale.yaml b/tests/e2e/manifests/chi/test-052-keeper-rescale.yaml new file mode 100644 index 000000000..1589718a5 --- /dev/null +++ b/tests/e2e/manifests/chi/test-052-keeper-rescale.yaml @@ -0,0 +1,17 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallation" +metadata: + name: test-052-keeper-rescale +spec: + useTemplates: + - name: clickhouse-version + configuration: + zookeeper: + nodes: + - host: keeper-test-052-chk + port: 2181 + clusters: + - name: default + layout: + shardsCount: 1 + replicasCount: 2 \ No newline at end of file diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-24.8.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-24.8.yaml new file mode 100644 index 000000000..4307f0daa --- /dev/null +++ b/tests/e2e/manifests/chit/tpl-clickhouse-24.8.yaml @@ -0,0 +1,17 @@ +apiVersion: "clickhouse.altinity.com/v1" +kind: "ClickHouseInstallationTemplate" + +metadata: + name: clickhouse-version +spec: + defaults: + templates: + podTemplate: default + templates: + podTemplates: + - name: default + spec: + containers: + - name: clickhouse-pod + image: clickhouse/clickhouse-server:24.8 + imagePullPolicy: IfNotPresent diff --git a/tests/e2e/manifests/chit/tpl-clickhouse-stable.yaml b/tests/e2e/manifests/chit/tpl-clickhouse-stable.yaml index b053a5995..ba5ee4d0d 100644 --- a/tests/e2e/manifests/chit/tpl-clickhouse-stable.yaml +++ b/tests/e2e/manifests/chit/tpl-clickhouse-stable.yaml @@ -14,6 +14,6 @@ spec: containers: - name: clickhouse-pod # image: clickhouse/clickhouse-server:23.8.8.21.altinitystable - image: clickhouse/clickhouse-server:24.3 + image: clickhouse/clickhouse-server:24.8.5.115 imagePullPolicy: IfNotPresent diff --git a/tests/e2e/manifests/chk/test-052-chk-rescale-1.yaml b/tests/e2e/manifests/chk/test-052-chk-rescale-1.yaml new file mode 100644 index 000000000..117a0b9e5 --- /dev/null +++ b/tests/e2e/manifests/chk/test-052-chk-rescale-1.yaml @@ -0,0 +1,30 @@ +apiVersion: "clickhouse-keeper.altinity.com/v1" +kind: "ClickHouseKeeperInstallation" +metadata: + name: test-052-chk +spec: + defaults: + templates: + podTemplate: default + volumeClaimTemplate: default + templates: + podTemplates: + - name: default + spec: + containers: + - name: clickhouse-keeper + imagePullPolicy: IfNotPresent + image: "clickhouse/clickhouse-keeper:24.10" + volumeClaimTemplates: + - name: default + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + configuration: + clusters: + - name: keeper + layout: + replicasCount: 1 diff --git a/tests/e2e/manifests/chk/test-052-chk-rescale-3.yaml b/tests/e2e/manifests/chk/test-052-chk-rescale-3.yaml new file mode 100644 index 000000000..44022cb4f --- /dev/null +++ b/tests/e2e/manifests/chk/test-052-chk-rescale-3.yaml @@ -0,0 +1,30 @@ +apiVersion: "clickhouse-keeper.altinity.com/v1" +kind: "ClickHouseKeeperInstallation" +metadata: + name: test-052-chk +spec: + defaults: + templates: + podTemplate: default + volumeClaimTemplate: default + templates: + podTemplates: + - name: default + spec: + containers: + - name: clickhouse-keeper + imagePullPolicy: IfNotPresent + image: "clickhouse/clickhouse-keeper:24.10" + volumeClaimTemplates: + - name: default + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 1Gi + configuration: + clusters: + - name: keeper + layout: + replicasCount: 3 diff --git a/tests/e2e/manifests/chopconf/test-050-chopconf.yaml b/tests/e2e/manifests/chopconf/test-050-chopconf.yaml index 23dde4dc0..889e5c9c5 100644 --- a/tests/e2e/manifests/chopconf/test-050-chopconf.yaml +++ b/tests/e2e/manifests/chopconf/test-050-chopconf.yaml @@ -3,6 +3,14 @@ kind: "ClickHouseOperatorConfiguration" metadata: name: "test-050-chopconf" spec: + metrics: + labels: + exclude: + - exclude_this_label + - exclude_this_annotation label: exclude: - - exclude_this_label \ No newline at end of file + - exclude_this_label + annotation: + exclude: + - exclude_this_annotation \ No newline at end of file diff --git a/tests/e2e/run_tests_local.sh b/tests/e2e/run_tests_local.sh index 88976dcb5..c9795b31a 100755 --- a/tests/e2e/run_tests_local.sh +++ b/tests/e2e/run_tests_local.sh @@ -32,7 +32,7 @@ function select_test_goal() { echo "Having specified explicitly: ${specified_goal}" return 0 else - echo "What would you like to start. Possible options:" + echo "What would you like to start? Possible options:" echo " 1 - test operator" echo " 2 - test keeper" echo " 3 - test metrics" @@ -132,7 +132,7 @@ if [[ ! -z "${MINIKUBE_PRELOAD_IMAGES}" ]]; then clickhouse/clickhouse-server:23.8 clickhouse/clickhouse-server:latest altinity/clickhouse-server:22.8.15.25.altinitystable - docker.io/zookeeper:3.8.3 + docker.io/zookeeper:3.8.4 " for image in ${IMAGES}; do docker pull -q ${image} && \ diff --git a/tests/e2e/steps.py b/tests/e2e/steps.py index d1c9adf7e..6a88fc008 100644 --- a/tests/e2e/steps.py +++ b/tests/e2e/steps.py @@ -152,31 +152,42 @@ def check_metrics_monitoring( self, operator_namespace, operator_pod, - expect_pattern, + expect_pattern = "", + expect_metric = "", + expect_labels = "", container="metrics-exporter", port="8888", max_retries=7 ): - with Then(f"metrics-exporter /metrics endpoint result should contain {expect_pattern}"): + with Then(f"metrics-exporter /metrics endpoint result should contain {expect_pattern} {expect_metric} {expect_labels}"): for i in range(1, max_retries): url_cmd = util.make_http_get_request("127.0.0.1", port, "/metrics") out = kubectl.launch( f"exec {operator_pod} -c {container} -- {url_cmd}", ns=operator_namespace, ) - # print(out) + if expect_metric != "": + lines = [m for m in out.splitlines() if m.startswith(expect_metric)] + if len(lines) > 0: + metric = lines[0] + print(metric) + expected_pattern_found = expect_labels in metric + else: + expected_pattern_found = False + break - rx = re.compile(expect_pattern, re.MULTILINE) - matches = rx.findall(out) - expected_pattern_found = False + if expect_pattern != "": + rx = re.compile(expect_pattern, re.MULTILINE) + matches = rx.findall(out) + expected_pattern_found = False - if matches: - expected_pattern_found = True + if matches: + expected_pattern_found = True - if expected_pattern_found: - break + if expected_pattern_found: + break - with Then("Not ready. Wait for " + str(i * 5) + " seconds"): - time.sleep(i * 5) + with Then("Not ready. Wait for " + str(i * 5) + " seconds"): + time.sleep(i * 5) assert expected_pattern_found, error() \ No newline at end of file diff --git a/tests/e2e/test_keeper.py b/tests/e2e/test_keeper.py index db58e1341..ac31ea246 100644 --- a/tests/e2e/test_keeper.py +++ b/tests/e2e/test_keeper.py @@ -106,8 +106,7 @@ def check_zk_root_znode(chi, keeper_type, pod_count, retry_count=15): "zookeeper": "2", "zookeeper-operator": "3", "clickhouse-keeper": "2", - "clickhouse-keeper_with_chk": "2", - "CHK": "2", + "chk": "1", } if expected_out[keeper_type] != out.strip(" \t\r\n") and i + 1 < retry_count: with Then(f"{keeper_type} system.zookeeper not ready, wait {(i + 1) * 3} sec"): @@ -358,10 +357,10 @@ def test_clickhouse_keeper_rescale(self): @Requirements(RQ_SRS_026_ClickHouseOperator_CustomResource_Kind_ClickHouseKeeperInstallation("1.0")) def test_clickhouse_keeper_rescale_chk(self): test_keeper_rescale_outline( - keeper_type="clickhouse-keeper_with_chk", + keeper_type="chk", pod_for_insert_data="chi-test-cluster-for-zk-default-0-1-0", - keeper_manifest_1_node="clickhouse-keeper-1-node-for-test-only.yaml", - keeper_manifest_3_node="clickhouse-keeper-3-node-for-test-only.yaml", + keeper_manifest_1_node="clickhouse-keeper-1-node-for-test.yaml", + keeper_manifest_3_node="clickhouse-keeper-3-node-for-test.yaml", ) @@ -428,7 +427,7 @@ def test_keeper_probes_outline( min_bytes_for_compact_part=10485760, parts_to_delay_insert=1000000, parts_to_throw_insert=1000000, - max_parts_in_total=1000000; + max_parts_in_total=1000000; """, ) with Then("Insert data to keeper_bench for make zookeeper workload"): diff --git a/tests/e2e/test_operator.py b/tests/e2e/test_operator.py index 9eb25de71..6f8992a8f 100644 --- a/tests/e2e/test_operator.py +++ b/tests/e2e/test_operator.py @@ -200,103 +200,6 @@ def test_007(self): delete_test_namespace() -@TestCheck -def test_operator_upgrade(self, manifest, service, version_from, version_to=None, shell=None): - if version_to is None: - version_to = current().context.operator_version - with Given(f"clickhouse-operator from {version_from}"): - util.install_operator_version(version_from) - time.sleep(15) - - chi = yaml_manifest.get_name(util.get_full_path(manifest, True)) - cluster = chi - - kubectl.create_and_check( - manifest=manifest, - check={ - "object_counts": { - "statefulset": 2, - "pod": 2, - "service": 3, - }, - "do_not_delete": 1, - }, - ) - start_time = kubectl.get_field("pod", f"chi-{chi}-{cluster}-0-0-0", ".status.startTime") - - with Then("Create tables"): - for h in [f"chi-{chi}-{cluster}-0-0-0", f"chi-{chi}-{cluster}-1-0-0"]: - clickhouse.query( - chi, - "CREATE TABLE IF NOT EXISTS test_local (a UInt32) Engine = Log", - host=h, - ) - clickhouse.query(chi, "INSERT INTO test_local SELECT 1", host=h) - - trigger_event = threading.Event() - - with When("I create new shells"): - shell_1 = get_shell() - shell_2 = get_shell() - shell_3 = get_shell() - - Check("run query until receive stop event", test=run_select_query, parallel=True)( - host=service, - user="test_009", - password="test_009", - query="select count() from cluster('{cluster}', system.one)", - res1="2", - res2="1", - trigger_event=trigger_event, - shell=shell_1 - ) - - Check("Check that cluster definition does not change during restart", test=check_remote_servers, parallel=True)( - chi=chi, - shards=2, - trigger_event=trigger_event, - shell=shell_2 - ) - - try: - with When(f"upgrade operator to {version_to}"): - util.install_operator_version(version_to, shell=shell_3) - time.sleep(15) - - kubectl.wait_chi_status(chi, "Completed", shell=shell_3) - kubectl.wait_objects(chi, {"statefulset": 2, "pod": 2, "service": 3}, shell=shell_3) - - finally: - trigger_event.set() - join() - - with Then("I recreate shell"): - shell = get_shell() - self.context.shell = shell - - with Then("Check that table is here"): - tables = clickhouse.query(chi, "SHOW TABLES") - assert "test_local" in tables - out = clickhouse.query(chi, "SELECT count() FROM test_local") - assert out == "1" - - with Then("ClickHouse pods should not be restarted during upgrade"): - new_start_time = kubectl.get_field("pod", f"chi-{chi}-{cluster}-0-0-0", ".status.startTime") - if start_time != new_start_time: - kubectl.launch(f"describe chi -n {self.context.test_namespace} {chi}") - kubectl.launch( - # In my env "pod/: prefix is already returned by $(kubectl get pods -o name -n {current().context.operator_namespace} | grep clickhouse-operator) - # f"logs -n {current().context.operator_namespace} pod/$(kubectl get pods -o name -n {current().context.operator_namespace} | grep clickhouse-operator) -c clickhouse-operator" - f"logs -n {current().context.operator_namespace} $(kubectl get pods -o name -n {current().context.operator_namespace} | grep clickhouse-operator) -c clickhouse-operator" - ) - assert start_time == new_start_time, error( - f"{start_time} != {new_start_time}, pod restarted after operator upgrade" - ) - - with Finally("I clean up"): - with By("deleting chi"): - kubectl.delete_chi(chi) - def wait_operator_restart(chi, wait_objects, shell=None): with When("Restart operator"): @@ -344,6 +247,8 @@ def test_operator_restart(self, manifest, service, version=None): }, ) + wait_for_cluster(chi, cluster, 2) + with Then("Create tables"): for h in [f"chi-{chi}-{cluster}-0-0-0", f"chi-{chi}-{cluster}-1-0-0"]: clickhouse.query( @@ -356,7 +261,6 @@ def test_operator_restart(self, manifest, service, version=None): "CREATE TABLE IF NOT EXISTS test_dist as test_local Engine = Distributed('{cluster}', default, test_local, a)", host=h, ) - wait_for_cluster(chi, cluster, 2) trigger_event = threading.Event() @@ -565,6 +469,101 @@ def test_008_3(self): delete_test_namespace() +@TestCheck +def test_operator_upgrade(self, manifest, service, version_from, version_to=None, shell=None): + if version_to is None: + version_to = current().context.operator_version + with Given(f"clickhouse-operator from {version_from}"): + current().context.operator_version = version_from + create_shell_namespace_clickhouse_template() + + chi = yaml_manifest.get_name(util.get_full_path(manifest, True)) + cluster = chi + + kubectl.create_and_check( + manifest=manifest, + check={ + "object_counts": { + "statefulset": 2, + "pod": 2, + "service": 3, + }, + "do_not_delete": 1, + }, + ) + start_time = kubectl.get_field("pod", f"chi-{chi}-{cluster}-0-0-0", ".status.startTime") + + with Then("Create tables"): + for h in [f"chi-{chi}-{cluster}-0-0-0", f"chi-{chi}-{cluster}-1-0-0"]: + clickhouse.query( + chi, + "CREATE TABLE IF NOT EXISTS test_local (a UInt32) Engine = Log", + host=h, + ) + clickhouse.query(chi, "INSERT INTO test_local SELECT 1", host=h) + + trigger_event = threading.Event() + + with When("I create new shells"): + shell_1 = get_shell() + shell_2 = get_shell() + shell_3 = get_shell() + + Check("run query until receive stop event", test=run_select_query, parallel=True)( + host=service, + user="test_009", + password="test_009", + query="select count() from cluster('{cluster}', system.one)", + res1="2", + res2="1", + trigger_event=trigger_event, + shell=shell_1 + ) + + Check("Check that cluster definition does not change during restart", test=check_remote_servers, parallel=True)( + chi=chi, + shards=2, + trigger_event=trigger_event, + shell=shell_2 + ) + + with When(f"upgrade operator to {version_to}"): + util.install_operator_version(version_to) + time.sleep(15) + + kubectl.wait_chi_status(chi, "Completed") + kubectl.wait_objects(chi, {"statefulset": 2, "pod": 2, "service": 3}) + + trigger_event.set() + join() + + with Then("I recreate shell"): + shell = get_shell() + self.context.shell = shell + + with Then("Check that table is here"): + tables = clickhouse.query(chi, "SHOW TABLES") + assert "test_local" in tables + out = clickhouse.query(chi, "SELECT count() FROM test_local") + assert out == "1" + + with Then("ClickHouse pods should not be restarted during upgrade"): + new_start_time = kubectl.get_field("pod", f"chi-{chi}-{cluster}-0-0-0", ".status.startTime") + if start_time != new_start_time: + kubectl.launch(f"describe chi -n {self.context.test_namespace} {chi}") + kubectl.launch( + # In my env "pod/: prefix is already returned by $(kubectl get pods -o name -n {current().context.operator_namespace} | grep clickhouse-operator) + # f"logs -n {current().context.operator_namespace} pod/$(kubectl get pods -o name -n {current().context.operator_namespace} | grep clickhouse-operator) -c clickhouse-operator" + f"logs -n {current().context.operator_namespace} $(kubectl get pods -o name -n {current().context.operator_namespace} | grep clickhouse-operator) -c clickhouse-operator" + ) + assert start_time == new_start_time, error( + f"{start_time} != {new_start_time}, pod restarted after operator upgrade" + ) + + with Finally("I clean up"): + delete_test_namespace() + + @TestScenario @Name("test_009_1. Test operator upgrade") @Requirements(RQ_SRS_026_ClickHouseOperator_Managing_UpgradingOperator("1.0")) @@ -573,8 +572,6 @@ def test_009_1(self, version_from="0.23.7", version_to=None): if version_to is None: version_to = self.context.operator_version - create_shell_namespace_clickhouse_template() - with Check("Test simple chi for operator upgrade"): test_operator_upgrade( manifest="manifests/chi/test-009-operator-upgrade-1.yaml", @@ -583,9 +580,6 @@ def test_009_1(self, version_from="0.23.7", version_to=None): version_to=version_to, ) - with Finally("I clean up"): - delete_test_namespace() - @TestScenario @Name("test_009_2. Test operator upgrade") @@ -594,8 +588,6 @@ def test_009_2(self, version_from="0.23.7", version_to=None): if version_to is None: version_to = self.context.operator_version - create_shell_namespace_clickhouse_template() - with Check("Test advanced chi for operator upgrade"): test_operator_upgrade( manifest="manifests/chi/test-009-operator-upgrade-2.yaml", @@ -604,9 +596,6 @@ def test_009_2(self, version_from="0.23.7", version_to=None): version_to=version_to, ) - with Finally("I clean up"): - delete_test_namespace() - @TestScenario @Name("test_010. Test zookeeper initialization") @@ -614,7 +603,6 @@ def test_009_2(self, version_from="0.23.7", version_to=None): def test_010(self): create_shell_namespace_clickhouse_template() - util.set_operator_version(current().context.operator_version) util.require_keeper(keeper_type=self.context.keeper_type) kubectl.create_and_check( @@ -635,6 +623,41 @@ def test_010(self): with Finally("I clean up"): delete_test_namespace() +@TestScenario +@Name("test_010_1. Test zookeeper initialization AFTER starting a cluster") +def test_010_1(self): + create_shell_namespace_clickhouse_template() + chi = "test-010-zkroot" + + kubectl.create_and_check( + manifest="manifests/chi/test-010-zkroot.yaml", + check={ + "apply_templates": { + current().context.clickhouse_template, + }, + "do_not_delete": 1, + "chi_status": "InProgress" + }, + ) + + with Then("Wait 60 seconds for operator to start creating ZooKeeper root"): + time.sleep(60) + + # with Then("CHI should be in progress with no pods created yet"): + # assert kubectl.get_chi_status(chi) == "InProgress" + # assert kubectl.get_count("pod", chi = chi) == 0 + + util.require_keeper(keeper_type=self.context.keeper_type) + + kubectl.wait_chi_status(chi, "Completed") + + with And("ClickHouse should not complain regarding zookeeper path"): + out = clickhouse.query_with_error("test-010-zkroot", "select path from system.zookeeper where path = '/' limit 1") + assert "/" == out + + with Finally("I clean up"): + delete_test_namespace() + def get_user_xml_from_configmap(chi, user): users_xml = kubectl.get("configmap", f"chi-{chi}-common-usersd")["data"]["chop-generated-users.xml"] @@ -1261,14 +1284,14 @@ def get_shards_from_remote_servers(chi, cluster, shell=None): return chi_shards -def wait_for_cluster(chi, cluster, num_shards, num_replicas=0, pwd="", force_wait = False): +def wait_for_cluster(chi, cluster, num_shards, num_replicas=0, pwd="", force_wait=False): with Given(f"Cluster {cluster} is properly configured"): - if current().context.operator_version >= "0.24" and force_wait == False: + if current().context.operator_version >= "0.24" and force_wait is False: print(f"operator {current().context.operator_version} does not require extra wait, skipping check") else: with By(f"remote_servers have {num_shards} shards"): assert num_shards == get_shards_from_remote_servers(chi, cluster) - with By(f"ClickHouse recognizes {num_shards} shards in the cluster"): + with By(f"ClickHouse recognizes {num_shards} shards in the cluster {cluster}"): for shard in range(num_shards): shards = "" for i in range(1, 10): @@ -1286,22 +1309,41 @@ def wait_for_cluster(chi, cluster, num_shards, num_replicas=0, pwd="", force_wai assert shards == str(num_shards) if num_replicas > 0: - with By(f"ClickHouse recognizes {num_replicas} replicas in the cluster"): - for replica in range(num_replicas): - replicas = "" - for i in range(1, 10): - replicas = clickhouse.query( - chi, - f"select uniq(replica_num) from system.clusters where cluster ='{cluster}'", - host=f"chi-{chi}-{cluster}-0-{replica}", - pwd=pwd, - with_error=True, + with By(f"ClickHouse recognizes {num_replicas} replicas shard in the cluster {cluster}"): + for shard in range(num_shards): + for replica in range(num_replicas): + replicas = "" + for i in range(1, 10): + replicas = clickhouse.query( + chi, + f"select uniq(replica_num) from system.clusters where cluster ='{cluster}'", + host=f"chi-{chi}-{cluster}-{shard}-{replica}", + pwd=pwd, + with_error=True, ) - if replicas == str(num_replicas): - break - with Then("Not ready. Wait for " + str(i * 5) + " seconds"): - time.sleep(i * 5) - assert replicas == str(num_replicas) + if replicas == str(num_replicas): + break + with Then(f"Not ready. {replicas}/{num_replicas} replicas Wait for " + str(i * 5) + " seconds"): + time.sleep(i * 5) + assert replicas == str(num_replicas) + num_hosts = num_shards * num_replicas + with By(f"ClickHouse recognizes {num_hosts} hosts in the cluster {cluster}"): + for shard in range(num_shards): + for replica in range(num_replicas): + hosts = "" + for i in range(1, 10): + hosts = clickhouse.query( + chi, + f"select count() from system.clusters where cluster ='{cluster}'", + host=f"chi-{chi}-{cluster}-{shard}-{replica}", + pwd=pwd, + with_error=True, + ) + if hosts == str(num_hosts): + break + with Then(f"Not ready. {hosts}/{num_hosts} hosts Wait for " + str(i * 5) + " seconds"): + time.sleep(i * 5) + assert hosts == str(num_hosts) @TestScenario @@ -1394,12 +1436,14 @@ def test_014_0(self): "CREATE TABLE test_replicated_014.test_replicated_014 (a Int8) Engine = ReplicatedMergeTree ORDER BY tuple()", ] - wait_for_cluster(chi_name, cluster, n_shards) + wait_for_cluster(chi_name, cluster, n_shards, 1) with Then("Create schema objects"): for q in create_ddls: clickhouse.query(chi_name, q, host=f"chi-{chi_name}-{cluster}-0-0") + # Give some time for replication to catch up + time.sleep(10) with Given("Replicated tables are created on a first replica and data is inserted"): for table in replicated_tables: if table != "test_atomic_014.test_mv2_014": @@ -1461,6 +1505,34 @@ def check_schema_propagation(replicas): ) assert out == "1" + with And("Replicated database should have correct uuid, so new tables are automatically created"): + import time + + new_table = "test_replicated_014_" + str(int(time.time())) + new_table_ddl = f"CREATE TABLE test_replicated_014.{new_table} (a Int8) Engine = ReplicatedMergeTree ORDER BY tuple()" + with Then(f"Create {new_table} on one node only"): + clickhouse.query(chi_name, new_table_ddl) + + # Give some time for replication to catch up + time.sleep(10) + + for replica in replicas: + for shard in shards: + host=f"chi-{chi_name}-{cluster}-{shard}-{replica}" + out = clickhouse.query( + chi_name, + f"SELECT uuid FROM system.databases where name = 'test_replicated_014'", + host=host, + ) + print(f"{host} database uuid: {out}") + print(f"Checking {new_table}") + out = clickhouse.query( + chi_name, + f"SELECT count() FROM system.tables WHERE name = '{new_table}'", + host=host, + ) + assert out == "1" + with And("Replicated table should have the data"): for replica in replicas: for shard in shards: @@ -1527,9 +1599,7 @@ def check_schema_propagation(replicas): kubectl.launch(f"delete pod {self.context.keeper_type}-0") time.sleep(1) - with Then( - f"try insert into the table while {self.context.keeper_type} offline table should be in readonly mode" - ): + with Then(f"try insert into the table while {self.context.keeper_type} offline table should be in readonly mode"): out = clickhouse.query_with_error(chi_name, "SET insert_keeper_max_retries=0; INSERT INTO test_local_014 VALUES(2)") assert "Table is in readonly mode" in out @@ -1559,28 +1629,44 @@ def check_schema_propagation(replicas): time.sleep(10) check_schema_propagation([1]) + with When("Remove shard"): + manifest = "manifests/chi/test-014-0-replication-2-1.yaml" + chi_name = yaml_manifest.get_name(util.get_full_path(manifest)) + kubectl.create_and_check( + manifest=manifest, + check={ + "pod_count": 2, + "do_not_delete": 1, + }, + timeout=600, + ) + with Then("Shard should be deleted in ZooKeeper"): + out = clickhouse.query_with_error( + chi_name, + f"SELECT count() FROM system.zookeeper WHERE path ='/clickhouse/{cluster}/tables/1/default'", + ) + note(f"Found {out} replicated tables in {self.context.keeper_type}") + # FIXME: it fails + # assert "DB::Exception: No node" in out or out == "0" + with When("Delete chi"): kubectl.delete_chi("test-014-replication") - with Then( - f"Tables should be deleted in {self.context.keeper_type}. We can test it re-creating the chi and checking {self.context.keeper_type} contents" - ): - manifest = "manifests/chi/test-014-0-replication-1.yaml" - kubectl.create_and_check( - manifest=manifest, - check={ - "pod_count": 2, - "pdb": {"default": 1}, - "do_not_delete": 1, - }, + manifest = "manifests/chi/test-014-0-replication-1.yaml" + kubectl.create_and_check( + manifest=manifest, + check={ + "pod_count": 2, + "do_not_delete": 1, + }, + ) + with Then("Tables are deleted in ZooKeeper"): + out = clickhouse.query_with_error( + chi_name, + f"SELECT count() FROM system.zookeeper WHERE path ='/clickhouse/{cluster}/tables/0/default'", ) - with Then("Tables are deleted in ZooKeeper"): - out = clickhouse.query_with_error( - chi_name, - f"SELECT count() FROM system.zookeeper WHERE path ='/clickhouse/{chi_name}/tables/0/default'", - ) - note(f"Found {out} replicated tables in {self.context.keeper_type}") - assert "DB::Exception: No node" in out or out == "0" + note(f"Found {out} replicated tables in {self.context.keeper_type}") + assert "DB::Exception: No node" in out or out == "0" with Finally("I clean up"): delete_test_namespace() @@ -2584,27 +2670,45 @@ def test_024(self): }, ) - def checkAnnotations(annotation, value): + def check_annotations(annotation, value, allow_to_fail_for_pvc=False): with Then(f"Pod annotation {annotation}={value} should populated from a podTemplate"): - assert kubectl.get_field("pod", "chi-test-024-default-0-0-0", f".metadata.annotations.podtemplate/{annotation}") == value + have = kubectl.get_field("pod", "chi-test-024-default-0-0-0", f".metadata.annotations.podtemplate/{annotation}") + print(f"pod annotation have: {have}") + print(f"pod annotation need: {value}") + assert have == value with And(f"Service annotation {annotation}={value} should be populated from a serviceTemplate"): - assert kubectl.get_field("service", "clickhouse-test-024", f".metadata.annotations.servicetemplate/{annotation}") == value + have = kubectl.get_field("service", "clickhouse-test-024", f".metadata.annotations.servicetemplate/{annotation}") + print(f"service annotation have: {have}") + print(f"service annotation need: {value}") + assert have == value with And(f"PVC annotation {annotation}={value} should be populated from a volumeTemplate"): - assert kubectl.get_field("pvc", "-l clickhouse.altinity.com/chi=test-024", f".metadata.annotations.pvc/{annotation}") == value + have = kubectl.get_field("pvc", "-l clickhouse.altinity.com/chi=test-024", f".metadata.annotations.pvc/{annotation}") + print(f"pvc annotation have: {have}") + print(f"pvc annotation need: {value}") + assert allow_to_fail_for_pvc or (have == value) with And(f"Pod annotation {annotation}={value} should populated from a CHI"): - assert kubectl.get_field("pod", "chi-test-024-default-0-0-0", f".metadata.annotations.chi/{annotation}") == value + have = kubectl.get_field("pod", "chi-test-024-default-0-0-0", f".metadata.annotations.chi/{annotation}") + print(f"pod annotation have: {have}") + print(f"pod annotation need: {value}") + assert have == value with And(f"Service annotation {annotation}={value} should be populated from a CHI"): - assert kubectl.get_field("service", "clickhouse-test-024", f".metadata.annotations.chi/{annotation}") == value + have = kubectl.get_field("service", "clickhouse-test-024", f".metadata.annotations.chi/{annotation}") + print(f"service annotation have: {have}") + print(f"service annotation need: {value}") + assert have == value with And(f"PVC annotation {annotation}={value} should be populated from a CHI"): - assert kubectl.get_field("pvc", "-l clickhouse.altinity.com/chi=test-024", f".metadata.annotations.chi/{annotation}") == value + have = kubectl.get_field("pvc", "-l clickhouse.altinity.com/chi=test-024", f".metadata.annotations.chi/{annotation}") + print(f"pvc annotation have: {have}") + print(f"pvc annotation need: {value}") + assert allow_to_fail_for_pvc or (have == value) - checkAnnotations("test", "test") + check_annotations("test", "test") with And("Service annotation macros should be resolved"): assert ( @@ -2632,8 +2736,8 @@ def checkAnnotations(annotation, value): "do_not_delete": 1, }, ) - checkAnnotations("test", "test-2") - checkAnnotations("test-2", "test-2") + check_annotations("test", "test-2") + check_annotations("test-2", "test-2") with When("Revert template annotations to original values"): kubectl.create_and_check( @@ -2643,10 +2747,9 @@ def checkAnnotations(annotation, value): "do_not_delete": 1, }, ) - checkAnnotations("test", "test") - # with Then("Annotation test-2 should be removed"): - # TODO. Does not work for services yet - # checkAnnotations("test-2", "") + check_annotations("test", "test") + with Then("Annotation test-2 should be removed"): + check_annotations("test-2", "", allow_to_fail_for_pvc=True) with Finally("I clean up"): delete_test_namespace() @@ -3304,19 +3407,8 @@ def test_032(self): create_shell_namespace_clickhouse_template() util.require_keeper(keeper_type=self.context.keeper_type) - create_table = """ - CREATE TABLE test_local_032 ON CLUSTER 'default' (a UInt32) - Engine = ReplicatedMergeTree('/clickhouse/{installation}/tables/{shard}/{database}/{table}', '{replica}') - PARTITION BY tuple() - ORDER BY a - """.replace( - "\r", "" - ).replace( - "\n", "" - ) manifest = "manifests/chi/test-032-rescaling.yaml" - chi = yaml_manifest.get_name(util.get_full_path(manifest)) kubectl.create_and_check( @@ -3341,16 +3433,18 @@ def test_032(self): # remote_servers = kubectl.get("configmap", f"chi-{chi}-common-configd")["data"]["chop-generated-remote_servers.xml"] # print(remote_servers) wait_for_cluster(chi, 'default', 2, 2) - time.sleep(60) with Given("Create replicated and distributed tables"): - clickhouse.query(chi, create_table) + clickhouse.query( + chi, + "CREATE TABLE test_local_032 ON CLUSTER 'default' (a UInt32) Engine = ReplicatedMergeTree() PARTITION BY tuple() ORDER BY a", + ) clickhouse.query( chi, "CREATE TABLE test_distr_032 ON CLUSTER 'default' AS test_local_032 Engine = Distributed('default', default, test_local_032, a%2)", ) clickhouse.query(chi, f"INSERT INTO test_distr_032 select * from numbers({numbers})") - time.sleep(60) + time.sleep(10) with Then("Distributed table is created on all nodes"): cnt = clickhouse.query(chi_name=chi, sql="select count() from cluster('all-sharded', system.tables) where name='test_distr_032'") @@ -3367,7 +3461,6 @@ def test_032(self): with When("I create new shells"): shell_1 = get_shell() shell_2 = get_shell() - shell_3 = get_shell() Check("run query until receive stop event", test=run_select_query, parallel=True)( host="clickhouse-test-032-rescaling", @@ -3392,10 +3485,6 @@ def test_032(self): kubectl.create_and_check( manifest="manifests/chi/test-032-rescaling-2.yaml", check={ - "apply_templates": { - self.context.clickhouse_template, - "manifests/chit/tpl-persistent-volume-100Mi.yaml", - }, "object_counts": { "statefulset": 4, "pod": 4, @@ -3403,8 +3492,7 @@ def test_032(self): }, "do_not_delete": 1, }, - timeout=int(1000), - shell=shell_3 + timeout=900, ) trigger_event.set() @@ -4012,7 +4100,6 @@ def test_040(self): "pod_volumes": { "/var/lib/clickhouse", }, - "pod_image": current().context.clickhouse_version, "do_not_delete": 1, "chi_status": "InProgress", }, @@ -4573,8 +4660,8 @@ def test_048(self): """Check clickhouse-operator support ClickHouseKeeperInstallation with PVC in keeper manifest.""" create_shell_namespace_clickhouse_template() - util.require_keeper(keeper_type="CHK", - keeper_manifest="clickhouse-keeper-3-node-for-test-only-version-24.yaml") + util.require_keeper(keeper_type="chk", + keeper_manifest="clickhouse-keeper-3-node-for-test-only.yaml") manifest = f"manifests/chi/test-048-clickhouse-keeper.yaml" chi = yaml_manifest.get_name(util.get_full_path(manifest)) cluster = "default" @@ -4586,29 +4673,7 @@ def test_048(self): "do_not_delete": 1, }, ) - with When("I create replicated table"): - create_table = """ - CREATE TABLE test_local_048 ON CLUSTER 'default' (a UInt32) - Engine = ReplicatedMergeTree('/clickhouse/{installation}/tables/{shard}/{database}/{table}', '{replica}') - PARTITION BY tuple() - ORDER BY a - """.replace( - "\r", "" - ).replace( - "\n", "" - ) - clickhouse.query(chi, create_table) - - numbers = 100 - with And("I insert data in the replicated table"): - clickhouse.query(chi, f"INSERT INTO test_local_048 select * from numbers({numbers})") - - with Then("Check replicated table on host 0 has all rows"): - out = clickhouse.query(chi, "SELECT count(*) from test_local_048", host=f"chi-{chi}-{cluster}-0-0-0") - assert out == f"{numbers}", error() - with Then("Check replicated table on host 1 has all rows"): - out = clickhouse.query(chi, "SELECT count(*) from test_local_048", host=f"chi-{chi}-{cluster}-0-1-0") - assert out == f"{numbers}", error() + check_replication(chi, {0,1}, 1) with Finally("I clean up"): delete_test_namespace() @@ -4621,13 +4686,13 @@ def test_049(self): when clickhouse-keeper defined with ClickHouseKeeperInstallation.""" create_shell_namespace_clickhouse_template() - util.require_keeper(keeper_type="CHK", - keeper_manifest="clickhouse-keeper-3-node-for-test-only-version-24.yaml") + util.require_keeper(keeper_type="chk", + keeper_manifest="clickhouse-keeper-3-node-for-test-only.yaml") manifest = f"manifests/chi/test-049-clickhouse-keeper-upgrade.yaml" chi = yaml_manifest.get_name(util.get_full_path(manifest)) cluster = "default" - keeper_version_from = "24.3.5.46" - keeper_version_to = "24.8.5.115" + keeper_version_from = "24.8" + keeper_version_to = "24.9" with Given("CHI with 2 replicas"): kubectl.create_and_check( manifest=manifest, @@ -4637,30 +4702,11 @@ def test_049(self): }, ) - with When("I create replicated table"): - create_table = """ - CREATE TABLE test_local_049 ON CLUSTER 'default' (a UInt32) - Engine = ReplicatedMergeTree('/clickhouse/{installation}/tables/{shard}/{database}/{table}', '{replica}') - PARTITION BY tuple() - ORDER BY a - """.replace( - "\r", "" - ).replace( - "\n", "" - ) - clickhouse.query(chi, create_table) - - with And("I insert data in the replicated table"): - clickhouse.query(chi, f"INSERT INTO test_local_049 select 1") - - with Then("Check replicated table has data on both nodes"): - for replica in {0,1}: - out = clickhouse.query(chi, "SELECT count(*) from test_local_049", host=f"chi-{chi}-{cluster}-0-{replica}-0") - assert out == "1", error() + check_replication(chi, {0,1}, 1) with When(f"I check clickhouse-keeper version is {keeper_version_from}"): assert keeper_version_from in \ - kubectl.get_field('pod', 'chk-clickhouse-keeper-test-only-0-0-0', '.spec.containers[0].image'), error() + kubectl.get_field('pod', 'chk-clickhouse-keeper-test-0-0-0', '.spec.containers[0].image'), error() with Then(f"I change keeper version to {keeper_version_to}"): cmd = f"""patch chk clickhouse-keeper --type='json' --patch='[{{"op":"replace","path":"/spec/templates/podTemplates/0/spec/containers/0/image","value":"clickhouse/clickhouse-keeper:{keeper_version_to}"}}]'""" @@ -4672,20 +4718,11 @@ def test_049(self): kubectl.wait_chk_status('clickhouse-keeper', 'Completed') with When(f"I check clickhouse-keeper version is changed to {keeper_version_to}"): - assert keeper_version_to in \ - kubectl.get_field('pod', 'chk-clickhouse-keeper-test-only-0-0-0', '.spec.containers[0].image'), error() - assert keeper_version_to in \ - kubectl.get_field('pod', 'chk-clickhouse-keeper-test-only-0-1-0', '.spec.containers[0].image'), error() - assert keeper_version_to in \ - kubectl.get_field('pod', 'chk-clickhouse-keeper-test-only-0-2-0', '.spec.containers[0].image'), error() + kubectl.wait_field('pod', 'chk-clickhouse-keeper-test-0-0-0', '.spec.containers[0].image', f'clickhouse/clickhouse-keeper:{keeper_version_to}', retries=5) + kubectl.wait_field('pod', 'chk-clickhouse-keeper-test-0-1-0', '.spec.containers[0].image', f'clickhouse/clickhouse-keeper:{keeper_version_to}', retries=5) + kubectl.wait_field('pod', 'chk-clickhouse-keeper-test-0-2-0', '.spec.containers[0].image', f'clickhouse/clickhouse-keeper:{keeper_version_to}', retries=5) - with And("I insert data in the replicated table after clickhouse-keeper upgrade"): - clickhouse.query(chi, f"INSERT INTO test_local_049 select 2", timeout=600) - - with Then("Check replicated table has data on both nodes"): - for replica in {0,1}: - out = clickhouse.query(chi, "SELECT count(*) from test_local_049", host=f"chi-{chi}-{cluster}-0-{replica}-0") - assert out == "2", error() + check_replication(chi, {0,1}, 2) with Finally("I clean up"): delete_test_namespace() @@ -4714,20 +4751,39 @@ def test_050(self): }, ) - def test_labels(chi, label, value): + def test_labels(chi, type, key, value): + + with Then(f"Pod {type} {key}={value} should populated from CHI"): + assert kubectl.get_field("pod", f"-l clickhouse.altinity.com/chi={chi}", f".metadata.{type}s.{key}") == value + + with And(f"Service {type} {key}={value} should populated from CHI"): + assert kubectl.get_field("service", f"-l clickhouse.altinity.com/chi={chi}", f".metadata.{type}s.{key}") == value - with Then(f"Pod label {label}={value} should populated from CHI"): - assert kubectl.get_field("pod", f"-l clickhouse.altinity.com/chi={chi}", f".metadata.labels.{label}") == value + with And(f"PVC {type} {key}={value} should populated from CHI"): + assert kubectl.get_field("pvc", f"-l clickhouse.altinity.com/chi={chi}", f".metadata.{type}s.{key}") == value - with And(f"Service label {label}={value} should populated from CHI"): - assert kubectl.get_field("service", f"-l clickhouse.altinity.com/chi={chi}", f".metadata.labels.{label}") == value + test_labels(chi, "label", "include_this_label", "test-050-label") - with And(f"PVC label {label}={value} should populated from CHI"): - assert kubectl.get_field("pvc", f"-l clickhouse.altinity.com/chi={chi}", f".metadata.labels.{label}") == value + test_labels(chi, "label", "exclude_this_label", "") - test_labels(chi, "include_this_label", "test-050") + test_labels(chi, "annotation", "include_this_annotation", "test-050-annotation") - test_labels(chi, "exclude_this_label", "") + test_labels(chi, "annotation", "exclude_this_annotation", "") + + + with Then("Check that exposed metrics do not have labels and annotations that are excluded"): + operator_namespace=current().context.operator_namespace + out = kubectl.launch("get pods -l app=clickhouse-operator", ns=operator_namespace).splitlines()[1] + operator_pod = re.split(r"[\t\r\n\s]+", out)[0] + + # chi_clickhouse_metric_VersionInteger{chi="test-050",exclude_this_annotation="test-050-annotation",hostname="chi-test-050-default-0-0.test-050-e1884706-9a94-11ef-a786-367ddacfe5fd.svc.cluster.local",include_this_annotation="test-050-annotation",include_this_label="test-050-label",namespace="test-050-e1884706-9a94-11ef-a786-367ddacfe5fd"} + expect_labels = f"chi=\"test-050\",hostname=\"chi-test-050-default-0-0.{operator_namespace}.svc.cluster.local\",include_this_annotation=\"test-050-annotation\",include_this_label=\"test-050-label\"" + check_metrics_monitoring( + operator_namespace = operator_namespace, + operator_pod=operator_pod, + expect_metric="chi_clickhouse_metric_VersionInteger", + expect_labels=expect_labels + ) with Finally("I clean up"): delete_test_namespace() @@ -4769,17 +4825,7 @@ def test_051(self): }, ) - with When("I create replicated table"): - create_table = "CREATE TABLE test_local_051 ON CLUSTER 'default' (a UInt32) Engine = ReplicatedMergeTree ORDER BY a" - clickhouse.query(chi, create_table) - - with And("I insert data in the replicated table"): - clickhouse.query(chi, f"INSERT INTO test_local_051 select 1") - - with Then("Check replicated table has data on both nodes"): - for replica in {0,1}: - out = clickhouse.query(chi, "SELECT count(*) from test_local_051", host=f"chi-{chi}-{cluster}-0-{replica}-0") - assert out == "1", error() + check_replication(chi, {0,1}, 1, "test_local_051") with When(f"upgrade operator to {version_to}"): util.install_operator_version(version_to) @@ -4824,13 +4870,7 @@ def test_051(self): out = clickhouse.query(chi, host=host, sql="SELECT count(*) from system.replicas where is_readonly") assert out == "0", error() - with And("I insert data in the replicated table"): - clickhouse.query(chi, f"INSERT INTO test_local_051 select 2") - - with Then("Check replicated table has data on both nodes"): - for replica in {0,1}: - out = clickhouse.query(chi, "SELECT count(*) from test_local_051", host=f"chi-{chi}-{cluster}-0-{replica}-0") - assert out == "2", error() + check_replication(chi, {0,1}, 2, "test_local_051") with Finally("I clean up"): delete_test_namespace() @@ -4869,17 +4909,22 @@ def test_051_1(self): }, ) - with When("I create replicated table"): - create_table = "CREATE TABLE test_local_051 ON CLUSTER 'default' (a UInt32) Engine = ReplicatedMergeTree ORDER BY a" - clickhouse.query(chi, create_table) + check_replication(chi, {0,1}, 1) - with And("I insert data in the replicated table"): - clickhouse.query(chi, f"INSERT INTO test_local_051 select 1") + with Then("Unattach old CHK resources"): + kubectl.launch(f"patch sts {chk} -p " + """\'{"metadata":{"ownerReferences":null}}\'""") + kubectl.launch(f"patch cm {chk} -p " + """\'{"metadata":{"ownerReferences":null}}\'""") + kubectl.launch(f"patch service {chk} -p " + """\'{"metadata":{"ownerReferences":null}}\'""") + kubectl.launch(f"patch service {chk}-headless -p " + """\'{"metadata":{"ownerReferences":null}}\'""") + kubectl.launch(f"label pod -lapp={chk} app-") + kubectl.launch(f"label sts -lapp={chk} app-") - with Then("Check replicated table has data on both nodes"): - for replica in {0,1}: - out = clickhouse.query(chi, "SELECT count(*) from test_local_051", host=f"chi-{chi}-{cluster}-0-{replica}-0") - assert out == "1", error() + with Then("Confirm that statefulset and pod are still running if we delete chk"): + kubectl.delete_kind("chk", chk) + assert kubectl.get_field("pod", "test-051-chk-0", ".status.phase") == "Running" + assert kubectl.get_count("sts", "test-051-chk") == 1 + assert kubectl.get_count("cm", "test-051-chk") == 1 + assert kubectl.get_count("service", "test-051-chk") == 1 old_pvc = "both-paths-test-051-chk-0" pv = kubectl.get_pv_name(old_pvc) @@ -4889,7 +4934,8 @@ def test_051_1(self): kubectl.launch(f"patch pv {pv}" + """ -p \'{"spec":{"persistentVolumeReclaimPolicy":"Retain"}}\'""") with Then("Delete old Keeper resources"): - kubectl.delete_kind("chk", chk) + kubectl.delete_kind("sts", "test-051-chk") + kubectl.delete_kind("pod", "test-051-chk-0") kubectl.delete_kind("pvc", old_pvc) with Then("Unmount PV from old PVC"): @@ -4926,17 +4972,118 @@ def test_051_1(self): time.sleep(10) assert out == "1", error() - with And("I insert data in the replicated table"): - clickhouse.query(chi, f"INSERT INTO test_local_051 select 2") + check_replication(chi, {0,1}, 2) + + with Finally("I clean up"): + delete_test_namespace() + +@TestScenario +@Name("test_052. Clickhouse-keeper scale-up/scale-down") +def test_052(self): + """Check that clickhouse-operator support scale-up/scale-down without service interruption""" + + create_shell_namespace_clickhouse_template() - with Then("Check replicated table has data on both nodes"): - for replica in {0,1}: - out = clickhouse.query(chi, "SELECT count(*) from test_local_051", host=f"chi-{chi}-{cluster}-0-{replica}-0") - assert out == "2", error() + chi_manifest = "manifests/chi/test-052-keeper-rescale.yaml" + chk_manifest_1 = "manifests/chk/test-052-chk-rescale-1.yaml" + chk_manifest_3 = "manifests/chk/test-052-chk-rescale-3.yaml" + chi = yaml_manifest.get_name(util.get_full_path(chi_manifest)) + chk = yaml_manifest.get_name(util.get_full_path(chk_manifest_1)) + + cluster = "default" + + with Given("Install CHK"): + kubectl.create_and_check( + manifest=chk_manifest_1, kind="chk", + check={ + "pod_count": 1, + "do_not_delete": 1, + }, + ) + + with Given("CHI with 2 replicas"): + kubectl.create_and_check( + manifest=chi_manifest, + check={ + "pod_count": 2, + "do_not_delete": 1, + }, + ) + + check_replication(chi, {0,1}, 1) + + with Given("Rescale CHK to 3 replicas"): + kubectl.create_and_check( + manifest=chk_manifest_3, kind="chk", + check={ + "pod_count": 1, + "do_not_delete": 1, + }, + ) + + check_replication(chi, {0,1}, 2) + + with Then("Kill first pod to switch the leader"): + kubectl.launch(f"delete pod chk-test-052-chk-keeper-0-0-0") + time.sleep(10) + + # with Then("Force leader to be on the first node only"): + # kubectl.create_and_check( + # manifest="manifests/chk/test-052-chk-rescale-1.1.yaml", kind="chk", + # check={ + # "pod_count": 3, + # "do_not_delete": 1, + # }, + # ) + + # check_replication(chi, {0,1}, 3) + + + # with Then("Remove other nodes from the raft configuration"): + # kubectl.create_and_check( + # manifest="manifests/chk/test-052-chk-rescale-1.2.yaml", kind="chk", + # check={ + # "do_not_delete": 1, + # }, + # ) + + # check_replication(chi, {0,1}, 4) + + + with Then("Rescale CHK back to 1 replica"): + kubectl.create_and_check( + manifest="manifests/chk/test-052-chk-rescale-1.yaml", kind="chk", + check={ + "pod_count": 1, + "do_not_delete": 1, + }, + ) + + check_replication(chi, {0,1}, 5) with Finally("I clean up"): delete_test_namespace() + +def check_replication(chi, replicas, token, table = ''): + cluster = clickhouse.query(chi, "select substitution from system.macros where macro = 'cluster'") + if table == '': + table = chi.replace('-','_') + + wait_for_cluster(chi, cluster, 1, len(replicas)) + + with When("Create a replicated table if not exists"): + clickhouse.query(chi, f"CREATE TABLE IF NOT EXISTS {table} ON CLUSTER '{cluster}' (a UInt32) Engine = ReplicatedMergeTree ORDER BY a") + + with And("I insert data in the replicated table"): + clickhouse.query(chi, f"INSERT INTO {table} select {token}", timeout=300) + + with Then("Check replicated table has data on both nodes"): + for replica in replicas: + out = clickhouse.query(chi, f"SELECT a from {table} where a={token}", host=f"chi-{chi}-{cluster}-0-{replica}-0") + assert out == f"{token}", error() + + @TestModule @Name("e2e.test_operator") @Requirements(RQ_SRS_026_ClickHouseOperator_CustomResource_APIVersion("1.0"), diff --git a/tests/e2e/util.py b/tests/e2e/util.py index 7dcb1b299..97aa98691 100644 --- a/tests/e2e/util.py +++ b/tests/e2e/util.py @@ -83,7 +83,7 @@ def require_keeper(keeper_manifest="", keeper_type=settings.keeper_type, force_i if keeper_type == "clickhouse-keeper": keeper_manifest = "clickhouse-keeper-1-node-256M-for-test-only.yaml" if keeper_manifest == "" else keeper_manifest keeper_manifest = f"../../deploy/clickhouse-keeper/clickhouse-keeper-manually/{keeper_manifest}" - if keeper_type == "CHK" or keeper_type == "clickhouse-keeper_with_chk": + if keeper_type == "chk": keeper_manifest = ( "clickhouse-keeper-1-node-for-test-only.yaml" if keeper_manifest == "" else keeper_manifest ) @@ -102,16 +102,14 @@ def require_keeper(keeper_manifest="", keeper_type=settings.keeper_type, force_i expected_docs = { "zookeeper": 5 if "scaleout-pvc" in keeper_manifest else 4, "clickhouse-keeper": 7, - "clickhouse-keeper_with_chk": 2, - "CHK": 2, + "chk": 2, "zookeeper-operator": 3 if "probes" in keeper_manifest else 1, } expected_pod_prefix = { "zookeeper": "zookeeper", "zookeeper-operator": "zookeeper", "clickhouse-keeper": "clickhouse-keeper", - "clickhouse-keeper_with_chk": "chk-clickhouse-keeper-test-only-0", - "CHK": "chk-clickhouse-keeper-test-only-0" + "chk": "chk-clickhouse-keeper-test-0" } assert ( docs_count == expected_docs[keeper_type] @@ -119,7 +117,7 @@ def require_keeper(keeper_manifest="", keeper_type=settings.keeper_type, force_i with Given(f"Install {keeper_type} {keeper_nodes} nodes"): kubectl.apply(get_full_path(keeper_manifest, lookup_in_host=False)) for pod_num in range(keeper_nodes): - if keeper_type == "CHK" or keeper_type == "clickhouse-keeper_with_chk" : + if keeper_type == "chk": pod_name = f"{expected_pod_prefix[keeper_type]}-{pod_num}-0" else: pod_name = f"{expected_pod_prefix[keeper_type]}-{pod_num}" @@ -129,6 +127,10 @@ def require_keeper(keeper_manifest="", keeper_type=settings.keeper_type, force_i kubectl.wait_pod_status(pod_name, "Running") kubectl.wait_container_status(pod_name, "true") + if keeper_type == "chk": + kubectl.wait_chk_status("clickhouse-keeper", 'Completed') + + def wait_clickhouse_cluster_ready(chi): with Given("All expected pods present in system.clusters"): @@ -165,9 +167,9 @@ def install_clickhouse_and_keeper( if keeper_type == "zookeeper": keeper_manifest = "zookeeper-1-node-1GB-for-tests-only.yaml" if keeper_type == "clickhouse-keeper": - keeper_manifest = "clickhouse-keeper-1-node-256M-for-test-only.yaml" - if keeper_type == "clickhouse-keeper_with_chk" or keeper_type == "CHK": - keeper_manifest = "clickhouse-keeper-1-node-for-test-only.yaml" + keeper_manifest = "clickhouse-keeper-1-node-256M-for-test.yaml" + if keeper_type == "chk": + keeper_manifest = "clickhouse-keeper-1-node-for-test.yaml" if keeper_type == "zookeeper-operator": keeper_manifest = "zookeeper-operator-1-node.yaml" @@ -296,7 +298,7 @@ def install_operator_if_not_exist( def install_operator_version(version, shell=None): - if version == current().context.operator_version: + if version == current().context.operator_version or version == "dev": manifest = get_full_path(current().context.clickhouse_operator_install_manifest) manifest = f"cat {manifest}" else: diff --git a/tests/image/build_docker.sh b/tests/image/build_docker.sh index ea9372437..43aa020d9 100755 --- a/tests/image/build_docker.sh +++ b/tests/image/build_docker.sh @@ -12,7 +12,7 @@ CLICKHOUSE_IMAGE=${CLICKHOUSE_IMAGE:="clickhouse/clickhouse-server:23.8"} CLICKHOUSE_IMAGE_OLD=${CLICKHOUSE_IMAGE_OLD:="clickhouse/clickhouse-server:23.3"} CLICKHOUSE_IMAGE_LATEST=${CLICKHOUSE_IMAGE_LATEST:="clickhouse/clickhouse-server:latest"} CLICKHOUSE_OPERATOR_TESTS_IMAGE=${CLICKHOUSE_OPERATOR_TESTS_IMAGE:="registry.gitlab.com/altinity-public/container-images/clickhouse-operator-test-runner:latest"} -ZOOKEEPER_IMAGE=${ZOOKEEPER_IMAGE:="zookeeper:3.8.3"} +ZOOKEEPER_IMAGE=${ZOOKEEPER_IMAGE:="zookeeper:3.8.4"} K8S_VERSION=${K8S_VERSION:=1.28.5} MINIKUBE_PRELOADED_TARBALL="preloaded-images-k8s-v18-v${K8S_VERSION}-docker-overlay2-amd64.tar.lz4"