Deploy on existing OpenShift cluster

This guide describes how to deploy cloudscale.ch CCM on an existing cluster.

Step-by-step guide

  1. Ensure nodes are registered with cloudscale.ch provider id

    If not already present add the following machine config which customizes the kubelet systemd service on all cluster nodes. As described in github.com/openshift/infrastructure-provider-onboarding-guide/blob/main/docs/platform-external/installing.md.

    Applying the MachineConfig causes node reboots. 
    spec:
  config:
    ignition:
      version: 3.2.0
    storage:
      files:
        - mode: 493 # 0755
          path: /usr/local/bin/appuio-kubelet-providerid
          contents:
            inline: |
              #!/bin/bash
              set -e -o pipefail
              NODECONF=/etc/systemd/system/kubelet.service.d/20-appuio-providerid.conf
              if [ -e "\${NODECONF}" ]; then
                echo "Not replacing existing \${NODECONF}"
                exit 0
              fi
              PROVIDERID=$(curl -sL http://169.254.169.254/openstack/2017-02-22/meta_data.json | \
                jq -r .meta.cloudscale_uuid)
              cat >"\${NODECONF}" <<EOF
              [Service]
              Environment="KUBELET_PROVIDERID=cloudscale://\${PROVIDERID}"
              EOF
    systemd:
      units:
        - name: appuio-kubelet-providerid.service
          enabled: true
          contents: |
            [Unit]
            Description=Fetch provide id from metadata service
            After=NetworkManager-wait-online.service
            Before=kubelet.service
            [Service]
            ExecStart=/usr/local/bin/appuio-kubelet-providerid
            Type=oneshot
            [Install]
            WantedBy=network-online.target

  2. Check if provider id is present on nodes

    for n in $(kubectl get nodes -oname); do
  PROVIDERID=$(oc -n syn-debug-nodes --as=cluster-admin debug $n -- chroot /host cat /etc/systemd/system/kubelet.service.d/20-appuio-providerid.conf 2>&1 | grep PROVIDERID | sed -e 's/^Environment="KUBELET_PROVIDERID=\([^"]\+\)"$/\1/g')
  echo kubectl --as=cluster-admin patch $n --type=merge -p "{\"spec\":{\"providerID\":\"${PROVIDERID}\"}}";
done

  3. Patch nodes with provider id

    for n in $(kubectl get nodes -oname); do
  PROVIDERID=$(oc -n syn-debug-nodes --as=cluster-admin debug $n -- chroot /host cat /etc/systemd/system/kubelet.service.d/20-appuio-providerid.conf 2>&1 | grep PROVIDERID | sed -e 's/^Environment="KUBELET_PROVIDERID=\([^"]\+\)"$/\1/g')
  kubectl --as=cluster-admin patch $n --type=merge -p "{\"spec\":{\"providerID\":\"${PROVIDERID}\"}}";
done
kubectl get no -ocustom-columns='NAME:.metadata.name,PROVIDER_ID:.spec.providerID'
    NAME           PROVIDER_ID
infra-8344     cloudscale://1b04b1fb-d6c8-4108-92f4-f64f8c1ca960
infra-87c9     cloudscale://2813282a-2a88-461d-a927-4ad96bf4360a
infra-eba2     cloudscale://22fd8f7d-0eb8-4d10-82d7-e35df65e62b8
master-3b79    cloudscale://b7597a27-0129-40cc-9bd8-ac19f5d3deb6
master-6cc2    cloudscale://556ca8bd-37a3-4a48-8412-8eafe90c606a
master-edc2    cloudscale://48255c88-ad2b-4242-a4ba-74332f16b264
storage-17fd   cloudscale://2584f7f1-a2cf-4598-903b-80ff6acc1dd0
storage-534a   cloudscale://e2c96748-7d24-45e7-b345-8b83d7dc21bf
storage-c4e4   cloudscale://a8358646-fe30-4fb6-bffb-6971863286d3
worker-10db    cloudscale://f2a2cc9b-aae9-4674-ae64-de4bd3cfb1db
worker-3b6c    cloudscale://671553ab-2243-4075-b35b-0592aa9d2299
worker-52c6    cloudscale://a6255254-46bf-4c4f-8638-9eb26559814e
worker-8e76    cloudscale://a0ba2200-bfc3-435a-b733-8607b6ee4190

  4. Deploy component-cloudscale-cloud-controller-manager

  5. Patch infrastructure config

    This step triggers node reboots to apply the Kubelet flag --cloud-provider=external. 
    kubectl --as cluster-admin patch infrastructure.config  cluster --type=merge -p '{"spec":{"platformSpec":{"external":{"platformName":"cloudscale.ch"},"type":"External"}}}'
infrastructure.config.openshift.io/cluster patched
    kubectl proxy &
curl -XPATCH -H"Content-Type: application/merge-patch+json" http://localhost:8001/apis/config.openshift.io/v1/infrastructures/cluster/status -d '{"status":{"platform":"External","platformStatus":{"external":{"cloudControllerManager":{"state":"External"}},"type":"External"}}}'

  6. Taint nodes with node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule

    kubectl --as cluster-admin taint node --all node.cloudprovider.kubernetes.io/uninitialized=true:NoSchedule

  7. Check if instance-type is applied

    kubectl get nodes -ocustom-columns='NAME:.metadata.name,INSTANCE_TYPE:.metadata.labels.node\.kubernetes\.io/instance-type'
    NAME           INSTANCE_TYPE
infra-8344     flex-24-6
infra-87c9     flex-24-6
infra-eba2     flex-24-6
master-3b79    flex-24-6
master-6cc2    flex-24-6
master-edc2    flex-24-6
storage-17fd   flex-8-2
storage-534a   flex-8-2
storage-c4e4   flex-8-2
worker-10db    flex-16-4
worker-3b6c    flex-16-4
worker-52c6    flex-16-4
worker-8e76    flex-16-4