Migrate the router floating IP from Puppet LBs to cloudscale LBaaS

# For example: https://api.syn.vshn.net
# IMPORTANT: do NOT add a trailing `/`. Commands below will fail.
export COMMODORE_API_URL=<lieutenant-api-endpoint>
export COMMODORE_API_TOKEN=<lieutenant-api-token>

export CLUSTER_ID=<lieutenant-cluster-id> # Looks like: c-<something>
export TENANT_ID=$(curl -sH "Authorization: Bearer ${COMMODORE_API_TOKEN}" ${COMMODORE_API_URL}/clusters/${CLUSTER_ID} | jq -r .tenant)

# From https://git.vshn.net/-/profile/personal_access_tokens, "api" scope is sufficient
export GITLAB_TOKEN=<gitlab-api-token>
export GITLAB_USER=<gitlab-user-name>

export VAULT_ADDR=https://vault-prod.syn.vshn.net
vault login -method=ldap username=<your.name>

export HIERADATA_REPO_SECRET=$(vault kv get \
  -format=json "clusters/kv/lbaas/hieradata_repo_token" | jq '.data.data')
export HIERADATA_REPO_USER=$(echo "${HIERADATA_REPO_SECRET}" | jq -r '.user')
export HIERADATA_REPO_TOKEN=$(echo "${HIERADATA_REPO_SECRET}" | jq -r '.token')

cloudscale_token=$(vault kv get \
  -format=json "clusters/kv/${TENANT_ID}/${CLUSTER_ID}/cloudscale")
export CLOUDSCALE_API_TOKEN=$(echo $cloudscale_token | jq -r '.data.data.token')

floaty_key=$(vault kv get \
  -format=json "clusters/kv/${TENANT_ID}/${CLUSTER_ID}/floaty")
export TF_VAR_lb_cloudscale_api_secret=$(echo $floaty_key | jq -r '.data.data.iam_secret')

commodore catalog compile "${CLUSTER_ID}"

kubectl --as=cluster-admin -n syn patch apps root --type=json \
  -p '[{"op":"replace", "path":"/spec/syncPolicy", "value": {}}]'

pushd inventory/classes/${TENANT_ID}

yq eval -i '.applications += ["cloudscale-loadbalancer-controller"]' ${CLUSTER_ID}.yml
yq eval -i ".parameters.openshift4_terraform.terraform_variables.allocate_router_vip_for_lb_controller = true" \
  ${CLUSTER_ID}.yml

git commit -a -m "Prepare ingress migration to cloudscale LBaaS for ${CLUSTER_ID}"
git push

popd

commodore catalog compile ${CLUSTER_ID}

kubectl --as=cluster-admin apply -f catalog/manifests/cloudscale-loadbalancer-controller/00_namespace.yaml
kapitan refs --reveal --refs-path catalog/refs -f catalog/manifests/cloudscale-loadbalancer-controller/10_secrets.yaml | \
  kubectl --as=cluster-admin apply -f -
kubectl --as=cluster-admin apply -Rf catalog/manifests/cloudscale-loadbalancer-controller/10_kustomize
kubectl -n appuio-cloudscale-loadbalancer-controller \
  wait --for condition=available \
  deploy cloudscale-loadbalancer-controller-controller-manager

yq 'del(.spec.floatingIPAddresses)' catalog/manifests/cloudscale-loadbalancer-controller/20_loadbalancers.yaml | \
  kubectl --as=cluster-admin apply -f -

export LB_IP=""
while [ "$LB_IP" == "" ]; do
  export LB_IP=$(kubectl --as=cluster-admin -n appuio-cloudscale-loadbalancer-controller \
    get loadbalancer ingress -oyaml | \
    yq '(.status.virtualIPAddresses[]|select(.address|contains("2a06")|not)|.address)//""')
  if [ "$LB_IP" == "" ]; then
    echo -n "."
    sleep 5
  fi
done && echo -e "\nLoadbalancer available at ${LB_IP}"

export APPS_DOMAIN=$(kapitan inventory -t cluster --inventory-backend=reclass-rs | \
  yq .parameters.openshift.appsDomain)
curl --resolve console-openshift-console.${APPS_DOMAIN}:80:${LB_IP} \
  http://console-openshift-console.${APPS_DOMAIN} -vI (1)

cat <<EOF > ./terraform.env
CLOUDSCALE_API_TOKEN
TF_VAR_ignition_bootstrap
TF_VAR_lb_cloudscale_api_secret
TF_VAR_control_vshn_net_token
GIT_AUTHOR_NAME
GIT_AUTHOR_EMAIL
HIERADATA_REPO_TOKEN
EOF

# Set terraform image and tag to be used
tf_image=$(\
  yq eval ".parameters.openshift4_terraform.images.terraform.image" \
  dependencies/openshift4-terraform/class/defaults.yml)
tf_tag=$(\
  yq eval ".parameters.openshift4_terraform.images.terraform.tag" \
  dependencies/openshift4-terraform/class/defaults.yml)

# Generate the terraform alias
base_dir=$(pwd)
alias terraform='touch .terraformrc; docker run -it --rm \
  -e REAL_UID=$(id -u) \
  -e TF_CLI_CONFIG_FILE=/tf/.terraformrc \
  --env-file ${base_dir}/terraform.env \
  -w /tf \
  -v $(pwd):/tf \
  --ulimit memlock=-1 \
  "${tf_image}:${tf_tag}" /tf/terraform.sh'

export GITLAB_REPOSITORY_URL=$(curl -sH "Authorization: Bearer $(commodore fetch-token)" ${COMMODORE_API_URL}/clusters/${CLUSTER_ID} | jq -r '.gitRepo.url' | sed 's|ssh://||; s|/|:|')
export GITLAB_REPOSITORY_NAME=${GITLAB_REPOSITORY_URL##*/}
export GITLAB_CATALOG_PROJECT_ID=$(curl -sH "Authorization: Bearer ${GITLAB_TOKEN}" "https://git.vshn.net/api/v4/projects?simple=true&search=${GITLAB_REPOSITORY_NAME/.git}" | jq -r ".[] | select(.ssh_url_to_repo == \"${GITLAB_REPOSITORY_URL}\") | .id")
export GITLAB_STATE_URL="https://git.vshn.net/api/v4/projects/${GITLAB_CATALOG_PROJECT_ID}/terraform/state/cluster"

pushd catalog/manifests/openshift4-terraform/

terraform init \
  "-backend-config=address=${GITLAB_STATE_URL}" \
  "-backend-config=lock_address=${GITLAB_STATE_URL}/lock" \
  "-backend-config=unlock_address=${GITLAB_STATE_URL}/lock" \
  "-backend-config=username=${GITLAB_USER}" \
  "-backend-config=password=${GITLAB_TOKEN}" \
  "-backend-config=lock_method=POST" \
  "-backend-config=unlock_method=DELETE" \
  "-backend-config=retry_wait_min=5"

terraform state mv "module.cluster.module.lb.cloudscale_floating_ip.router_vip[0]" \
  "module.cluster.cloudscale_floating_ip.router_vip[0]"

terraform refresh
export INGRESS_FLOATING_IP=$(terraform output -raw router_vip)

declare -a LB_FQDNS
for id in 1 2; do
  LB_FQDNS[$id]=$(terraform state show "module.cluster.module.lb.cloudscale_server.lb[$(expr $id - 1)]" | grep fqdn | awk '{print $2}' | tr -d ' "\r\n')
done

for lb in "${LB_FQDNS[@]}"; do
  ssh $lb sudo puppetctl stop "Migrating router floating IP"
done

for lb in "${LB_FQDNS[@]}"; do
  ssh $lb sudo sed -i "/${INGRESS_FLOATING_IP}/d" /etc/floaty/global.yaml
  ssh $lb sudo systemctl restart keepalived
done

terraform apply

popd

pushd inventory/classes/${TENANT_ID}

yq eval -i '.parameters.openshift.cloudscale.ingress_floating_ip_v4 = "'$INGRESS_FLOATING_IP'"' \
  ${CLUSTER_ID}.yml

git commit -a -m "Migrate ingress floating IP to cloudscale LBaaS for ${CLUSTER_ID}"
git push

popd

commodore catalog compile ${CLUSTER_ID} --push -i

kubectl --as=cluster-admin -n syn patch apps root --type=json \
  -p '[{
    "op":"replace",
    "path":"/spec/syncPolicy",
    "value": {"automated": {"prune": true, "selfHeal": true}}
  }]'

export AUTH_HEADER="Authorization: Bearer ${TF_VAR_lb_cloudscale_api_secret}"
while [ "$(curl -sH"$AUTH_HEADER" https://api.cloudscale.ch/v1/floating-ips/${INGRESS_FLOATING_IP} | jq -r '.load_balancer')" == "null" ]; do
  echo -n '.'
  sleep 1
done && echo -e "\nFloating IP attached to LBaaS instance"

curl https://console-openshift-console.${APPS_DOMAIN} -vI

for lb in "${LB_FQDNS[@]}"; do
  ssh $lb sudo puppetctl start
done

curl https://console-openshift-console.${APPS_DOMAIN} -vI