Create a Kubernetes cluster on Openstack with Kubeadm

    Your Kubernetes control-plane has initialized successfully!

    To start using your cluster, you need to run the following as a regular user:

        mkdir -p $HOME/.kube
        sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
        sudo chown $(id -u):$(id -g) $HOME/.kube/config

    Alternatively, if you are the root user, you can run:

        export KUBECONFIG=/etc/kubernetes/admin.conf

    You should now deploy a pod network to the cluster.
    Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
        https://kubernetes.io/docs/concepts/cluster-administration/addons/

    You can now join any number of control-plane nodes by copying certificate authorities
    and service account keys on each node and then running the following as root:

        kubeadm join 10.172.89.94:6443 --token q1zxoz.17z6x5lxwo9kklbw \
        --discovery-token-ca-cert-hash sha256:2f10e19e0d019eee3a2083e62b9831f44dcc22097999de2728955374afb17d26 \
        --control-plane

    Then you can join any number of worker nodes by running the following on each as root:

    kubeadm join 10.172.89.94:6443 --token q1zxoz.17z6x5lxwo9kklbw \
    --discovery-token-ca-cert-hash sha256:2f10e19e0d019eee3a2083e62b9831f44dcc22097999de2728955374afb17d26

    # https://github.com/kubernetes/ingress-nginx/tree/main/charts/ingress-nginx

    helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
    helm repo update

    # Download Chart locally and then deploy
    helm pull  ingress-nginx/ingress-nginx --version 4.1.4 --untar

    # Update values.yml to enable hostPort in order `host port` in order to expose the ctrl on  port 80 & 443
        ## Use host ports 80 and 443
        ## Disabled by default
        hostPort:
            # -- Enable 'hostPort' or not
            enabled: true
            ports:
            # -- 'hostPort' http port
            http: 80
            # -- 'hostPort' https port
            https: 443

    # Deploy
    helm  --kubeconfig ../.config/kube_certif install ingress-ctrl-nginx -n ingress-ctrl --create-namespace   ingress-nginx/

    # in container section of deplyment, you can see
        name: controller
        ports:
        - containerPort: 80
        hostPort: 80
        name: http
        protocol: TCP
        - containerPort: 443
        hostPort: 443
        name: https
        protocol: TCP

    helm repo add cpo https://kubernetes.github.io/cloud-provider-openstack
    helm repo update

    # values file
    cat > cinder-csi_values.yaml <<EOF
    storageClass:
    enabled: true
    delete:
        isDefault: true
        allowVolumeExpansion: true
    retain:
        isDefault: false
        allowVolumeExpansion: true
    secret:
    enabled: true
    create: true
    name: csi-cloud-config
    data:
        cloud-config: |-
        [Global]
        auth-url="{{ openstack cloud API endpoint }}"
        username="{{ openstack cloud username }}"
        password="{{ openstack cloud password }}"
        domain-name=Default
        region="{{ openstack cloud project name }}"
        tenant-name="{{ openstack cloud region name }}"
    EOF

    # check chart version
    helm search repo cpo/openstack-cinder-csi --versions
    NAME                        CHART VERSION   APP VERSION DESCRIPTION                   
    cpo/openstack-cinder-csi    2.2.0           v1.24.0     Cinder CSI Chart for OpenStack
    cpo/openstack-cinder-csi    2.1.0           v1.23.0     Cinder CSI Chart for OpenStack
    cpo/openstack-cinder-csi    1.4.9           v1.22.0     Cinder CSI Chart for OpenStack
    cpo/openstack-cinder-csi    1.4.8           latest      Cinder CSI Chart for OpenStack
    cpo/openstack-cinder-csi    1.3.9           v1.21.0     Cinder CSI Chart for OpenStack

    # version 2.2.0 don't work for me, error: still connecting to unix:///csi/csi.sock
    # K8s: Server Version: version.Info{Major:"1", Minor:"24"

    # Deploy
    helm install cinder-csi cpo/openstack-cinder-csi --values components/cinder-csi_values.yaml -n kube-system --atomic --version 1.4.9

    # https://github.com/sighupio/permission-manager/blob/master/docs/installation.md

    kubectl create ns permission-manager
    kubectl apply -f https://github.com/sighupio/permission-manager/releases/download/v1.7.1-rc1/deploy.yml -n  permission-manager