We recently changed the storage backend of the clusters and hence introduced a new storage class. The question now is, how do we migrate the existing storage to use the new storage class?
Luckily there is already a tool which supports storage migration: pv-migrate :
is a CLI tool/kubectl plugin to easily migrate the contents of one Kubernetes PersistentVolumeClaim
to another.
Migration Procedure In order to be able to migrate an existing volume, we may need to ensure that the volume is not used.
Create a new namespace e.g. volume-mig
where we will place temporary pvc for the migration process
Get the pv used for the pvc we want to migrate
Before we proceed, you want to ensure that the pv has the reclaim policy set to retain , so unclaimed pv’s are not removed
Create a new pvc $PVC_NAME-new.yaml
in the volume-mig
namespace.
apiVersion : v1
kind : PersistentVolumeClaim
metadata :
name : <storage-name>-new
namespace : volume-mig
spec :
accessModes :
- ReadWriteOnce
resources :
requests :
storage : <size>
storageClassName : <target storageclass>
Apply $PVC_NAME-new.yaml
and grab the pv name.
Prepare the deployment $PVC_NAME-old.yaml
for a pvc with the old volume (from 2.) an the volume-mig
- we don’t apply it yet
)
apiVersion : v1
kind : PersistentVolumeClaim
metadata :
name : <storage-name>-old
namespace : volume-mig
spec :
accessModes :
- ReadWriteOnce
resources :
requests :
storage : <size>
volumeName : <pv-name>
storageClassName : <origin storageclass>
Prepare the deployment $PVC_NAME.yaml
for a pvc with the new volume (from 5.) an the volume-mig
- we don’t apply it yet
)
apiVersion : v1
kind : PersistentVolumeClaim
metadata :
name : <storage-name>-old
namespace : volume-mig
spec :
accessModes :
- ReadWriteOnce
resources :
requests :
storage : <size>
volumeName : <pv-name from 5.>
storageClassName : <target storageclass>
Scale down the pod which is using the volume to be migrated (pv from 2.). If the workload is managed by an operator, the operator has also to be scaled down.
Delete the pvc of the pv you want to migrate
Remove the claimref
in the pv (from 2.)
Deploy $PVC_NAME-old.yaml
Run pv-migrate to migrate the content from the old volume (2.) to the new one (5.)
Once the migration is done, the pvc for the old and the new volume have to be removed.
The same applies to the claimref
on both volumes - actually important is to remove it on the new volume, cause we will use this for the original workload.
Deploy $PVC_NAME.yaml
which will re-create the pvc we deleted in step 9. The main difference is, that it is using the new volume (based on the new storage class)
Scale up the workload back - same applies for the operator if applicable.
Script it The work is very tedious and error-prone, so I wrapped it in a little script, to automate the whole thing.
#!/usr/bin/env bash -e
STORAGECLASS_OLD= "hpe-csi"
STORAGECLASS= "vsphere-csi-sc"
MIG_NS= "volume-mig"
DEBUG= 0
SCALE_OPERATOR= 0
ERR_MSG= "ERROR, please check log"
trap 'ret=$?; printf "%s\n" "$ERR_MSG" >&2; exit "$ret"' ERR
# Function: Print a help message.
usage() {
echo "Migrate volume from '-s STROAGECLASS' to '-t STORAGECLASS'"
echo "-c CONTEXT kubectl context" 1>&2
echo "-n NAMESPACE namespace in which you want to migrated" 1>&2
echo "-s STORAGECLASS to migrate from (source)" 1>&2
echo "-t STORAGECLASS to migrate to (target)" 1>&2
echo "-d Show debug information" 1>&2
}
# Function: Exit with error.
exit_abnormal() {
usage
exit 1
}
text_separator() {
ch= "-"
len= "80"
if [ " $#" -eq 2 ] ; then
ch= $1
elif [ " $#" -gt 2 ] ; then
ch= $1
len= $2
fi
printf '%*s\n' " $len" | tr ' ' " $ch"
}
create_mig_pvc_old() {
local PVC_NAME= $1
local PV_NAME= $2
local STORAGE= $3
local YAML_FILE= $TARGET/$PVC_NAME-old.yaml
echo "[Info] Create pvc ' $PVC_NAME-old' with the old volume in the temp ns ' $MIG_NS'"
touch $YAML_FILE
CREATE_PVC_CMD= "cat $YAML_FILE_ORIG | yq 'del(.status)|del(.metadata.finalizers)|del(.metadata.resourceVersion)|del(.metadata.uid)|del(.metadata.annotations)|del(.metadata.creationTimestamp)|.spec.storageClassName=\" $STORAGECLASS_OLD\"|.metadata.namespace=\" $MIG_NS\"|.spec.volumeName=\" $PV_NAME\"|.metadata.name=\" $PVC_NAME-old\"' > $YAML_FILE"
if [ $DEBUG -eq 1 ] ; then
echo -e "[Debug] Create PVC $PVC_NAME-new:\n $CREATE_PVC_CMD"
fi
bash -c " $CREATE_PVC_CMD"
kubectl apply -f $YAML_FILE -n $MIG_NS --context $CONTEXT
sleep 5
}
create_mig_pvc_new() {
local PVC_NAME= $1
local STORAGE= $2
local YAML_FILE= $TARGET/$PVC_NAME-new.yaml
echo "[Info] Deploy a new pvc ' $PVC_NAME' in the temp ns ' $MIG_NS'"
touch $YAML_FILE
CREATE_PVC_CMD= "cat $YAML_FILE_ORIG | yq 'del(.status)|del(.metadata.finalizers)|del(.metadata.resourceVersion)|del(.metadata.uid)|del(.metadata.annotations)|del(.metadata.creationTimestamp)|del(.spec.volumeName)|.spec.storageClassName=\" $STORAGECLASS\"|.metadata.namespace=\" $MIG_NS\"|.metadata.name=\" $PVC_NAME-new\"' > $YAML_FILE"
if [ $DEBUG -eq 1 ] ; then
echo -e "[Debug] Create PVC $PVC_NAME-new:\n $CREATE_PVC_CMD"
fi
bash -c " $CREATE_PVC_CMD"
kubectl apply -f $YAML_FILE -n $MIG_NS --context $CONTEXT
sleep 5
}
create_new_pvc() {
local PVC_NAME= $1
local PV_NAME= $2
local STORAGE= $3
local YAML_FILE= $TARGET/$PVC_NAME.yaml
echo "[Info] Deploy the pvc ' $PVC_NAME' in the namespace ' $NAMESPACE'"
touch $YAML_FILE
CREATE_PVC_CMD= "cat $YAML_FILE_ORIG | yq 'del(.status)|del(.metadata.finalizers)|del(.metadata.resourceVersion)|del(.metadata.uid)|del(.metadata.annotations)|del(.metadata.creationTimestamp)|del(.spec.volumeName)|.spec.storageClassName=\" $STORAGECLASS\"|.metadata.namespace=\" $NAMESPACE\"|.spec.volumeName=\" $PV_NAME\"|.metadata.name=\" $PVC_NAME\"' > $YAML_FILE"
if [ $DEBUG -eq 1 ] ; then
echo -e "[Debug] Create PVC $PVC_NAME:\n $CREATE_PVC_CMD"
fi
bash -c " $CREATE_PVC_CMD"
kubectl apply -f $YAML_FILE -n $NAMESPACE --context $CONTEXT
sleep 5
}
get_original_pvc() {
local PVC_NAME= $1
YAML_FILE_ORIG= $TARGET/$PVC_NAME-orig.yaml
touch $YAML_FILE_ORIG
CREATE_PVC_CMD= "kubectl get pvc $PVC_NAME -n $NAMESPACE --context $CONTEXT -o yaml > $YAML_FILE_ORIG"
bash -c " $CREATE_PVC_CMD"
}
do_migrate() {
echo "[Info] Perform migration with pv-migrate"
local PVC_NAME= $1
CMD_MIGRATE= "pv-migrate migrate $PVC_NAME-old -n $MIG_NS -c $CONTEXT $PVC_NAME-new -N $MIG_NS -C $CONTEXT"
echo -e "[Debug] command for migratiion:\n $CMD_MIGRATE"
bash -c " $CMD_MIGRATE"
}
scale_workload() {
local WORKLOAD= $1
local KIND= $2
local REPLICAS= $3
echo "[Info] Scaling ${ KIND} ' ${ WORKLOAD} ' to ${ REPLICAS} "
if [ -z " $WORKLOAD" ] || [ -z " $KIND" ] || [ -z " $REPLICAS" ] ; then
echo "[Error] Missing parameters, aborting"
echo " WORKLOAD= $WORKLOAD"
echo " KIND= $KIND"
echo " REPLICAS= $REPLICAS"
exit 1
fi
local text_to_display= "Scale workload (y/n)?"
CMD_SCALE= "kubectl scale $KIND $WORKLOAD --replicas= $REPLICAS -n $NAMESPACE --context $CONTEXT"
if [ $DEBUG -eq 1 ] ; then
echo $CMD_SCALE
fi
while [[ " $CONTINUE" != "y" ]] && [[ " $CONTINUE" != "n" ]] ; do
echo -e $text_to_display
read CONTINUE
done
if [[ " $CONTINUE" == "y" ]] ; then
bash -c " $CMD_SCALE"
fi
CONTINUE= ""
sleep 5 # TODO: Remove
}
delete_pvc() {
local PVC_NAME= $1
local CMD_DELETE_PVC= "kubectl delete pvc $PVC_NAME -n $NAMESPACE --context $CONTEXT"
if [ $DEBUG -eq 1 ] ; then
echo $CMD_DELETE_PVC
fi
bash -c " $CMD_DELETE_PVC"
}
remove_claimref() {
local PV_NAME= $1
local NAMESPACE= $2
local YAML_FILE= $TARGET/$PV_NAME.yaml
echo "[Info] Remove ClaimRef from volume ' ${ PV_NAME} ' in ns ' $NAMESPACE'"
#CMD_REMOVE_CLAIMREF="kubectl get pv $PV_NAME -n $NAMESPACE --context $CONTEXT -o yaml | yq 'del(.spec.claimRef)' | yq 'del(.status)' > $YAML_FILE"
CMD_REMOVE_CLAIMREF= "kubectl patch pv $PV_NAME -n $NAMESPACE --context $CONTEXT -p '{\"spec\":{\"claimRef\": null}}'"
if [ $DEBUG -eq 1 ] ; then
echo $CMD_REMOVE_CLAIMREF
fi
bash -c " $CMD_REMOVE_CLAIMREF"
#bash -c "kubectl apply -f $YAML_FILE -n $NAMESPACE --context $CONTEXT"
}
delete_mig_pvcs() {
echo "[Info] Delete the pvcs in the temp ns ' $MIG_NS'"
local PVC_NAME= $1
kubectl delete pvc " $PVC_NAME-old" -n " $MIG_NS" --context " $CONTEXT"
kubectl delete pvc " $PVC_NAME-new" -n " $MIG_NS" --context " $CONTEXT"
}
if ! command -v yq &> /dev/null
then
echo "yq could not be found, but is required for this script to run."
echo "BINARY=yq_linux_amd64 && VERSION=v4.25.1 && wget https://github.com/mikefarah/yq/releases/download/ ${ VERSION} / ${ BINARY} .tar.gz -O - | tar xz && sudo mv ${ BINARY} /usr/bin/yq"
exit
fi
if ! command -v pv-migrate &> /dev/null
then
#pv-migrate_v0.12.1_linux_arm64.tar.gz
echo "pv-migrate could not be found, but is required for this script to run."
echo "BINARY=linux_x86_64 && VERSION=v0.12.1 && wget https://github.com/utkuozdemir/pv-migrate/releases/download/\${VERSION}/pv-migrate_\${VERSION}_\${BINARY}.tar.gz -O - | tar xz && sudo mv pv-migrat /usr/bin/pv-migrate"
exit
fi
if ! command -v kubectl &> /dev/null
then
echo "kubectl could not be found, but is required for this script to run."
exit
fi
while getopts "hc:n:ado" options; do
case ${ options} in
c )
CONTEXT= ${ OPTARG}
echo "Context ' $CONTEXT' selected"
;;
n )
NAMESPACE= ${ OPTARG}
echo "Namespace ' $NAMESPACE' selected"
;;
s )
STORAGECLASS_OLD= ${ OPTARG}
echo "Source storage class ' $STORAGECLASS_OLD'"
;;
5 )
STORAGECLASS= ${ OPTARG}
echo "Target storage class ' $STORAGECLASS'"
;;
d )
DEBUG= 1
;;
h) exit_abnormal;;
\? ) exit_abnormal;;
*) exit_abnormal;;
esac
done
if [ ! " $NAMESPACE" ] || ! [ " $CONTEXT" ] ; then
echo -e "arguments -c and -n must be provided"
exit_abnormal
fi
if [ ! " $STORAGECLASS" ] || ! [ " $STORAGECLASS_OLD" ] ; then
echo -e "arguments -s and -t must be provided"
exit_abnormal
fi
TARGET= "./tmp/ $CONTEXT"
if [[ ! -d " $TARGET" ]] ; then
mkdir -p $TARGET
fi
text_separator
CMD_GET_PVCS= "kubectl get pvc -n $NAMESPACE --context $CONTEXT -o=jsonpath=\"{range .items[?(@.spec.storageClassName=='hpe-csi')]}{.metadata.name}{' '}{end}\""
LIST_OF_PVC= $( bash -c " $CMD_GET_PVCS" )
PS3= "Select which of the PVCs you want to migrate from ' $STORAGECLASS_OLD' to ' $STORAGECLASS': "
select PVC_NAME in $LIST_OF_PVC "quit" ; do
case $PVC_NAME in
quit)
break
;;
*)
echo -e "Selected pvc: $PVC_NAME\n"
CMD_GET_PVC= "kubectl get pvc $PVC_NAME -n $NAMESPACE --context $CONTEXT -o=jsonpath=\"{.spec.volumeName}{' '}{.metadata.labels}\""
if [ $DEBUG -eq 1 ] ; then
echo -e "[Debug] Get PVC $PVC_NAME:\n $CMD_GET_PVC"
fi
PVC=( $( bash -c " $CMD_GET_PVC" ) )
if [ $DEBUG -eq 1 ] ; then
echo -e "[Debug] Result:\n ${ PVC[@]} \n"
fi
CMD_GET_PV_DETAILS= "kubectl get pv ${ PVC[0]} --context $CONTEXT -o=jsonpath=\"{.spec.capacity.storage}{' '}{.spec.persistentVolumeReclaimPolicy}\""
if [ $DEBUG -eq 1 ] ; then
echo -e "[Debug] Get PV details:\n $CMD_GET_PV_DETAILS"
fi
PV_DETAILS=( $( bash -c " $CMD_GET_PV_DETAILS" ) )
if [ $DEBUG -eq 1 ] ; then
echo -e "[Debug] Result:\n ${ PV_DETAILS} \n"
fi
text_separator
if [[ " ${ PV_DETAILS[1]} " == "Retain" ]] ; then
echo "[Info] Starting with migration of VolumeName ' $PVC'"
text_separator
echo "[Info] Check if operators are present. Operators related to the workloads using ' $PVC' shall be scaled down"
CMD_GET_OPERATORS= "kubectl get sts,deploy -n $NAMESPACE --context $CONTEXT -o=json | jq -r '.items[] | select(.metadata.name| test(\".*operator\")) | \"\\(.kind)/\\(.metadata.name)\"'"
if [ $DEBUG -eq 1 ] ; then
echo -e "[Debug] Get operators:\n ${ CMD_GET_OPERATORS} "
fi
LIST_OF_OPERATORS= $( bash -c " $CMD_GET_OPERATORS" )
PS3= "Select operators to scale down, use skip if workload using the pvc is not managed by an operator"
select OPERATOR_NAME in $LIST_OF_OPERATORS "skip" ; do
case $OPERATOR_NAME in
skip)
break
;;
*)
SCALE_OPERATOR= 1
echo "[Info] Operator selected: $OPERATOR_NAME"
CMD_GET_OP_DETAILS= "kubectl get $OPERATOR_NAME -n $NAMESPACE --context $CONTEXT -o=jsonpath=\"{.metadata.name}{' '}{.spec.replicas}{' '}{.kind}{' '}{.spec.template.metadata.labels.app\.kubernetes\.io/managed-by}{' '}{.metadata.labels.app\.kubernetes\.io/instance}\""
if [ $DEBUG -eq 1 ] ; then
echo -e "[Debug] Get operator details from operator workload:\n ${ CMD_GET_OP_DETAILS} "
fi
OPERATORS_DETAILS=( $( bash -c " ${ CMD_GET_OP_DETAILS} " ) )
if [ $DEBUG -eq 1 ] ; then
echo -e "[Debug] Result:\n ${ OPERATORS_DETAILS[@]} "
fi
break
;;
esac
done
CMD_GET_WORKLOADS= "kubectl get sts,deploy -n $NAMESPACE --context $CONTEXT -o=jsonpath=\"{range .items[*]}{.kind}{'/'}{.metadata.name}{' '}{end}\""
if [ $DEBUG -eq 1 ] ; then
echo -e "[Debug] Get workloads:\n ${ CMD_GET_WORKLOADS} "
fi
LIST_OF_WORKLOADS= $( bash -c " $CMD_GET_WORKLOADS" )
PS3= "Select workload to which the PVC ' $PVC_NAME' belongs to: "
select WORKLOAD_NAME in $LIST_OF_WORKLOADS "quit" ; do
case $WORKLOAD_NAME in
quit)
break
;;
*)
echo "[Info] Workload manually selected: $WORKLOAD_NAME"
CMD_GET_WL_DETAILS= "kubectl get $WORKLOAD_NAME -n $NAMESPACE --context $CONTEXT -o=jsonpath=\"{.metadata.name}{' '}{.spec.replicas}{' '}{.kind}{' '}{.spec.template.metadata.labels.app\.kubernetes\.io/managed-by}{' '}{.metadata.labels.app\.kubernetes\.io/instance}\""
if [ $DEBUG -eq 1 ] ; then
echo -e "[Debug] Get workload details from manually selected workload:\n ${ CMD_GET_WL_DETAILS} "
fi
DEPLOYMENT_DETAILS=( $( bash -c " ${ CMD_GET_WL_DETAILS} " ) )
if [ $DEBUG -eq 1 ] ; then
echo -e "[Debug] Result:\n ${ DEPLOYMENT_DETAILS[@]} "
fi
break
;;
esac
done
get_original_pvc $PVC_NAME
create_mig_pvc_new $PVC_NAME ${ PV_DETAILS[0]}
CMD_GET_PV_NEW= "kubectl get pvc $PVC_NAME-new -n $MIG_NS --context $CONTEXT -o=jsonpath=\"{.spec.volumeName}\""
if [ $DEBUG -eq 1 ] ; then
echo -e "[Debug] Get new pv:\n $CMD_GET_PV_NEW"
fi
PV_NEW= $( bash -c " $CMD_GET_PV_NEW" )
if [ $DEBUG -eq 1 ] ; then
echo -e "[Debug] Result:\n $PV_NEW"
fi
text_separator
if [ $SCALE_OPERATOR -eq 1 ] ; then
scale_workload ${ OPERATORS_DETAILS[0]} ${ OPERATORS_DETAILS[2]} 0
fi
scale_workload ${ DEPLOYMENT_DETAILS[0]} ${ DEPLOYMENT_DETAILS[2]} 0
text_separator
delete_pvc $PVC_NAME
remove_claimref ${ PVC[0]} $NAMESPACE
echo "[Info] Get status of PV ' ${ PVC[0]} '"
CMD_GET_PV_STATUS= "kubectl get pv ${ PVC[0]} --context $CONTEXT -o=jsonpath=\"{.status.phase}\""
PV_STATUS= $( bash -c " $CMD_GET_PV_STATUS" )
if [ $DEBUG -eq 1 ] ; then
echo "[Info] Status of ${ PVC[0]} is ' $PV_STATUS'"
fi
while [[ $PV_STATUS != "Available" ]] ; do
echo "[ERROR] ${ PVC[0]} is ' $PV_STATUS', but must be in status 'Available'"
sleep 5
PV_STATUS= $( bash -c " $CMD_GET_PV_STATUS" )
done
create_mig_pvc_old $PVC_NAME $PVC ${ PV_DETAILS[0]}
do_migrate $PVC_NAME
# get pv of migrated disk
CMD_GET_PVC= "kubectl get pvc $PVC_NAME-new -n $MIG_NS --context $CONTEXT -o=jsonpath=\"{.spec.volumeName}{' '}{.metadata.labels.app}\""
PVC=( $( bash -c " $CMD_GET_PVC" ) )
delete_mig_pvcs $PVC_NAME
remove_claimref ${ PVC[0]} $NAMESPACE
create_new_pvc $PVC_NAME ${ PVC[0]} ${ PV_DETAILS[0]}
#### Restore state
text_separator
scale_workload ${ DEPLOYMENT_DETAILS[0]} ${ DEPLOYMENT_DETAILS[2]} ${ DEPLOYMENT_DETAILS[1]}
if [ $SCALE_OPERATOR -eq 1 ] ; then
scale_workload ${ OPERATORS_DETAILS[0]} ${ OPERATORS_DETAILS[2]} ${ OPERATORS_DETAILS[1]}
fi
else
echo -e "[ERROR] persistentVolumeReclaimPolicy has to be 'Retain'"
echo -e " $PVC will not be migrated\n"
fi
;;
esac
SCALE_OPERATOR= 0
break
done