diff --git a/build/ci/jenkins/QaPR.groovy b/build/ci/jenkins/QaPR.groovy new file mode 100644 index 0000000000..8273a2400d --- /dev/null +++ b/build/ci/jenkins/QaPR.groovy @@ -0,0 +1,226 @@ +#!/usr/bin/env groovy + +int total_timeout_minutes = 120 +int e2e_timeout_seconds = 70 * 60 +def imageTag='' +int case_timeout_seconds = 10 * 60 +def chart_version='2.4.25' +pipeline { + options { + timestamps() + timeout(time: total_timeout_minutes, unit: 'MINUTES') + buildDiscarder logRotator(artifactDaysToKeepStr: '30') + parallelsAlwaysFailFast() + preserveStashes(buildCount: 5) + + } + agent { + kubernetes { + label 'milvus-e2e-test-pipeline' + inheritFrom 'default' + defaultContainer 'main' + yamlFile 'build/ci/jenkins/pod/qa/rte.yaml' + customWorkspace '/home/jenkins/agent/workspace' + } + } + environment { + PROJECT_NAME = 'milvus' + SEMVER = "${BRANCH_NAME.contains('/') ? BRANCH_NAME.substring(BRANCH_NAME.lastIndexOf('/') + 1) : BRANCH_NAME}" + DOCKER_BUILDKIT = 1 + ARTIFACTS = "${env.WORKSPACE}/_artifacts" + DOCKER_CREDENTIALS_ID = "f0aacc8e-33f2-458a-ba9e-2c44f431b4d2" + TARGET_REPO = "milvusdb" + // CI_DOCKER_CREDENTIAL_ID = "ci-docker-registry" + CI_DOCKER_CREDENTIAL_ID = "qa-ci-docker-registry" + MILVUS_HELM_NAMESPACE = "milvus-ci" + DISABLE_KIND = true + // HUB = 'registry.milvus.io/milvus' + HUB = 'harbor.zilliz.cc/milvus-ci' + JENKINS_BUILD_ID = "${env.BUILD_ID}" + CI_MODE="pr" + MIRROR_URL="http://10.201.20.246:5000" + } + + stages { + stage ('Build'){ + steps { + container('main') { + dir ('build'){ + sh './set_docker_mirror.sh' + } + dir ('tests/scripts') { + script { + sh 'printenv' + def date = sh(returnStdout: true, script: 'date +%Y%m%d').trim() + def gitShortCommit = sh(returnStdout: true, script: 'git rev-parse --short HEAD').trim() + imageTag="${env.BRANCH_NAME}-${date}-${gitShortCommit}" + withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){ + sh """ + TAG="${imageTag}" \ + ./e2e-k8s.sh \ + --skip-export-logs \ + --skip-install \ + --skip-cleanup \ + --skip-setup \ + --skip-test + """ + + // stash imageTag info for rebuild install & E2E Test only + sh "echo ${imageTag} > imageTag.txt" + stash includes: 'imageTag.txt', name: 'imageTag' + + } + } + } + } + } + } + + + stage('Install & E2E Test') { + matrix { + axes { + axis { + name 'MILVUS_SERVER_TYPE' + values 'standalone', 'distributed' + } + axis { + name 'MILVUS_CLIENT' + values 'pymilvus' + } + } + + stages { + stage('Install') { + steps { + container('main') { + dir ('tests/scripts') { + script { + sh 'printenv' + def clusterEnabled = "false" + if ("${MILVUS_SERVER_TYPE}" == 'distributed') { + clusterEnabled = "true" + } + + if ("${MILVUS_CLIENT}" == "pymilvus") { + if ("${imageTag}"==''){ + dir ("imageTag"){ + try{ + unstash 'imageTag' + imageTag=sh(returnStdout: true, script: 'cat imageTag.txt | tr -d \'\n\r\'') + }catch(e){ + print "No Image Tag info remained ,please rerun build to build new image." + exit 1 + } + } + } + withCredentials([usernamePassword(credentialsId: "${env.CI_DOCKER_CREDENTIAL_ID}", usernameVariable: 'CI_REGISTRY_USERNAME', passwordVariable: 'CI_REGISTRY_PASSWORD')]){ + sh """ + MILVUS_CLUSTER_ENABLED=${clusterEnabled} \ + TAG=${imageTag}\ + ./e2e-k8s.sh \ + --skip-export-logs \ + --skip-cleanup \ + --skip-setup \ + --skip-test \ + --skip-build \ + --skip-build-image \ + --install-extra-arg "--set etcd.persistence.storageClass=local-path \ + --set minio.persistence.storageClass=local-path \ + --set etcd.metrics.enabled=false \ + --set etcd.metrics.podMonitor.enabled=false\ + --set metrics.serviceMonitor.enabled=true \ + --version ${chart_version} \ + -f values/pr.yaml" + """ + } + } else { + error "Error: Unsupported Milvus client: ${MILVUS_CLIENT}" + } + } + } + } + } + } + stage('E2E Test'){ + agent { + kubernetes { + label 'milvus-e2e-test-pr' + inheritFrom 'default' + defaultContainer 'main' + yamlFile 'build/ci/jenkins/pod/qa/rte.yaml' + customWorkspace '/home/jenkins/agent/workspace' + } + } + steps { + container('pytest') { + dir ('tests/scripts') { + script { + def release_name=sh(returnStdout: true, script: './get_release_name.sh') + def clusterEnabled = 'false' + if ("${MILVUS_SERVER_TYPE}" == "distributed") { + clusterEnabled = "true" + } + if ("${MILVUS_CLIENT}" == "pymilvus") { + sh """ + MILVUS_HELM_RELEASE_NAME="${release_name}" \ + MILVUS_HELM_NAMESPACE="milvus-ci" \ + MILVUS_CLUSTER_ENABLED="${clusterEnabled}" \ + TEST_TIMEOUT="${e2e_timeout_seconds}" \ + ./ci_e2e.sh "-n 6 -x --tags L0 L1 --timeout ${case_timeout_seconds}" + """ + + } else { + error "Error: Unsupported Milvus client: ${MILVUS_CLIENT}" + } + } + } + } + } + + } + } + post{ + always { + container('pytest'){ + dir("${env.ARTIFACTS}") { + sh "tar -zcvf artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz /tmp/ci_logs/test --remove-files || true" + archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${MILVUS_CLIENT}-pytest-logs.tar.gz ", allowEmptyArchive: true + } + } + container('main') { + dir ('tests/scripts') { + script { + def release_name=sh(returnStdout: true, script: './get_release_name.sh') + sh "./qa/uninstall_milvus.sh --release-name ${release_name}" + sh "./qa/ci_logs.sh --log-dir /ci-logs --artifacts-name ${env.ARTIFACTS}/artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs \ + --release-name ${release_name}" + dir("${env.ARTIFACTS}") { + archiveArtifacts artifacts: "artifacts-${PROJECT_NAME}-${MILVUS_SERVER_TYPE}-${SEMVER}-${env.BUILD_NUMBER}-${MILVUS_CLIENT}-e2e-logs.tar.gz", allowEmptyArchive: true + } + } + } + } + } + } + } + + } + } + post{ + unsuccessful { + container('jnlp') { + dir ('tests/scripts') { + script { + def authorEmail = sh(returnStdout: true, script: './get_author_email.sh ') + emailext subject: '$DEFAULT_SUBJECT', + body: '$DEFAULT_CONTENT', + recipientProviders: [developers(), culprits()], + replyTo: '$DEFAULT_REPLYTO', + to: "${authorEmail},devops@zilliz.com" + } + } + } + } + } +} \ No newline at end of file diff --git a/build/ci/jenkins/pod/qa/rte.yaml b/build/ci/jenkins/pod/qa/rte.yaml new file mode 100644 index 0000000000..031e0fecd5 --- /dev/null +++ b/build/ci/jenkins/pod/qa/rte.yaml @@ -0,0 +1,66 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus-e2e + namespace: jenkins +spec: + enableServiceLinks: false + containers: + - name: main + image: harbor.zilliz.cc/dockerhub/milvusdb/krte:20211213-dcc15e9 + env: + - name: DOCKER_IN_DOCKER_ENABLED + value: "true" + - name: DOCKER_VOLUME_DIRECTORY + value: "/mnt/disk/.docker" + tty: true + securityContext: + privileged: true + args: ["cat"] + resources: + limits: + cpu: "6" + memory: 12Gi + requests: + cpu: "0.5" + memory: 5Gi + volumeMounts: + - mountPath: /docker-graph + name: docker-graph + - mountPath: /var/lib/docker + name: docker-root + - mountPath: /lib/modules + name: modules + readOnly: true + - mountPath: /sys/fs/cgroup + name: cgroup + - mountPath: /mnt/disk/.docker + name: build-cache + subPath: docker-volume + - name: pytest + image: harbor.zilliz.cc/dockerhub/milvusdb/pytest:20220118-3eb5cc4 + resources: + limits: + cpu: "6" + memory: 12Gi + requests: + cpu: "0.5" + memory: 5Gi + volumes: + - emptyDir: {} + name: docker-graph + - emptyDir: {} + name: docker-root + - hostPath: + path: /tmp/krte/cache + type: DirectoryOrCreate + name: build-cache + - hostPath: + path: /lib/modules + type: Directory + name: modules + - hostPath: + path: /sys/fs/cgroup + type: Directory + name: cgroup diff --git a/build/set_docker_mirror.sh b/build/set_docker_mirror.sh index fbdcccddc2..fb4c0acc2d 100755 --- a/build/set_docker_mirror.sh +++ b/build/set_docker_mirror.sh @@ -21,7 +21,8 @@ set -e # Use nexus as docker mirror registry -MIRROR_URL="http://nexus-nexus-repository-manager-docker-5000.nexus:5000" +MIRROR_URL="${MIRROR_URL:-http://nexus-nexus-repository-manager-docker-5000.nexus:5000}" +#MIRROR_URL="http://nexus-nexus-repository-manager-docker-5000.nexus:5000" # Add registry mirror config into docker daemon set_daemon_json_file(){ diff --git a/tests/scripts/qa/ci_logs.sh b/tests/scripts/qa/ci_logs.sh new file mode 100755 index 0000000000..4271e17c35 --- /dev/null +++ b/tests/scripts/qa/ci_logs.sh @@ -0,0 +1,84 @@ +#!/bin/bash + +# Licensed to the LF AI & Data foundation under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Exit immediately for non zero status +set -e +# Check unset variables +# set -u +# Print commands +set -x + +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +ROOT="$( cd -P "$( dirname "$SOURCE" )/../.." && pwd )" + +while (( "$#" )); do + case "$1" in + + --release-name) + RELEASE_NAME=$2 + shift 2 + ;; + + --artifacts-name) + ARTIFACTS_NAME=$2 + shift 2 + ;; + + --log-dir) + LOG_DIR=$2 + shift 2 + ;; + + -h|--help) + { set +x; } 2>/dev/null + HELP=" +Usage: + $0 [flags] [Arguments] + + --log-dir Log Path + + --artifacts-name Artifacts Name + + --release-name Milvus helm release name + + -h or --help Print help information + + +Use \"$0 --help\" for more information about a given command. +" + echo -e "${HELP}" ; exit 0 + ;; + -*) + echo "Error: Unsupported flag $1" >&2 + exit 1 + ;; + *) # preserve positional arguments + PARAMS+=("$1") + shift + ;; + esac +done + +RELEASE_NAME=${RELEASE_NAME:-milvus-testing} + +tar -zcvf ${ARTIFACTS_NAME:-artifacts}.tar.gz ${RELEASE_NAME}/* \ No newline at end of file diff --git a/tests/scripts/qa/uninstall_milvus.sh b/tests/scripts/qa/uninstall_milvus.sh new file mode 100755 index 0000000000..dd16a42920 --- /dev/null +++ b/tests/scripts/qa/uninstall_milvus.sh @@ -0,0 +1,112 @@ +#!/bin/bash + +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. + +# Exit immediately for non zero status +set -e +# Print commands +# set -x + +while (( "$#" )); do + case "$1" in + + --release-name) + RELEASE_NAME=$2 + shift 2 + ;; + + -h|--help) + { set +x; } 2>/dev/null + HELP=" +Usage: + $0 [flags] [Arguments] + + --release-name Milvus helm release name + + -h or --help Print help information + + +Use \"$0 --help\" for more information about a given command. +" + echo -e "${HELP}" ; exit 0 + ;; + -*) + echo "Error: Unsupported flag $1" >&2 + exit 1 + ;; + *) # preserve positional arguments + PARAMS+=("$1") + shift + ;; + esac +done + +MILVUS_HELM_RELEASE_NAME="${MILVUS_HELM_RELEASE_NAME:-milvus-testing}" +MILVUS_HELM_NAMESPACE="${MILVUS_HELM_NAMESPACE:-default}" + +if [[ -n "${RELEASE_NAME:-}" ]]; then + MILVUS_HELM_RELEASE_NAME="${RELEASE_NAME}" + # List pod list before uninstall + kubectl get pods -n ${MILVUS_HELM_NAMESPACE} -o wide | grep "${MILVUS_HELM_RELEASE_NAME}-" + # Show restart pods last terminated reason + # restart_pods=$(kubectl get pods -n ${MILVUS_HELM_NAMESPACE} | grep "${MILVUS_HELM_RELEASE_NAME}-" | grep 'ago)' | awk '{print $1}') + + # for restart_pod in ${restart_pods} + # do + # reason=$(kubectl get pod ${restart_pod} -n milvus-ci -o json | jq .status.containerStatuses[0].lastState.terminated.reason ) + # restart_count=$(kubectl get pod ${restart_pod} -n milvus-ci -o json | jq .status.containerStatuses[0].restartCount ) + # echo "${restart_pod} restarts ${restart_count}, last terminateed reason is ${reason}" + # done + +fi +mkdir ${MILVUS_HELM_RELEASE_NAME} +all_pods=$(kubectl get pods -n ${MILVUS_HELM_NAMESPACE} | grep "${MILVUS_HELM_RELEASE_NAME}-" | awk '{print $1}') +for pod_name in ${all_pods} +do + kubectl logs ${pod_name} -n ${MILVUS_HELM_NAMESPACE} --all-containers=true > ${MILVUS_HELM_RELEASE_NAME}/${pod_name}.log + kubectl logs ${pod_name} -n ${MILVUS_HELM_NAMESPACE} --all-containers=true --previous=true > ${MILVUS_HELM_RELEASE_NAME}/${pod_name}_previous.log +done + + +# Uninstall Milvus Helm Release +# Do not exit with error when release not found so that the script can also be used to clean up related pvc even helm release has been uninstalled already +helm uninstall -n "${MILVUS_HELM_NAMESPACE}" "${MILVUS_HELM_RELEASE_NAME}" || true + +MILVUS_LABELS1="app.kubernetes.io/instance=${MILVUS_HELM_RELEASE_NAME}" +MILVUS_LABELS2="release=${MILVUS_HELM_RELEASE_NAME}" + +# Clean up pvc +kubectl delete pvc --wait -n "${MILVUS_HELM_NAMESPACE}" $(kubectl get pvc -n "${MILVUS_HELM_NAMESPACE}" -l "${MILVUS_LABELS1}" -o jsonpath='{range.items[*]}{.metadata.name} ') || true +kubectl delete pvc --wait -n "${MILVUS_HELM_NAMESPACE}" $(kubectl get pvc -n "${MILVUS_HELM_NAMESPACE}" -l "${MILVUS_LABELS2}" -o jsonpath='{range.items[*]}{.metadata.name} ') || true + +# Add check & delete pvc again in case pvc need time to be deleted +# clean_label_pvc(){ +# local label=${1?label expected as first argument.} + +# for i in {1..10} +# do +# PVC=$(kubectl get pvc -n "${MILVUS_HELM_NAMESPACE}" -l "${label}" -o jsonpath='{range.items[*]}{.metadata.name} ') +# STATUS=$(echo ${PVC} | wc -w ) +# echo "status is ${STATUS}" +# if [ $STATUS == 0 ]; then +# break +# else +# sleep 5 +# echo "PVCs are ${PVC}" +# kubectl delete pvc --wait -n "${MILVUS_HELM_NAMESPACE}" ${PVC} +# fi +# done +# } + +# echo "Check & Delete Persistent Volumes" +# clean_label_pvc ${MILVUS_LABELS1} +# clean_label_pvc ${MILVUS_LABELS2} \ No newline at end of file