From 4462899f085519f445351cbd98a0516b8f48ce2b Mon Sep 17 00:00:00 2001 From: Sergii Tkachenko Date: Mon, 6 May 2024 16:11:31 -0700 Subject: buildscripts: simplify PSM interop Kokoro buildscripts (#11121) (#11167) Integrates the new features of the the Kokoro PSM Interop install library introduced in grpc/psm-interop#73. Nearly all common functionality was moved from per-language/per-branch PSM Interop build scripts to [psm_interop_kokoro_lib.sh](https://github.com/grpc/psm-interop/blob/main/.kokoro/psm_interop_kokoro_lib.sh): 1. The list of tests in the each test suite 2. Per-test-suite flag customization 3. `run_test` methods 4. `build_docker_images_if_needed` methods 5. Generic `build_test_app_docker_images` methods (simple docker build + docker push + docker tag). grpc-java is one exception, as it doesn't run docker directly, but a cloudbuild flow. Now all PSM Interop jobs share the same buildscripts by all test suites: 1. buildscript that invokes the test: `psm-interop-test-{language}.sh` (configured as `build_file` in the build cfg) 2. buildscript that builds the xDS test client/server and publishes them as a Docker image: `psm-interop-build-{language}.sh` (conventional name called from `psm_interop_kokoro_lib.sh`) `psm-interop-test-{language}.sh`: 1. Sets `GRPC_LANGUAGE`, `BUILD_SCRIPT_DIR` environment variables. 2. Downloads the shared `psm_interop_kokoro_lib.sh` from the main branch of the psm-interop repo. 3. Sources `psm-interop-build-{language}.sh` 4. Calls `psm::run "${PSM_TEST_SUITE}"` (`PSM_TEST_SUITE` configured in the cfg file). `psm-interop-build-{language}.sh`: 1. Defines `psm::lang::build_docker_images` which is called from `psm_interop_kokoro_lib.sh`. 2. Invokes any repo-specific logic. 3. May use `psm::build::docker_images_generic` for generic Docker build, tag, push, or provide implement its own build/publish method. References: - b/288578634 - See the full list of the new features at grpc/psm-interop#73. - Additional fixes to the shared lib: grpc/psm-interop#78, grpc/psm-interop#79 --- buildscripts/kokoro/psm-interop-build-java.sh | 90 +++++++++++++ buildscripts/kokoro/psm-interop-test-java.sh | 27 ++++ buildscripts/kokoro/psm-security.cfg | 6 +- buildscripts/kokoro/psm-security.sh | 185 ------------------------- buildscripts/kokoro/xds_k8s_lb.cfg | 6 +- buildscripts/kokoro/xds_k8s_lb.sh | 186 -------------------------- buildscripts/kokoro/xds_url_map.cfg | 6 +- buildscripts/kokoro/xds_url_map.sh | 178 ------------------------ 8 files changed, 132 insertions(+), 552 deletions(-) create mode 100755 buildscripts/kokoro/psm-interop-build-java.sh create mode 100755 buildscripts/kokoro/psm-interop-test-java.sh delete mode 100755 buildscripts/kokoro/psm-security.sh delete mode 100755 buildscripts/kokoro/xds_k8s_lb.sh delete mode 100755 buildscripts/kokoro/xds_url_map.sh diff --git a/buildscripts/kokoro/psm-interop-build-java.sh b/buildscripts/kokoro/psm-interop-build-java.sh new file mode 100755 index 000000000..8c7a970ef --- /dev/null +++ b/buildscripts/kokoro/psm-interop-build-java.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash +# Copyright 2024 gRPC authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +set -eo pipefail + +# This file defines psm::lang::build_docker_images, which is directly called +# from psm_interop_kokoro_lib.sh. + +# Used locally. +readonly BUILD_APP_PATH="interop-testing/build/install/grpc-interop-testing" + +####################################### +# Builds the test app using gradle and smoke-checks its binaries +# Globals: +# SRC_DIR Absolute path to the source repo. +# BUILD_APP_PATH +# Arguments: +# None +# Outputs: +# Writes the output of xds-test-client and xds-test-server --help to stderr +####################################### +_build_java_test_app() { + psm::tools::log "Building Java test app" + cd "${SRC_DIR}" + + set -x + GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" \ + ./gradlew --no-daemon grpc-interop-testing:installDist -x test \ + -PskipCodegen=true -PskipAndroid=true --console=plain + set +x + + psm::tools::log "Test-run grpc-java PSM interop binaries" + psm::tools::run_ignore_exit_code "${SRC_DIR}/${BUILD_APP_PATH}/bin/xds-test-client" --help + psm::tools::run_ignore_exit_code "${SRC_DIR}/${BUILD_APP_PATH}/bin/xds-test-server" --help +} + +####################################### +# Builds test app Docker images and pushes them to GCR +# Globals: +# BUILD_APP_PATH +# SERVER_IMAGE_NAME: Test server Docker image name +# CLIENT_IMAGE_NAME: Test client Docker image name +# GIT_COMMIT: SHA-1 of git commit being built +# TESTING_VERSION: version branch under test, f.e. v1.42.x, master +# Arguments: +# None +# Outputs: +# Writes the output of `gcloud builds submit` to stdout, stderr +####################################### +psm::lang::build_docker_images() { + local java_build_log="${BUILD_LOGS_ROOT}/build-lang-java.log" + _build_java_test_app |& tee "${java_build_log}" + + psm::tools::log "Building Java xDS interop test app Docker images" + local docker_dir="${SRC_DIR}/buildscripts/xds-k8s" + local build_dir + build_dir="$(mktemp -d)" + + # Copy Docker files, log properties, and the test app to the build dir + { + cp -v "${docker_dir}/"*.Dockerfile "${build_dir}" + cp -v "${docker_dir}/"*.properties "${build_dir}" + cp -rv "${SRC_DIR}/${BUILD_APP_PATH}" "${build_dir}" + } >> "${java_build_log}" + + + # cloudbuild.yaml substitution variables + local substitutions="" + substitutions+="_SERVER_IMAGE_NAME=${SERVER_IMAGE_NAME}," + substitutions+="_CLIENT_IMAGE_NAME=${CLIENT_IMAGE_NAME}," + substitutions+="COMMIT_SHA=${GIT_COMMIT}," + substitutions+="BRANCH_NAME=${TESTING_VERSION}," + + # Run Google Cloud Build + gcloud builds submit "${build_dir}" \ + --config="${docker_dir}/cloudbuild.yaml" \ + --substitutions="${substitutions}" \ + | tee -a "${java_build_log}" +} diff --git a/buildscripts/kokoro/psm-interop-test-java.sh b/buildscripts/kokoro/psm-interop-test-java.sh new file mode 100755 index 000000000..f249d579d --- /dev/null +++ b/buildscripts/kokoro/psm-interop-test-java.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +set -eo pipefail + +# Input parameters to psm:: methods of the install script. +readonly GRPC_LANGUAGE="java" +readonly BUILD_SCRIPT_DIR="$(dirname "$0")" + +# Used locally. +readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/psm-interop/${TEST_DRIVER_BRANCH:-main}/.kokoro/psm_interop_kokoro_lib.sh" + +psm::lang::source_install_lib() { + echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}" + local install_lib + # Download to a tmp file. + install_lib="$(mktemp -d)/psm_interop_kokoro_lib.sh" + curl -s --retry-connrefused --retry 5 -o "${install_lib}" "${TEST_DRIVER_INSTALL_SCRIPT_URL}" + # Checksum. + if command -v sha256sum &> /dev/null; then + echo "Install script checksum:" + sha256sum "${install_lib}" + fi + source "${install_lib}" +} + +psm::lang::source_install_lib +source "${BUILD_SCRIPT_DIR}/psm-interop-build-${GRPC_LANGUAGE}.sh" +psm::run "${PSM_TEST_SUITE}" diff --git a/buildscripts/kokoro/psm-security.cfg b/buildscripts/kokoro/psm-security.cfg index 508f4dbe9..76d2c5785 100644 --- a/buildscripts/kokoro/psm-security.cfg +++ b/buildscripts/kokoro/psm-security.cfg @@ -1,7 +1,7 @@ # Config file for internal CI # Location of the continuous shell script in repository. -build_file: "grpc-java/buildscripts/kokoro/psm-security.sh" +build_file: "grpc-java/buildscripts/kokoro/psm-interop-test-java.sh" timeout_mins: 240 action { @@ -11,3 +11,7 @@ action { strip_prefix: "artifacts" } } +env_vars { + key: "PSM_TEST_SUITE" + value: "security" +} diff --git a/buildscripts/kokoro/psm-security.sh b/buildscripts/kokoro/psm-security.sh deleted file mode 100755 index 2f37e2a95..000000000 --- a/buildscripts/kokoro/psm-security.sh +++ /dev/null @@ -1,185 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# Constants -readonly GITHUB_REPOSITORY_NAME="grpc-java" -readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/psm-interop/${TEST_DRIVER_BRANCH:-main}/.kokoro/psm_interop_kokoro_lib.sh" -## xDS test server/client Docker images -readonly SERVER_IMAGE_NAME="us-docker.pkg.dev/grpc-testing/psm-interop/java-server" -readonly CLIENT_IMAGE_NAME="us-docker.pkg.dev/grpc-testing/psm-interop/java-client" -readonly FORCE_IMAGE_BUILD="${FORCE_IMAGE_BUILD:-0}" -readonly BUILD_APP_PATH="interop-testing/build/install/grpc-interop-testing" - -####################################### -# Builds the test app using gradle and smoke-checks its binaries -# Globals: -# SRC_DIR -# BUILD_APP_PATH -# Arguments: -# None -# Outputs: -# Writes the output of xds-test-client and xds-test-server --help to stderr -####################################### -build_java_test_app() { - echo "Building Java test app" - cd "${SRC_DIR}" - GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" \ - ./gradlew --no-daemon grpc-interop-testing:installDist -x test \ - -PskipCodegen=true -PskipAndroid=true --console=plain - - # Test-run binaries - run_ignore_exit_code "${SRC_DIR}/${BUILD_APP_PATH}/bin/xds-test-client" --help - run_ignore_exit_code "${SRC_DIR}/${BUILD_APP_PATH}/bin/xds-test-server" --help -} - -####################################### -# Builds test app Docker images and pushes them to GCR -# Globals: -# BUILD_APP_PATH -# SERVER_IMAGE_NAME: Test server Docker image name -# CLIENT_IMAGE_NAME: Test client Docker image name -# GIT_COMMIT: SHA-1 of git commit being built -# TESTING_VERSION: version branch under test, f.e. v1.42.x, master -# Arguments: -# None -# Outputs: -# Writes the output of `gcloud builds submit` to stdout, stderr -####################################### -build_test_app_docker_images() { - echo "Building Java xDS interop test app Docker images" - local docker_dir="${SRC_DIR}/buildscripts/xds-k8s" - local build_dir - build_dir="$(mktemp -d)" - # Copy Docker files, log properties, and the test app to the build dir - cp -v "${docker_dir}/"*.Dockerfile "${build_dir}" - cp -v "${docker_dir}/"*.properties "${build_dir}" - cp -rv "${SRC_DIR}/${BUILD_APP_PATH}" "${build_dir}" - # Pick a branch name for the built image - local branch_name='experimental' - if is_version_branch "${TESTING_VERSION}"; then - branch_name="${TESTING_VERSION}" - fi - # Run Google Cloud Build - gcloud builds submit "${build_dir}" \ - --config "${docker_dir}/cloudbuild.yaml" \ - --substitutions "_SERVER_IMAGE_NAME=${SERVER_IMAGE_NAME},_CLIENT_IMAGE_NAME=${CLIENT_IMAGE_NAME},COMMIT_SHA=${GIT_COMMIT},BRANCH_NAME=${branch_name}" - # TODO(sergiitk): extra "cosmetic" tags for versioned branches, e.g. v1.34.x - # TODO(sergiitk): do this when adding support for custom configs per version -} - -####################################### -# Builds test app and its docker images unless they already exist -# Globals: -# SERVER_IMAGE_NAME: Test server Docker image name -# CLIENT_IMAGE_NAME: Test client Docker image name -# GIT_COMMIT: SHA-1 of git commit being built -# FORCE_IMAGE_BUILD -# Arguments: -# None -# Outputs: -# Writes the output to stdout, stderr -####################################### -build_docker_images_if_needed() { - # Check if images already exist - server_tags="$(gcloud_gcr_list_image_tags "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}")" - printf "Server image: %s:%s\n" "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}" - echo "${server_tags:-Server image not found}" - - client_tags="$(gcloud_gcr_list_image_tags "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}")" - printf "Client image: %s:%s\n" "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}" - echo "${client_tags:-Client image not found}" - - # Build if any of the images are missing, or FORCE_IMAGE_BUILD=1 - if [[ "${FORCE_IMAGE_BUILD}" == "1" || -z "${server_tags}" || -z "${client_tags}" ]]; then - build_java_test_app - build_test_app_docker_images - else - echo "Skipping Java test app build" - fi -} - -####################################### -# Executes the test case -# Globals: -# TEST_DRIVER_FLAGFILE: Relative path to test driver flagfile -# KUBE_CONTEXT: The name of kubectl context with GKE cluster access -# TEST_XML_OUTPUT_DIR: Output directory for the test xUnit XML report -# SERVER_IMAGE_NAME: Test server Docker image name -# CLIENT_IMAGE_NAME: Test client Docker image name -# GIT_COMMIT: SHA-1 of git commit being built -# TESTING_VERSION: version branch under test: used by the framework to -# determine the supported PSM features. -# Arguments: -# Test case name -# Outputs: -# Writes the output of test execution to stdout, stderr -# Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml -####################################### -run_test() { - # Test driver usage: - # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage - local test_name="${1:?Usage: run_test test_name}" - local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}" - mkdir -pv "${out_dir}" - set -x - python -m "tests.${test_name}" \ - --flagfile="${TEST_DRIVER_FLAGFILE}" \ - --kube_context="${KUBE_CONTEXT}" \ - --server_image="${SERVER_IMAGE_NAME}:${GIT_COMMIT}" \ - --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \ - --testing_version="${TESTING_VERSION}" \ - --force_cleanup \ - --collect_app_logs \ - --log_dir="${out_dir}" \ - --xml_output_file="${out_dir}/sponge_log.xml" \ - |& tee "${out_dir}/sponge_log.log" -} - -####################################### -# Main function: provision software necessary to execute tests, and run them -# Globals: -# KOKORO_ARTIFACTS_DIR -# GITHUB_REPOSITORY_NAME -# SRC_DIR: Populated with absolute path to the source repo -# TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing -# the test driver -# TEST_DRIVER_FULL_DIR: Populated with the path to the test driver source code -# TEST_DRIVER_FLAGFILE: Populated with relative path to test driver flagfile -# TEST_XML_OUTPUT_DIR: Populated with the path to test xUnit XML report -# GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build -# GIT_COMMIT: Populated with the SHA-1 of git commit being built -# GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built -# KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access -# Arguments: -# None -# Outputs: -# Writes the output of test execution to stdout, stderr -####################################### -main() { - local script_dir - script_dir="$(dirname "$0")" - - # Source the test driver from the master branch. - echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}" - source /dev/stdin <<< "$(curl -s "${TEST_DRIVER_INSTALL_SCRIPT_URL}")" - - activate_gke_cluster GKE_CLUSTER_PSM_SECURITY - - set -x - if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then - kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}" - else - local_setup_test_driver "${script_dir}" - fi - build_docker_images_if_needed - # Run tests - cd "${TEST_DRIVER_FULL_DIR}" - local failed_tests=0 - test_suites=("baseline_test" "security_test" "authz_test") - for test in "${test_suites[@]}"; do - run_test $test || (( ++failed_tests )) - done - echo "Failed test suites: ${failed_tests}" -} - -main "$@" diff --git a/buildscripts/kokoro/xds_k8s_lb.cfg b/buildscripts/kokoro/xds_k8s_lb.cfg index 10ea2d43b..191bd49e3 100644 --- a/buildscripts/kokoro/xds_k8s_lb.cfg +++ b/buildscripts/kokoro/xds_k8s_lb.cfg @@ -1,7 +1,7 @@ # Config file for internal CI # Location of the continuous shell script in repository. -build_file: "grpc-java/buildscripts/kokoro/xds_k8s_lb.sh" +build_file: "grpc-java/buildscripts/kokoro/psm-interop-test-java.sh" timeout_mins: 180 action { @@ -11,3 +11,7 @@ action { strip_prefix: "artifacts" } } +env_vars { + key: "PSM_TEST_SUITE" + value: "lb" +} diff --git a/buildscripts/kokoro/xds_k8s_lb.sh b/buildscripts/kokoro/xds_k8s_lb.sh deleted file mode 100755 index 284427368..000000000 --- a/buildscripts/kokoro/xds_k8s_lb.sh +++ /dev/null @@ -1,186 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# Constants -readonly GITHUB_REPOSITORY_NAME="grpc-java" -readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/psm-interop/${TEST_DRIVER_BRANCH:-main}/.kokoro/psm_interop_kokoro_lib.sh" -## xDS test server/client Docker images -readonly SERVER_IMAGE_NAME="us-docker.pkg.dev/grpc-testing/psm-interop/java-server" -readonly CLIENT_IMAGE_NAME="us-docker.pkg.dev/grpc-testing/psm-interop/java-client" -readonly FORCE_IMAGE_BUILD="${FORCE_IMAGE_BUILD:-0}" -readonly BUILD_APP_PATH="interop-testing/build/install/grpc-interop-testing" - -####################################### -# Builds the test app using gradle and smoke-checks its binaries -# Globals: -# SRC_DIR -# BUILD_APP_PATH -# Arguments: -# None -# Outputs: -# Writes the output of xds-test-client and xds-test-server --help to stderr -####################################### -build_java_test_app() { - echo "Building Java test app" - cd "${SRC_DIR}" - GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" \ - ./gradlew --no-daemon grpc-interop-testing:installDist -x test \ - -PskipCodegen=true -PskipAndroid=true --console=plain - - # Test-run binaries - run_ignore_exit_code "${SRC_DIR}/${BUILD_APP_PATH}/bin/xds-test-client" --help - run_ignore_exit_code "${SRC_DIR}/${BUILD_APP_PATH}/bin/xds-test-server" --help -} - -####################################### -# Builds test app Docker images and pushes them to GCR -# Globals: -# BUILD_APP_PATH -# SERVER_IMAGE_NAME: Test server Docker image name -# CLIENT_IMAGE_NAME: Test client Docker image name -# GIT_COMMIT: SHA-1 of git commit being built -# TESTING_VERSION: version branch under test, f.e. v1.42.x, master -# Arguments: -# None -# Outputs: -# Writes the output of `gcloud builds submit` to stdout, stderr -####################################### -build_test_app_docker_images() { - echo "Building Java xDS interop test app Docker images" - local docker_dir="${SRC_DIR}/buildscripts/xds-k8s" - local build_dir - build_dir="$(mktemp -d)" - # Copy Docker files, log properties, and the test app to the build dir - cp -v "${docker_dir}/"*.Dockerfile "${build_dir}" - cp -v "${docker_dir}/"*.properties "${build_dir}" - cp -rv "${SRC_DIR}/${BUILD_APP_PATH}" "${build_dir}" - # Pick a branch name for the built image - local branch_name='experimental' - if is_version_branch "${TESTING_VERSION}"; then - branch_name="${TESTING_VERSION}" - fi - # Run Google Cloud Build - gcloud builds submit "${build_dir}" \ - --config "${docker_dir}/cloudbuild.yaml" \ - --substitutions "_SERVER_IMAGE_NAME=${SERVER_IMAGE_NAME},_CLIENT_IMAGE_NAME=${CLIENT_IMAGE_NAME},COMMIT_SHA=${GIT_COMMIT},BRANCH_NAME=${branch_name}" - # TODO(sergiitk): extra "cosmetic" tags for versioned branches, e.g. v1.34.x - # TODO(sergiitk): do this when adding support for custom configs per version -} - -####################################### -# Builds test app and its docker images unless they already exist -# Globals: -# SERVER_IMAGE_NAME: Test server Docker image name -# CLIENT_IMAGE_NAME: Test client Docker image name -# GIT_COMMIT: SHA-1 of git commit being built -# FORCE_IMAGE_BUILD -# Arguments: -# None -# Outputs: -# Writes the output to stdout, stderr -####################################### -build_docker_images_if_needed() { - # Check if images already exist - server_tags="$(gcloud_gcr_list_image_tags "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}")" - printf "Server image: %s:%s\n" "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}" - echo "${server_tags:-Server image not found}" - - client_tags="$(gcloud_gcr_list_image_tags "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}")" - printf "Client image: %s:%s\n" "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}" - echo "${client_tags:-Client image not found}" - - # Build if any of the images are missing, or FORCE_IMAGE_BUILD=1 - if [[ "${FORCE_IMAGE_BUILD}" == "1" || -z "${server_tags}" || -z "${client_tags}" ]]; then - build_java_test_app - build_test_app_docker_images - else - echo "Skipping Java test app build" - fi -} - -####################################### -# Executes the test case -# Globals: -# TEST_DRIVER_FLAGFILE: Relative path to test driver flagfile -# KUBE_CONTEXT: The name of kubectl context with GKE cluster access -# SECONDARY_KUBE_CONTEXT: The name of kubectl context with secondary GKE cluster access, if any -# TEST_XML_OUTPUT_DIR: Output directory for the test xUnit XML report -# SERVER_IMAGE_NAME: Test server Docker image name -# CLIENT_IMAGE_NAME: Test client Docker image name -# GIT_COMMIT: SHA-1 of git commit being built -# Arguments: -# Test case name -# Outputs: -# Writes the output of test execution to stdout, stderr -# Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml -####################################### -run_test() { - # Test driver usage: - # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage - local test_name="${1:?Usage: run_test test_name}" - local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}" - mkdir -pv "${out_dir}" - set -x - python -m "tests.${test_name}" \ - --flagfile="${TEST_DRIVER_FLAGFILE}" \ - --kube_context="${KUBE_CONTEXT}" \ - --secondary_kube_context="${SECONDARY_KUBE_CONTEXT}" \ - --server_image="${SERVER_IMAGE_NAME}:${GIT_COMMIT}" \ - --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \ - --testing_version="${TESTING_VERSION}" \ - --force_cleanup \ - --collect_app_logs \ - --log_dir="${out_dir}" \ - --xml_output_file="${out_dir}/sponge_log.xml" \ - |& tee "${out_dir}/sponge_log.log" -} - -####################################### -# Main function: provision software necessary to execute tests, and run them -# Globals: -# KOKORO_ARTIFACTS_DIR -# GITHUB_REPOSITORY_NAME -# SRC_DIR: Populated with absolute path to the source repo -# TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing -# the test driver -# TEST_DRIVER_FULL_DIR: Populated with the path to the test driver source code -# TEST_DRIVER_FLAGFILE: Populated with relative path to test driver flagfile -# TEST_XML_OUTPUT_DIR: Populated with the path to test xUnit XML report -# GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build -# GIT_COMMIT: Populated with the SHA-1 of git commit being built -# GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built -# KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access -# Arguments: -# None -# Outputs: -# Writes the output of test execution to stdout, stderr -####################################### -main() { - local script_dir - script_dir="$(dirname "$0")" - - # Source the test driver from the master branch. - echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}" - source /dev/stdin <<< "$(curl -s "${TEST_DRIVER_INSTALL_SCRIPT_URL}")" - - activate_gke_cluster GKE_CLUSTER_PSM_LB - activate_secondary_gke_cluster GKE_CLUSTER_PSM_LB - - set -x - if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then - kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}" - else - local_setup_test_driver "${script_dir}" - fi - build_docker_images_if_needed - # Run tests - cd "${TEST_DRIVER_FULL_DIR}" - local failed_tests=0 - test_suites=("api_listener_test" "change_backend_service_test" "failover_test" "remove_neg_test" "round_robin_test" "affinity_test" "outlier_detection_test" "custom_lb_test") - for test in "${test_suites[@]}"; do - run_test $test || (( ++failed_tests )) - done - echo "Failed test suites: ${failed_tests}" -} - -main "$@" diff --git a/buildscripts/kokoro/xds_url_map.cfg b/buildscripts/kokoro/xds_url_map.cfg index 1fa6c0141..3e27164fe 100644 --- a/buildscripts/kokoro/xds_url_map.cfg +++ b/buildscripts/kokoro/xds_url_map.cfg @@ -1,7 +1,7 @@ # Config file for internal CI # Location of the continuous shell script in repository. -build_file: "grpc-java/buildscripts/kokoro/xds_url_map.sh" +build_file: "grpc-java/buildscripts/kokoro/psm-interop-test-java.sh" timeout_mins: 90 action { @@ -11,3 +11,7 @@ action { strip_prefix: "artifacts" } } +env_vars { + key: "PSM_TEST_SUITE" + value: "url_map" +} diff --git a/buildscripts/kokoro/xds_url_map.sh b/buildscripts/kokoro/xds_url_map.sh deleted file mode 100755 index 3760fcd22..000000000 --- a/buildscripts/kokoro/xds_url_map.sh +++ /dev/null @@ -1,178 +0,0 @@ -#!/usr/bin/env bash -set -eo pipefail - -# Constants -readonly GITHUB_REPOSITORY_NAME="grpc-java" -readonly TEST_DRIVER_INSTALL_SCRIPT_URL="https://raw.githubusercontent.com/${TEST_DRIVER_REPO_OWNER:-grpc}/psm-interop/${TEST_DRIVER_BRANCH:-main}/.kokoro/psm_interop_kokoro_lib.sh" -## xDS test client Docker images -readonly SERVER_IMAGE_NAME="us-docker.pkg.dev/grpc-testing/psm-interop/java-server" -readonly CLIENT_IMAGE_NAME="us-docker.pkg.dev/grpc-testing/psm-interop/java-client" -readonly FORCE_IMAGE_BUILD="${FORCE_IMAGE_BUILD:-0}" -readonly BUILD_APP_PATH="interop-testing/build/install/grpc-interop-testing" - -####################################### -# Builds the test app using gradle and smoke-checks its binaries -# Globals: -# SRC_DIR -# BUILD_APP_PATH -# Arguments: -# None -# Outputs: -# Writes the output of xds-test-client and xds-test-server --help to stderr -####################################### -build_java_test_app() { - echo "Building Java test app" - cd "${SRC_DIR}" - GRADLE_OPTS="-Dorg.gradle.jvmargs='-Xmx1g'" \ - ./gradlew --no-daemon grpc-interop-testing:installDist -x test \ - -PskipCodegen=true -PskipAndroid=true --console=plain - - # Test-run binaries - run_ignore_exit_code "${SRC_DIR}/${BUILD_APP_PATH}/bin/xds-test-client" --help - run_ignore_exit_code "${SRC_DIR}/${BUILD_APP_PATH}/bin/xds-test-server" --help -} - -####################################### -# Builds test app Docker images and pushes them to GCR -# Globals: -# BUILD_APP_PATH -# SERVER_IMAGE_NAME: Test server Docker image name -# CLIENT_IMAGE_NAME: Test client Docker image name -# GIT_COMMIT: SHA-1 of git commit being built -# TESTING_VERSION: version branch under test, f.e. v1.42.x, master -# Arguments: -# None -# Outputs: -# Writes the output of `gcloud builds submit` to stdout, stderr -####################################### -build_test_app_docker_images() { - echo "Building Java xDS interop test app Docker images" - local docker_dir="${SRC_DIR}/buildscripts/xds-k8s" - local build_dir - build_dir="$(mktemp -d)" - # Copy Docker files, log properties, and the test app to the build dir - cp -v "${docker_dir}/"*.Dockerfile "${build_dir}" - cp -v "${docker_dir}/"*.properties "${build_dir}" - cp -rv "${SRC_DIR}/${BUILD_APP_PATH}" "${build_dir}" - # Pick a branch name for the built image - local branch_name='experimental' - if is_version_branch "${TESTING_VERSION}"; then - branch_name="${TESTING_VERSION}" - fi - # Run Google Cloud Build - gcloud builds submit "${build_dir}" \ - --config "${docker_dir}/cloudbuild.yaml" \ - --substitutions "_SERVER_IMAGE_NAME=${SERVER_IMAGE_NAME},_CLIENT_IMAGE_NAME=${CLIENT_IMAGE_NAME},COMMIT_SHA=${GIT_COMMIT},BRANCH_NAME=${branch_name}" - # TODO(sergiitk): extra "cosmetic" tags for versioned branches, e.g. v1.34.x - # TODO(sergiitk): do this when adding support for custom configs per version -} - -####################################### -# Builds test app and its docker images unless they already exist -# Globals: -# SERVER_IMAGE_NAME: Test server Docker image name -# CLIENT_IMAGE_NAME: Test client Docker image name -# GIT_COMMIT: SHA-1 of git commit being built -# FORCE_IMAGE_BUILD -# Arguments: -# None -# Outputs: -# Writes the output to stdout, stderr -####################################### -build_docker_images_if_needed() { - # Check if images already exist - server_tags="$(gcloud_gcr_list_image_tags "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}")" - printf "Server image: %s:%s\n" "${SERVER_IMAGE_NAME}" "${GIT_COMMIT}" - echo "${server_tags:-Server image not found}" - - client_tags="$(gcloud_gcr_list_image_tags "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}")" - printf "Client image: %s:%s\n" "${CLIENT_IMAGE_NAME}" "${GIT_COMMIT}" - echo "${client_tags:-Client image not found}" - - # Build if any of the images are missing, or FORCE_IMAGE_BUILD=1 - if [[ "${FORCE_IMAGE_BUILD}" == "1" || -z "${server_tags}" || -z "${client_tags}" ]]; then - build_java_test_app - build_test_app_docker_images - else - echo "Skipping Java test app build" - fi -} - -####################################### -# Executes the test case -# Globals: -# TEST_DRIVER_FLAGFILE: Relative path to test driver flagfile -# KUBE_CONTEXT: The name of kubectl context with GKE cluster access -# TEST_XML_OUTPUT_DIR: Output directory for the test xUnit XML report -# CLIENT_IMAGE_NAME: Test client Docker image name -# GIT_COMMIT: SHA-1 of git commit being built -# TESTING_VERSION: version branch under test: used by the framework to -# determine the supported PSM features. -# Arguments: -# Test case name -# Outputs: -# Writes the output of test execution to stdout, stderr -# Test xUnit report to ${TEST_XML_OUTPUT_DIR}/${test_name}/sponge_log.xml -####################################### -run_test() { - # Test driver usage: - # https://github.com/grpc/grpc/tree/master/tools/run_tests/xds_k8s_test_driver#basic-usage - local test_name="${1:?Usage: run_test test_name}" - local out_dir="${TEST_XML_OUTPUT_DIR}/${test_name}" - mkdir -pv "${out_dir}" - set -x - python -m "tests.${test_name}" \ - --flagfile="${TEST_DRIVER_FLAGFILE}" \ - --flagfile="config/url-map.cfg" \ - --kube_context="${KUBE_CONTEXT}" \ - --client_image="${CLIENT_IMAGE_NAME}:${GIT_COMMIT}" \ - --testing_version="${TESTING_VERSION}" \ - --collect_app_logs \ - --log_dir="${out_dir}" \ - --xml_output_file="${out_dir}/sponge_log.xml" \ - |& tee "${out_dir}/sponge_log.log" -} - -####################################### -# Main function: provision software necessary to execute tests, and run them -# Globals: -# KOKORO_ARTIFACTS_DIR -# GITHUB_REPOSITORY_NAME -# SRC_DIR: Populated with absolute path to the source repo -# TEST_DRIVER_REPO_DIR: Populated with the path to the repo containing -# the test driver -# TEST_DRIVER_FULL_DIR: Populated with the path to the test driver source code -# TEST_DRIVER_FLAGFILE: Populated with relative path to test driver flagfile -# TEST_XML_OUTPUT_DIR: Populated with the path to test xUnit XML report -# GIT_ORIGIN_URL: Populated with the origin URL of git repo used for the build -# GIT_COMMIT: Populated with the SHA-1 of git commit being built -# GIT_COMMIT_SHORT: Populated with the short SHA-1 of git commit being built -# KUBE_CONTEXT: Populated with name of kubectl context with GKE cluster access -# Arguments: -# None -# Outputs: -# Writes the output of test execution to stdout, stderr -####################################### -main() { - local script_dir - script_dir="$(dirname "$0")" - - # Source the test driver from the master branch. - echo "Sourcing test driver install script from: ${TEST_DRIVER_INSTALL_SCRIPT_URL}" - source /dev/stdin <<< "$(curl -s "${TEST_DRIVER_INSTALL_SCRIPT_URL}")" - - activate_gke_cluster GKE_CLUSTER_PSM_BASIC - - set -x - if [[ -n "${KOKORO_ARTIFACTS_DIR}" ]]; then - kokoro_setup_test_driver "${GITHUB_REPOSITORY_NAME}" - else - local_setup_test_driver "${script_dir}" - fi - build_docker_images_if_needed - # Run tests - cd "${TEST_DRIVER_FULL_DIR}" - run_test url_map || echo "Failed url_map test" -} - -main "$@" -- cgit v1.2.3