From 4f951789feb94e9e84f522621afbb8aeef68a176 Mon Sep 17 00:00:00 2001 From: Austin Shalit Date: Fri, 6 Dec 2019 17:46:29 -0500 Subject: [PATCH] Build testbench tests online inorder to improve speed (#2144) * Attempt to build testbench tests online inorder to improve speed * Fix contianer reference * Start to remove jenkins shell script * Change job names * Remove sshpass * Remove teststand code * Copy test results back * Fix build by using athena container * Fail if any command fails * Remove jenkins test script * Remove name argument * Fix param count * Add build display name * Fix scp to copy into dir * Update display names * Update stage name * Fix test results scp * Create local test report dir * Remove commented out old code * Remove force pseudo-terminal allocation * Remove extra variables * Update readme * Remove old test runs * Update license header --- azure-pipelines-testbench.yaml | 69 +++++++++++++--- test-scripts/README.md | 24 +++--- test-scripts/config.sh | 4 +- test-scripts/deploy-and-run-test-on-robot.sh | 47 +++++------ test-scripts/jenkins-run-tests-get-results.sh | 79 ------------------- test-scripts/run-tests-on-robot.sh | 16 ++-- 6 files changed, 100 insertions(+), 139 deletions(-) delete mode 100755 test-scripts/jenkins-run-tests-get-results.sh diff --git a/azure-pipelines-testbench.yaml b/azure-pipelines-testbench.yaml index c87ba40c77..67cd75c234 100644 --- a/azure-pipelines-testbench.yaml +++ b/azure-pipelines-testbench.yaml @@ -6,13 +6,20 @@ trigger: include: - master -jobs: -- job: TestBench - pool: RoboRioConnections - timeoutInMinutes: 60 - workspace: - clean: all - steps: +stages: +- stage: Build + jobs: + - job: IntegrationTests + displayName: Integration Tests + pool: + vmImage: 'Ubuntu 16.04' + + container: + image: wpilib/roborio-cross-ubuntu:2020-18.04 + + timeoutInMinutes: 0 + + steps: - task: Gradle@2 condition: and(succeeded(), not(startsWith(variables['Build.SourceBranch'], 'refs/tags/v'))) inputs: @@ -24,23 +31,61 @@ jobs: tasks: 'copyWpilibJIntegrationTestJarToOutput copyWpilibCTestLibrariesToOutput' options: '-Ponlylinuxathena -PbuildServer' - - task: ShellScript@2 - displayName: Run Tests & Get Results + - task: PublishPipelineArtifact@0 inputs: - scriptPath: test-scripts/jenkins-run-tests-get-results.sh + artifactName: 'Integration Tests' + targetPath: 'build/integrationTestFiles' + +- stage: TestBench + displayName: Test Bench + jobs: + - job: Cpp + displayName: C++ + pool: RoboRioConnections + timeoutInMinutes: 30 + workspace: + clean: all + steps: + - task: DownloadPipelineArtifact@0 + inputs: + artifactName: 'Integration Tests' + targetPath: 'build/integrationTestFiles' + + - task: ShellScript@2 + displayName: Run C++ Tests + inputs: + scriptPath: test-scripts/deploy-and-run-test-on-robot.sh + args: 'cpp -A "--gtest_output=xml:/home/admin/testResults/cppreport.xml"' - task: PublishTestResults@2 displayName: Publish C++ Test Results inputs: testResultsFormat: 'JUnit' - testResultsFiles: 'cpp*.xml' + testResultsFiles: '*.xml' testRunTitle: 'C++ Test Report' searchFolder: '$(System.DefaultWorkingDirectory)/test-reports' + - job: Java + pool: RoboRioConnections + timeoutInMinutes: 30 + workspace: + clean: all + steps: + - task: DownloadPipelineArtifact@0 + inputs: + artifactName: 'Integration Tests' + targetPath: 'build/integrationTestFiles' + + - task: ShellScript@2 + displayName: Run Java Tests + inputs: + scriptPath: test-scripts/deploy-and-run-test-on-robot.sh + args: 'java' + - task: PublishTestResults@2 displayName: Publish Java Test Results inputs: testResultsFormat: 'JUnit' - testResultsFiles: 'java*.xml' + testResultsFiles: '*.xml' testRunTitle: 'Java Test Report' searchFolder: '$(System.DefaultWorkingDirectory)/test-reports' diff --git a/test-scripts/README.md b/test-scripts/README.md index a96764ed68..d09cb8eab2 100644 --- a/test-scripts/README.md +++ b/test-scripts/README.md @@ -1,17 +1,19 @@ -# WPILIB TEST SCRIPTS +# WPILib Test Scripts + ## Overview + These test scripts are designed to allow the user of the WPILib test framework to quickly and easily deploy and run their tests on the WPI roboRIO. -In order for the automated test system to work there is a driverstation onboard the roboRIO that handles a queue of users waiting to use the driver station. All of the interaction with this queue is handled internally by test scripts contained within this folder. - -If you deploy code to the test stand using the Eclipse plugins, you _must_ remove the build artifacts in `/home/lvuser`, or you will break tests. +If you deploy code to the test stand using GradleRIO, you _must_ remove the build artifacts in `/home/lvuser`, or you will break the test stand. ## roboRIO Setup -The roboRIO on the test bench must be updated everytime NI releases a new image. +The roboRIO on the test bench must be updated every time NI releases a new image. -1. [Use the roboRIO Imaging Tool to format the roboRIO with the lastest image.](https://wpilib.screenstepslive.com/s/4485/m/13503/l/144984-imaging-your-roborio) -2. [Install Java on the roboRIO.](https://wpilib.screenstepslive.com/s/4485/m/13503/l/599747-installing-java-8-on-the-roborio-using-the-frc-roborio-java-installer-java-only) -3. SFTP the [teststand, netconsole, and libstdc++ ipk files](https://users.wpi.edu/~phplenefisch/ipk/) on to the roboRIO. -4. ssh on to the roboRIO as the admin user (ex: `ssh admin@roboRIO-190-FRC.local`) -5. Use opkg to install the ipk files (ex: `opkg install teststand_1.2-1_armv7a-vfp.ipk`) -6. Reboot the roboRIO +1. [Use the roboRIO Imaging Tool to format the roboRIO with the lastest image.](https://frcdocs.wpi.edu/en/latest/docs/getting-started/getting-started-frc-control-system/imaging-your-roborio.html) +2. Set a static ip on the roboRIO web dashboard to `10.1.90.2` +2. Install Java on the roboRIO + 1. [Download the JRE from Maven.](https://frcmaven.wpi.edu/artifactory/list/release/edu/wpi/first/jdk/) + 2. Transfer the JRE ipk to the roboRIO with scp: `scp admin@roboRIO-190-FRC.local:/tmp/frcjre.ipk` + 3. Install the JRE: `opkg install /tmp/frcjre.ipk` + 4. Remove the ipk file: `rm /tmp/frcjre.ipk` +3. Reboot the roboRIO diff --git a/test-scripts/config.sh b/test-scripts/config.sh index 48eeda4d14..be2587ad0b 100755 --- a/test-scripts/config.sh +++ b/test-scripts/config.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash #*----------------------------------------------------------------------------*# -#* Copyright (c) FIRST 2014. All Rights Reserved. *# +#* Copyright (c) 2014-2019 FIRST. All Rights Reserved. *# #* Open Source Software - may be modified and shared by FRC teams. The code *# #* must be accompanied by the FIRST BSD license file in the root directory of *# -#* the project. *# +#* the project. *# #*----------------------------------------------------------------------------*# # If this is changed, update the .gitignore diff --git a/test-scripts/deploy-and-run-test-on-robot.sh b/test-scripts/deploy-and-run-test-on-robot.sh index f023a18e0e..ca5f59fb72 100755 --- a/test-scripts/deploy-and-run-test-on-robot.sh +++ b/test-scripts/deploy-and-run-test-on-robot.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash #*----------------------------------------------------------------------------*# -#* Copyright (c) FIRST 2014. All Rights Reserved. *# +#* Copyright (c) 2014-2019 FIRST. All Rights Reserved. *# #* Open Source Software - may be modified and shared by FRC teams. The code *# #* must be accompanied by the FIRST BSD license file in the root directory of *# -#* the project. *# +#* the project. *# #*----------------------------------------------------------------------------*# # Configurable variables @@ -19,7 +19,7 @@ DEFAULT_DESTINATION_RUN_TEST_SCRIPT=${DEFAULT_DESTINATION_DIR}/${DEFAULT_LOCAL_R usage="$(basename "$0") [-h] (java|cpp) [-A] [arg] [arg]... A script designed to run the integration tests. -This script should only be run on the roborio. +This script should only be run on the computer connected to the roboRIO. Where: -h Show this help text. -A Disable language recomended arguments. @@ -30,20 +30,26 @@ Where: LANGUAGE=none LOCAL_TEST_FILE=none DESTINATION_TEST_FILE=none -LIBRARY_FILES=none TEST_RUN_ARGS="" +DESTINATION_TEST_RESULTS=none +LOCAL_TEST_RESULTS=none -# Begin searching for options from the third paramater on + +# Begin searching for options from the second paramater on PARAM_ARGS=${@:2} if [[ "$1" = java ]]; then LANGUAGE=$1 LOCAL_TEST_FILE=$DEFAULT_LOCAL_JAVA_TEST_FILE DESTINATION_TEST_FILE=$DEFAULT_DESTINATION_JAVA_TEST_FILE + DESTINATION_TEST_RESULTS=$DEFAULT_DESTINATION_JAVA_TEST_RESULTS + LOCAL_TEST_RESULTS=$DEFAULT_LOCAL_JAVA_TEST_RESULT elif [[ "$1" = cpp ]]; then LANGUAGE=$1 LOCAL_TEST_FILE=$DEFAULT_LOCAL_CPP_TEST_FILE DESTINATION_TEST_FILE=$DEFAULT_DESTINATION_CPP_TEST_FILE + DESTINATION_TEST_RESULTS=$DEFAULT_DESTINATION_CPP_TEST_RESULTS + LOCAL_TEST_RESULTS=$DEFAULT_LOCAL_CPP_TEST_RESULT elif [[ "$1" = "-h" ]]; then printf "Usage:\n" echo "$usage" @@ -65,25 +71,14 @@ TEST_RUN_ARGS="${@:2}" shopt -s huponexit -SCP_TEST_SCRIPT="scp config.sh ${DEFAULT_LOCAL_RUN_TEST_SCRIPT} ${ROBOT_ADDRESS}:/${DEFAULT_DESTINATION_DIR}" -SSH_CHMOD_AND_MAKE_TEMP_TEST_DIR="ssh -t ${ROBOT_ADDRESS} \"chmod a+x ${DEFAULT_DESTINATION_RUN_TEST_SCRIPT}; mkdir ${DEFAULT_TEST_SCP_DIR}; touch ${DESTINATION_TEST_FILE}\"" -SCP_TEST_PROGRAM="scp ${LOCAL_TEST_FILE} ${ROBOT_ADDRESS}:${DESTINATION_TEST_FILE}" -SSH_RUN_TESTS="ssh -t ${ROBOT_ADDRESS} ${DEFAULT_DESTINATION_RUN_TEST_SCRIPT} ${LANGUAGE} $(whoami) -d ${DEFAULT_TEST_SCP_DIR} ${TEST_RUN_ARGS}" -SCP_NATIVE_LIBRARIES="scp ${DEFAULT_LIBRARY_NATIVE_FILES}/* ${ROBOT_ADDRESS}:${DEFAULT_LIBRARY_NATIVE_DESTINATION}" -CONFIG_NATIVE_LIBRARIES="ssh -t ${ADMIN_ROBOT_ADDRESS} ldconfig" +# Fail if any command fails +set -e -if [ $(which sshpass) ]; then - sshpass -p "" ${SCP_NATIVE_LIBRARIES} - sshpass -p "" ${CONFIG_NATIVE_LIBRARIES} - sshpass -p "" ${SCP_TEST_SCRIPT} - sshpass -p "" ${SSH_CHMOD_AND_MAKE_TEMP_TEST_DIR} - sshpass -p "" ${SCP_TEST_PROGRAM} - sshpass -p "" ${SSH_RUN_TESTS} -else - eval ${SCP_NATIVE_LIBRARIES} - eval ${CONFIG_NATIVE_LIBRARIES} - eval ${SCP_TEST_SCRIPT} - eval ${SSH_CHMOD_AND_MAKE_TEMP_TEST_DIR} - eval ${SCP_TEST_PROGRAM} - eval ${SSH_RUN_TESTS} -fi +ssh ${ROBOT_ADDRESS} "rm -R ${DEFAULT_DESTINATION_TEST_RESULTS_DIR}; mkdir ${DEFAULT_DESTINATION_TEST_RESULTS_DIR}" +scp ${DEFAULT_LIBRARY_NATIVE_FILES}/* ${ROBOT_ADDRESS}:${DEFAULT_LIBRARY_NATIVE_DESTINATION} +ssh ${ADMIN_ROBOT_ADDRESS} ldconfig +scp config.sh ${DEFAULT_LOCAL_RUN_TEST_SCRIPT} ${ROBOT_ADDRESS}:/${DEFAULT_DESTINATION_DIR} +ssh ${ROBOT_ADDRESS} "chmod a+x ${DEFAULT_DESTINATION_RUN_TEST_SCRIPT}; mkdir ${DEFAULT_TEST_SCP_DIR}; touch ${DESTINATION_TEST_FILE}" +scp ${LOCAL_TEST_FILE} ${ROBOT_ADDRESS}:${DESTINATION_TEST_FILE} +ssh ${ROBOT_ADDRESS} ${DEFAULT_DESTINATION_RUN_TEST_SCRIPT} ${LANGUAGE} -d ${DEFAULT_TEST_SCP_DIR} ${TEST_RUN_ARGS} +mkdir ${DEFAULT_LOCAL_TEST_RESULTS_DIR}; scp ${ROBOT_ADDRESS}:${DESTINATION_TEST_RESULTS} ${LOCAL_TEST_RESULTS} diff --git a/test-scripts/jenkins-run-tests-get-results.sh b/test-scripts/jenkins-run-tests-get-results.sh deleted file mode 100755 index 8981b4d78c..0000000000 --- a/test-scripts/jenkins-run-tests-get-results.sh +++ /dev/null @@ -1,79 +0,0 @@ -#!/usr/bin/env bash -#*----------------------------------------------------------------------------*# -#* Copyright (c) FIRST 2014. All Rights Reserved. *# -#* Open Source Software - may be modified and shared by FRC teams. The code *# -#* must be accompanied by the FIRST BSD license file in the root directory of *# -#* the project. *# -#*----------------------------------------------------------------------------*# - -# Configurable variables -source config.sh - -( -# Wait for lock -printf "Getting exclusive lock for RIO execution...\n" -flock -x 200 || exit 1 - -# Ensure the teststand is dead -SSH_STOP_TESTSTAND="ssh -t ${ROBOT_ADDRESS} sh -c '/etc/init.d/teststand stop; sleep 1'" -if [ $(which sshpass) ]; then - sshpass -p "" ${SSH_STOP_TESTSTAND} -else - eval ${SSH_STOP_TESTSTAND} -fi - -# If there are already test results in the repository then remove them -if [[ -e ${DEFAULT_LOCAL_TEST_RESULTS_DIR} ]]; then - rm -R ${DEFAULT_LOCAL_TEST_RESULTS_DIR} -fi - -# Make the directory where the tests should live -mkdir ${DEFAULT_LOCAL_TEST_RESULTS_DIR} 2>/dev/null - -# Remove the preivous test results from the the robot -SSH_REMOVE_OLD_TEST_RESULTS="ssh -t ${ROBOT_ADDRESS} rm -R ${DEFAULT_DESTINATION_TEST_RESULTS_DIR}; mkdir ${DEFAULT_DESTINATION_TEST_RESULTS_DIR}" -if [ $(which sshpass) ]; then - sshpass -p "" ${SSH_REMOVE_OLD_TEST_RESULTS} -else - eval ${SSH_REMOVE_OLD_TEST_RESULTS} -fi - -printf "Running cpp test\n" - -# Run the C++ Tests -./deploy-and-run-test-on-robot.sh cpp -A "--gtest_output=xml:${DEFAULT_DESTINATION_CPP_TEST_RESULTS}" - -# Retrive the C++ Test Results -SCP_GET_CPP_TEST_RESULT="scp ${ROBOT_ADDRESS}:${DEFAULT_DESTINATION_CPP_TEST_RESULTS} ${DEFAULT_LOCAL_CPP_TEST_RESULT}" -if [ $(which sshpass) ]; then - sshpass -p "" ${SCP_GET_CPP_TEST_RESULT} -else - eval ${SCP_GET_CPP_TEST_RESULT} -fi - -sleep 10 - -# Run the Java Tests -./deploy-and-run-test-on-robot.sh java - -# Retrive the Java Test Results -SCP_GET_JAVA_TEST_RESULT="scp ${ROBOT_ADDRESS}:${DEFAULT_DESTINATION_JAVA_TEST_RESULTS} ${DEFAULT_LOCAL_JAVA_TEST_RESULT}" -if [ $(which sshpass) ]; then - sshpass -p "" ${SCP_GET_JAVA_TEST_RESULT} -else - eval ${SCP_GET_JAVA_TEST_RESULT} -fi - -# Make sure that we got test results back. -if [ ! -e ${DEFAULT_LOCAL_CPP_TEST_RESULT} ]; then - echo "There are no results from the C++ tests; they must have failed." - exit 100 -fi - -if [ ! -e ${DEFAULT_LOCAL_JAVA_TEST_RESULT} ]; then - echo "There are no results from the Java tests; they must have failed." - exit 101 -fi - -# The mutex is released when this program exits -) 200>/var/lock/jenkins.rio.exclusivelock diff --git a/test-scripts/run-tests-on-robot.sh b/test-scripts/run-tests-on-robot.sh index 67d54f20db..96d10c007e 100755 --- a/test-scripts/run-tests-on-robot.sh +++ b/test-scripts/run-tests-on-robot.sh @@ -1,9 +1,9 @@ #!/usr/bin/env bash #*----------------------------------------------------------------------------*# -#* Copyright (c) FIRST 2014. All Rights Reserved. *# +#* Copyright (c) 2014-2019 FIRST. All Rights Reserved. *# #* Open Source Software - may be modified and shared by FRC teams. The code *# #* must be accompanied by the FIRST BSD license file in the root directory of *# -#* the project. *# +#* the project. *# #*----------------------------------------------------------------------------*# # This file is intended to be run in the DEFAULT_TEST_DIR on the roboRIO. @@ -17,12 +17,11 @@ source config.sh DEFAULT_TEST_DIR=${DEFAULT_DESTINATION_DIR} DEFAULT_PATH_TO_JRE=/usr/local/frc/JRE/bin/java -usage="$(basename "$0") [-h] (java|cpp) name [-d test_dir] [-A] [arg] [arg]... +usage="$(basename "$0") [-h] (java|cpp) [-d test_dir] [-A] [arg] [arg]... A script designed to run the integration tests. This script should only be run on the roborio. Where: -h Show this help text - name The name of the user trying to run the tests (used for driver station) -d The directory where the tests have been placed. This is done to prevent overwriting an already running program. This scrip will automatically move the test into the ${DEFAULT_TEST_DIR} @@ -42,12 +41,11 @@ fi LANGUAGE=none TEST_FILE=none -NAME=$2 TEST_DIR="$DEFAULT_TEST_DIR" -# Begin searching for options from the third paramater on -PARAM_ARGS=${@:3} +# Begin searching for options from the second paramater on +PARAM_ARGS=${@:2} # Where the test arguments start -TEST_RUN_ARGS=${@:3} +TEST_RUN_ARGS=${@:2} RUN_WITH_DEFAULT_ARGS=true DEFAULT_ARGS="" @@ -70,7 +68,7 @@ else exit 1 fi -PARAM_COUNTER=2 +PARAM_COUNTER=1 printf "Param Args ${PARAM_ARGS}\n" # Check for optional paramaters