CTest:Buildserver

Introduction

In a normal CTest installation you have a central repository for your sourcecode. Each client has a special configuration to know how to get the source, build and test it. After finishing the results will be sent to central CDash. This configuration has to be set up at each of the clients. If you create a new branch in your repository and want to get it built by all clients then you must change the configuration at each of the clients.

This solution adds the possibility to manage this in a central place (at the BuildMaster).

How it works

Each BuildSlave asks the BuildMaster for a list of buildnames. Then the BuildSlave downloads a CTestScript.cmake for each buildname and executes it.

Requirements

• CMake/CTest
• wget
• webserver with PHP support

Installation

Buildmaster

Change into your document root directory of your webserver and create a direcotry "CDash", if it does not exesit already. Save the following content as generateCTestScript.php in the CDash direcotory:

 @$site = $_GET["site"];
 @$build_name = $_GET["buildname"];
 
 // Checks
 if(!isset($site))
   {
   echo "Not a valid site!";
   return;
   }
 
 // THIS CONFIGURATION SHOULD COME FROM THE DATABASE
 $config_from_database = array(
         "BUILDSERVER1" => array(
                 "Test1" => array(
                         "@CTS_CMAKE_GENERATOR@" => "Unix Makefiles")),
         "BUILDSERVER2" => array(
                 "Test1" => array(
                         "@CTS_CMAKE_GENERATOR@" => "Unix Makefiles")),
         "BUILDSERVER3" => array(
                 "Test1" => array(
                         "@CTS_CMAKE_GENERATOR@" => "NMake Makefiles"),
                 "Test2" => array(
                         "@CTS_CMAKE_GENERATOR@" => "NMake Makefiles",
                         "@CTS_REPOSITORY_URL@" => "http://VERSIONCONTROL/svn/test2/trunk"),
                 "Test3" => array(
                         "@CTS_CMAKE_GENERATOR@" => "NMake Makefiles",
                         "@CTS_REPOSITORY_URL@" => "http://VERSIONCONTROL/svn/test3/branches/next_release")),
         "BUILDSERVER4" => array(
                 "Test1" => array(
                         "@CTS_CMAKE_GENERATOR@" => "NMake Makefiles")));
 
 
 // no build given, so Generate overviewlist
 if(!isset($build_name))
   {
   $list = array_keys($config_from_database[$site]);
 
   header('Content-Type: text/plain');
   echo implode("\n", $list);
   return;
   }
 
 // DEFAULT SETTINGS
 $script_parameter = array(
         "@CTS_SITE@" => $site,
         "@CTS_PROJECT_NAME@" => "Test",
         "@CTS_BUILD_NAME@" => $build_name,
         "@CTS_NIGHTLY_START_TIME@" => "22:00:00 UTC",
         "@CTS_CMAKE_GENERATOR@" => "Unix Makefiles",
         "@CTS_BUILD_CONFIGURATION@" => "Release",
         "@CTS_CTEST_MODEL@" => "Continuous",
         "@CTS_BUILD_OPTIONS@" => "-DEXAMPLE_OPTION=1",
         "@CTS_REPOSITORY_URL@" => "http://VERSIONCONTROL/svn/test1");
 
 // template.txt SHOULD ALSO COME FROM THE DATABASE
 $ctestscript = strtr(strtr(implode(file("template.txt")), $config_from_database[$site][$build_name]), $script_parameter);
 
 header('Vary: User-Agent');
 if(ob_get_contents())
   echo "Some data has already been output";
 if(isset($_SERVER['HTTP_USER_AGENT']) && strpos($_SERVER['HTTP_USER_AGENT'],'MSIE'))
   header('Content-Type: application/force-download');
 else
   header('Content-Type: application/octet-stream');
 if(headers_sent())
   echo "Some data has already been output to browser";
 
 header("Content-Disposition: attachment; filename=\"CTestScript.cmake\"");
 header("Content-Transfer-Encoding: binary");
 header("Content-Length: ".strlen($ctestscript));
 echo $ctestscript;
 
 ?>

The following is an example for the template.txt, which also does a upload of a generated NSIS setup:

 set(CTEST_SITE "@CTS_SITE@")
 set(CTEST_PROJECT_NAME "@CTS_PROJECT_NAME@")
 set(CTEST_BUILD_NAME "@CTS_BUILD_NAME@")
 set(CTEST_NIGHTLY_START_TIME "@CTS_NIGHTLY_START_TIME@")
 set(CTEST_CMAKE_GENERATOR "@CTS_CMAKE_GENERATOR@")
 set(CTEST_BUILD_CONFIGURATION "@CTS_BUILD_CONFIGURATION@")
 set(MODEL "@CTS_CTEST_MODEL@")
 
 set(BUILD_OPTIONS "@CTS_BUILD_OPTIONS@")
 set(REPOSITORY_URL "@CTS_REPOSITORY_URL@")
 
 
 find_package(Subversion)
 
 set(CTEST_SOURCE_DIRECTORY "${CTEST_SCRIPT_DIRECTORY}/source")
 set(CTEST_BINARY_DIRECTORY "${CTEST_SCRIPT_DIRECTORY}/build")
 
 set(CTEST_UPDATE_COMMAND ${Subversion_SVN_EXECUTABLE})
 
 if(NOT DEFINED CTEST_DROP_METHOD)
 	set(CTEST_DROP_METHOD "http")
 endif()
 
 if(CTEST_DROP_METHOD STREQUAL "http")
 	set(CTEST_DROP_SITE "BUILDMASTER")
 	set(CTEST_DROP_LOCATION
 		"/CDash/submit.php?project=${CTEST_PROJECT_NAME}")
 	set(CTEST_DROP_BUILD_LOCATION
 		"/CDash/uploadbuild.php?project=${CTEST_PROJECT_NAME}")
 	set(CTEST_TRIGGER_SITE "")
 endif()
 
 
 string(COMPARE NOTEQUAL "${MODEL}" "Continuous" SUBMIT_ALWAYS)
 set(NEED_REPOSITORY_CHECKOUT 1)
 
 if(EXISTS "${CTEST_SOURCE_DIRECTORY}")
 	subversion_wc_info("${CTEST_SOURCE_DIRECTORY}" REPOSITORY)
 
 	string(COMPARE
 		NOTEQUAL
 		"${REPOSITORY_URL}"
 		"${REPOSITORY_WC_URL}"
 		NEED_REPOSITORY_CHECKOUT)
 
 	if(${NEED_REPOSITORY_CHECKOUT})
 		file(REMOVE_RECURSE "${CTEST_SOURCE_DIRECTORY}")
 	endif()
 else()
 	set(NEED_REPOSITORY_CHECKOUT 1)
 endif()
 
 if(${NEED_REPOSITORY_CHECKOUT})
 	set(CTEST_CHECKOUT_COMMAND
 		"${CTEST_UPDATE_COMMAND} co ${REPOSITORY_URL} \"${CTEST_SOURCE_DIRECTORY}\" -r HEAD")
 else()
 	set(CTEST_CHECKOUT_COMMAND "${CTEST_UPDATE_COMMAND} update")
 endif()
 
 
 set(CTEST_CONFIGURE_COMMAND
 	"${CMAKE_COMMAND} -DCMAKE_BUILD_TYPE=${CTEST_BUILD_CONFIGURATION} ${BUILD_OPTIONS} \"-G${CTEST_CMAKE_GENERATOR}\" \"${CTEST_SOURCE_DIRECTORY}\"")
 
 ctest_start(${MODEL})
 
 if(${CTEST_SCRIPT_ARG} MATCHES ${MODEL})
 	ctest_update(SOURCE
 		"${CTEST_SOURCE_DIRECTORY}"
 		RETURN_VALUE
 		NUMBER_FILES)
 
 	if(${SUBMIT_ALWAYS}
 		OR
 		${NEED_REPOSITORY_CHECKOUT}
 		OR
 		${NUMBER_FILES}
 		GREATER
 		0)
 		ctest_configure(BUILD "${CTEST_BINARY_DIRECTORY}")
 		ctest_build(BUILD "${CTEST_BINARY_DIRECTORY}")
 		ctest_test(BUILD "${CTEST_BINARY_DIRECTORY}")
 		ctest_submit()
 
 		execute_process(COMMAND
 			"cpack"
 			WORKING_DIRECTORY
 			${CTEST_BINARY_DIRECTORY}
 			RESULT_VARIABLE
 			cpackResult
 			OUTPUT_VARIABLE
 			cpackLog
 			ERROR_VARIABLE
 			cpackLog)
 		file(WRITE ${CTEST_BINARY_DIRECTORY}/Testing/cpack.log "${cpackLog}")
 		file(GLOB
 			UPLOAD_FILES
 			"build/*.deb" "build/*.exe")
 		foreach(_currentArg ${UPLOAD_FILES})
 			get_filename_component(_fn ${_currentArg} NAME)
 			execute_process(COMMAND
 				wget
 				"${CTEST_DROP_METHOD}://${CTEST_DROP_SITE}${CTEST_DROP_BUILD_LOCATION}&fn=${_fn}"
 				"--post-file=${_currentArg}"
 				"-o${CTEST_BINARY_DIRECTORY}/Testing/upload.log"
 				"-q")
 			file(REMOVE "${CTEST_BINARY_DIRECTORY}/${_currentArg}")
 		endforeach()
 	endif()
 endif()

Linux BuildSlave

Create a a directory "/var/builds" and save the following shell script as make.sh in this directory

 cd /var/builds
 site=BUILDSLAVE1
 
 wget http://BUILDMASTER/CDash/generateCTestScript.php?site=$site\&buildtype=$1 -O list.txt -q
 
 for d in $(ls -d */); do
   SKIP_DELETE=0
 
   for p in $(cat list.txt); do
     if [ $p = ${d%%/} ]; then
       SKIP_DELETE=1
     fi
   done
 
   if [ $SKIP_DELETE = 0 ]; then
     rm -r ${d%%/}
   fi
 done
 
 for p in $(cat list.txt); do
   if [ ! -d $p ]; then
     mkdir $p
   fi
 
   if [ ! -e $p.lock ]; then
     touch $p.lock
 
     cd $p
     wget http://BUILDMASTER/CDash/generateCTestScript.php?site=$site\&buildname=$p -O CTestScript.cmake -q
     ctest -S CTestScript.cmake,$1
     cd ..
 
     rm -f $p.lock
   fi
 done
 
 rm list.txt

Finaly you have to create a cron job (see CTest Scripting) for each buildtype (Nightly, Continuous), which will execute this shell script:

• For a nightly build create a job to execute "/var/builds/make.sh Nightly" at your CTEST_NIGHTLY_START_TIME.
• To setup a continuous build create a job, which will execute "/var/builds/make.sh Continuous" every minute.

Windows BuildSlave

Create a a directory "C:\builds" and save the following batch script as make.bat in this directory

 @echo off
 set site=BUILDSLAVE3
 
 call "C:\Program Files\Microsoft Visual Studio 9.0\VC\bin\vcvars32.bat"
 
 if not exist %1 mkdir %1
 pushd %1
 
 wget http://BUILDMASTER/CDash/generateCTestScript.php?site=%site%^&buildtype=%1 -O list.txt -q
 
 for /D %%d in (*) do (
    if exist skip_delete.flag del skip_delete.flag
       for /f %%p in (list.txt) do (
          if %%p == %%d echo 0 > skip_delete.flag
       )
 
    if not exist skip_delete.flag rmdir /S /Q %%d
 )
 
 if exist skip_delete.flag del skip_delete.flag
 
 for /f %%p in (list.txt) do (
    if not exist %%p mkdir %%p
 
    if not exist %%p.lock (
       echo 0 > %%p.lock
 
       pushd %%p
       wget http://BUILDMASTER/CDash/generateCTestScript.php?site=%site%^&buildname=%%p -O CTestScript.cmake -q
       ctest -S CTestScript.cmake,%1
       popd
 
       del %%p.lock
    )
 )
 
 del list.txt
 popd

Finaly you have to create a sheduled task (see CTest Scripting) for each buildtype (Nightly, Continuous), which will execute this batch file:

• For a nightly build create a task to execute "C:\builds\make.bat Nightly" at your CTEST_NIGHTLY_START_TIME.
• To setup a continuous build create a task, which will execute "C:\builds\make.bat Continuous" every minute.

---

CMake: [Welcome | Site Map]