diff options
author | Liam McLoughlin <hexxeh@hexxeh.net> | 2011-07-08 03:44:09 +0100 |
---|---|---|
committer | Liam McLoughlin <hexxeh@hexxeh.net> | 2011-07-08 03:44:09 +0100 |
commit | 9d971158f5fb4888f8ac33540c85801a6064bcda (patch) | |
tree | 6d3832c0160cb4bee165de0c3918c19176072883 | |
parent | Fix minimal config and incorrect cachedkernel logic (diff) | |
download | gentoaster-9d971158f5fb4888f8ac33540c85801a6064bcda.tar.gz gentoaster-9d971158f5fb4888f8ac33540c85801a6064bcda.tar.bz2 gentoaster-9d971158f5fb4888f8ac33540c85801a6064bcda.zip |
Added basic Gearman worker/client
Client is only for testing purposes, real client will be part of WebUI
Worker needs better error handling, does work in current form though
Added progress reporting hack in create_image.sh, going to replace this with a better system soon
-rw-r--r-- | .gitignore | 5 | ||||
-rw-r--r-- | client.php | 33 | ||||
-rwxr-xr-x | create_image.sh | 100 | ||||
-rw-r--r-- | daemon.php | 61 |
4 files changed, 149 insertions, 50 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f8cf34d --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +move_test.sh +res/bzImage +res/kernelmodules/ +res/portage-latest.tar.bz2 +res/stage3.tar.bz2 diff --git a/client.php b/client.php new file mode 100644 index 0000000..0110d3b --- /dev/null +++ b/client.php @@ -0,0 +1,33 @@ +<?php + + // Gentoaster build daemon client + // Licensed under GPL v3, see COPYING file + + $client= new GearmanClient(); + $client->addServer(); + + echo "Sending job\n"; + + do { + $result = $client->do("invoke_image_build", file_get_contents("configs/minimal.ini")); + switch($client->returnCode()) { + case GEARMAN_WORK_DATA: + break; + case GEARMAN_WORK_STATUS: + list($numerator, $denominator)= $client->doStatus(); + $progress = ceil($numerator/$denominator*100); + echo "Status: $progress% complete\n"; + break; + case GEARMAN_SUCCESS: + echo "Job finished!\n"; + echo $result; + break; + default: + echo "Return code: " . $client->returnCode() . "\n"; + exit; + } + } + + while($client->returnCode() != GEARMAN_SUCCESS); + +?> diff --git a/create_image.sh b/create_image.sh index 03466b1..3cee693 100755 --- a/create_image.sh +++ b/create_image.sh @@ -14,8 +14,11 @@ hash qemu-img 2>&- || { echo >&2 "Gentoaster requires qemu-img, but it's not ins hash extlinux 2>&- || { echo >&2 "Gentoaster requires extlinux, but it's not installed."; exit 1; } hash sfdisk 2>&- || { echo >&2 "Gentoaster requires sfdisk, but it's not installed."; exit 1; } +# Figure out where we are +RUNNING_DIRECTORY=$(cd `dirname $0` && pwd) + # Load shflags to parse flags -. ./lib/shflags +. ${RUNNING_DIRECTORY}/lib/shflags DEFINE_string 'config' '' 'configuration to build from' DEFINE_string 'proxy' '' 'HTTP proxy to use for emerges' @@ -30,7 +33,7 @@ if [ -z ${FLAGS_config} ]; then fi # Parse the configuration we took as an arg -source parse_config.sh ${FLAGS_config} +source ${RUNNING_DIRECTORY}/parse_config.sh ${FLAGS_config} # Generate a few helper variables using the configuration file @@ -40,7 +43,7 @@ IMAGE_BYTES=$(( ${IMAGE_MEGABYTES} * 1024 * 1024 )) IMAGES_OUTPUT_PATH=`pwd` IMAGE_WORK_PATH="${IMAGES_OUTPUT_PATH}/${BUILD_ID}" LOG_FILE="${IMAGE_WORK_PATH}/log.txt" -TOOL_RES_PATH=`pwd`/res +TOOL_RES_PATH=${RUNNING_DIRECTORY}/res NUM_JOBS=$(( `grep -c processor /proc/cpuinfo`+1 )) STAGE3_URL="http://distribution.hexxeh.net/gentoo/stage3-i686-latest.tar.bz2" @@ -48,34 +51,33 @@ PORTAGE_URL="http://distribution.hexxeh.net/gentoo/portage-latest.tar.bz2" BINHOST_URL="http://tinderbox.dev.gentoo.org/default-linux/x86" EMERGE_PROXY="${FLAGS_proxy}" -echo "Creating build working directory" +echo "Step 1: Creating build working directory" mkdir -p ${IMAGE_WORK_PATH} cd ${IMAGE_WORK_PATH} - -echo "Starting log at ${LOG_FILE}..." echo "" > ${LOG_FILE} # Create disk iamge BYTES_PER_CYLINDER=$(( 512*63*255 )) CYLINDERS=$(( ${IMAGE_BYTES}/${BYTES_PER_CYLINDER} )) REAL_IMAGE_BYTES=$(( (${CYLINDERS}+1)*${BYTES_PER_CYLINDER} )) -echo "Creating image ${IMAGE_NAME}, size ${REAL_IMAGE_BYTES} bytes" +echo "Step 2: Creating image ${IMAGE_NAME}, size ${REAL_IMAGE_BYTES} bytes" qemu-img create -f raw ${IMAGE_NAME} ${REAL_IMAGE_BYTES} &>> ${LOG_FILE} # Clean up old mounts cleanup_mounts() { - echo "Cleaning up old mounts" - umount -d ${IMAGE_WORK_PATH}/rootfs/dev/pts &>> ${LOG_FILE} - umount -d ${IMAGE_WORK_PATH}/rootfs/dev/shm &>> ${LOG_FILE} - umount -d ${IMAGE_WORK_PATH}/rootfs/dev &>> ${LOG_FILE} - umount -d ${IMAGE_WORK_PATH}/rootfs/proc &>> ${LOG_FILE} - umount -d ${IMAGE_WORK_PATH}/rootfs/boot &>> ${LOG_FILE} - umount -d ${IMAGE_WORK_PATH}/rootfs &>> ${LOG_FILE} + sleep 2 + umount -d -f ${IMAGE_WORK_PATH}/rootfs/dev/pts &>> ${LOG_FILE} + umount -d -f ${IMAGE_WORK_PATH}/rootfs/dev/shm &>> ${LOG_FILE} + umount -d -f ${IMAGE_WORK_PATH}/rootfs/dev &>> ${LOG_FILE} + umount -d -f ${IMAGE_WORK_PATH}/rootfs/proc &>> ${LOG_FILE} + umount -d -f ${IMAGE_WORK_PATH}/rootfs/boot &>> ${LOG_FILE} + umount -d -f ${IMAGE_WORK_PATH}/rootfs &>> ${LOG_FILE} + sleep 2 } cleanup_mounts # Create partition table -echo "Writing partition table" +echo "Step 3: Writing partition table" echo -e "\x55\xaa" | dd bs=1 count=2 seek=510 of=${IMAGE_NAME} conv=notrunc &>> ${LOG_FILE} LOOP_DEV_IMAGE=`losetup -f` losetup --sizelimit ${REAL_IMAGE_BYTES} ${LOOP_DEV_IMAGE} ${IMAGE_NAME} &>> ${LOG_FILE} @@ -89,7 +91,7 @@ sleep 2 losetup -d ${LOOP_DEV_IMAGE} # Mounting new root and boot -echo "Creating filesystems" +echo "Step 4: Creating filesystems" LOOP_DEV_BOOT=`losetup -f` losetup -o $(( 512 * 2048 )) --sizelimit $(( ${BOOT_MEGABYTES} * 1024 * 1024 )) ${LOOP_DEV_BOOT} ${IMAGE_NAME} @@ -109,7 +111,7 @@ mkswap ${LOOP_DEV_SWAP} &>> ${LOG_FILE} sleep 2 losetup -d ${LOOP_DEV_SWAP} -echo "Mounting fileystems" +echo "Step 5: Mounting fileystems" mkdir -p rootfs mount -o loop,offset=$(( ( 512 * 2048 ) + ( ${BOOT_MEGABYTES} * 1024 * 1024 ) + ( ${SWAP_MEGABYTES} * 1024 * 1024 ) )) ${IMAGE_NAME} rootfs @@ -125,7 +127,7 @@ then wget ${STAGE3_URL} -O ${TOOL_RES_PATH}/stage3.tar.bz2 &>> ${LOG_FILE} fi -echo "Extracting Stage 3" +echo "Step 6: Extracting Stage 3" tar jxf ${TOOL_RES_PATH}/stage3.tar.bz2 &>> ${LOG_FILE} if [ ! -f ${TOOL_RES_PATH}/portage-latest.tar.bz2 ]; @@ -134,17 +136,17 @@ then wget ${PORTAGE_URL} -O ${TOOL_RES_PATH}/portage-latest.tar.bz2 &>> ${LOG_FILE} fi -echo "Extracting Portage snapshot" +echo "Step 7: Extracting Portage snapshot" cd usr tar jxf ${TOOL_RES_PATH}/portage-latest.tar.bz2 &>> ${LOG_FILE} -echo "Setting up chroot" +echo "Step 8: Setting up chroot" cd .. mount -t proc /proc proc mount --rbind /dev dev cp -L /etc/resolv.conf etc/resolv.conf -echo "Setting up make.conf" +echo "Step 9: Setting up make.conf" mkdir -p usr/portage/packages echo "PORTAGE_BINHOST=\"${BINHOST_URL}\"" >> etc/make.conf echo "PKGDIR=\"/usr/portage/packages\"" >> etc/make.conf @@ -158,67 +160,66 @@ if [[ ${OUTPUT_FORMAT} = "vbox" ]]; then fi if [ -n $EMERGE_PROXY ]; then - echo "Enabling HTTP proxy" + echo "Enabling HTTP proxy" &>> ${LOG_FILE} echo "http_proxy=\"${EMERGE_PROXY}\"" >> etc/make.conf fi -echo "Setting up package.use" +echo "Step 10: Setting up package.use" mkdir -p etc/portage echo ${PACKAGE_USE} >> etc/portage/package.use -echo "Setting up package.accept_keywords" +echo "Step 11: Setting up package.accept_keywords" echo ${PACKAGE_ACCEPT_KEYWORDS} >> etc/portage/package.accept_keywords -echo "Entering chroot" +echo "Step 12: Chroot setup" linux32 chroot . env-update &>> ${LOG_FILE} - -echo "Loading profile" linux32 chroot . /bin/bash /etc/profile &>> ${LOG_FILE} -echo "Setting timezone to ${TIMEZONE}" +echo "Step 13: Setting timezone to ${TIMEZONE}" linux32 chroot . cp /usr/share/zoneinfo/${TIMEZONE} /etc/localtime &>> ${LOG_FILE} -echo "Setting hostname to ${HOSTNAME}" +echo "Step 14: Setting hostname to ${HOSTNAME}" linux32 chroot . /bin/bash -c "echo 127.0.0.1 ${HOSTNAME}.local ${HOSTNAME} localhost > /etc/hosts" &>> ${LOG_FILE} -echo "Copying new fstab" +echo "Step 15: Copying new fstab" cp ${TOOL_RES_PATH}/fstab etc/fstab &>> ${LOG_FILE} -echo "Setting up networking" +echo "Step 16: Setting up networking" echo 'config_eth0=( "dhcp" )' > etc/conf.d/net cp etc/init.d/net.lo etc/init.d/net.eth0 linux32 chroot . rc-update add net.eth0 default &>> ${LOG_FILE} +echo "Step 17: Setting up kernel" # If we got the flag, used a cached kernel to reduce build times for testing -if [[ ${FLAGS_cachedkernel} = ${FLAGS_true} ]]; then - echo "Using cached kernel" +if [[ ${FLAGS_cachedkernel} -eq ${FLAGS_TRUE} ]]; then + echo "Using cached kernel" &>> ${LOG_FILE} cp ${TOOL_RES_PATH}/bzImage boot/kernel cp -R ${TOOL_RES_PATH}/kernelmodules/* lib/modules/ else - echo "Downloading/installing kernel sources" + echo "Downloading/installing kernel sources" &>> ${LOG_FILE} linux32 chroot . emerge gentoo-sources &>> ${LOG_FILE} - echo "Copying kernel configuration" + echo "Copying kernel configuration" &>> ${LOG_FILE} cp ${TOOL_RES_PATH}/kernelconfig usr/src/linux/.config - echo "Building kernel" + echo "Building kernel" &>> ${LOG_FILE} linux32 chroot . make -C /usr/src/linux -j${NUM_JOBS} &>> ${LOG_FILE} - echo "Installing kernel" + echo "Installing kernel" &>> ${LOG_FILE} linux32 chroot . make -C /usr/src/linux modules_install &>> ${LOG_FILE} usr/src/linux/arch/i386/boot/bzImage boot/kernel &>> ${LOG_FILE} fi -echo "Setting root password" +echo "Step 18: Setting root password" linux32 chroot . /bin/bash -c "echo 'root:${ROOT_PASSWORD}' | chpasswd" &>> ${LOG_FILE} -echo "Processing packages list" +echo "Step 19: Processing packages list" for PACKAGE in ${PACKAGES_LIST}; do - echo "Installing ${PACKAGE}" + echo "Installing ${PACKAGE}" &>> ${LOG_FILE} linux32 chroot . emerge --jobs=${NUM_JOBS} ${PACKAGE} &>> ${LOG_FILE} done -echo "Adding default user" +echo "Step 20: Adding default user" linux32 chroot . useradd -g users -G lp,wheel,audio,cdrom,portage -m ${DEFAULT_USERNAME} linux32 chroot . /bin/bash -c "echo '${DEFAULT_USERNAME}:${DEFAULT_PASSWORD}' | chpasswd" &>> ${LOG_FILE} @@ -231,32 +232,31 @@ then linux32 chroot . usermod -a vboxguest ${DEFAULT_USERNAME} fi -echo "Cleaning up make.conf" +echo "Step 21: Cleaning up make.conf" if [ -n ${EMERGE_PROXY} ]; then sed -i '/http_proxy/ d' etc/make.conf fi sed -i '/MAKEOPTS/ d' etc/make.conf -echo "Installing extlinux" -extlinux --heads 255 --sectors 63 --install boot -dd if=/usr/lib/extlinux/mbr.bin of=../${IMAGE_NAME} conv=notrunc +echo "Step 22: Installing extlinux" +extlinux --heads 255 --sectors 63 --install boot &>> ${LOG_FILE} +dd if=/usr/lib/extlinux/mbr.bin of=../${IMAGE_NAME} conv=notrunc &>> ${LOG_FILE} cp ${TOOL_RES_PATH}/extlinux.conf boot/ cd .. cleanup_mounts - case "${OUTPUT_FORMAT}" in "raw" ) - echo "Already in raw format, not converting" + echo "Already in raw format, not converting" &>> ${LOG_FILE} IMAGE_OUT="${BUILD_ID}.image" ;; "vbox" ) - echo "Converting image from RAW to VDI" + echo "Converting image from RAW to VDI" &>> ${LOG_FILE} qemu-img convert -O vdi ${IMAGE_NAME} ${BUILD_ID}.vdi rm -rf ${IMAGE_NAME} IMAGE_OUT="${BUILD_ID}.vdi" ;; "vmware" ) - echo "Converting image from RAW to VMDK" + echo "Converting image from RAW to VMDK" &>> ${LOG_FILE} qemu-img convert -O vmdk ${IMAGE_NAME} ${BUILD_ID}.vmdk rm -rf ${IMAGE_NAME} IMAGE_OUT="${BUILD_ID}.vmdk" @@ -266,6 +266,6 @@ mv ${IMAGE_OUT} ${IMAGES_OUTPUT_PATH}/${IMAGE_OUT} mv ${LOG_FILE} ${IMAGES_OUTPUT_PATH}/${BUILD_ID}.log rm -rf ${IMAGE_WORK_PATH} -echo "Image build completed!" +echo "Step 23: Image build completed!" echo "Your image is here: ${IMAGES_OUTPUT_PATH}/${IMAGE_OUT}" echo "Your log file is here: ${IMAGES_OUTPUT_PATH}/${BUILD_ID}.log" diff --git a/daemon.php b/daemon.php new file mode 100644 index 0000000..deeb0d6 --- /dev/null +++ b/daemon.php @@ -0,0 +1,61 @@ +<?php + + // Gentoaster build daemon worker + // Licensed under GPL v3, see COPYING file + + $configurations_path = "/var/www/gentoaster"; + $gentoaster_path = "/usr/share/gentoaster"; + $tool_name = "create_image.sh"; + + // DO NOT EDIT BELOW THIS LINE + + $progress_magic = 23; + + $worker = new GearmanWorker(); + $worker->addServer(); + $worker->addFunction("invoke_image_build", "image_build"); + while ($worker->work()); + + function image_build($job) { + global $configurations_path, $gentoaster_path, $tool_name, $progress_magic; + + echo "Got job ID ".$job->handle(). "(".md5($job->handle()).")\n"; + + $configuration_string = $job->workload(); + $configuration_array = parse_ini_string($configuration_string); + + if($configuration_array !== FALSE) { + $build_id = $configuration_array["BUILD_ID"]; + $build_path = $configurations_path."/".$build_id; + mkdir($build_path); + + if(is_writable($build_path)) { + chdir($build_path); + file_put_contents("config.ini", $configuration_string); + $tool_args = "--config config.ini --cachedkernel"; + $process_handle = popen($gentoaster_path."/".$tool_name." ".$tool_args, "r"); + + $nonstatus_output = ""; + + while(!feof($process_handle)) { + $progress_line = fgets($process_handle); + preg_match("/Step (.+):/", $progress_line, $matches); + if(sizeof($matches) > 0) { + $job->sendStatus($matches[1], $progress_magic); + } else { + $nonstatus_output .= $progress_line; + } + } + + pclose($process_handle); + + return $nonstatus_output; + } else { + echo "Configured build path is not writable"; + } + } else { + echo "Configuration string is not valid"; + } + } + +?> |