diff options
author | Peter Wilmott <p@p8952.info> | 2015-02-07 22:03:07 +0000 |
---|---|---|
committer | Peter Wilmott <p@p8952.info> | 2015-02-13 00:09:49 +0000 |
commit | c90672132096e8695ce17b034d77e12a945244fb (patch) | |
tree | cb200744a09bdde5b26bdbf6f5048f06211734c3 | |
parent | Add repoman tidy task to clear entries with all targets enabled (diff) | |
download | ruby-tinderbox-c90672132096e8695ce17b034d77e12a945244fb.tar.gz ruby-tinderbox-c90672132096e8695ce17b034d77e12a945244fb.tar.bz2 ruby-tinderbox-c90672132096e8695ce17b034d77e12a945244fb.zip |
Replace Vagrant with Docker
This replaces the underlying build system with one based around docker.
This is for the following reasons:
1. Portability - Builds can be run on cloud services without spinning up
a second VM.
2. Reduced Overhead - There is a non-negligible increase in compile
times when you are running in Docker vs VirtualBox.
3. Coolness - Docker is cool, I wanted an excuse to use it.
-rw-r--r-- | .gitignore | 11 | ||||
-rw-r--r-- | Dockerfile | 10 | ||||
-rw-r--r-- | README.md | 44 | ||||
-rw-r--r-- | Vagrantfile | 21 | ||||
-rw-r--r-- | cache/.keep | 0 | ||||
-rwxr-xr-x | conf/get_stage3.sh | 11 | ||||
-rw-r--r-- | web/Gemfile | 3 | ||||
-rw-r--r-- | web/Gemfile.lock | 15 | ||||
-rw-r--r-- | web/Rakefile | 50 | ||||
-rw-r--r-- | web/app.rb | 3 | ||||
-rw-r--r-- | web/lib/ci.rb | 27 | ||||
-rw-r--r-- | web/lib/models.rb | 2 | ||||
-rw-r--r-- | web/lib/repoman.rb | 43 |
13 files changed, 151 insertions, 89 deletions
@@ -1,14 +1,11 @@ -*.box -*.sqlite3 -*.tar.xz -.vagrant/ +!cache/.keep +!web/ci-logs/.keep +!web/repo-logs/.keep +cache/ gentoo-x86/ -logs/ ruby-overlay/ web/.bundle/ web/ci-logs/ web/repo-logs/ -!web/ci-logs/.keep -!web/repo-logs/.keep web/var/log/*.log web/var/run/*.pid diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..35ed4da --- /dev/null +++ b/Dockerfile @@ -0,0 +1,10 @@ +FROM scratch +MAINTAINER Peter Wilmott <p@p8952.info> +ADD cache/stage3-amd64.tar.bz2 / +RUN mkdir /ruby-tinderbox +ADD conf/provision.sh /ruby-tinderbox/ +ADD conf/make.conf /ruby-tinderbox/ +ADD tinder.sh /ruby-tinderbox/ +ADD repoman.sh /ruby-tinderbox/ +RUN sed -i -e 's/sudo //g' /ruby-tinderbox/provision.sh +RUN /ruby-tinderbox/provision.sh @@ -1,20 +1,46 @@ # Ruby Tinderbox -Scripts for testing Ruby related packages in Gentoo Linux. +Framework for testing Ruby related packages in Gentoo Linux. ## Usage - vagrant up - vagrant ssh - cd /vagrant - ./tinder.sh category/package-version-revision - +Build the docker base image: + + ./conf/get_stage3.sh + docker build -t gentoo/ruby-tinderbox . + +Run the tinderbox scripts against a single package: + + docker run gentoo/ruby-tinderbox /ruby-tinderbox/tinder.sh category/package-version-revision + +Run the tinderbox scripts against multiple packages: + + docker run gentoo/ruby-tinderbox /ruby-tinderbox/tinder.sh \ + category/package-version-revision \ + category/package-version-revision \ + category/package-version-revision + +By default the tinderbox script will build binary packages which are discarded +when the container is removed. If you test multiple packages at once these will +be reused where possible for each of the packages. + +If you want to take advantage of binary packages after a container has been +removed you can persist them by bind mounting a directory on the host to +`/usr/portage/packages`. + + docker run -v /tmp/bincache:/usr/portage/packages gentoo/ruby-tinderbox \ + /ruby-tinderbox/tinder.sh category/package-version-revision + # [Ruby Stats](http://ruby-stats.p8952.info/) -Web interface and build server built on top of the above mentioned scripts. +Web interface and build server built on top of the Ruby Tinderbox framework. + +## Usage # License -Ruby Tinderbox and the bundled Ruby Stats are both [licensed under the AGPL](https://github.com/p8952/ruby-tinderbox/blob/master/LICENSE). +Ruby Tinderbox and Ruby Stats are both [licensed under the AGPL](https://github.com/p8952/ruby-tinderbox/blob/master/LICENSE). -Some of the [javascript used by Ruby Stats](https://github.com/p8952/ruby-tinderbox/tree/master/web/public/js) is licensed under the Mozilla Public License and MIT licenses. This is noted in the headers of the relevent files. +Some of the [javascript used by Ruby Stats](https://github.com/p8952/ruby-tinderbox/tree/master/web/public/js) +is licensed under the Mozilla Public License and MIT licenses. This is noted in +the headers of the relevant files. diff --git a/Vagrantfile b/Vagrantfile deleted file mode 100644 index fbeb970..0000000 --- a/Vagrantfile +++ /dev/null @@ -1,21 +0,0 @@ -Vagrant.configure(2) do |config| - config.vm.box = 'gentoo-amd64' - - config.vm.provider :virtualbox do |vbox| - vbox.cpus = 2 - vbox.memory = 2048 - end - - config.vm.provider :aws do |aws, override| - config.vm.box_url = 'https://github.com/mitchellh/vagrant-aws/raw/master/dummy.box' - config.vm.synced_folder '.', '/vagrant', type: 'rsync', rsync__exclude: ['gentoo-x86/', 'web/'], :rsync_excludes => ['gentoo-x86/', 'web/'] - aws.ami = 'ami-a355d3d4' - aws.instance_type = 't2.micro' - aws.region = 'eu-west-1' - aws.keypair_name = 'AWS-Key' - override.ssh.username = 'ec2-user' - override.ssh.private_key_path = '~/.ssh/AWS-Key.pem' - end - - config.vm.provision "shell", path: "conf/provision.sh" -end diff --git a/cache/.keep b/cache/.keep new file mode 100644 index 0000000..e69de29 --- /dev/null +++ b/cache/.keep diff --git a/conf/get_stage3.sh b/conf/get_stage3.sh new file mode 100755 index 0000000..da04ec2 --- /dev/null +++ b/conf/get_stage3.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +set -o errexit -o nounset -o pipefail +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +DIST_MIRROR="http://mirror.bytemark.co.uk/gentoo/" +LATEST_STAGE3=$(curl -s $DIST_MIRROR/releases/amd64/autobuilds/latest-stage3-amd64.txt | tail -1 | awk '{print $1}') +STAGE3_URI="$DIST_MIRROR/releases/amd64/autobuilds/$LATEST_STAGE3" + +if [[ ! -f "$SCRIPT_DIR/../cache/stage3-amd64.tar.bz2" ]]; then + curl -o "$SCRIPT_DIR/../cache/stage3-amd64.tar.bz2" $STAGE3_URI +fi diff --git a/web/Gemfile b/web/Gemfile index b86a535..61aa55d 100644 --- a/web/Gemfile +++ b/web/Gemfile @@ -1,5 +1,7 @@ source 'https://rubygems.org' +gem 'archive-tar-minitar' +gem 'docker-api' gem 'gems' gem 'net-scp' gem 'net-ssh' @@ -8,7 +10,6 @@ gem 'pmap' gem 'puma' gem 'sequel' gem 'sinatra' -gem 'vagrant_rbapi' group :development do gem 'minitest' diff --git a/web/Gemfile.lock b/web/Gemfile.lock index d009a6d..66308a8 100644 --- a/web/Gemfile.lock +++ b/web/Gemfile.lock @@ -1,14 +1,21 @@ GEM remote: https://rubygems.org/ specs: + archive-tar-minitar (0.5.2) ast (2.0.0) astrolabe (1.3.0) parser (>= 2.2.0.pre.3, < 3.0) celluloid (0.16.0) timers (~> 4.0.0) + docker-api (1.17.0) + archive-tar-minitar + excon (>= 0.38.0) + json + excon (0.44.1) ffi (1.9.6) gems (0.8.3) hitimes (1.2.2) + json (1.8.2) listen (2.8.5) celluloid (>= 0.15.2) rb-fsevent (>= 0.9.3) @@ -30,7 +37,6 @@ GEM rack-test (0.6.3) rack (>= 1.0) rainbow (2.0.0) - rake (10.4.2) rb-fsevent (0.9.4) rb-inotify (0.9.5) ffi (>= 0.5.0) @@ -51,15 +57,13 @@ GEM tilt (1.4.1) timers (4.0.1) hitimes - vagrant_rbapi (0.0.2) - net-scp - net-ssh - rake PLATFORMS ruby DEPENDENCIES + archive-tar-minitar + docker-api gems minitest net-scp @@ -72,4 +76,3 @@ DEPENDENCIES rubocop sequel sinatra - vagrant_rbapi diff --git a/web/Rakefile b/web/Rakefile index 3f3dfab..0140c84 100644 --- a/web/Rakefile +++ b/web/Rakefile @@ -1,10 +1,23 @@ require_relative 'app' task default: 'test' + +desc 'Run the test suite' task :test do Dir.glob('./test/test_*.rb') { |f| require f } end +desc 'Run : update_packages => run_ci_untested => update_ci => run_repoman => update_repoman' +task :nightly do + Rake::Task['db:update_packages'].invoke + Rake::Task['docker:setup'].invoke + Rake::Task['docker:run_ci_untested'].invoke + Rake::Task['db:update_ci'].invoke + Rake::Task['docker:run_repoman'].invoke + Rake::Task['db:update_repoman'].invoke + Rake::Task['docker:teardown'].invoke +end + namespace :db do DB.loggers << Logger.new($stdout) @@ -13,49 +26,64 @@ namespace :db do Sequel::Migrator.run(DB, 'db/migrations') end + desc 'Update the packages database with new versions and targets' task :update_packages do update_packages end + desc 'Update the build database with logfiles from ci-logs/' task :update_ci do update_ci end + desc 'Clear the build database' task :clear_ci do clear_ci end + desc 'Update the repoman database with logfiles from repo-logs/' task :update_repoman do update_repoman end - task :tidy_repoman do - tidy_repoman - end - + desc 'Clear the repoman database' task :clear_repoman do clear_repoman end end -namespace :vm do - provisioner = ENV['provisioner'] - provisioner = 'virtualbox' if provisioner.nil? +namespace :docker do + desc 'Build a docker image to use with subsequent tasks' + task :setup do + Docker.options[:read_timeout] = 36_000 + Docker.options[:write_timeout] = 36_000 + docker_path = File.dirname(File.expand_path(File.dirname(__FILE__))) + @docker_image = Docker::Image.build_from_dir(docker_path) + end + + desc 'Remove a previously built docker image' + task :teardown do + @docker_image.remove + end + desc 'Build and test all packages' task :run_ci_all do - run_ci(:all, provisioner) + run_ci(@docker_image, :all) end + desc 'Build and test a fixed number of packages (num_of_packages=5)' num_of_packages = ENV['num_of_packages'].to_i task :run_ci_some do - run_ci(num_of_packages, provisioner) + run_ci(@docker_image, num_of_packages) end + desc 'Build and test all untested packages and their reverse dependencies' task :run_ci_untested do - run_ci(:untested, provisioner) + run_ci(@docker_image, :untested) end + desc 'Run repoman against the current and next targets for all packages' task :run_repoman do - run_repoman(provisioner) + run_repoman(@docker_image) end end @@ -1,3 +1,5 @@ +require 'archive/tar/minitar' +require 'docker' require 'gems' require 'logger' require 'net/scp' @@ -5,7 +7,6 @@ require 'net/ssh' require 'pmap' require 'sequel' require 'sinatra/base' -require 'vagrant_rbapi' require_relative 'lib/ci' require_relative 'lib/helpers' diff --git a/web/lib/ci.rb b/web/lib/ci.rb index 64d649e..153cc2f 100644 --- a/web/lib/ci.rb +++ b/web/lib/ci.rb @@ -1,4 +1,4 @@ -def run_ci(num_of_packages, provisioner) +def run_ci(docker_image, num_of_packages) packages = [] Package.order { [category, lower(name), version] }.each do |package| packages << package[:identifier] @@ -32,17 +32,22 @@ def run_ci(num_of_packages, provisioner) packages = packages.sample(num_of_packages) end - exit if packages.empty? + packages = packages.uniq + packages.each do |package| + docker_container = docker_image.run("/ruby-tinderbox/tinder.sh #{package}") + docker_container.wait(36_000) - begin - vagrant_path = File.dirname(File.dirname(File.expand_path(File.dirname(__FILE__)))) - vagrant = Vagrant_Rbapi.new(vagrant_path) - vagrant.up(provisioner) - sleep 5 while vagrant.status != 'running' - vagrant.ssh('sudo /vagrant/tinder.sh ' + packages.join(' ')) - vagrant.scp(:download, true, '/vagrant/ci-logs', 'web') - ensure - vagrant.destroy + tar = Tempfile.new('tar') + File.open(tar, 'w') do |file| + docker_container.copy('/ruby-tinderbox/ci-logs') do |chunk| + file.write(chunk) + end + end + Archive::Tar::Minitar.unpack(tar, File.dirname(File.expand_path(File.dirname(__FILE__)))) + tar.close + tar.unlink + + docker_container.delete end end diff --git a/web/lib/models.rb b/web/lib/models.rb index ff0b3ee..667e54f 100644 --- a/web/lib/models.rb +++ b/web/lib/models.rb @@ -1,4 +1,4 @@ -DB = Sequel.connect(ENV['DATABASE_URL'], max_connections: 12 , pool_timeout: 60) +DB = Sequel.connect(ENV['DATABASE_URL'], max_connections: 12, pool_timeout: 60) class Package < Sequel::Model end diff --git a/web/lib/repoman.rb b/web/lib/repoman.rb index 8287837..ae512fd 100644 --- a/web/lib/repoman.rb +++ b/web/lib/repoman.rb @@ -1,4 +1,4 @@ -def run_repoman(provisioner) +def run_repoman(docker_image) packages = [] Package.order { [category, lower(name), version] }.each do |package| target = '' @@ -6,19 +6,13 @@ def run_repoman(provisioner) target = package[:r20_target] unless package[:r20_target] == 'nil' target = package[:r21_target] unless package[:r21_target] == 'nil' target = package[:r22_target] unless package[:r22_target] == 'nil' - if target.empty? - Repoman.where(package_id: package[:identifier]).delete - next - end + next if target.empty? next_target = '' next_target = 'ruby20' if target == 'ruby19' next_target = 'ruby21' if target == 'ruby20' next_target = 'ruby22' if target == 'ruby21' - if next_target.empty? - Repoman.where(package_id: package[:identifier]).delete - next - end + next if next_target.empty? category = package[:category] name = package[:name] @@ -28,18 +22,27 @@ def run_repoman(provisioner) packages << "#{category} #{name} #{version}#{revision} #{target} #{next_target}" end - packages = "'" + packages.join("' '") + "'" + packages = packages.uniq + packages.each do |package| + package = "'" + package + "'" + end + packages = packages.unshift('/ruby-tinderbox/repoman.sh') + + docker_container = docker_image.run(packages) + docker_container.wait(36_000) - begin - vagrant_path = File.dirname(File.dirname(File.expand_path(File.dirname(__FILE__)))) - vagrant = Vagrant_Rbapi.new(vagrant_path) - vagrant.up(provisioner) - sleep 5 while vagrant.status != 'running' - vagrant.ssh('sudo /vagrant/repoman.sh ' + packages) - vagrant.scp(:download, true, '/vagrant/repo-logs', 'web') - ensure - vagrant.destroy + tar = Tempfile.new('tar') + File.open(tar, 'w') do |file| + docker_container.copy('/ruby-tinderbox/repo-logs') do |chunk| + file.write(chunk) + puts chunk + end end + Archive::Tar::Minitar.unpack(tar, File.dirname(File.expand_path(File.dirname(__FILE__)))) + tar.close + tar.unlink + + docker_container.delete end def update_repoman @@ -78,9 +81,7 @@ def update_repoman next_log: next_log ) end -end -def tidy_repoman Package.order { [category, lower(name), version] }.each do |package| target = '' target = package[:r19_target] unless package[:r19_target] == 'nil' |