adds client api integration tests for runc using bash w/bats

Makefile

@@ -40,6 +40,12 @@ dbuild: runctestimage
 	docker cp $(RUNC_INSTANCE):$(RUNC_BUILD_PATH) .
 	docker rm $(RUNC_INSTANCE)
 
+integration: runctestimage
+	docker run -e TESTFLAGS -t --privileged --rm -v $(CURDIR):/go/src/$(PROJECT) $(RUNC_TEST_IMAGE) make localintegration
+
+localintegration:
+	bats tests/integration${TESTFLAGS}
+
 install:
 	install -D -m0755 runc /usr/local/sbin/runc
 

script/test_Dockerfile

@@ -18,16 +18,25 @@ RUN apt-get update && apt-get install -y \
     python-minimal \
     --no-install-recommends
 
+# install bats 
+RUN cd /tmp \
+    && git clone https://github.com/sstephenson/bats.git \
+    && cd bats \
+    && git reset --hard 03608115df2071fff4eaaff1605768c275e5f81f \
+    && ./install.sh /usr/local
+
 # install criu
 ENV CRIU_VERSION 1.7
 RUN mkdir -p /usr/src/criu \
-        && curl -sSL https://github.com/xemul/criu/archive/v${CRIU_VERSION}.tar.gz | tar -v -C /usr/src/criu/ -xz --strip-components=1 \
-        && cd /usr/src/criu \
-        && make install-criu
+    && curl -sSL https://github.com/xemul/criu/archive/v${CRIU_VERSION}.tar.gz | tar -v -C /usr/src/criu/ -xz --strip-components=1 \
+    && cd /usr/src/criu \
+    && make install-criu
 
 # setup a playground for us to spawn containers in
-RUN mkdir /busybox && \
-    curl -sSL 'https://github.com/jpetazzo/docker-busybox/raw/buildroot-2014.11/rootfs.tar' | tar -xC /busybox
+RUN mkdir /busybox \
+    && mkdir /testdata \
+    && curl -o /testdata/busybox.tar -sSL 'https://github.com/jpetazzo/docker-busybox/raw/buildroot-2014.11/rootfs.tar' \
+    && tar -C /busybox -xf /testdata/busybox.tar 
 
 COPY script/tmpmount /
 WORKDIR /go/src/github.com/opencontainers/runc

tests/integration/README.md

@@ -0,0 +1,83 @@
+# runc Integration Tests
+
+Integration tests provide end-to-end testing of runc.
+
+Note that integration tests do **not** replace unit tests.
+
+As a rule of thumb, code should be tested thoroughly with unit tests.
+Integration tests on the other hand are meant to test a specific feature end
+to end.
+
+Integration tests are written in *bash* using the
+[bats](https://github.com/sstephenson/bats) framework.
+
+## Running integration tests
+
+The easiest way to run integration tests is with Docker:
+```
+$ make integration
+```
+Alternatively, you can run integration tests directly on your host through make:
+```
+$ sudo make localintegration
+```
+Or you can just run them directly using bats 
+```
+$ sudo bats tests/integration
+```
+To run a single test bucket:
+```
+$ make integration TESTFLAGS="/checkpoint.bats"
+```
+
+
+To run them on your host, you will need to setup a development environment plus
+[bats](https://github.com/sstephenson/bats#installing-bats-from-source) 
+For example:
+```
+$ cd ~/go/src/github.com 
+$ git clone https://github.com/sstephenson/bats.git 
+$ cd bats 
+$ ./install.sh /usr/local
+```
+
+> **Note**: There are known issues running the integration tests using 
+> **devicemapper** as a storage driver, make sure that your docker daemon 
+> is using **aufs** if you want to successfully run the integration tests.
+
+## Writing integration tests
+
+[helper functions]
+(https://github.com/opencontainers/runc/blob/master/test/integration/helpers.bash)
+are provided in order to facilitate writing tests.
+
+```sh
+#!/usr/bin/env bats
+
+# This will load the helpers.
+load helpers
+
+# setup is called at the beginning of every test.
+function setup() {
+  # see functions teardown_hello and setup_hello in helpers.bash, used to
+  # create a pristine environment for running your tests
+  teardown_hello
+  setup_hello
+}
+
+# teardown is called at the end of every test.
+function teardown() {
+  teardown_hello
+}
+
+@test "this is a simple test" {
+  run "$RUNC" start containerid
+  # "run" automatically populates $status, $output and $lines.
+  # Please refer to bats documentation to find out more.
+  [ "$status" -eq 0 ]
+
+  # check expected output
+  [[ "${output}" == *"Hello"* ]]
+}
+
+```

tests/integration/checkpoint.bats

@@ -0,0 +1,61 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+  teardown_busybox
+  setup_busybox
+}
+
+function teardown() {
+  teardown_busybox
+}
+
+@test "checkpoint and restore" {
+  if [ ! -e "$CRIU" ] ; then
+    skip
+  fi
+
+  # criu does not work with external terminals so..
+  # setting terminal and root:readonly: to false
+  sed -i 's;"terminal": true;"terminal": false;' config.json  
+  sed -i 's;"readonly": true;"readonly": false;' config.json 
+  sed -i 's/"sh"/"sh","-c","while :; do date; sleep 1; done"/' config.json
+
+  (  
+    # start busybox (not detached) 
+    run "$RUNC" start test_busybox
+    [ "$status" -eq 0 ]
+  ) &
+
+  # check state
+  wait_for_container 15 1 test_busybox
+
+  run "$RUNC" state test_busybox
+  [ "$status" -eq 0 ]
+  [[ "${output}" == *"running"* ]]
+
+  # checkpoint the running container 
+  run "$RUNC" --criu "$CRIU" checkpoint test_busybox
+  # if you are having problems getting criu to work uncomment the following dump:
+  #cat /run/opencontainer/containers/test_busybox/criu.work/dump.log
+  [ "$status" -eq 0 ]
+
+  # after checkpoint busybox is no longer running
+  run "$RUNC" state test_busybox
+  [ "$status" -ne 0 ]
+
+  # restore from checkpoint
+  (
+    run "$RUNC" --criu "$CRIU" restore test_busybox
+    [ "$status" -eq 0 ]
+  ) &
+
+  # check state
+  wait_for_container 15 1 test_busybox
+
+  # busybox should be back up and running
+  run "$RUNC" state test_busybox
+  [ "$status" -eq 0 ]
+  [[ "${output}" == *"running"* ]]
+}

tests/integration/debug.bats

@@ -0,0 +1,70 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+  teardown_hello
+  setup_hello
+}
+
+function teardown() {
+  teardown_hello
+}
+
+@test "global --debug" {
+  # start hello-world
+  run "$RUNC" --debug start test_hello
+  echo "${output}"
+  [ "$status" -eq 0 ]
+}
+
+@test "global --debug to --log" {
+  # start hello-world
+  run "$RUNC" --log log.out --debug start test_hello
+  [ "$status" -eq 0 ]
+
+  # check output does not include debug info
+  [[ "${output}" != *"level=debug"* ]]
+
+  # check log.out was generated 
+  [ -e log.out ]
+
+  # check expected debug output was sent to log.out
+  run cat log.out
+  [ "$status" -eq 0 ]
+  [[ "${output}" == *"level=debug"* ]]
+}
+
+@test "global --debug to --log --log-format 'text'" {
+  # start hello-world
+  run "$RUNC" --log log.out --log-format "text" --debug start test_hello
+  [ "$status" -eq 0 ]
+
+  # check output does not include debug info
+  [[ "${output}" != *"level=debug"* ]]
+
+  # check log.out was generated 
+  [ -e log.out ]
+
+  # check expected debug output was sent to log.out
+  run cat log.out
+  [ "$status" -eq 0 ]
+  [[ "${output}" == *"level=debug"* ]]
+}
+
+@test "global --debug to --log --log-format 'json'" {
+  # start hello-world
+  run "$RUNC" --log log.out --log-format "json" --debug start test_hello
+  [ "$status" -eq 0 ]
+
+  # check output does not include debug info
+  [[ "${output}" != *"level=debug"* ]]
+
+  # check log.out was generated 
+  [ -e log.out ]
+
+  # check expected debug output was sent to log.out
+  run cat log.out
+  [ "$status" -eq 0 ]
+  [[ "${output}" == *'"level":"debug"'* ]]
+}

tests/integration/delete.bats

@@ -0,0 +1,33 @@
+#!/usr/bin/env bats
+
+load helpers
+
+function setup() {
+  teardown_busybox
+  setup_busybox
+}
+
+function teardown() {
+  teardown_busybox
+}
+
+@test "runc delete" {
+  # start busybox detached
+  run "$RUNC" start -d --console /dev/pts/ptmx test_busybox
+  [ "$status" -eq 0 ]
+
+  # check state
+  wait_for_container 15 1 test_busybox
+
+  testcontainer test_busybox running
+
+  run "$RUNC" kill test_busybox KILL
+  # wait for busybox to be in the destroyed state
+  retry 10 1 eval "'$RUNC' state test_busybox | grep -q 'destroyed'" 
+
+  # delete test_busybox
+  run "$RUNC" delete test_busybox
+
+  run "$RUNC" state test_busybox
+  [ "$status" -ne 0 ]
+}