diff --git a/.gitignore b/.gitignore index 5dd1af0..87cd122 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,5 @@ obj/* bin/* -tools/* data/*.model others/* latex/* @@ -13,3 +12,6 @@ example/*.xml example/model/*.xml *.dat log +config.mk +Makefile +tools/rapidxml* diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..87d5e7a --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "tools/libcumatrix"] + path = tools/libcumatrix + url = https://github.com/MnO2/libcumatrix.git +[submodule "tools/utility"] + path = tools/utility + url = https://github.com/MnO2/utility.git diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..9388556 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,31 @@ +# Use a build matrix to do two builds in parallel: +# one using CMake, and one using make. +language: cpp + +# Cache Ubuntu apt packages. +cache: apt + +compiler: gcc + +before_install: + - export NUM_THREADS=4 + - export SCRIPTS=./scripts/travis + +install: + - sudo -E $SCRIPTS/travis_install.sh + +before_script: + - export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/lib:/usr/local/cuda/lib64 + - export PATH=$PATH:/usr/local/cuda/bin + +script: $SCRIPTS/travis_build_and_test.sh + +notifications: +# Emails are sent to the committer's git-configured email address by default, +# but only if they have access to the repository. To enable Travis on your +# public fork of Caffe, just go to travis-ci.org and flip the switch on for +# your Caffe fork. To configure your git email address, use: +# git config --global user.email me@example.com + email: + on_success: always + on_failure: always diff --git a/Makefile b/Makefile.in similarity index 62% rename from Makefile rename to Makefile.in index 62d5cf8..d2f379d 100755 --- a/Makefile +++ b/Makefile.in @@ -1,10 +1,12 @@ -CC=gcc -CXX=g++ +BASEPROJ?=$(shell pwd) + +include config.mk + +CFG_NVCC+= -arch=sm_21 -w #-Xcompiler "-Wall" CFLAGS= -NVCC=nvcc -arch=sm_21 -w #-Xcompiler "-Wall" -BOTON_UTIL_ROOT=tools/utility/ -CUMATRIX_ROOT=tools/libcumatrix/ +BOTON_UTIL_ROOT=tools/utility +CUMATRIX_ROOT=tools/libcumatrix INCLUDE= -I ./ \ -I include/ \ @@ -35,22 +37,27 @@ EXECUTABLES=nn-train\ EXECUTABLES:=$(addprefix bin/, $(EXECUTABLES)) -.PHONY: debug all o3 dump_nrv ctags clean +.PHONY: debug all o3 dump_nrv ctags clean depndencies test + +all: dependencies $(EXECUTABLES) ctags + +dependencies: + $(MAKE) -C tools/libcumatrix/ BASEPROJ=$(BASEPROJ) + $(MAKE) -C tools/utility/ BASEPROJ=$(BASEPROJ) -all: $(EXECUTABLES) ctags o3: CFLAGS+=-O3 o3: all debug: CFLAGS+=-g -DDEBUG debug: all -dump_nrv: NVCC+=-Xcompiler "-fdump-tree-nrv" all +dump_nrv: CFG_NVCC+=-Xcompiler "-fdump-tree-nrv" all dump_nrv: all vpath %.h include/ vpath %.cpp src/ vpath %.cu src/ -OBJDIR:=obj/ +OBJDIR:=obj OBJ:=$(addprefix $(OBJDIR)/, $(addsuffix .o,$(basename $(SOURCES)))) $(OBJDIR): @@ -58,22 +65,22 @@ $(OBJDIR): LIBRARY=-lpbar -lcumatrix CUDA_LIBRARY=-lcuda -lcudart -lcublas -LIBRARY_PATH=-L$(BOTON_UTIL_ROOT)/lib/ -L$(CUMATRIX_ROOT)/lib -L/usr/local/cuda/lib64 +LIBRARY_PATH=-L$(BOTON_UTIL_ROOT)/lib/ -L$(CUMATRIX_ROOT)/lib -L$(CFG_CUDA_LIBPATH) $(EXECUTABLES): bin/% : $(OBJ) $(OBJDIR)/%.o - $(CXX) -o $@ $(CFLAGS) -std=c++0x $(INCLUDE) $^ $(LIBRARY_PATH) $(LIBRARY) $(CUDA_LIBRARY) + $(CFG_CXX) -o $@ $(CFLAGS) -std=c++0x $(INCLUDE) $^ $(LIBRARY_PATH) $(LIBRARY) $(CUDA_LIBRARY) # +==============================+ # +===== Other Phony Target =====+ # +==============================+ $(OBJDIR)/%.o: %.cpp - $(CXX) $(CPPFLAGS) -std=c++0x -o $@ -c $< + $(CFG_CXX) $(CPPFLAGS) -std=c++0x -o $@ -c $< $(OBJDIR)/%.o: %.cu include/%.h - $(NVCC) $(CFLAGS) $(INCLUDE) -o $@ -c $< + $(CFG_NVCC) $(CFLAGS) $(INCLUDE) -o $@ -c $< $(OBJDIR)/%.d: %.cpp | $(OBJDIR) - @$(CXX) -MM $(CPPFLAGS) $< > $@.$$$$; \ + @$(CFG_CXX) -MM $(CPPFLAGS) $< > $@.$$$$; \ sed 's,\($*\)\.o[ :]*,$(OBJDIR)/\1.o $@ : ,g' < $@.$$$$ > $@;\ rm -f $@.$$$$ @@ -81,5 +88,13 @@ $(OBJDIR)/%.d: %.cpp | $(OBJDIR) ctags: @if command -v ctags >/dev/null 2>&1; then ctags -R --langmap=C:+.cu *; fi + clean: + $(MAKE) clean -C tools/libcumatrix/ BASEPROJ=$(BASEPROJ) + $(MAKE) clean -C tools/utility/ BASEPROJ=$(BASEPROJ) rm -rf $(EXECUTABLES) $(OBJDIR)/* + + +# Dummy make test for travis-ci +test: + $(shell exit 0) diff --git a/README.md b/README.md index 577b229..8a65f8e 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ libdnn 中文說明 ====== (See English version below) +[![Build Status](https://travis-ci.org/MnO2/libdnn.svg?branch=master)](https://travis-ci.org/MnO2/libdnn) + [libdnn](https://github.com/botonchou/libdnn) 是一個輕量、好讀、人性化的**深層學習**函式庫。由 C++ 和 CUDA 撰寫而成，目的是讓開發人員、研究人員、或任何有興趣的人都可以輕鬆體驗並駕馭深層學習所帶來的威力。 詳細的教學和使用說明，請參考[Wiki](https://github.com/botonchou/libdnn/wiki) diff --git a/configure b/configure new file mode 100755 index 0000000..754485d --- /dev/null +++ b/configure @@ -0,0 +1,385 @@ +#!/bin/bash + +test_cmd() { + "$@" + local status=$? + + if [ $status -ne 0 ] + then err "are you using the gnu version?" + else msg "test $@ OK" + fi +} + + +msg() { + echo "$CFG_SELF: $1" +} + +step_msg() { + msg + msg "$1" + msg +} + +warn() { + echo "$CFG_SELF: WARNING: $1" +} + +err() { + echo "$CFG_SELF: error: $1" + exit 1 +} + +need_cmd() { + if command -v $1 >/dev/null 2>&1 + then msg "found $1" + else err "need $1" + fi +} + + +probe() { + local V=$1 + shift + local P + local T + + for P + do + T=$(command -v $P 2>&1) + if [ $? -eq 0 ] + then + VER0=$($P --version 2>/dev/null | head -1 \ + | sed -e 's/[^0-9]*\([vV]\?[0-9.]\+[^ ]*\).*/\1/' ) + if [ $? -eq 0 -a "x${VER0}" != "x" ] + then + VER="($VER0)" + else + VER="" + fi + break + else + VER="" + T="" + fi + done + + eval $V=\$T + putvar $V "$VER" +} + +probe_need() { + local V=$1 + probe $* + eval VV=\$$V + + if [ -z "$VV" ] + then + err "needed, but unable to find any of: $*" + fi +} + +putvar() { + local T + eval T=\$$1 + eval TLEN=\${#$1} + if [ $TLEN -gt 35 ] + then + printf "$CFG_SELF: %-20s := %.35s ...\n" $1 "$T" + else + printf "$CFG_SELF: %-20s := %s %s\n" $1 "$T" "$2" + fi + printf "%-20s := %s\n" $1 "$T" >>config.tmp +} + +move_if_changed() { + if cmp -s $1 $2 + then + msg "leaving $2 unchanged" + else + msg "mv $1 $2" + mv -f $1 $2 + chmod u-w $2 # make moved artifact read-only + fi +} + +copy_if_changed() { + if cmp -s $1 $2 + then + msg "leaving $2 unchanged" + else + msg "cp $1 $2" + cp -f $1 $2 + chmod u-w $2 # make copied artifact read-only + fi +} + + +validate_opt () { + for arg in $CFG_CONFIGURE_ARGS + do + isArgValid=0 + for option in $BOOL_OPTIONS + do + if test --disable-$option = $arg + then + isArgValid=1 + fi + if test --enable-$option = $arg + then + isArgValid=1 + fi + done + for option in $VAL_OPTIONS + do + if echo "$arg" | grep -q -- "--$option=" + then + isArgValid=1 + fi + done + if [ "$arg" = "--help" ] + then + echo + echo "No more help available for Configure options," + echo "check the Wiki or join our IRC channel" + break + else + if test $isArgValid -eq 0 + then + err "Option '$arg' is not recognized" + fi + fi + done +} + + +opt_core() { + BOOL_OPTIONS="$BOOL_OPTIONS $2" + + local SAVE=$1 + local OP=$2 + local DEFAULT=$3 + shift + shift + shift + local DOC="$*" + local FLAG="" + + if [ $DEFAULT -eq 0 ] + then + FLAG="enable" + else + FLAG="disable" + DOC="don't $DOC" + fi + + if [ $HELP -eq 0 ] + then + for arg in $CFG_CONFIGURE_ARGS + do + if [ "$arg" = "--${FLAG}-${OP}" ] + then + OP=$(echo $OP | tr 'a-z-' 'A-Z_') + FLAG=$(echo $FLAG | tr 'a-z' 'A-Z') + local V="CFG_${FLAG}_${OP}" + eval $V=1 + if [ "$SAVE" = "save" ] + then + putvar $V + fi + fi + done + else + if [ ! -z "$META" ] + then + OP="$OP=<$META>" + fi + printf " --%-30s %s\n" "$FLAG-$OP" "$DOC" + fi +} + +opt_nosave() { + opt_core nosave "$@" +} + +opt() { + opt_core save "$@" +} + +CFG_OSTYPE=$(uname -s) +CFG_CPUTYPE=$(uname -m) + + +case $CFG_OSTYPE in + Linux) + CFG_OSTYPE=unknown-linux-gnu + ;; + Darwin) + CFG_OSTYPE=apple-darwin + ;; + MSYS*) + CFG_OSTYPE=pc-windows-gnu + ;; + *) + err "unknown OS type: $CFG_OSTYPE" + ;; +esac + + +if [ $CFG_OSTYPE = apple-darwin -a $CFG_CPUTYPE = i386 ] +then + if sysctl hw.optional.x86_64 | grep -q ': 1' + then + CFG_CPUTYPE=x86_64 + fi +fi + + + +DEFAULT_BUILD="${CFG_CPUTYPE}-${CFG_OSTYPE}" + +CFG_SRC_DIR="$(cd $(dirname $0) && pwd)/" +CFG_BUILD_DIR="$(pwd)/" +CFG_SELF="$0" +CFG_CONFIGURE_ARGS="$@" + +OPTIONS="" +HELP=0 +if [ "$1" = "--help" ] +then + HELP=1 + shift + echo + echo "Usage: $CFG_SELF [options]" + echo + echo "Options:" + echo +else + msg "recreating config.tmp" + echo '' >config.tmp + + step_msg "processing $CFG_SELF args" +fi + +BOOL_OPTIONS="" +VAL_OPTIONS="" + +opt ccache 0 "invoke gcc/clang via ccache to reuse object files between builds" + + +if [ $HELP -eq 1 ] +then + echo + exit 0 +fi + +# Validate Options +step_msg "validating $CFG_SELF args" +validate_opt + + + +msg "looking for required commands" +need_cmd mkdir +need_cmd printf + +step_msg "looking for needed program" + +probe_need CFG_NVCC nvcc +probe_need CFG_GIT git +probe_need CFG_WGET wget +probe_need CFG_UNZIP unzip +probe_need CFG_MAKE make +probe_need CFG_CTAGS ctags + +probe CFG_CCACHE ccache + + +if [ $CFG_OSTYPE = apple-darwin ] +then + CFG_OSX_GCC_VERSION=$(g++ --version 2>&1 | grep "Apple LLVM version") + + if [ $? -eq 0 ] + then + step_msg "on OS X 10.9, g++ is an alias of clang. Which is not good since CUDA is only compatible with g++, trying to probe other path instead" + + #probe for homebrew installed g++-4.9 + probe_need CFG_GCC gcc-4.9 + probe_need CFG_GCXX g++-4.9 + + else + if [ $(g++ --version 2>&1 | grep -c ' 4\.[0-6]') -ne 0 ]; then + err "older g++ found, please install a newer version of gcc" + fi + + probe_need CFG_GCC gcc + probe_need CFG_GCXX g++ + fi + + CFG_CUDA_LIBPATH=/usr/local/cuda/lib + putvar CFG_CUDA_LIBPATH + +elif [ $CFG_OSTYPE = unknown-linux-gnu ] +then + probe_need CFG_GCC gcc + probe_need CFG_GCXX g++ + + CFG_CUDA_LIBPATH=/usr/local/cuda/lib64 + putvar CFG_CUDA_LIBPATH +fi + + +msg "testing ctags is gnu" +test_cmd ctags -R + + +if [ ! -z "$CFG_ENABLE_CCACHE" ] +then + if [ -z "$CFG_CC" ] + then + if [ -z "$CFG_CCACHE" ] + then + err "ccache requested but not found" + fi + + CFG_CC="ccache $CFG_GCC" + CFG_CXX="ccache $CFG_GCXX" + else + CFG_CC=$CFG_GCC + CFG_CXX=$CFG_GCXX + fi +else + CFG_CC=$CFG_GCC + CFG_CXX=$CFG_GCXX +fi + +putvar CFG_CC +putvar CFG_CXX + +mkdir -p tools/ bin/ obj/ + +$CFG_WGET http://nchc.dl.sourceforge.net/project/rapidxml/rapidxml/rapidxml%201.13/rapidxml-1.13.zip -P tools/ +$CFG_UNZIP -o tools/rapidxml-1.13.zip -d tools + +$CFG_GIT submodule update --init --recursive + +#if [ "$1" == "--zip-only" ]; then + #$CFG_WGET https://codeload.github.com/botonchou/utility/zip/master -O tools/utility.zip + #$CFG_UNZIP tools/utility.zip -d tools/ && mv tools/utility-master tools/utility + + #$CFG_WGET https://codeload.github.com/botonchou/libcumatrix/zip/master -O tools/libcumatrix.zip + #$CFG_UNZIP tools/libcumatrix.zip -d tools/ && mv tools/libcumatrix-master tools/libcumatrix +#else + #printf "\33[34m[Info]\33[0m You can specify \33[33m--zip-only\33[0m if git clone is too slow\n" + #$CFG_GIT clone https://github.com/botonchou/utility.git tools/utility + #$CFG_GIT clone https://github.com/botonchou/libcumatrix.git tools/libcumatrix +#fi + + +copy_if_changed Makefile.in ./Makefile +move_if_changed config.tmp config.mk +rm -f config.tmp + + +msg "libdnn configuration complete" +step_msg "please type 'make' to build the library" diff --git a/install-sh b/install-sh deleted file mode 100755 index 3f7cac0..0000000 --- a/install-sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -mkdir -p tools bin/ obj/ - - -wget http://nchc.dl.sourceforge.net/project/rapidxml/rapidxml/rapidxml%201.13/rapidxml-1.13.zip -P tools/ -cd tools/ && unzip rapidxml-1.13.zip && cd - - -if [ "$1" == "--zip-only" ]; then - wget https://codeload.github.com/botonchou/utility/zip/master -O tools/utility.zip - unzip tools/utility.zip -d tools/ && mv tools/utility-master tools/utility - - wget https://codeload.github.com/botonchou/libcumatrix/zip/master -O tools/libcumatrix.zip - unzip tools/libcumatrix.zip -d tools/ && mv tools/libcumatrix-master tools/libcumatrix -else - printf "\33[34m[Info]\33[0m You can specify \33[33m--zip-only\33[0m if git clone is too slow\n" - git clone https://github.com/botonchou/utility.git tools/utility - git clone https://github.com/botonchou/libcumatrix.git tools/libcumatrix -fi - -# Make dependencies first -cd tools/utility && make o3 && cd - -cd tools/libcumatrix && make o3 && cd - - -# Make !! -make o3 diff --git a/scripts/travis/travis_build_and_test.sh b/scripts/travis/travis_build_and_test.sh new file mode 100755 index 0000000..4b9fca2 --- /dev/null +++ b/scripts/travis/travis_build_and_test.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Script called by Travis to do a CPU-only build of and test Caffe. + +set -e +MAKE="make --jobs=$NUM_THREADS --keep-going" + +./configure +$MAKE all +$MAKE test diff --git a/scripts/travis/travis_install.sh b/scripts/travis/travis_install.sh new file mode 100755 index 0000000..58a3f78 --- /dev/null +++ b/scripts/travis/travis_install.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -e + +MAKE="make --jobs=$NUM_THREADS" + +apt-get -y update +apt-get install \ + wget git curl unzip ctags + + +CUDA_URL=http://developer.download.nvidia.com/compute/cuda/repos/ubuntu1204/x86_64/cuda-repo-ubuntu1204_6.5-14_amd64.deb +CUDA_FILE=/tmp/cuda_install.deb +curl $CUDA_URL -o $CUDA_FILE +dpkg -i $CUDA_FILE +rm -f $CUDA_FILE + +apt-get -y update +apt-get -y install cuda-core-6-5 cuda-cublas-6-5 cuda-cublas-dev-6-5 cuda-cudart-6-5 cuda-cudart-dev-6-5 cuda-curand-6-5 cuda-curand-dev-6-5 cuda + +ln -s /usr/local/cuda-6.5 /usr/local/cuda diff --git a/tools/libcumatrix b/tools/libcumatrix new file mode 160000 index 0000000..3214a58 --- /dev/null +++ b/tools/libcumatrix @@ -0,0 +1 @@ +Subproject commit 3214a58fe87f75c0b5ece1acca2b149a4d60a4ee diff --git a/tools/utility b/tools/utility new file mode 160000 index 0000000..18f9496 --- /dev/null +++ b/tools/utility @@ -0,0 +1 @@ +Subproject commit 18f9496d502d8034fb3ce9f975b413da8e931a33