diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..f08214bb --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,127 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: build + +on: + push: + branches: + - 'release/**' + release: + types: + - created + +jobs: + wheel: + + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + + matrix: + include: + # MacOS + - python: '3.6' + os: macos-latest + - python: '3.7' + os: macos-latest + - python: '3.8' + os: macos-latest + - python: '3.9' + os: macos-latest + # Windows + # - python: '3.6' + # os: windows-latest + #- python: '3.7' + # os: windows-latest + #- python: '3.8' + # os: windows-latest + #- python: '3.9' + # os: windows-latest + + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Cache .a files + uses: actions/cache@v2 + with: + key: ${{ runner.os }} + path: | + src/rocksdb/libsnappy.a + src/rocksdb/liblz4.a + src/rocksdb/libbz2.a + src/rocksdb/libzstd.a + src/rocksdb/libz.a + src/rocksdb/librocksdb.a + + - name: Setup python${{ matrix.python }} + uses: actions/setup-python@v2 + with: + python-version: "${{ matrix.python }}" + + - name: Make static library files + run: make + + - name: Install requires + run: python -m pip install cython twine wheel + + - name: Build wheel for python "${{ matrix.python }}" + run: python setup.py bdist_wheel + + - name: Publishing to pypi + run: twine upload --skip-existing --disable-progress-bar dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} + + + linux-wheels: + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v2 + with: + submodules: recursive + + - name: Cache .a files + uses: actions/cache@v2 + with: + key: ${{ runner.os }} + path: | + src/rocksdb/libsnappy.a + src/rocksdb/liblz4.a + src/rocksdb/libbz2.a + src/rocksdb/libzstd.a + src/rocksdb/libz.a + src/rocksdb/librocksdb.a + + - name: Make static library files + uses: docker://quay.io/pypa/manylinux2014_x86_64 + with: + args: /usr/bin/make + + - name: Install requires + run: python -m pip install cython twine wheel + + - name: Building manylinux2014 wheels + uses: docker://quay.io/pypa/manylinux2014_x86_64 + with: + args: /bin/bash scripts/make-wheels.sh + + - name: Setup python${{ matrix.python }} + uses: actions/setup-python@v2 + with: + python-version: "3.9" + + - name: Install requires + run: python -m pip install twine + + - name: Publishing to pypi + run: twine upload --skip-existing --disable-progress-bar dist/*.whl + env: + TWINE_USERNAME: __token__ + TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..d3e6e2ca --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "src/rocksdb"] + path = src/rocksdb + url = https://github.com/facebook/rocksdb.git diff --git a/MANIFEST.in b/MANIFEST.in index fc0223ff..e324b5de 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ +include README.rst include rocksdb/cpp/*.hpp recursive-include rocksdb *.pxd recursive-include rocksdb *.pyx diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..e8988541 --- /dev/null +++ b/Makefile @@ -0,0 +1,44 @@ +all: src/rocksdb/librocksdb.a + +JOBS=8 + +src/rocksdb/librocksdb.a: + make \ + -e EXTRA_CXXFLAGS="-fPIC" \ + -e EXTRA_CFLAGS="-fPIC" \ + -C src/rocksdb \ + -j $(JOBS) \ + libsnappy.a \ + liblz4.a \ + libbz2.a \ + libzstd.a \ + libz.a + + (cd src/rocksdb && mkdir -p build && cd build && cmake \ + -DWITH_SNAPPY=1 \ + -DWITH_LZ4=1 \ + -DWITH_ZLIB=1 \ + -DWITH_ZSTD=1 \ + -DWITH_GFLAGS=0 \ + -DROCKSDB_BUILD_SHARED=0 \ + -DWITH_TOOLS=0 \ + -DWITH_BENCHMARK_TOOLS=0 \ + -DWITH_CORE_TOOLS=0 \ + -DWITH_JEMALLOC=0 \ + -DCMAKE_BUILD_TYPE=Release \ + -DSnappy_INCLUDE_DIRS=../snappy-1.1.8/ \ + -DSnappy_LIBRARIES=../snappy-1.1.8/build \ + -Dlz4_INCLUDE_DIRS=../lz4-1.9.3/lib \ + -Dlz4_LIBRARIES=../lz4-1.9.3/lib \ + -Dzstd_INCLUDE_DIRS=../zstd-1.4.9/lib \ + -Dzstd_LIBRARIES=../zstd-1.4.9/lib \ + -DZLIB_INCLUDE_DIR=../zlib-1.2.11 \ + -DZLIB_LIBRARY=./zlib-1.2.11 \ + -DCMAKE_CXX_FLAGS="-fPIC -I../snappy-1.1.8/build -I../zstd-1.4.9/lib/dictBuilder" \ + .. && make -j $(JOBS)) + + cp src/rocksdb/build/librocksdb.a src/rocksdb/librocksdb.a + +clean: + rm -rf src/rocksdb/build + make -C src/rocksdb clean diff --git a/README.rst b/README.rst index d46de8a2..f7b3a630 100644 --- a/README.rst +++ b/README.rst @@ -4,15 +4,15 @@ :target: https://anaconda.org/conda-forge/python-rocksdb Note -========= +==== The original pyrocksdb (https://pypi.python.org/pypi/pyrocksdb/0.4) has not been updated for long time. I update pyrocksdb to support the latest rocksdb. Please open issues in github if you have any problem. News (2019/04/18) -========= -Currently I am refactoring the code, and more features like TTL are coming soon. And the installation with cmake will be much more easily. +================= +Currently I am refactoring the code, and more features like TTL are coming soon. And the installation with cmake will be much more easily. News (2019/04/19) -========= +================= I have created a new branch(https://github.com/twmht/python-rocksdb/tree/pybind11) which provides the basic functions (`put`, `get` and `delete`) now. And the installtion is much more easily! you can try it if you encounter any installtion issues in the current version of `python-rocksdb`. The branch is under development and will be released to PypI after I migrate most of the existing features. diff --git a/setup.py b/setup.py index 6b4dfc45..fd0ea73a 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,23 @@ import platform +import os from setuptools import setup from setuptools import find_packages from setuptools import Extension +try: + from Cython.Build import cythonize +except ImportError: + def cythonize(extensions): + return extensions -extra_compile_args = [ + SOURCES = ['rocksdb/_rocksdb.cpp'] +else: + SOURCES = ['rocksdb/_rocksdb.pyx'] + +EXTRA_COMPILE_ARGS = [ '-std=c++11', - '-O3', + '-fPIC', + '-Os', '-Wall', '-Wextra', '-Wconversion', @@ -14,15 +25,49 @@ '-fno-rtti', ] +LIBRARIES = ['rocksdb', 'snappy', 'bz2', 'z', 'lz4'] +EXTRA_OBJECTS = [] +EXTRA_LINK_ARGS = [] +INCLUDE_DIRS = [] + + if platform.system() == 'Darwin': - extra_compile_args += ['-mmacosx-version-min=10.7', '-stdlib=libc++'] + EXTRA_COMPILE_ARGS += ['-mmacosx-version-min=10.7', '-stdlib=libc++'] + EXTRA_LINK_ARGS += ['-Wl,-s'] + +if platform.system() == 'Linux': + EXTRA_LINK_ARGS += ['-Wl,--strip-all'] + + +STATIC_LIBRARIES = [os.path.join("src", "rocksdb", item) for item in [ + "librocksdb.a", + "libbz2.a", + "liblz4.a", + "libsnappy.a", + "libz.a", + "libzstd.a", +]] + +if all(map(os.path.exists, STATIC_LIBRARIES)): + LIBRARIES = [] + EXTRA_OBJECTS = STATIC_LIBRARIES + INCLUDE_DIRS = [ + os.path.join("src", "rocksdb", "bzip2-1.0.8"), + os.path.join("src", "rocksdb", "zstd-1.4.9", "lib"), + os.path.join("src", "rocksdb", "zlib-1.2.11"), + os.path.join("src", "rocksdb", "snappy-1.1.8"), + os.path.join("src", "rocksdb", "snappy-1.1.8", "build"), + os.path.join("src", "rocksdb", "lz4-1.9.3", "lib"), + os.path.join("src", "rocksdb", "include"), + ] setup( name="python-rocksdb", version='0.7.0', + keywords=['rocksdb', 'static', 'build'], description="Python bindings for RocksDB", - keywords='rocksdb', + long_description=open("README.rst").read(), author='Ming Hsuan Tu', author_email="qrnnis2623891@gmail.com", url="https://github.com/twmht/python-rocksdb", @@ -31,17 +76,20 @@ install_requires=['setuptools>=25'], package_dir={'rocksdb': 'rocksdb'}, packages=find_packages('.'), - ext_modules=[Extension( + ext_modules=cythonize([Extension( 'rocksdb._rocksdb', - ['rocksdb/_rocksdb.pyx'], - extra_compile_args=extra_compile_args, + SOURCES, + extra_compile_args=EXTRA_COMPILE_ARGS, language='c++', - libraries=['rocksdb', 'snappy', 'bz2', 'z', 'lz4'], - )], + libraries=LIBRARIES, + include_dirs=INCLUDE_DIRS, + extra_objects=EXTRA_OBJECTS, + extra_link_args=EXTRA_LINK_ARGS, + )]), extras_require={ "doc": ['sphinx_rtd_theme', 'sphinx'], "test": ['pytest'], }, - include_package_data=True, + include_package_data=False, zip_safe=False, )