Add static binaries for package

.github/workflows/build.yml

@@ -0,0 +1,127 @@
+# This workflow will install Python dependencies, run tests and lint with a single version of Python
+# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
+
+name: build
+
+on:
+  push:
+    branches:
+      - 'release/**'
+  release:
+    types:
+      - created
+
+jobs:
+  wheel:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+
+      matrix:
+        include:
+          # MacOS
+          - python: '3.6'
+            os: macos-latest
+          - python: '3.7'
+            os: macos-latest
+          - python: '3.8'
+            os: macos-latest
+          - python: '3.9'
+            os: macos-latest
+          # Windows
+          # - python: '3.6'
+          #  os: windows-latest
+          #- python: '3.7'
+          #  os: windows-latest
+          #- python: '3.8'
+          #  os: windows-latest
+          #- python: '3.9'
+          #  os: windows-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - name: Cache .a files
+        uses: actions/cache@v2
+        with:
+          key: ${{ runner.os }}
+          path: |
+            src/rocksdb/libsnappy.a
+            src/rocksdb/liblz4.a
+            src/rocksdb/libbz2.a
+            src/rocksdb/libzstd.a
+            src/rocksdb/libz.a
+            src/rocksdb/librocksdb.a
+
+      - name: Setup python${{ matrix.python }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: "${{ matrix.python }}"
+
+      - name: Make static library files
+        run: make
+
+      - name: Install requires
+        run: python -m pip install cython twine wheel
+
+      - name: Build wheel for python "${{ matrix.python }}"
+        run: python setup.py bdist_wheel
+
+      - name: Publishing to pypi
+        run: twine upload --skip-existing --disable-progress-bar dist/*.whl
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
+
+
+  linux-wheels:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          submodules: recursive
+
+      - name: Cache .a files
+        uses: actions/cache@v2
+        with:
+          key: ${{ runner.os }}
+          path: |
+            src/rocksdb/libsnappy.a
+            src/rocksdb/liblz4.a
+            src/rocksdb/libbz2.a
+            src/rocksdb/libzstd.a
+            src/rocksdb/libz.a
+            src/rocksdb/librocksdb.a
+
+      - name: Make static library files
+        uses: docker://quay.io/pypa/manylinux2014_x86_64
+        with:
+          args: /usr/bin/make
+
+      - name: Install requires
+        run: python -m pip install cython twine wheel
+
+      - name: Building manylinux2014 wheels
+        uses: docker://quay.io/pypa/manylinux2014_x86_64
+        with:
+          args: /bin/bash scripts/make-wheels.sh
+
+      - name: Setup python${{ matrix.python }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.9"
+
+      - name: Install requires
+        run: python -m pip install twine
+
+      - name: Publishing to pypi
+        run: twine upload --skip-existing --disable-progress-bar dist/*.whl
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "src/rocksdb"]
+	path = src/rocksdb
+	url = https://github.com/facebook/rocksdb.git

MANIFEST.in

@@ -1,3 +1,4 @@
+include README.rst
 include rocksdb/cpp/*.hpp
 recursive-include rocksdb *.pxd
 recursive-include rocksdb *.pyx

Makefile

@@ -0,0 +1,44 @@
+all: src/rocksdb/librocksdb.a
+
+JOBS=8
+
+src/rocksdb/librocksdb.a:
+	make \
+		-e EXTRA_CXXFLAGS="-fPIC" \
+		-e EXTRA_CFLAGS="-fPIC" \
+		-C src/rocksdb \
+		-j $(JOBS) \
+			libsnappy.a \
+			liblz4.a \
+			libbz2.a \
+			libzstd.a \
+			libz.a
+
+	(cd src/rocksdb && mkdir -p build && cd build && cmake \
+		-DWITH_SNAPPY=1 \
+		-DWITH_LZ4=1 \
+		-DWITH_ZLIB=1 \
+		-DWITH_ZSTD=1 \
+		-DWITH_GFLAGS=0 \
+		-DROCKSDB_BUILD_SHARED=0 \
+		-DWITH_TOOLS=0 \
+		-DWITH_BENCHMARK_TOOLS=0 \
+		-DWITH_CORE_TOOLS=0 \
+		-DWITH_JEMALLOC=0 \
+		-DCMAKE_BUILD_TYPE=Release \
+		-DSnappy_INCLUDE_DIRS=../snappy-1.1.8/ \
+		-DSnappy_LIBRARIES=../snappy-1.1.8/build \
+		-Dlz4_INCLUDE_DIRS=../lz4-1.9.3/lib \
+		-Dlz4_LIBRARIES=../lz4-1.9.3/lib \
+		-Dzstd_INCLUDE_DIRS=../zstd-1.4.9/lib \
+		-Dzstd_LIBRARIES=../zstd-1.4.9/lib \
+		-DZLIB_INCLUDE_DIR=../zlib-1.2.11 \
+		-DZLIB_LIBRARY=./zlib-1.2.11 \
+		-DCMAKE_CXX_FLAGS="-fPIC -I../snappy-1.1.8/build -I../zstd-1.4.9/lib/dictBuilder" \
+		.. && make -j $(JOBS))
+
+	cp src/rocksdb/build/librocksdb.a src/rocksdb/librocksdb.a
+
+clean:
+	rm -rf src/rocksdb/build
+	make -C src/rocksdb clean

README.rst

@@ -4,15 +4,15 @@
    :target: https://anaconda.org/conda-forge/python-rocksdb
 
 Note
-=========
+====
 The original pyrocksdb (https://pypi.python.org/pypi/pyrocksdb/0.4) has not been updated for long time. I update pyrocksdb to support the latest rocksdb. Please open issues in github if you have any problem.
 
 News (2019/04/18)
-=========
-Currently I am refactoring the code, and more features like TTL are coming soon. And the installation with cmake will be much more easily. 
+=================
+Currently I am refactoring the code, and more features like TTL are coming soon. And the installation with cmake will be much more easily.
 
 News (2019/04/19)
-=========
+=================
 I have created a new branch(https://github.com/twmht/python-rocksdb/tree/pybind11) which provides the basic functions (`put`, `get` and `delete`) now. And the installtion is much more easily!  you can try it if you encounter any installtion issues in the current version of `python-rocksdb`.
 
 The branch is under development and will be released to PypI after I migrate most of the existing features.

setup.py

@@ -1,28 +1,73 @@
 import platform
+import os
 from setuptools import setup
 from setuptools import find_packages
 from setuptools import Extension
 
+try:
+    from Cython.Build import cythonize
+except ImportError:
+    def cythonize(extensions):
+        return extensions
 
-extra_compile_args = [
+    SOURCES = ['rocksdb/_rocksdb.cpp']
+else:
+    SOURCES = ['rocksdb/_rocksdb.pyx']
+
+EXTRA_COMPILE_ARGS = [
     '-std=c++11',
-    '-O3',
+    '-fPIC',
+    '-Os',
     '-Wall',
     '-Wextra',
     '-Wconversion',
     '-fno-strict-aliasing',
     '-fno-rtti',
 ]
 
+LIBRARIES = ['rocksdb', 'snappy', 'bz2', 'z', 'lz4']
+EXTRA_OBJECTS = []
+EXTRA_LINK_ARGS = []
+INCLUDE_DIRS = []
+
+
 if platform.system() == 'Darwin':
-    extra_compile_args += ['-mmacosx-version-min=10.7', '-stdlib=libc++']
+    EXTRA_COMPILE_ARGS += ['-mmacosx-version-min=10.7', '-stdlib=libc++']
+    EXTRA_LINK_ARGS += ['-Wl,-s']
+
+if platform.system() == 'Linux':
+    EXTRA_LINK_ARGS += ['-Wl,--strip-all']
+
+
+STATIC_LIBRARIES = [os.path.join("src", "rocksdb", item) for item in [
+    "librocksdb.a",
+    "libbz2.a",
+    "liblz4.a",
+    "libsnappy.a",
+    "libz.a",
+    "libzstd.a",
+]]
+
+if all(map(os.path.exists, STATIC_LIBRARIES)):
+    LIBRARIES = []
+    EXTRA_OBJECTS = STATIC_LIBRARIES
+    INCLUDE_DIRS = [
+        os.path.join("src", "rocksdb", "bzip2-1.0.8"),
+        os.path.join("src", "rocksdb", "zstd-1.4.9", "lib"),
+        os.path.join("src", "rocksdb", "zlib-1.2.11"),
+        os.path.join("src", "rocksdb", "snappy-1.1.8"),
+        os.path.join("src", "rocksdb", "snappy-1.1.8", "build"),
+        os.path.join("src", "rocksdb", "lz4-1.9.3", "lib"),
+        os.path.join("src", "rocksdb", "include"),
+    ]
 
 
 setup(
     name="python-rocksdb",
     version='0.7.0',
+    keywords=['rocksdb', 'static', 'build'],
     description="Python bindings for RocksDB",
-    keywords='rocksdb',
+    long_description=open("README.rst").read(),
     author='Ming Hsuan Tu',
     author_email="qrnnis2623891@gmail.com",
     url="https://github.com/twmht/python-rocksdb",
@@ -31,17 +76,20 @@
     install_requires=['setuptools>=25'],
     package_dir={'rocksdb': 'rocksdb'},
     packages=find_packages('.'),
-    ext_modules=[Extension(
+    ext_modules=cythonize([Extension(
         'rocksdb._rocksdb',
-        ['rocksdb/_rocksdb.pyx'],
-        extra_compile_args=extra_compile_args,
+        SOURCES,
+        extra_compile_args=EXTRA_COMPILE_ARGS,
         language='c++',
-        libraries=['rocksdb', 'snappy', 'bz2', 'z', 'lz4'],
-    )],
+        libraries=LIBRARIES,
+        include_dirs=INCLUDE_DIRS,
+        extra_objects=EXTRA_OBJECTS,
+        extra_link_args=EXTRA_LINK_ARGS,
+    )]),
     extras_require={
         "doc": ['sphinx_rtd_theme', 'sphinx'],
         "test": ['pytest'],
     },
-    include_package_data=True,
+    include_package_data=False,
     zip_safe=False,
 )