Prepare CRAN release

.Rbuildignore

@@ -12,3 +12,6 @@ src/vendor/*
 ^CRAN-SUBMISSION$
 ^src/rust/vendor$
 ^src/rust/vendor-authors.R$
+^src/rust/target$
+^src/Makevars$
+^src/Makevars\.win$

.gitignore

@@ -1,3 +1,5 @@
 .Rproj.user
 docs
 src/rust/vendor
+src/Makevars
+src/Makevars.win

DESCRIPTION

@@ -1,6 +1,6 @@
 Package: rtiktoken
 Title: A Byte-Pair-Encoding (BPE) Tokenizer for OpenAI's Large Language Models
-Version: 0.0.6
+Version: 0.0.7
 Authors@R: c(
     person("David", "Zimmermann-Kollenda", , "david_j_zimmermann@hotmail.com", role = c("aut", "cre")),
     person("Roger Zurawicki", role = "aut", comment = "tiktoken-rs Rust library"),
@@ -16,6 +16,8 @@ SystemRequirements: Cargo (Rust's package manager), rustc >= 1.65.0
 Encoding: UTF-8
 Roxygen: list(markdown = TRUE)
 RoxygenNote: 7.3.2
-Config/rextendr/version: 0.3.1
+Config/rextendr/version: 0.3.1.9001
 Config/testthat/edition: 3
 Config/rtiktoken/MSRV: 1.65.0
+Depends: 
+    R (>= 4.2)

NEWS.md

@@ -1,15 +1,19 @@
-# rtiktoken 0.5
-
-* switch to rextendr defaults for configure and makevars
-
-# rtiktoken 0.3, 0.4
-
-* fix build for CRAN
-
-# rtiktoken 0.2
-
-* fix Notes and Warning for CRAN release
-
-# rtiktoken 0.1
-
-* Initial functionality & tests
+# rtiktoken 0.0.7
+
+* update extendr version and templates to address non API changes and symbols testing in R 4.5
+
+# rtiktoken 0.5
+
+* switch to rextendr defaults for configure and makevars
+
+# rtiktoken 0.3, 0.4
+
+* fix build for CRAN
+
+# rtiktoken 0.2
+
+* fix Notes and Warning for CRAN release
+
+# rtiktoken 0.1
+
+* Initial functionality & tests

configure

@@ -1,3 +1,3 @@
 #!/usr/bin/env sh
 : "${R_HOME=`R RHOME`}"
-"${R_HOME}/bin/Rscript" tools/msrv.R 
+"${R_HOME}/bin/Rscript" tools/config.R 

configure.win

@@ -1,2 +1,2 @@
 #!/usr/bin/env sh
-"${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" tools/msrv.R
+"${R_HOME}/bin${R_ARCH_BIN}/Rscript.exe" tools/config.R

src/Makevars.in

@@ -0,0 +1,46 @@
+TARGET_DIR = ./rust/target
+LIBDIR = $(TARGET_DIR)/@LIBDIR@
+STATLIB = $(LIBDIR)/librtiktoken.a
+PKG_LIBS = -L$(LIBDIR) -lrtiktoken
+
+all: $(SHLIB) rust_clean
+
+.PHONY: $(STATLIB)
+
+$(SHLIB): $(STATLIB)
+
+CARGOTMP = $(CURDIR)/.cargo
+VENDOR_DIR = $(CURDIR)/vendor
+
+
+# RUSTFLAGS appends --print=native-static-libs to ensure that 
+# the correct linkers are used. Use this for debugging if need. 
+#
+# CRAN note: Cargo and Rustc versions are reported during
+# configure via tools/msrv.R.
+#
+# When the NOT_CRAN flag is *not* set, the vendor.tar.xz, if present,
+# is unzipped and used for offline compilation.
+$(STATLIB):
+
+	# Check if NOT_CRAN is false and unzip vendor.tar.xz if so
+	if [ "$(NOT_CRAN)" != "true" ]; then \
+		if [ -f ./rust/vendor.tar.xz ]; then \
+			tar xf rust/vendor.tar.xz && \
+			mkdir -p $(CARGOTMP) && \
+			cp rust/vendor-config.toml $(CARGOTMP)/config.toml; \
+		fi; \
+	fi
+
+	export CARGO_HOME=$(CARGOTMP) && \
+	export PATH="$(PATH):$(HOME)/.cargo/bin" && \
+	RUSTFLAGS="$(RUSTFLAGS) --print=native-static-libs" cargo build @CRAN_FLAGS@ --lib @PROFILE@ --manifest-path=./rust/Cargo.toml --target-dir $(TARGET_DIR)
+
+	# Always clean up CARGOTMP
+	rm -Rf $(CARGOTMP);
+
+rust_clean: $(SHLIB)
+	rm -Rf $(CARGOTMP) $(VENDOR_DIR) @CLEAN_TARGET@
+
+clean:
+	rm -Rf $(SHLIB) $(STATLIB) $(OBJECTS) $(TARGET_DIR)

src/Makevars.win.in

@@ -0,0 +1,41 @@
+TARGET = $(subst 64,x86_64,$(subst 32,i686,$(WIN)))-pc-windows-gnu
+
+TARGET_DIR = ./rust/target
+LIBDIR = $(TARGET_DIR)/$(TARGET)/@LIBDIR@
+STATLIB = $(LIBDIR)/librtiktoken.a
+PKG_LIBS = -L$(LIBDIR) -lrtiktoken -lws2_32 -ladvapi32 -luserenv -lbcrypt -lntdll
+
+all: $(SHLIB) rust_clean
+
+.PHONY: $(STATLIB)
+
+$(SHLIB): $(STATLIB)
+
+CARGOTMP = $(CURDIR)/.cargo
+VENDOR_DIR = vendor
+
+$(STATLIB):
+	mkdir -p $(TARGET_DIR)/libgcc_mock
+	touch $(TARGET_DIR)/libgcc_mock/libgcc_eh.a
+
+	if [ "$(NOT_CRAN)" != "true" ]; then \
+		if [ -f ./rust/vendor.tar.xz ]; then \
+			tar xf rust/vendor.tar.xz && \
+			mkdir -p $(CARGOTMP) && \
+			cp rust/vendor-config.toml $(CARGOTMP)/config.toml; \
+		fi; \
+	fi
+
+	# Build the project using Cargo with additional flags
+	export CARGO_HOME=$(CARGOTMP) && \
+	export LIBRARY_PATH="$(LIBRARY_PATH);$(CURDIR)/$(TARGET_DIR)/libgcc_mock" && \
+	RUSTFLAGS="$(RUSTFLAGS) --print=native-static-libs" cargo build @CRAN_FLAGS@ --target=$(TARGET) --lib @PROFILE@ --manifest-path=rust/Cargo.toml --target-dir=$(TARGET_DIR)
+
+	# Always clean up CARGOTMP
+	rm -Rf $(CARGOTMP);
+
+rust_clean: $(SHLIB)
+	rm -Rf $(CARGOTMP) $(VENDOR_DIR) @CLEAN_TARGET@
+
+clean:
+	rm -Rf $(SHLIB) $(STATLIB) $(OBJECTS) $(TARGET_DIR)