diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 952060a..344794e 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -28,3 +28,7 @@ jobs: run: mvn --batch-mode deploy env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + OSSRH_USERNAME: ${{ secrets.OSSRH_USERNAME }} + OSSRH_TOKEN: ${{ secrets.OSSRH_TOKEN }} + MAVEN_GPG_PASSPHRASE: ${{ secrets.MAVEN_GPG_PASSPHRASE }} + MAVEN_GPG_KEY: ${{ secrets.MAVEN_GPG_KEY }} diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0131ece..1af8c08 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -31,9 +31,9 @@ jobs: with: python-version: '3.12' - - name: Install test file generator requirements + - name: Install reference file generator requirements run: | - pip install asdf numpy + pip install asdf numpy lz4 - name: Locate Python id: locate-python diff --git a/asdf-compression-commons-compress/pom.xml b/asdf-compression-commons-compress/pom.xml index cd990af..85b6a1a 100644 --- a/asdf-compression-commons-compress/pom.xml +++ b/asdf-compression-commons-compress/pom.xml @@ -10,7 +10,7 @@ asdf-compression-commons-compress asdf-compression-commons-compress - Support for BZip2 and LZ4 Frame compressed blocks + Support for BZip2 and LZ4 compressed blocks diff --git a/asdf-compression-commons-compress/src/main/java/org/asdfformat/asdf/io/compression/Lz4Compressor.java b/asdf-compression-commons-compress/src/main/java/org/asdfformat/asdf/io/compression/Lz4Compressor.java new file mode 100644 index 0000000..37e0f13 --- /dev/null +++ b/asdf-compression-commons-compress/src/main/java/org/asdfformat/asdf/io/compression/Lz4Compressor.java @@ -0,0 +1,46 @@ +package org.asdfformat.asdf.io.compression; + +import org.apache.commons.compress.compressors.lz4.BlockLZ4CompressorInputStream; +import org.asdfformat.asdf.io.util.IOUtils; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class Lz4Compressor implements Compressor { + public static final byte[] IDENTIFIER = {108, 122, 52, 0}; // 'lz4' + padding + + @Override + public byte[] getIdentifier() { + return IDENTIFIER; + } + + @Override + public long decompress(final ByteBuffer inputBuffer, final ByteBuffer outputBuffer) throws IOException { + long bytesDecompressed = 0L; + + while (inputBuffer.hasRemaining()) { + inputBuffer.order(ByteOrder.BIG_ENDIAN); + final int lz4BlockLength = inputBuffer.getInt() - 4; + if (lz4BlockLength < 0) { + throw new RuntimeException("LZ4 block length > " + Integer.MAX_VALUE + " not supported"); + } + + // Discard the uncompressed data size written by the Python LZ4 bindings: + inputBuffer.getInt(); + + final ByteBuffer lz4BlockInputBuffer = inputBuffer.duplicate(); + lz4BlockInputBuffer.limit(inputBuffer.position() + lz4BlockLength); + + try (final ByteBufferInputStream byteBufferInputStream = new ByteBufferInputStream(lz4BlockInputBuffer); + final BlockLZ4CompressorInputStream blockLZ4CompressorInputStream = new BlockLZ4CompressorInputStream(byteBufferInputStream) + ) { + bytesDecompressed += IOUtils.transferTo(blockLZ4CompressorInputStream, outputBuffer); + } + + inputBuffer.position(inputBuffer.position() + lz4BlockLength); + } + + return bytesDecompressed; + } +} diff --git a/asdf-compression-commons-compress/src/test/java/org/asdfformat/asdf/io/compression/Lz4CompressorReferenceTest.java b/asdf-compression-commons-compress/src/test/java/org/asdfformat/asdf/io/compression/Lz4CompressorReferenceTest.java new file mode 100644 index 0000000..46bec19 --- /dev/null +++ b/asdf-compression-commons-compress/src/test/java/org/asdfformat/asdf/io/compression/Lz4CompressorReferenceTest.java @@ -0,0 +1,35 @@ +package org.asdfformat.asdf.io.compression; + +import org.asdfformat.asdf.Asdf; +import org.asdfformat.asdf.AsdfFile; +import org.asdfformat.asdf.io.compression.testing.CommonsCompressReferenceFileType; +import org.asdfformat.asdf.ndarray.DoubleNdArray; +import org.asdfformat.asdf.standard.AsdfStandardType; +import org.asdfformat.asdf.testing.ReferenceFileUtils; +import org.junit.jupiter.api.Tag; +import org.junitpioneer.jupiter.cartesian.CartesianTest; + +import java.io.IOException; +import java.nio.file.Path; + +import static org.asdfformat.asdf.io.compression.testing.TestCategories.REFERENCE_TESTS; +import static org.junit.jupiter.api.Assertions.assertEquals; + +@Tag(REFERENCE_TESTS) +public class Lz4CompressorReferenceTest { + @CartesianTest + public void test1d( + @CartesianTest.Enum(value = CommonsCompressReferenceFileType.class, names = {"NDARRAY_COMPRESSED_LZ4"}) final CommonsCompressReferenceFileType referenceFileType, + @CartesianTest.Enum(AsdfStandardType.class) final AsdfStandardType asdfStandardType + ) throws IOException { + final Path path = ReferenceFileUtils.getPath(referenceFileType, asdfStandardType.getVersion()); + + try (final AsdfFile asdfFile = Asdf.open(path)) { + final DoubleNdArray doubleNdArray = asdfFile.getTree().get("arr").asNdArray().asDoubleNdArray(); + + for (int i = 0; i < 10; i++) { + assertEquals(doubleNdArray.get(i), i); + } + } + } +} diff --git a/asdf-compression-commons-compress/src/test/java/org/asdfformat/asdf/io/compression/testing/CommonsCompressReferenceFileType.java b/asdf-compression-commons-compress/src/test/java/org/asdfformat/asdf/io/compression/testing/CommonsCompressReferenceFileType.java new file mode 100644 index 0000000..3a6d8de --- /dev/null +++ b/asdf-compression-commons-compress/src/test/java/org/asdfformat/asdf/io/compression/testing/CommonsCompressReferenceFileType.java @@ -0,0 +1,23 @@ +package org.asdfformat.asdf.io.compression.testing; + +import org.asdfformat.asdf.testing.ReferenceFile; + +import java.io.IOException; +import java.io.InputStream; + +public enum CommonsCompressReferenceFileType implements ReferenceFile { + NDARRAY_COMPRESSED_LZ4, + ; + + @Override + public String getName() { + return name(); + } + + @Override + public InputStream openScript() throws IOException { + return CommonsCompressReferenceFileType.class.getResourceAsStream( + String.format("/test-file-scripts/%s.py", name().toLowerCase()) + ); + } +} diff --git a/asdf-compression-commons-compress/src/test/java/org/asdfformat/asdf/io/compression/testing/TestCategories.java b/asdf-compression-commons-compress/src/test/java/org/asdfformat/asdf/io/compression/testing/TestCategories.java new file mode 100644 index 0000000..e21c501 --- /dev/null +++ b/asdf-compression-commons-compress/src/test/java/org/asdfformat/asdf/io/compression/testing/TestCategories.java @@ -0,0 +1,5 @@ +package org.asdfformat.asdf.io.compression.testing; + +public class TestCategories { + public static final String REFERENCE_TESTS = "reference-tests"; +} diff --git a/asdf-compression-commons-compress/src/test/resources/test-file-scripts/ndarray_compressed_lz4.py b/asdf-compression-commons-compress/src/test/resources/test-file-scripts/ndarray_compressed_lz4.py new file mode 100644 index 0000000..2e3407f --- /dev/null +++ b/asdf-compression-commons-compress/src/test/resources/test-file-scripts/ndarray_compressed_lz4.py @@ -0,0 +1,3 @@ +af["arr"] = np.arange(0, 10, dtype=np.float64) + +af.set_array_compression(af["arr"], "lz4") diff --git a/asdf-core/src/main/java/org/asdfformat/asdf/io/compression/Compressors.java b/asdf-core/src/main/java/org/asdfformat/asdf/io/compression/Compressors.java index 4ae4ae8..d0549b9 100644 --- a/asdf-core/src/main/java/org/asdfformat/asdf/io/compression/Compressors.java +++ b/asdf-core/src/main/java/org/asdfformat/asdf/io/compression/Compressors.java @@ -22,6 +22,11 @@ public class Compressors { .identifier("bzp2".getBytes(StandardCharsets.UTF_8)) .moduleName("asdf-compression-commons-compress") .build(), + OptionalCompressor.builder() + .className("org.asdfformat.asdf.io.compression.Lz4Compressor") + .identifier(new byte[] {108, 122, 52, 0}) + .moduleName("asdf-compression-commons-compress") + .build(), OptionalCompressor.builder() .className("org.asdfformat.asdf.io.compression.Lz4FrameCompressor") .identifier("lz4f".getBytes(StandardCharsets.UTF_8)) diff --git a/asdf-core/src/main/java/org/asdfformat/asdf/testing/ReferenceFile.java b/asdf-core/src/main/java/org/asdfformat/asdf/testing/ReferenceFile.java new file mode 100644 index 0000000..9585264 --- /dev/null +++ b/asdf-core/src/main/java/org/asdfformat/asdf/testing/ReferenceFile.java @@ -0,0 +1,9 @@ +package org.asdfformat.asdf.testing; + +import java.io.IOException; +import java.io.InputStream; + +public interface ReferenceFile { + String getName(); + InputStream openScript() throws IOException; +} diff --git a/asdf-core/src/test/java/org/asdfformat/asdf/testing/TestFiles.java b/asdf-core/src/main/java/org/asdfformat/asdf/testing/ReferenceFileUtils.java similarity index 71% rename from asdf-core/src/test/java/org/asdfformat/asdf/testing/TestFiles.java rename to asdf-core/src/main/java/org/asdfformat/asdf/testing/ReferenceFileUtils.java index 74015fe..3214ea3 100644 --- a/asdf-core/src/test/java/org/asdfformat/asdf/testing/TestFiles.java +++ b/asdf-core/src/main/java/org/asdfformat/asdf/testing/ReferenceFileUtils.java @@ -15,7 +15,7 @@ import java.util.Map; import java.util.Optional; -public class TestFiles { +public class ReferenceFileUtils { private static final String PYTHON_PATH = System.getenv("ASDF_JAVA_TESTS_PYTHON_PATH"); private static final Path TEST_FILE_GENERATOR_PY_PATH = getTestFileGeneratorPyPath(); @@ -26,8 +26,8 @@ private static Path getTestFileGeneratorPyPath() { final Path path = file.toPath(); try ( - final InputStream inputStream = Optional.ofNullable(TestFiles.class.getResourceAsStream("/generation/test_file_generator.py")) - .orElseThrow(() -> new RuntimeException("Missing generation/test_file_generator.py")); + final InputStream inputStream = Optional.ofNullable(ReferenceFileUtils.class.getResourceAsStream("/testing/reference_file_generator.py")) + .orElseThrow(() -> new RuntimeException("Missing testing/reference_file_generator.py")); final OutputStream outputStream = Files.newOutputStream(path, StandardOpenOption.CREATE) ) { IOUtils.transferTo(inputStream, outputStream); @@ -38,24 +38,24 @@ private static Path getTestFileGeneratorPyPath() { private static final Map TEST_FILES = new HashMap<>(); - public static Path getPath(final TestFileType testFileType, final Version asdfStandardVersion) { - final String key = makeKey(testFileType, asdfStandardVersion); + public static Path getPath(final ReferenceFile testfile, final Version asdfStandardVersion) { + final String key = makeKey(testfile, asdfStandardVersion); if (!TEST_FILES.containsKey(key)) { - TEST_FILES.put(key, generateTestFile(testFileType, asdfStandardVersion)); + TEST_FILES.put(key, generateTestFile(testfile, asdfStandardVersion)); } return TEST_FILES.get(key); } @SneakyThrows(IOException.class) - private static Path generateTestFile(final TestFileType testFileType, final Version asdfStandardVersion) { - try (final InputStream scriptInputStream = TestFiles.class.getResourceAsStream(testFileType.getScriptResourceName())) { + private static Path generateTestFile(final ReferenceFile referenceFile, final Version asdfStandardVersion) { + try (final InputStream scriptInputStream = referenceFile.openScript()) { if (scriptInputStream == null) { - throw new RuntimeException("Missing generator script for " + testFileType); + throw new RuntimeException("Missing generator script for " + referenceFile.getName()); } - final File file = File.createTempFile(testFileType + "-", ".asdf"); + final File file = File.createTempFile(referenceFile.getName() + "-", ".asdf"); file.deleteOnExit(); final Path path = file.toPath(); @@ -75,7 +75,7 @@ private static Path generateTestFile(final TestFileType testFileType, final Vers } } - private static String makeKey(final TestFileType testFileType, final Version asdfStandardVersion) { - return String.format("%s:%s", testFileType, asdfStandardVersion); + private static String makeKey(final ReferenceFile referenceFile, final Version asdfStandardVersion) { + return String.format("%s:%s", referenceFile, asdfStandardVersion); } } diff --git a/asdf-core/src/test/resources/generation/test_file_generator.py b/asdf-core/src/main/resources/testing/reference_file_generator.py similarity index 100% rename from asdf-core/src/test/resources/generation/test_file_generator.py rename to asdf-core/src/main/resources/testing/reference_file_generator.py diff --git a/asdf-core/src/test/java/org/asdfformat/asdf/ndarray/NdArrayCompressedZlibReferenceTest.java b/asdf-core/src/test/java/org/asdfformat/asdf/ndarray/NdArrayCompressedZlibReferenceTest.java index 773e2d4..89c536e 100644 --- a/asdf-core/src/test/java/org/asdfformat/asdf/ndarray/NdArrayCompressedZlibReferenceTest.java +++ b/asdf-core/src/test/java/org/asdfformat/asdf/ndarray/NdArrayCompressedZlibReferenceTest.java @@ -3,8 +3,8 @@ import org.asdfformat.asdf.Asdf; import org.asdfformat.asdf.AsdfFile; import org.asdfformat.asdf.standard.AsdfStandardType; -import org.asdfformat.asdf.testing.TestFileType; -import org.asdfformat.asdf.testing.TestFiles; +import org.asdfformat.asdf.testing.CoreReferenceFileType; +import org.asdfformat.asdf.testing.ReferenceFileUtils; import org.junit.jupiter.api.Tag; import org.junitpioneer.jupiter.cartesian.CartesianTest; @@ -18,10 +18,10 @@ public class NdArrayCompressedZlibReferenceTest { @CartesianTest public void test1d( - @CartesianTest.Enum(value = TestFileType.class, names = {"NDARRAY_COMPRESSED_ZLIB"}) final TestFileType testFileType, + @CartesianTest.Enum(value = CoreReferenceFileType.class, names = {"NDARRAY_COMPRESSED_ZLIB"}) final CoreReferenceFileType coreTestFileType, @CartesianTest.Enum(AsdfStandardType.class) final AsdfStandardType asdfStandardType ) throws IOException { - final Path path = TestFiles.getPath(testFileType, asdfStandardType.getVersion()); + final Path path = ReferenceFileUtils.getPath(coreTestFileType, asdfStandardType.getVersion()); try (final AsdfFile asdfFile = Asdf.open(path)) { final DoubleNdArray doubleNdArray = asdfFile.getTree().get("arr").asNdArray().asDoubleNdArray(); diff --git a/asdf-core/src/test/java/org/asdfformat/asdf/ndarray/NdArrayFloat64ReferenceTest.java b/asdf-core/src/test/java/org/asdfformat/asdf/ndarray/NdArrayFloat64ReferenceTest.java index 764a8a9..b94b803 100644 --- a/asdf-core/src/test/java/org/asdfformat/asdf/ndarray/NdArrayFloat64ReferenceTest.java +++ b/asdf-core/src/test/java/org/asdfformat/asdf/ndarray/NdArrayFloat64ReferenceTest.java @@ -3,8 +3,8 @@ import org.asdfformat.asdf.Asdf; import org.asdfformat.asdf.AsdfFile; import org.asdfformat.asdf.standard.AsdfStandardType; -import org.asdfformat.asdf.testing.TestFileType; -import org.asdfformat.asdf.testing.TestFiles; +import org.asdfformat.asdf.testing.CoreReferenceFileType; +import org.asdfformat.asdf.testing.ReferenceFileUtils; import org.junit.jupiter.api.Tag; import org.junitpioneer.jupiter.cartesian.CartesianTest; @@ -18,10 +18,10 @@ public class NdArrayFloat64ReferenceTest { @CartesianTest public void test1d( - @CartesianTest.Enum(value = TestFileType.class, names = {"NDARRAY_FLOAT64_1D_BLOCK_BIG", "NDARRAY_FLOAT64_1D_BLOCK_LITTLE", "NDARRAY_FLOAT64_1D_INLINE"}) final TestFileType testFileType, + @CartesianTest.Enum(value = CoreReferenceFileType.class, names = {"NDARRAY_FLOAT64_1D_BLOCK_BIG", "NDARRAY_FLOAT64_1D_BLOCK_LITTLE", "NDARRAY_FLOAT64_1D_INLINE"}) final CoreReferenceFileType coreTestFileType, @CartesianTest.Enum(AsdfStandardType.class) final AsdfStandardType asdfStandardType ) throws IOException { - final Path path = TestFiles.getPath(testFileType, asdfStandardType.getVersion()); + final Path path = ReferenceFileUtils.getPath(coreTestFileType, asdfStandardType.getVersion()); try (final AsdfFile asdfFile = Asdf.open(path)) { final DoubleNdArray doubleNdArray = asdfFile.getTree().get("arr").asNdArray().asDoubleNdArray(); diff --git a/asdf-core/src/test/java/org/asdfformat/asdf/ndarray/NdArrayStructuredReferenceTest.java b/asdf-core/src/test/java/org/asdfformat/asdf/ndarray/NdArrayStructuredReferenceTest.java index 1971bd0..abf1698 100644 --- a/asdf-core/src/test/java/org/asdfformat/asdf/ndarray/NdArrayStructuredReferenceTest.java +++ b/asdf-core/src/test/java/org/asdfformat/asdf/ndarray/NdArrayStructuredReferenceTest.java @@ -3,8 +3,8 @@ import org.asdfformat.asdf.Asdf; import org.asdfformat.asdf.AsdfFile; import org.asdfformat.asdf.standard.AsdfStandardType; -import org.asdfformat.asdf.testing.TestFileType; -import org.asdfformat.asdf.testing.TestFiles; +import org.asdfformat.asdf.testing.CoreReferenceFileType; +import org.asdfformat.asdf.testing.ReferenceFileUtils; import org.junit.jupiter.api.Tag; import org.junitpioneer.jupiter.cartesian.CartesianTest; @@ -18,10 +18,10 @@ public class NdArrayStructuredReferenceTest { @CartesianTest public void test1d( - @CartesianTest.Enum(value = TestFileType.class, names = {"NDARRAY_STRUCTURED_1D_BLOCK"}) final TestFileType testFileType, + @CartesianTest.Enum(value = CoreReferenceFileType.class, names = {"NDARRAY_STRUCTURED_1D_BLOCK"}) final CoreReferenceFileType coreTestFileType, @CartesianTest.Enum(AsdfStandardType.class) final AsdfStandardType asdfStandardType ) throws IOException { - final Path path = TestFiles.getPath(testFileType, asdfStandardType.getVersion()); + final Path path = ReferenceFileUtils.getPath(coreTestFileType, asdfStandardType.getVersion()); try (final AsdfFile asdfFile = Asdf.open(path)) { final TupleNdArray tupleNdArray = asdfFile.getTree().getNdArray("arr").asTupleNdArray(); diff --git a/asdf-core/src/test/java/org/asdfformat/asdf/testing/CoreReferenceFileType.java b/asdf-core/src/test/java/org/asdfformat/asdf/testing/CoreReferenceFileType.java new file mode 100644 index 0000000..be1ac4a --- /dev/null +++ b/asdf-core/src/test/java/org/asdfformat/asdf/testing/CoreReferenceFileType.java @@ -0,0 +1,25 @@ +package org.asdfformat.asdf.testing; + +import java.io.IOException; +import java.io.InputStream; + +public enum CoreReferenceFileType implements ReferenceFile { + NDARRAY_COMPRESSED_ZLIB, + NDARRAY_FLOAT64_1D_BLOCK_BIG, + NDARRAY_FLOAT64_1D_BLOCK_LITTLE, + NDARRAY_FLOAT64_1D_INLINE, + NDARRAY_STRUCTURED_1D_BLOCK, + ; + + @Override + public String getName() { + return name(); + } + + @Override + public InputStream openScript() throws IOException { + return CoreReferenceFileType.class.getResourceAsStream( + String.format("/reference-file-scripts/%s.py", name().toLowerCase()) + ); + } +} diff --git a/asdf-core/src/test/java/org/asdfformat/asdf/testing/TestFileType.java b/asdf-core/src/test/java/org/asdfformat/asdf/testing/TestFileType.java deleted file mode 100644 index b800b39..0000000 --- a/asdf-core/src/test/java/org/asdfformat/asdf/testing/TestFileType.java +++ /dev/null @@ -1,14 +0,0 @@ -package org.asdfformat.asdf.testing; - -public enum TestFileType { - NDARRAY_COMPRESSED_ZLIB, - NDARRAY_FLOAT64_1D_BLOCK_BIG, - NDARRAY_FLOAT64_1D_BLOCK_LITTLE, - NDARRAY_FLOAT64_1D_INLINE, - NDARRAY_STRUCTURED_1D_BLOCK, - ; - - public String getScriptResourceName() { - return String.format("/generation/scripts/%s.py", name().toLowerCase()); - } -} diff --git a/asdf-core/src/test/resources/generation/scripts/ndarray_compressed_zlib.py b/asdf-core/src/test/resources/reference-file-scripts/ndarray_compressed_zlib.py similarity index 100% rename from asdf-core/src/test/resources/generation/scripts/ndarray_compressed_zlib.py rename to asdf-core/src/test/resources/reference-file-scripts/ndarray_compressed_zlib.py diff --git a/asdf-core/src/test/resources/generation/scripts/ndarray_float64_1d_block_big.py b/asdf-core/src/test/resources/reference-file-scripts/ndarray_float64_1d_block_big.py similarity index 100% rename from asdf-core/src/test/resources/generation/scripts/ndarray_float64_1d_block_big.py rename to asdf-core/src/test/resources/reference-file-scripts/ndarray_float64_1d_block_big.py diff --git a/asdf-core/src/test/resources/generation/scripts/ndarray_float64_1d_block_little.py b/asdf-core/src/test/resources/reference-file-scripts/ndarray_float64_1d_block_little.py similarity index 100% rename from asdf-core/src/test/resources/generation/scripts/ndarray_float64_1d_block_little.py rename to asdf-core/src/test/resources/reference-file-scripts/ndarray_float64_1d_block_little.py diff --git a/asdf-core/src/test/resources/generation/scripts/ndarray_float64_1d_inline.py b/asdf-core/src/test/resources/reference-file-scripts/ndarray_float64_1d_inline.py similarity index 100% rename from asdf-core/src/test/resources/generation/scripts/ndarray_float64_1d_inline.py rename to asdf-core/src/test/resources/reference-file-scripts/ndarray_float64_1d_inline.py diff --git a/asdf-core/src/test/resources/generation/scripts/ndarray_structured_1d_block.py b/asdf-core/src/test/resources/reference-file-scripts/ndarray_structured_1d_block.py similarity index 100% rename from asdf-core/src/test/resources/generation/scripts/ndarray_structured_1d_block.py rename to asdf-core/src/test/resources/reference-file-scripts/ndarray_structured_1d_block.py diff --git a/pom.xml b/pom.xml index d9f5c57..4010a18 100644 --- a/pom.xml +++ b/pom.xml @@ -35,7 +35,7 @@ UTF-8 8 - 0.1-alpha-3 + 0.1-alpha-6 1.18.36 2.4 3.4.0 @@ -49,7 +49,7 @@ 3.1.4 2.2.1 3.11.2 - 1.5 + 3.2.4 3.5.3 0.7.0 5.11.4 @@ -187,6 +187,9 @@ sign + + bc +