From d5987d0b9b884265df2f51db81b733ed3970dece Mon Sep 17 00:00:00 2001 From: Shu Li Date: Sat, 15 Jul 2023 23:24:14 -0400 Subject: [PATCH 01/18] Fix "bytesize not defined" and add tests. --- src/data.jl | 2 +- test/data.jl | 132 +++++++++++++++++++++++++++++++++++++++++++++++ test/runtests.jl | 1 + 3 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 test/data.jl diff --git a/src/data.jl b/src/data.jl index d75ddec..c9f72cf 100644 --- a/src/data.jl +++ b/src/data.jl @@ -90,7 +90,7 @@ function updateheader!(d::MRCData; statistics=true) # misc h.mode = TYPE_TO_MODE[eltype(d)] - h.nsymbt = bytesize(extendedheader(d)) + h.nsymbt = sizeof(extendedheader(d)) h.nlabl = sum(s -> length(rstrip(s)) > 0, h.label) # update statistics diff --git a/test/data.jl b/test/data.jl new file mode 100644 index 0000000..f2ce3f4 --- /dev/null +++ b/test/data.jl @@ -0,0 +1,132 @@ +@testset "MRCData" begin + @testset "MRCData(header, extendedheader, data)" begin + h = MRCHeader() + eh = MRCExtendedHeader() + data = Array{Float32, 3}(undef, (2, 2, 2)) + d = MRCData(h, eh, data) + @test d.header === h + @test d.extendedheader === eh + @test d.data === data + @test eltype(d) === Float32 + @test ndims(d) === 3 + end + + @testset "MRCData(header, extendedheader)" begin + h = MRCHeader() + eh = MRCExtendedHeader() + d = MRCData(h, eh) + @test d.header === h + @test d.extendedheader === eh + @test ndims(d) == ndims(h) + @test size(d) == size(h) + end + + @testset "MRCData(size::NTuple{3,<:Integer})" begin + d = MRCData((10, 20, 30)) + @test size(d) == (10, 20, 30) + @test size(d.data) == (10, 20, 30) + @test ndims(d) == 3 + @test ndims(d.data) == 3 + + @testset "header(d::MRCData)" begin + h = header(d) + @test h.nx == 10 + @test h.ny == 20 + @test h.nz == 30 + end + end + + @testset "MRCData(size::Integer...)" begin + d1 = MRCData(10, 20, 30) + d2 = MRCData((10, 20, 30)) + @test size(d1) == size(d2) + end + + data = fill(1f0, (2, 2, 2)) + h = MRCHeader() + eh = MRCExtendedHeader() + d = MRCData(h, eh, data) + + @testset "getindex" begin + @test d[1, 1, 1] == 1.0f0 + @test d[2, 1, 1] == 1.0f0 + @test d[1, 2, 1] == 1.0f0 + @test d[2, 2, 1] == 1.0f0 + @test d[1, 1, 2] == 1.0f0 + @test d[2, 1, 2] == 1.0f0 + @test d[1, 2, 2] == 1.0f0 + @test d[2, 2, 2] == 1.0f0 + end + + @testset "setindex!" begin + d[1, 1, 1] = 10 + d[2, 1, 1] = 20 + d[1, 2, 1] = 30 + d[2, 2, 1] = 40 + d[1, 1, 2] = 50 + d[2, 1, 2] = 60 + d[1, 2, 2] = 70 + d[2, 2, 2] = 80 + @test d[1, 1, 1] == 10 + @test d[2, 1, 1] == 20 + @test d[1, 2, 1] == 30 + @test d[2, 2, 1] == 40 + @test d[1, 1, 2] == 50 + @test d[2, 1, 2] == 60 + @test d[1, 2, 2] == 70 + @test d[2, 2, 2] == 80 + end + + @testset "size" begin + @test size(d) == (2, 2, 2) + @test size(d, 1) == 2 + @test size(d, 2) == 2 + @test size(d, 3) == 2 + end + + @testset "ndims" begin + @test ndims(d) == 3 + end + + @testset "length" begin + @test length(d) == 8 + end + + @testset "iterate" begin + iter = iterate(d) + @test iter !== nothing && first(iter) == 10 + end + + @testset "lastindex" begin + @test lastindex(d) == 8 + end + + @testset "similar" begin + d_similar = similar(d) + @test typeof(d_similar) === typeof(d) + @test size(d_similar) == size(d) + @test eltype(d_similar) == eltype(d) + end + + @testset "IndexStyle" begin + @test Base.IndexStyle(typeof(d)) == Base.IndexStyle(typeof(d.data)) + end +end + +@testset "updateheader!(d::MRCData; statistics=true)" begin + data = fill(1f0, (2, 2, 2)) + h = MRCHeader() + @test h.nx == 0 + @test h.ny == 0 + @test h.nz == 0 + eh = MRCExtendedHeader() + d = MRCData(h, eh, data) + updateheader!(d) + @test d.header.dmin == 1.0f0 + @test d.header.dmax == 1.0f0 + @test d.header.dmean == 1.0f0 + @test d.header.rms == 0.0f0 + @test d.header.nx == 2 + @test d.header.ny == 2 + @test d.header.nz == 2 +end \ No newline at end of file diff --git a/test/runtests.jl b/test/runtests.jl index 0feedc6..2fdd44d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -5,4 +5,5 @@ using Test include("utils.jl") include("header.jl") include("io.jl") + include("data.jl") end From aa3ac84a5f3a95da063afb08c5d47794921a01f4 Mon Sep 17 00:00:00 2001 From: Shu Li Date: Sun, 16 Jul 2023 11:43:34 -0400 Subject: [PATCH 02/18] Format the codes. --- Project.toml | 2 +- test/data.jl | 50 +++++++++++++++++++++++++------------------------- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/Project.toml b/Project.toml index 27a3bd0..8183fb2 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "MRCFile" uuid = "c18c01d3-0ab9-49c3-bd2d-8f2e64a2b7a5" authors = ["Seth Axen "] -version = "0.1.0" +version = "0.1.1" [deps] CodecBzip2 = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd" diff --git a/test/data.jl b/test/data.jl index f2ce3f4..2125b06 100644 --- a/test/data.jl +++ b/test/data.jl @@ -1,22 +1,22 @@ @testset "MRCData" begin @testset "MRCData(header, extendedheader, data)" begin - h = MRCHeader() - eh = MRCExtendedHeader() - data = Array{Float32, 3}(undef, (2, 2, 2)) - d = MRCData(h, eh, data) + h = MRCHeader() + exh = MRCExtendedHeader() + data = Array{Float32,3}(undef, (2, 2, 2)) + d = MRCData(h, exh, data) @test d.header === h - @test d.extendedheader === eh + @test d.extendedheader === exh @test d.data === data @test eltype(d) === Float32 @test ndims(d) === 3 end @testset "MRCData(header, extendedheader)" begin - h = MRCHeader() - eh = MRCExtendedHeader() - d = MRCData(h, eh) + h = MRCHeader() + exh = MRCExtendedHeader() + d = MRCData(h, exh) @test d.header === h - @test d.extendedheader === eh + @test d.extendedheader === exh @test ndims(d) == ndims(h) @test size(d) == size(h) end @@ -35,17 +35,17 @@ @test h.nz == 30 end end - + @testset "MRCData(size::Integer...)" begin d1 = MRCData(10, 20, 30) d2 = MRCData((10, 20, 30)) @test size(d1) == size(d2) end - data = fill(1f0, (2, 2, 2)) - h = MRCHeader() - eh = MRCExtendedHeader() - d = MRCData(h, eh, data) + data = fill(1.0f0, (2, 2, 2)) + h = MRCHeader() + exh = MRCExtendedHeader() + d = MRCData(h, exh, data) @testset "getindex" begin @test d[1, 1, 1] == 1.0f0 @@ -57,7 +57,7 @@ @test d[1, 2, 2] == 1.0f0 @test d[2, 2, 2] == 1.0f0 end - + @testset "setindex!" begin d[1, 1, 1] = 10 d[2, 1, 1] = 20 @@ -76,27 +76,27 @@ @test d[1, 2, 2] == 70 @test d[2, 2, 2] == 80 end - + @testset "size" begin @test size(d) == (2, 2, 2) @test size(d, 1) == 2 @test size(d, 2) == 2 @test size(d, 3) == 2 end - + @testset "ndims" begin @test ndims(d) == 3 end - + @testset "length" begin @test length(d) == 8 end - + @testset "iterate" begin iter = iterate(d) @test iter !== nothing && first(iter) == 10 end - + @testset "lastindex" begin @test lastindex(d) == 8 end @@ -114,13 +114,13 @@ end @testset "updateheader!(d::MRCData; statistics=true)" begin - data = fill(1f0, (2, 2, 2)) - h = MRCHeader() + data = fill(1.0f0, (2, 2, 2)) + h = MRCHeader() @test h.nx == 0 @test h.ny == 0 @test h.nz == 0 - eh = MRCExtendedHeader() - d = MRCData(h, eh, data) + exh = MRCExtendedHeader() + d = MRCData(h, exh, data) updateheader!(d) @test d.header.dmin == 1.0f0 @test d.header.dmax == 1.0f0 @@ -129,4 +129,4 @@ end @test d.header.nx == 2 @test d.header.ny == 2 @test d.header.nz == 2 -end \ No newline at end of file +end From 7bd07361f4fd3cd276a290d28f58fb6db9e411db Mon Sep 17 00:00:00 2001 From: Shu Li Date: Sun, 16 Jul 2023 23:56:52 -0400 Subject: [PATCH 03/18] Add consistency tests and fix bugs. --- Project.toml | 1 + src/header.jl | 2 +- src/io.jl | 2 +- test/consistency.jl | 90 +++++++++++++++++++++++++++++++++++++++++++++ test/io.jl | 10 ++++- test/runtests.jl | 1 + 6 files changed, 102 insertions(+), 4 deletions(-) create mode 100644 test/consistency.jl diff --git a/Project.toml b/Project.toml index 8183fb2..2479dd8 100644 --- a/Project.toml +++ b/Project.toml @@ -9,6 +9,7 @@ CodecXz = "ba30903b-d9e8-5048-a5ec-d1f5b0d4b47b" CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Mmap = "a63ad114-7e13-5084-954f-fe012c677804" +PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" diff --git a/src/header.jl b/src/header.jl index 6bfee4c..270c437 100644 --- a/src/header.jl +++ b/src/header.jl @@ -111,7 +111,7 @@ function sizeoffield(name, type) end function bytestoentry(name, type, pointer) - name === :exttyp && return _unsafe_string(pointer, 4) + name === :exttyp && return unsafe_string(convert(Ptr{UInt8}, pointer), 4) name === :map && return _unsafe_string(pointer, 4) name === :label && return ntuple(Val(10)) do i return strip(_unsafe_string(pointer + (i - 1) * 80, 80), ' ') diff --git a/src/io.jl b/src/io.jl index d769ba1..b84d765 100644 --- a/src/io.jl +++ b/src/io.jl @@ -63,7 +63,7 @@ Read MRC file or stream, using a memory-mapped array to access the data. function read_mmap(io::IO, ::Type{MRCData}) head = read(io, MRCHeader) exthead = read(io, MRCExtendedHeader; header=head) - arraytype = Array{MRC.datatype(head),ndims(head)} + arraytype = Array{datatype(head),ndims(head)} data = Mmap.mmap(io, arraytype, size(head)[1:ndims(head)]) return MRCData(head, exthead, data) end diff --git a/test/consistency.jl b/test/consistency.jl new file mode 100644 index 0000000..e813cb3 --- /dev/null +++ b/test/consistency.jl @@ -0,0 +1,90 @@ +using Test +using PyCall + +py""" +import pip +pip.main(['install', 'mrcfile']) +pip.main(['install', 'numpy']) +""" + +# Load the Python module/package +mrcfile = pyimport("mrcfile") +numpy = pyimport("numpy") + +@testset "Consistency Test" begin + function map_test(map_jl::Array{Float32,3}, map_py::Array{Float32,3}) + permuted_py = permutedims(map_py, (3, 2, 1)) + @test map_jl == permuted_py + # @test map_jl == map_py + end + + function header_test(header_jl::MRCHeader, header_py::PyObject) + @test header_jl.nx == convert(Int32, header_py.nx) + @test header_jl.ny == convert(Int32, header_py.ny) + @test header_jl.nz == convert(Int32, header_py.nz) + @test header_jl.mode == convert(Int32, header_py.mode) + @test header_jl.nxstart == convert(Int32, header_py.nxstart) + @test header_jl.nystart == convert(Int32, header_py.nystart) + @test header_jl.nzstart == convert(Int32, header_py.nzstart) + @test header_jl.mx == convert(Int32, header_py.mx) + @test header_jl.my == convert(Int32, header_py.my) + @test header_jl.mz == convert(Int32, header_py.mz) + @test header_jl.cella_x == convert(Float32, header_py.cella.x) + @test header_jl.cella_y == convert(Float32, header_py.cella.y) + @test header_jl.cella_z == convert(Float32, header_py.cella.z) + @test header_jl.cellb_alpha == convert(Float32, header_py.cellb.alpha) + @test header_jl.cellb_beta == convert(Float32, header_py.cellb.beta) + @test header_jl.cellb_gamma == convert(Float32, header_py.cellb.gamma) + @test header_jl.mapc == convert(Int32, header_py.mapc) + @test header_jl.mapr == convert(Int32, header_py.mapr) + @test header_jl.maps == convert(Int32, header_py.maps) + @test header_jl.dmin == convert(Float32, header_py.dmin) + @test header_jl.dmax == convert(Float32, header_py.dmax) + @test header_jl.dmean == convert(Float32, header_py.dmean) + @test header_jl.ispg == convert(Int32, header_py.ispg) + @test header_jl.nsymbt == convert(Int32, header_py.nsymbt) + @test String([UInt8(b) for b in header_jl.extra1]) == convert(String, header_py.extra1.tobytes()) + @test header_jl.exttyp == header_py.exttyp.tostring() + @test header_jl.nversion == convert(Int32, header_py.nversion) + @test String([UInt8(b) for b in header_jl.extra2]) == header_py.extra2.tobytes() + @test header_jl.origin_x == convert(Float32, header_py.origin.x) + @test header_jl.origin_y == convert(Float32, header_py.origin.y) + @test header_jl.origin_z == convert(Float32, header_py.origin.z) + @test header_jl.map == header_py.map.tostring() + @test collect(header_jl.machst) == convert(Vector{UInt8}, header_py.machst) + @test header_jl.rms == convert(Float32, header_py.rms) + @test header_jl.nlabl == convert(Int32, header_py.nlabl) + @test strip.(collect(header_py.label)) == collect(header_jl.label) + end + + function exh_test(exh_jl::MRCExtendedHeader, exh_py::PyObject) + @test String([UInt8(b) for b in exh_jl.data]) == exh_py.tobytes() + end + + @testset "emd_3001.map" begin + emd3001_path = "$(@__DIR__)/testdata/emd_3001.map" + emd3001_jl = read("$(@__DIR__)/testdata/emd_3001.map", MRCData) + map_jl = emd3001_jl.data + header_jl = header(emd3001_jl) + exh_jl = extendedheader(emd3001_jl) + + emd3001_py = mrcfile.open("$(@__DIR__)/testdata/emd_3001.map"; mode="r") + data_copy = numpy.copy(emd3001_py.data) # Create a copy of the data + map_py = convert(Array{Float32,3}, data_copy) + header_py = emd3001_py.header + exh_py = emd3001_py.extended_header + emd3001_py.close() # Make sure the file is closed even if an error occurs + + @testset "map" begin + map_test(map_jl, map_py) + end + + @testset "header" begin + header_test(header_jl, header_py) + end + + @testset "extendedheader" begin + exh_test(exh_jl, exh_py) + end + end +end \ No newline at end of file diff --git a/test/io.jl b/test/io.jl index 94946df..f0fe4bf 100644 --- a/test/io.jl +++ b/test/io.jl @@ -33,7 +33,7 @@ @test h.ispg === Int32(4) @test h.nsymbt == sizeof(eh) == Int32(160) @test h.extra1 === Tuple(zeros(UInt8, 8)) - @test h.exttyp === "" + @test h.exttyp === "\0\0\0\0" @test h.nversion === Int32(0) @test h.extra2 === Tuple(zeros(UInt8, 84)) @test h.origin_x === Float32(0) @@ -80,7 +80,7 @@ @test h.ispg === Int32(1) @test h.nsymbt == sizeof(eh) == Int32(0) @test h.extra1 === Tuple(zeros(UInt8, 8)) - @test h.exttyp === "" + @test h.exttyp === "\0\0\0\0" @test h.nversion === Int32(0) @test h.extra2 === Tuple(zeros(UInt8, 84)) @test h.origin_x === Float32(0) @@ -96,6 +96,12 @@ end end +@testset "read_mmap" begin + emd3001 = read("$(@__DIR__)/testdata/emd_3001.map", MRCData) + emd3001mmap = read_mmap("$(@__DIR__)/testdata/emd_3001.map", MRCData) + @test emd3001 == emd3001mmap +end + @testset "write" begin @testset "buffer size has no effect on data" begin emd3001 = read("$(@__DIR__)/testdata/emd_3001.map", MRCData) diff --git a/test/runtests.jl b/test/runtests.jl index 2fdd44d..82a8ef7 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,4 +6,5 @@ using Test include("header.jl") include("io.jl") include("data.jl") + include("consistency.jl") end From 2f539e7a83318e5020767d7273673cb769a415d9 Mon Sep 17 00:00:00 2001 From: Shu Li Date: Sun, 16 Jul 2023 23:59:45 -0400 Subject: [PATCH 04/18] Format the codes. --- Project.toml | 2 +- test/consistency.jl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 2479dd8..9d72ea5 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "MRCFile" uuid = "c18c01d3-0ab9-49c3-bd2d-8f2e64a2b7a5" authors = ["Seth Axen "] -version = "0.1.1" +version = "0.1.2" [deps] CodecBzip2 = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd" diff --git a/test/consistency.jl b/test/consistency.jl index e813cb3..0824af3 100644 --- a/test/consistency.jl +++ b/test/consistency.jl @@ -87,4 +87,4 @@ numpy = pyimport("numpy") exh_test(exh_jl, exh_py) end end -end \ No newline at end of file +end From 032c368326bafa6caf2c7542800fac8c9d93a4c2 Mon Sep 17 00:00:00 2001 From: Shu Li Date: Mon, 17 Jul 2023 00:26:00 -0400 Subject: [PATCH 05/18] Move PyCall to [extras] --- Project.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Project.toml b/Project.toml index 9d72ea5..8a540dc 100644 --- a/Project.toml +++ b/Project.toml @@ -9,7 +9,6 @@ CodecXz = "ba30903b-d9e8-5048-a5ec-d1f5b0d4b47b" CodecZlib = "944b1d66-785c-5afd-91f1-9de20f533193" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Mmap = "a63ad114-7e13-5084-954f-fe012c677804" -PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" TranscodingStreams = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" @@ -22,6 +21,7 @@ julia = "^1" [extras] Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" [targets] -test = ["Test"] +test = ["Test", "PyCall"] From 02eef7e3931f23d6a63ab4bcaf70ad68fbb92af2 Mon Sep 17 00:00:00 2001 From: Shu Li Date: Mon, 17 Jul 2023 01:11:20 -0400 Subject: [PATCH 06/18] Using Conda to install packages for tests. --- Project.toml | 5 +++-- test/consistency.jl | 8 +++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/Project.toml b/Project.toml index 8a540dc..fce402f 100644 --- a/Project.toml +++ b/Project.toml @@ -20,8 +20,9 @@ TranscodingStreams = "0.9" julia = "^1" [extras] -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d" [targets] -test = ["Test", "PyCall"] +test = ["Test", "PyCall", "Conda"] diff --git a/test/consistency.jl b/test/consistency.jl index 0824af3..a387580 100644 --- a/test/consistency.jl +++ b/test/consistency.jl @@ -1,11 +1,9 @@ using Test using PyCall +using Conda -py""" -import pip -pip.main(['install', 'mrcfile']) -pip.main(['install', 'numpy']) -""" +Conda.pip_interop(true) +Conda.pip("install", ["mrcfile", "numpy"]) # Load the Python module/package mrcfile = pyimport("mrcfile") From 5e43b48b006fa6222e261714aac9ace8ae011f36 Mon Sep 17 00:00:00 2001 From: Shu Li Date: Mon, 17 Jul 2023 01:19:59 -0400 Subject: [PATCH 07/18] Using conda. --- test/consistency.jl | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/test/consistency.jl b/test/consistency.jl index a387580..85711ce 100644 --- a/test/consistency.jl +++ b/test/consistency.jl @@ -1,13 +1,12 @@ using Test -using PyCall using Conda +Conda.add("numpy") +Conda.add("mrcfile") -Conda.pip_interop(true) -Conda.pip("install", ["mrcfile", "numpy"]) - +using PyCall # Load the Python module/package -mrcfile = pyimport("mrcfile") numpy = pyimport("numpy") +mrcfile = pyimport("mrcfile") @testset "Consistency Test" begin function map_test(map_jl::Array{Float32,3}, map_py::Array{Float32,3}) From 3a7a88f8ea7bae18aaf4e6cae05f3a1e74e3454a Mon Sep 17 00:00:00 2001 From: Shu Li Date: Mon, 17 Jul 2023 04:51:01 -0400 Subject: [PATCH 08/18] Using CondPkg and PythonCall. --- .gitignore | 3 +- CondaPkg.toml | 3 ++ Project.toml | 6 +-- test/consistency.jl | 118 +++++++++++++++++++++++--------------------- 4 files changed, 71 insertions(+), 59 deletions(-) create mode 100644 CondaPkg.toml diff --git a/.gitignore b/.gitignore index aec6d51..0520531 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ *.jl.cov *.jl.mem /docs/build/ -Manifest.toml \ No newline at end of file +Manifest.toml +/.CondaPkg \ No newline at end of file diff --git a/CondaPkg.toml b/CondaPkg.toml new file mode 100644 index 0000000..eba2673 --- /dev/null +++ b/CondaPkg.toml @@ -0,0 +1,3 @@ +[deps] +mrcfile = "" +numpy = "" diff --git a/Project.toml b/Project.toml index fce402f..f6e6ad6 100644 --- a/Project.toml +++ b/Project.toml @@ -20,9 +20,9 @@ TranscodingStreams = "0.9" julia = "^1" [extras] -PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" -Conda = "8f4d0f93-b110-5947-807f-2305c1781a2d" +CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab" +PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" [targets] -test = ["Test", "PyCall", "Conda"] +test = ["Test", "CondaPkg", "PythonCall"] diff --git a/test/consistency.jl b/test/consistency.jl index 85711ce..eaac040 100644 --- a/test/consistency.jl +++ b/test/consistency.jl @@ -1,9 +1,7 @@ using Test -using Conda -Conda.add("numpy") -Conda.add("mrcfile") +using CondaPkg +using PythonCall -using PyCall # Load the Python module/package numpy = pyimport("numpy") mrcfile = pyimport("mrcfile") @@ -15,62 +13,62 @@ mrcfile = pyimport("mrcfile") # @test map_jl == map_py end - function header_test(header_jl::MRCHeader, header_py::PyObject) - @test header_jl.nx == convert(Int32, header_py.nx) - @test header_jl.ny == convert(Int32, header_py.ny) - @test header_jl.nz == convert(Int32, header_py.nz) - @test header_jl.mode == convert(Int32, header_py.mode) - @test header_jl.nxstart == convert(Int32, header_py.nxstart) - @test header_jl.nystart == convert(Int32, header_py.nystart) - @test header_jl.nzstart == convert(Int32, header_py.nzstart) - @test header_jl.mx == convert(Int32, header_py.mx) - @test header_jl.my == convert(Int32, header_py.my) - @test header_jl.mz == convert(Int32, header_py.mz) - @test header_jl.cella_x == convert(Float32, header_py.cella.x) - @test header_jl.cella_y == convert(Float32, header_py.cella.y) - @test header_jl.cella_z == convert(Float32, header_py.cella.z) - @test header_jl.cellb_alpha == convert(Float32, header_py.cellb.alpha) - @test header_jl.cellb_beta == convert(Float32, header_py.cellb.beta) - @test header_jl.cellb_gamma == convert(Float32, header_py.cellb.gamma) - @test header_jl.mapc == convert(Int32, header_py.mapc) - @test header_jl.mapr == convert(Int32, header_py.mapr) - @test header_jl.maps == convert(Int32, header_py.maps) - @test header_jl.dmin == convert(Float32, header_py.dmin) - @test header_jl.dmax == convert(Float32, header_py.dmax) - @test header_jl.dmean == convert(Float32, header_py.dmean) - @test header_jl.ispg == convert(Int32, header_py.ispg) - @test header_jl.nsymbt == convert(Int32, header_py.nsymbt) - @test String([UInt8(b) for b in header_jl.extra1]) == convert(String, header_py.extra1.tobytes()) - @test header_jl.exttyp == header_py.exttyp.tostring() - @test header_jl.nversion == convert(Int32, header_py.nversion) - @test String([UInt8(b) for b in header_jl.extra2]) == header_py.extra2.tobytes() - @test header_jl.origin_x == convert(Float32, header_py.origin.x) - @test header_jl.origin_y == convert(Float32, header_py.origin.y) - @test header_jl.origin_z == convert(Float32, header_py.origin.z) - @test header_jl.map == header_py.map.tostring() - @test collect(header_jl.machst) == convert(Vector{UInt8}, header_py.machst) - @test header_jl.rms == convert(Float32, header_py.rms) - @test header_jl.nlabl == convert(Int32, header_py.nlabl) - @test strip.(collect(header_py.label)) == collect(header_jl.label) + function header_test(header_jl::MRCHeader, header_py::Py) + @test Bool(header_jl.nx == header_py.nx) + @test Bool(header_jl.ny == header_py.ny) + @test Bool(header_jl.nz == header_py.nz) + @test Bool(header_jl.mode == header_py.mode) + @test Bool(header_jl.nxstart == header_py.nxstart) + @test Bool(header_jl.nystart == header_py.nystart) + @test Bool(header_jl.nzstart == header_py.nzstart) + @test Bool(header_jl.mx == header_py.mx) + @test Bool(header_jl.my == header_py.my) + @test Bool(header_jl.mz == header_py.mz) + @test Bool(header_jl.cella_x == header_py.cella.x) + @test Bool(header_jl.cella_y == header_py.cella.y) + @test Bool(header_jl.cella_z == header_py.cella.z) + @test Bool(header_jl.cellb_alpha == header_py.cellb.alpha) + @test Bool(header_jl.cellb_beta == header_py.cellb.beta) + @test Bool(header_jl.cellb_gamma == header_py.cellb.gamma) + @test Bool(header_jl.mapc == header_py.mapc) + @test Bool(header_jl.mapr == header_py.mapr) + @test Bool(header_jl.maps == header_py.maps) + @test Bool(header_jl.dmin == header_py.dmin) + @test Bool(header_jl.dmax == header_py.dmax) + @test Bool(header_jl.dmean == header_py.dmean) + @test Bool(header_jl.ispg == header_py.ispg) + @test Bool(header_jl.nsymbt == header_py.nsymbt) + @test [UInt8(b) for b in header_jl.extra1] == pyconvert(Vector{UInt8}, header_py.extra1.tobytes()) + @test [UInt8(b) for b in header_jl.exttyp] == pyconvert(Vector{UInt8}, header_py.exttyp.tobytes()) + @test Bool(header_jl.nversion == header_py.nversion) + @test ([UInt8(b) for b in header_jl.extra2]) == pyconvert(Vector{UInt8}, header_py.extra2.tobytes()) + @test Bool(header_jl.origin_x == header_py.origin.x) + @test Bool(header_jl.origin_y == header_py.origin.y) + @test Bool(header_jl.origin_z == header_py.origin.z) + @test [UInt8(b) for b in header_jl.map] == pyconvert(Vector{UInt8}, header_py.map.tobytes()) + @test [UInt8(b) for b in header_jl.machst] == pyconvert(Vector{UInt8}, header_py.machst.tobytes()) + @test Bool(header_jl.rms == header_py.rms) + @test Bool(header_jl.nlabl == header_py.nlabl) + @test join(header_jl.label) == + pyconvert(String, header_py.label.tostring().decode().strip()) end - function exh_test(exh_jl::MRCExtendedHeader, exh_py::PyObject) - @test String([UInt8(b) for b in exh_jl.data]) == exh_py.tobytes() + function exh_test(exh_jl::MRCExtendedHeader, exh_py::Py) + @test [UInt8(b) for b in exh_jl.data] == pyconvert(Vector{UInt8}, exh_py.tobytes()) end - @testset "emd_3001.map" begin - emd3001_path = "$(@__DIR__)/testdata/emd_3001.map" - emd3001_jl = read("$(@__DIR__)/testdata/emd_3001.map", MRCData) - map_jl = emd3001_jl.data - header_jl = header(emd3001_jl) - exh_jl = extendedheader(emd3001_jl) + function compare(map_path::String) + emd_jl = read(map_path, MRCData) + map_jl = emd_jl.data + header_jl = header(emd_jl) + exh_jl = extendedheader(emd_jl) - emd3001_py = mrcfile.open("$(@__DIR__)/testdata/emd_3001.map"; mode="r") - data_copy = numpy.copy(emd3001_py.data) # Create a copy of the data - map_py = convert(Array{Float32,3}, data_copy) - header_py = emd3001_py.header - exh_py = emd3001_py.extended_header - emd3001_py.close() # Make sure the file is closed even if an error occurs + emd_py = mrcfile.open(map_path; mode="r") + data_copy = numpy.copy(emd_py.data) # Create a copy of the data + map_py = pyconvert(Array{Float32,3}, data_copy) + header_py = emd_py.header + exh_py = emd_py.extended_header + emd_py.close() # Make sure the file is closed even if an error occurs @testset "map" begin map_test(map_jl, map_py) @@ -84,4 +82,14 @@ mrcfile = pyimport("mrcfile") exh_test(exh_jl, exh_py) end end + + @testset "emd_3001.map" begin + emd3001_path = "$(@__DIR__)/testdata/emd_3001.map" + compare(emd3001_path) + end + + @testset "emd_3197" begin + emd3197_path = "$(@__DIR__)/testdata/emd_3197.map" + compare(emd3197_path) + end end From d5bdac186de016b25ae004cc2b9a36e49bb2451b Mon Sep 17 00:00:00 2001 From: Shu Li Date: Mon, 17 Jul 2023 09:15:46 -0400 Subject: [PATCH 09/18] Update Project.toml Co-authored-by: Seth Axen --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index f6e6ad6..0002935 100644 --- a/Project.toml +++ b/Project.toml @@ -20,9 +20,9 @@ TranscodingStreams = "0.9" julia = "^1" [extras] -Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" CondaPkg = "992eb4ea-22a4-4c89-a5bb-47a3300528ab" PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] test = ["Test", "CondaPkg", "PythonCall"] From 3e6ab6b1e2b842fe179274327cdc7ec13788e968 Mon Sep 17 00:00:00 2001 From: Shu Li Date: Mon, 17 Jul 2023 09:15:58 -0400 Subject: [PATCH 10/18] Update Project.toml Co-authored-by: Seth Axen --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 0002935..68c47c3 100644 --- a/Project.toml +++ b/Project.toml @@ -25,4 +25,4 @@ PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" [targets] -test = ["Test", "CondaPkg", "PythonCall"] +test = ["CondaPkg", "PythonCall", "Test"] From 7c5db5908d59d955b395244f33e241d717bd102a Mon Sep 17 00:00:00 2001 From: Shu Li Date: Mon, 17 Jul 2023 09:16:59 -0400 Subject: [PATCH 11/18] Update test/consistency.jl Co-authored-by: Seth Axen --- test/consistency.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/consistency.jl b/test/consistency.jl index eaac040..ad1e73f 100644 --- a/test/consistency.jl +++ b/test/consistency.jl @@ -8,9 +8,11 @@ mrcfile = pyimport("mrcfile") @testset "Consistency Test" begin function map_test(map_jl::Array{Float32,3}, map_py::Array{Float32,3}) + # dimensions permuted due to row-major storage in Python and col-major in Julia + # see https://github.com/sethaxen/MRCFile.jl/issues/10 + @test_broken map_jl == map_py permuted_py = permutedims(map_py, (3, 2, 1)) @test map_jl == permuted_py - # @test map_jl == map_py end function header_test(header_jl::MRCHeader, header_py::Py) From 1cd1d5b7c6ef36853b794d564ed6fd4880ef6670 Mon Sep 17 00:00:00 2001 From: Shu Li Date: Mon, 17 Jul 2023 09:17:10 -0400 Subject: [PATCH 12/18] Update test/runtests.jl Co-authored-by: Seth Axen --- test/runtests.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/runtests.jl b/test/runtests.jl index 82a8ef7..aace6f0 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -6,5 +6,5 @@ using Test include("header.jl") include("io.jl") include("data.jl") - include("consistency.jl") + VERSION >= v"1.6" && include("consistency.jl") end From 7a5fc6a84d05d6840059e0576069d1052016dc68 Mon Sep 17 00:00:00 2001 From: Shu Li Date: Mon, 17 Jul 2023 09:27:47 -0400 Subject: [PATCH 13/18] Update consistency.jl --- test/consistency.jl | 144 ++++++++++++++++++++++---------------------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/test/consistency.jl b/test/consistency.jl index ad1e73f..970b455 100644 --- a/test/consistency.jl +++ b/test/consistency.jl @@ -6,92 +6,92 @@ using PythonCall numpy = pyimport("numpy") mrcfile = pyimport("mrcfile") -@testset "Consistency Test" begin - function map_test(map_jl::Array{Float32,3}, map_py::Array{Float32,3}) - # dimensions permuted due to row-major storage in Python and col-major in Julia - # see https://github.com/sethaxen/MRCFile.jl/issues/10 - @test_broken map_jl == map_py - permuted_py = permutedims(map_py, (3, 2, 1)) - @test map_jl == permuted_py - end +function compare_map(map_jl::Array{Float32,3}, map_py::Array{Float32,3}) + # dimensions permuted due to row-major storage in Python and col-major in Julia + # see https://github.com/sethaxen/MRCFile.jl/issues/10 + @test_broken map_jl == map_py + permuted_py = permutedims(map_py, (3, 2, 1)) + @test map_jl == permuted_py +end - function header_test(header_jl::MRCHeader, header_py::Py) - @test Bool(header_jl.nx == header_py.nx) - @test Bool(header_jl.ny == header_py.ny) - @test Bool(header_jl.nz == header_py.nz) - @test Bool(header_jl.mode == header_py.mode) - @test Bool(header_jl.nxstart == header_py.nxstart) - @test Bool(header_jl.nystart == header_py.nystart) - @test Bool(header_jl.nzstart == header_py.nzstart) - @test Bool(header_jl.mx == header_py.mx) - @test Bool(header_jl.my == header_py.my) - @test Bool(header_jl.mz == header_py.mz) - @test Bool(header_jl.cella_x == header_py.cella.x) - @test Bool(header_jl.cella_y == header_py.cella.y) - @test Bool(header_jl.cella_z == header_py.cella.z) - @test Bool(header_jl.cellb_alpha == header_py.cellb.alpha) - @test Bool(header_jl.cellb_beta == header_py.cellb.beta) - @test Bool(header_jl.cellb_gamma == header_py.cellb.gamma) - @test Bool(header_jl.mapc == header_py.mapc) - @test Bool(header_jl.mapr == header_py.mapr) - @test Bool(header_jl.maps == header_py.maps) - @test Bool(header_jl.dmin == header_py.dmin) - @test Bool(header_jl.dmax == header_py.dmax) - @test Bool(header_jl.dmean == header_py.dmean) - @test Bool(header_jl.ispg == header_py.ispg) - @test Bool(header_jl.nsymbt == header_py.nsymbt) - @test [UInt8(b) for b in header_jl.extra1] == pyconvert(Vector{UInt8}, header_py.extra1.tobytes()) - @test [UInt8(b) for b in header_jl.exttyp] == pyconvert(Vector{UInt8}, header_py.exttyp.tobytes()) - @test Bool(header_jl.nversion == header_py.nversion) - @test ([UInt8(b) for b in header_jl.extra2]) == pyconvert(Vector{UInt8}, header_py.extra2.tobytes()) - @test Bool(header_jl.origin_x == header_py.origin.x) - @test Bool(header_jl.origin_y == header_py.origin.y) - @test Bool(header_jl.origin_z == header_py.origin.z) - @test [UInt8(b) for b in header_jl.map] == pyconvert(Vector{UInt8}, header_py.map.tobytes()) - @test [UInt8(b) for b in header_jl.machst] == pyconvert(Vector{UInt8}, header_py.machst.tobytes()) - @test Bool(header_jl.rms == header_py.rms) - @test Bool(header_jl.nlabl == header_py.nlabl) - @test join(header_jl.label) == - pyconvert(String, header_py.label.tostring().decode().strip()) - end +function compare_header(header_jl::MRCHeader, header_py::Py) + @test Bool(header_jl.nx == header_py.nx) + @test Bool(header_jl.ny == header_py.ny) + @test Bool(header_jl.nz == header_py.nz) + @test Bool(header_jl.mode == header_py.mode) + @test Bool(header_jl.nxstart == header_py.nxstart) + @test Bool(header_jl.nystart == header_py.nystart) + @test Bool(header_jl.nzstart == header_py.nzstart) + @test Bool(header_jl.mx == header_py.mx) + @test Bool(header_jl.my == header_py.my) + @test Bool(header_jl.mz == header_py.mz) + @test Bool(header_jl.cella_x == header_py.cella.x) + @test Bool(header_jl.cella_y == header_py.cella.y) + @test Bool(header_jl.cella_z == header_py.cella.z) + @test Bool(header_jl.cellb_alpha == header_py.cellb.alpha) + @test Bool(header_jl.cellb_beta == header_py.cellb.beta) + @test Bool(header_jl.cellb_gamma == header_py.cellb.gamma) + @test Bool(header_jl.mapc == header_py.mapc) + @test Bool(header_jl.mapr == header_py.mapr) + @test Bool(header_jl.maps == header_py.maps) + @test Bool(header_jl.dmin == header_py.dmin) + @test Bool(header_jl.dmax == header_py.dmax) + @test Bool(header_jl.dmean == header_py.dmean) + @test Bool(header_jl.ispg == header_py.ispg) + @test Bool(header_jl.nsymbt == header_py.nsymbt) + @test [UInt8(b) for b in header_jl.extra1] == pyconvert(Vector{UInt8}, header_py.extra1.tobytes()) + @test [UInt8(b) for b in header_jl.exttyp] == pyconvert(Vector{UInt8}, header_py.exttyp.tobytes()) + @test Bool(header_jl.nversion == header_py.nversion) + @test ([UInt8(b) for b in header_jl.extra2]) == pyconvert(Vector{UInt8}, header_py.extra2.tobytes()) + @test Bool(header_jl.origin_x == header_py.origin.x) + @test Bool(header_jl.origin_y == header_py.origin.y) + @test Bool(header_jl.origin_z == header_py.origin.z) + @test [UInt8(b) for b in header_jl.map] == pyconvert(Vector{UInt8}, header_py.map.tobytes()) + @test [UInt8(b) for b in header_jl.machst] == pyconvert(Vector{UInt8}, header_py.machst.tobytes()) + @test Bool(header_jl.rms == header_py.rms) + @test Bool(header_jl.nlabl == header_py.nlabl) + @test join(header_jl.label) == + pyconvert(String, header_py.label.tostring().decode().strip()) +end - function exh_test(exh_jl::MRCExtendedHeader, exh_py::Py) - @test [UInt8(b) for b in exh_jl.data] == pyconvert(Vector{UInt8}, exh_py.tobytes()) - end +function compare_extendedheader(exh_jl::MRCExtendedHeader, exh_py::Py) + @test [UInt8(b) for b in exh_jl.data] == pyconvert(Vector{UInt8}, exh_py.tobytes()) +end - function compare(map_path::String) - emd_jl = read(map_path, MRCData) - map_jl = emd_jl.data - header_jl = header(emd_jl) - exh_jl = extendedheader(emd_jl) +function compare_mrcfile(map_path::String) + emd_jl = read(map_path, MRCData) + map_jl = emd_jl.data + header_jl = header(emd_jl) + exh_jl = extendedheader(emd_jl) - emd_py = mrcfile.open(map_path; mode="r") - data_copy = numpy.copy(emd_py.data) # Create a copy of the data - map_py = pyconvert(Array{Float32,3}, data_copy) - header_py = emd_py.header - exh_py = emd_py.extended_header - emd_py.close() # Make sure the file is closed even if an error occurs + emd_py = mrcfile.open(map_path; mode="r") + data_copy = numpy.copy(emd_py.data) # Create a copy of the data + map_py = pyconvert(Array{Float32,3}, data_copy) + header_py = emd_py.header + exh_py = emd_py.extended_header + emd_py.close() # Make sure the file is closed even if an error occurs - @testset "map" begin - map_test(map_jl, map_py) - end + @testset "map" begin + compare_map(map_jl, map_py) + end - @testset "header" begin - header_test(header_jl, header_py) - end + @testset "header" begin + compare_header(header_jl, header_py) + end - @testset "extendedheader" begin - exh_test(exh_jl, exh_py) - end + @testset "extendedheader" begin + compare_extendedheader(exh_jl, exh_py) end +end +@testset "Consistency Test" begin @testset "emd_3001.map" begin emd3001_path = "$(@__DIR__)/testdata/emd_3001.map" - compare(emd3001_path) + compare_mrcfile(emd3001_path) end @testset "emd_3197" begin emd3197_path = "$(@__DIR__)/testdata/emd_3197.map" - compare(emd3197_path) + compare_mrcfile(emd3197_path) end end From f41d85f4e827ee0084c8300461f35d856e682242 Mon Sep 17 00:00:00 2001 From: Shu Li Date: Mon, 17 Jul 2023 09:38:27 -0400 Subject: [PATCH 14/18] Update CondaPkg.toml. --- CondaPkg.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CondaPkg.toml b/CondaPkg.toml index eba2673..c681d56 100644 --- a/CondaPkg.toml +++ b/CondaPkg.toml @@ -1,3 +1,3 @@ [deps] -mrcfile = "" -numpy = "" +mrcfile = ">=1.2,<2.0" +numpy = ">=1.1,<2.0" From 90263804cbfc28e6fa88e5c481ada04501a7e70b Mon Sep 17 00:00:00 2001 From: Shu Li Date: Mon, 17 Jul 2023 15:45:04 -0400 Subject: [PATCH 15/18] Wrap MRCData in row-major order. --- src/data.jl | 31 ++++++++++++++++++++----------- src/header.jl | 2 +- src/io.jl | 16 ++++++++-------- test/consistency.jl | 10 +++------- test/data.jl | 25 +++++++++++++------------ test/header.jl | 2 +- test/io.jl | 4 ++-- 7 files changed, 48 insertions(+), 42 deletions(-) diff --git a/src/data.jl b/src/data.jl index c9f72cf..6270b97 100644 --- a/src/data.jl +++ b/src/data.jl @@ -25,24 +25,26 @@ Create an array of the specified size. struct MRCData{T<:Number,N,EH,D} <: AbstractArray{T,N} header::MRCHeader extendedheader::EH - data::D + original_data::D +end +function ReadMRCData(header, extendedheader, data::AbstractArray{T,N}) where {T,N} + return MRCData{T,N,typeof(extendedheader),typeof(data)}(header, extendedheader, data) end function MRCData(header, extendedheader, data::AbstractArray{T,N}) where {T,N} + data = permutedims(data, reverse(1:N)) return MRCData{T,N,typeof(extendedheader),typeof(data)}(header, extendedheader, data) end function MRCData(header=MRCHeader(), extendedheader=MRCExtendedHeader()) data_size = size(header) - data_length = prod(data_size) dtype = datatype(header) dims = ndims(header) - s = ntuple(i -> data_size[i], dims) - data = Array{dtype,dims}(undef, s) - return MRCData(header, extendedheader, data) + original_data = Array{dtype,dims}(undef, reverse(data_size)) + return ReadMRCData(header, extendedheader, original_data) end function MRCData(size::NTuple{3,<:Integer}) header = MRCHeader() for i in 1:3 - setproperty!(header, (:nx, :ny, :nz)[i], size[i]) + setproperty!(header, (:nz, :ny, :nx)[i], size[i]) end return MRCData(header) end @@ -62,6 +64,13 @@ Get extended header. """ extendedheader(d::MRCData) = d.extendedheader +""" + data(d::MRCData) -> AbstractArray + +Get header. +""" +data(d::MRCData) = parent(d) + """ updateheader!(data::MRCData; statistics = true) @@ -74,7 +83,7 @@ function updateheader!(d::MRCData; statistics=true) # update size s = size(d) for i in eachindex(s) - setproperty!(h, (:nx, :ny, :nz)[i], s[i]) + setproperty!(h, (:nz, :ny, :nx)[i], s[i]) end # update space group @@ -104,7 +113,7 @@ function updateheader!(d::MRCData; statistics=true) end # Array overloads -@inline Base.parent(d::MRCData) = d.data +@inline Base.parent(d::MRCData) = PermutedDimsArray(d.original_data, reverse(1:ndims(d.original_data))) @inline Base.getindex(d::MRCData, idx::Int...) = getindex(parent(d), idx...) @@ -134,21 +143,21 @@ end Return an iterator over map rows. """ -eachmaprow(d::MRCData) = eachslice(d; dims=header(d).mapr) +eachmaprow(d::MRCData) = eachslice(d; dims=ndims(d)-header(d).mapr+1) """ eachmapcol(d::MRCData) Return an iterator over columns. """ -eachmapcol(d::MRCData) = eachslice(d; dims=header(d).mapc) +eachmapcol(d::MRCData) = eachslice(d; dims=ndims(d)-header(d).mapc+1) """ eachmapsection(d::MRCData) Return an iterator over sections. """ -eachmapsection(d::MRCData) = eachslice(d; dims=header(d).maps) +eachmapsection(d::MRCData) = eachslice(d; dims=ndims(d)-header(d).maps+1) """ eachstackunit(d::MRCData) diff --git a/src/header.jl b/src/header.jl index 270c437..bd89365 100644 --- a/src/header.jl +++ b/src/header.jl @@ -136,7 +136,7 @@ end return offsets end -Base.size(h::MRCHeader) = (h.nx, h.ny, h.nz) +Base.size(h::MRCHeader) = (h.nz, h.ny, h.nx) Base.length(h::MRCHeader) = prod(size(h)) diff --git a/src/io.jl b/src/io.jl index b84d765..bbb4b54 100644 --- a/src/io.jl +++ b/src/io.jl @@ -19,9 +19,9 @@ function Base.read(io::IO, ::Type{T}; compress=:auto) where {T<:MRCData} header = read(newio, MRCHeader) extendedheader = read(newio, MRCExtendedHeader; header=header) d = MRCData(header, extendedheader) - read!(newio, d.data) + read!(newio, d.original_data) close(newio) - map!(bswaptoh(header.machst), d.data, d.data) + map!(bswaptoh(header.machst), d.original_data, d.original_data) return d end function Base.read(fn::AbstractString, ::Type{T}; compress=:auto) where {T<:MRCData} @@ -64,8 +64,8 @@ function read_mmap(io::IO, ::Type{MRCData}) head = read(io, MRCHeader) exthead = read(io, MRCExtendedHeader; header=head) arraytype = Array{datatype(head),ndims(head)} - data = Mmap.mmap(io, arraytype, size(head)[1:ndims(head)]) - return MRCData(head, exthead, data) + data = Mmap.mmap(io, arraytype, reverse(size(head))) + return ReadMRCData(head, exthead, data) end function read_mmap(path::AbstractString, T::Type{MRCData}) return open(path, "r") do io @@ -105,7 +105,7 @@ function Base.write( sz = write(newio, h) sz += write(newio, extendedheader(d)) T = datatype(h) - data = parent(d) + original_data = d.original_data fswap = bswapfromh(h.machst) buffer_size = div(buffer_size, sizeof(T)) if buffer === nothing @@ -120,15 +120,15 @@ function Base.write( # If `buffer` was provided as a parameter then `buffer_size` is redundant and # we must make sure that it matches `buffer`. buffer_size = length(buffer) - vlen = length(data) + vlen = length(original_data) vrem = vlen % buffer_size @inbounds @views begin if vrem != 0 - buffer[1:vrem] .= fswap.(T.(data[1:vrem])) + buffer[1:vrem] .= fswap.(T.(original_data[1:vrem])) sz += write(newio, buffer[1:vrem]) end for i in (vrem + 1):buffer_size:vlen - buffer .= fswap.(T.(data[i:(i + buffer_size - 1)])) + buffer .= fswap.(T.(original_data[i:(i + buffer_size - 1)])) sz += write(newio, buffer) end end diff --git a/test/consistency.jl b/test/consistency.jl index 970b455..efd24b7 100644 --- a/test/consistency.jl +++ b/test/consistency.jl @@ -6,12 +6,8 @@ using PythonCall numpy = pyimport("numpy") mrcfile = pyimport("mrcfile") -function compare_map(map_jl::Array{Float32,3}, map_py::Array{Float32,3}) - # dimensions permuted due to row-major storage in Python and col-major in Julia - # see https://github.com/sethaxen/MRCFile.jl/issues/10 - @test_broken map_jl == map_py - permuted_py = permutedims(map_py, (3, 2, 1)) - @test map_jl == permuted_py +function compare_map(map_jl::AbstractArray{Float32,3}, map_py::AbstractArray{Float32,3}) + @test map_jl == map_py end function compare_header(header_jl::MRCHeader, header_py::Py) @@ -60,7 +56,7 @@ end function compare_mrcfile(map_path::String) emd_jl = read(map_path, MRCData) - map_jl = emd_jl.data + map_jl = data(emd_jl) header_jl = header(emd_jl) exh_jl = extendedheader(emd_jl) diff --git a/test/data.jl b/test/data.jl index 2125b06..f44f4a8 100644 --- a/test/data.jl +++ b/test/data.jl @@ -2,11 +2,11 @@ @testset "MRCData(header, extendedheader, data)" begin h = MRCHeader() exh = MRCExtendedHeader() - data = Array{Float32,3}(undef, (2, 2, 2)) - d = MRCData(h, exh, data) + map_data = Array{Float32,3}(undef, (2, 2, 2)) + d = MRCData(h, exh, map_data) @test d.header === h @test d.extendedheader === exh - @test d.data === data + @test data(d) == map_data @test eltype(d) === Float32 @test ndims(d) === 3 end @@ -24,15 +24,16 @@ @testset "MRCData(size::NTuple{3,<:Integer})" begin d = MRCData((10, 20, 30)) @test size(d) == (10, 20, 30) - @test size(d.data) == (10, 20, 30) + @test size(data(d)) == (10, 20, 30) + @test size(d.original_data) == (30, 20, 10) @test ndims(d) == 3 - @test ndims(d.data) == 3 + @test ndims(d.original_data) == 3 @testset "header(d::MRCData)" begin h = header(d) - @test h.nx == 10 + @test h.nx == 30 @test h.ny == 20 - @test h.nz == 30 + @test h.nz == 10 end end @@ -42,10 +43,10 @@ @test size(d1) == size(d2) end - data = fill(1.0f0, (2, 2, 2)) + map_data = fill(1.0f0, (2, 2, 2)) h = MRCHeader() exh = MRCExtendedHeader() - d = MRCData(h, exh, data) + d = MRCData(h, exh, map_data) @testset "getindex" begin @test d[1, 1, 1] == 1.0f0 @@ -109,18 +110,18 @@ end @testset "IndexStyle" begin - @test Base.IndexStyle(typeof(d)) == Base.IndexStyle(typeof(d.data)) + @test Base.IndexStyle(typeof(d)) == Base.IndexStyle(typeof(d.original_data)) end end @testset "updateheader!(d::MRCData; statistics=true)" begin - data = fill(1.0f0, (2, 2, 2)) + map_data = fill(1.0f0, (2, 2, 2)) h = MRCHeader() @test h.nx == 0 @test h.ny == 0 @test h.nz == 0 exh = MRCExtendedHeader() - d = MRCData(h, exh, data) + d = MRCData(h, exh, map_data) updateheader!(d) @test d.header.dmin == 1.0f0 @test d.header.dmax == 1.0f0 diff --git a/test/header.jl b/test/header.jl index 6c08d29..2351e60 100644 --- a/test/header.jl +++ b/test/header.jl @@ -178,7 +178,7 @@ end @testset "size(::MRCHeader)" begin h = MRCHeader(; nx=10, ny=20, nz=30) - @test size(h) == (10, 20, 30) + @test size(h) == (30, 20, 10) end @testset "length(::MRCHeader)" begin diff --git a/test/io.jl b/test/io.jl index f0fe4bf..ff80059 100644 --- a/test/io.jl +++ b/test/io.jl @@ -8,9 +8,9 @@ h = header(emd3001) eh = extendedheader(emd3001) p = parent(emd3001) - @test h.nx === Int32(size(p)[1]) === Int32(73) + @test h.nx === Int32(size(p)[3]) === Int32(73) @test h.ny === Int32(size(p)[2]) === Int32(43) - @test h.nz === Int32(size(p)[3]) === Int32(25) + @test h.nz === Int32(size(p)[1]) === Int32(25) @test MRCFile.datatype(h.mode) === eltype(p) === Float32 @test h.nxstart === Int32(0) @test h.nystart === Int32(-21) From 0b6216eb1e8aab664437871be4d750f0ecc5e6e3 Mon Sep 17 00:00:00 2001 From: Shu Li Date: Mon, 17 Jul 2023 22:57:23 -0400 Subject: [PATCH 16/18] Change to row-major order and modify docs. --- Project.toml | 2 +- README.md | 15 +++++- docs/make.jl | 7 ++- docs/src/documentation.md | 102 ++++++++++++++++++++++++++++++++++++++ docs/src/examples.md | 32 ++++++++++++ docs/src/index.md | 8 +-- src/data.jl | 8 +-- 7 files changed, 160 insertions(+), 14 deletions(-) create mode 100644 docs/src/documentation.md create mode 100644 docs/src/examples.md mode change 100644 => 120000 docs/src/index.md diff --git a/Project.toml b/Project.toml index 68c47c3..b7816b5 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "MRCFile" uuid = "c18c01d3-0ab9-49c3-bd2d-8f2e64a2b7a5" authors = ["Seth Axen "] -version = "0.1.2" +version = "0.1.3" [deps] CodecBzip2 = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd" diff --git a/README.md b/README.md index 7e3cc04..cfd6592 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ # MRCFile.jl + ![Lifecycle](https://img.shields.io/badge/lifecycle-experimental-orange.svg) [![Build Status](https://github.com/sethaxen/MRCFile.jl/workflows/CI/badge.svg)](https://github.com/sethaxen/MRCFile.jl/actions?query=workflow%3ACI+branch%main) [![Coverage](https://codecov.io/gh/sethaxen/MRCFile.jl/branch/main/graph/badge.svg)](https://codecov.io/gh/sethaxen/MRCFile.jl) @@ -6,12 +7,16 @@ [![Dev](https://img.shields.io/badge/docs-dev-blue.svg)](https://sethaxen.github.io/MRCFile.jl/dev) [![Code Style: Blue](https://img.shields.io/badge/code%20style-blue-4495d1.svg)](https://github.com/invenia/BlueStyle) +## Description + MRCFile.jl implements the [MRC2014 format](https://www.ccpem.ac.uk/mrc_format/mrc2014.php) for storing image and volume data such as those produced by electron microscopy. It offers the ability to read, edit, and write MRC files, as well as utility functions for extracting useful information from the headers. The key type is `MRCData`, which contains the contents of the MRC file, accessible with `header` and `extendedheader`. `MRCData` is an `AbstractArray` whose elements are those of the data portion of the file and can be accessed or modified accordingly. +**Notice:** Since `1.1.3`, the `MRCData` is represent as row-major order to meet the requirement of [MRC2014 spec](https://www.ccpem.ac.uk/mrc_format/mrc2014.php). + ## Installation ```julia @@ -19,6 +24,14 @@ using Pkg Pkg.add("MRCFile") ``` +or install from the Julia package REPL, which can be accessed by pressing `]` from the Julia REPL: + +```julia +add MRCFile +``` + +See the [documentation](https://sethaxen.github.io/MRCFile.jl/stable/) for information on how to use MRCFile. + ## Example This example downloads a map of [TRPV1](https://www.emdataresource.org/EMD-5778) and animates slices taken through the map. @@ -50,7 +63,7 @@ gif(anim, "emd-$(emdid)_slices.gif", fps = 30) ![EMD-5778 slices](https://github.com/sethaxen/MRCFile.jl/blob/main/docs/src/assets/emd-5778_slices.gif) -# Reading a map as a memory-mapped array +## Reading a map as a memory-mapped array MRC files can be huge. It is convenient then to load their data as [memory-mapped arrays](https://docs.julialang.org/en/v1/stdlib/Mmap/). diff --git a/docs/make.jl b/docs/make.jl index c545d39..ff0b7bc 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -10,7 +10,12 @@ makedocs(; canonical="https://sethaxen.github.io/MRCFile.jl", assets=String[], ), - pages=["Home" => "index.md", "API" => "api.md"], + pages=[ + "Home" => "index.md", + "Documentation" => "documentation.md", + "Examples" => "examples.md", + "API" => "api.md", + ], ) deploydocs(; repo="github.com/sethaxen/MRCFile.jl", devbranch="main") diff --git a/docs/src/documentation.md b/docs/src/documentation.md new file mode 100644 index 0000000..8082f67 --- /dev/null +++ b/docs/src/documentation.md @@ -0,0 +1,102 @@ +# MRCFile documentation + +MRCFile.jl implements the [MRC2014 format](https://www.ccpem.ac.uk/mrc_format/mrc2014.php) for storing image and volume data such as those produced by electron microscopy. +It offers the ability to read, edit, and write MRC files, as well as utility functions for extracting useful information from the headers. + +The key type is `MRCData`, which contains the contents of the MRC file, accessible with `header` and `extendedheader`. +`MRCData` is an `AbstractArray` whose elements are those of the data portion of the file and can be accessed or modified accordingly. +The `data`, `header` and `extendedheader` are consistent with [mrcfile](https://github.com/ccpem/mrcfile). +Help on individual functions can be found in the API section or by using `?function name` from within Julia. + +## Opening MRC files + +MRC files can be opened using the `read()` or `read_mmap()` functions. These return an instance of the `MRCData` struct. + +```julia +using MRCFile + +dmap = read("/path/to/mrc/file.mrc", MRCData) +h = header(dmap) +exh = extendedheader(dmap) + +dmin, dmax = extrema(h) +drange = dmax - dmin +``` + +It is efficient to load the data as [memory-mapped arrays](https://docs.julialang.org/en/v1/stdlib/Mmap/). + +``` +# Open the file in memory-mapped mode +dmap = read_mmap("/path/to/mrc/file.mrc", MRCData) +``` + +## Handling compressed files + +All the functions above can also handle `.gz` or `.bz2` compressed files. + +```julia +using MRCFile + +dmap = read("/path/to/mrc/file.map.gz", MRCData) +# or +dmap = read("/path/to/mrc/file.map.bz2", MRCData) +``` + +## Write MRC file + +MRC files can be saved using the `write()` function. + +```julia +write("/path/to/mrc/file.mrc", dmap) +``` + +## Accessing the header + +The variables in header can be accessed using the following variable names, suppose `h=MRCHeader`: + +| Variable name | Access | Type | +| :-------------- | :------------ | :---------------- | +| NX | h.nx | Int32 | +| NY | h.ny | Int32 | +| NZ | h.nz | Int32 | +| MODE | h.mode | Int32 | +| NXSTART | h.nxstart | Int32 | +| NYSTART | h.nystart | Int32 | +| NZSTART | h.nzstart | Int32 | +| MX | h.mx | Int32 | +| MY | h.my | Int32 | +| MZ | h.mz | Int32 | +| CELLA_X | h.cella_x | Float32 | +| CELLA_Y | h.cella_y | Float32 | +| CELLA_Z | h.cella_z | Float32 | +| CELLB_alpha | h.cellb_alpha | Float32 | +| CELLB_beta | h.cellb_beta | Float32 | +| CELLB_gamma | h.cellb_gamma | Float32 | +| MAPC | h.mapc | Int32 | +| MAPR | h.mapc | Int32 | +| MAPS | h.mapc | Int32 | +| DMIN | h.dmin | Float32 | +| DMAX | h.dmax | Float32 | +| DMEAN | h.dmean | Float32 | +| ISPG | h.ispg | Int32 | +| NSYMBP | h.nsymbt | Int32 | +| EXTRA1 | h.extra1 | NTuple{8,UInt8} | +| EXYTYP | h.exttyp | String | +| NVERSION | h.nversion | Int32 | +| EXTRA2 | h.extra1 | NTuple{8,UInt8} | +| ORIGIN_X | h.origin_x | Float32 | +| ORIGIN_Y | h.origin_y | Float32 | +| ORIGIN_Z | h.origin_z | Float32 | +| MAP | h.map | String | +| MACHST | h.machst | NTuple{4,UInt8} | +| RMS | h.rms | Float32 | +| NLABL | h.nlabl | Int32 | +| LABEL | h.label | NTuple{10,String} | + +## Keeping the header and data in sync + +Update the header stored in `MRCData` from the data and its extended header. + +```julia +updateheader!(dmap) +``` diff --git a/docs/src/examples.md b/docs/src/examples.md new file mode 100644 index 0000000..0bac2e7 --- /dev/null +++ b/docs/src/examples.md @@ -0,0 +1,32 @@ +# MRCFile Example + +## Example 1 + +This example downloads a map of [TRPV1](https://www.emdataresource.org/EMD-5778) and animates slices taken through the map. + +To set-up this example, install FTPClient and Plots with + +```julia +using Pkg +Pkg.add("FTPClient") +Pkg.add("Plots") +``` + +```julia +using MRCFile, FTPClient, Plots + +emdid = 5778 # TRPV1 +ftp = FTP(hostname = "ftp.rcsb.org/pub/emdb/structures/EMD-$(emdid)/map") +dmap = read(download(ftp, "emd_$(emdid).map.gz"), MRCData) +close(ftp) +dmin, dmax = extrema(header(dmap)) +drange = dmax - dmin + +anim = @animate for xsection in eachmapsection(dmap) + plot(RGB.((xsection .- dmin) ./ drange)) +end + +gif(anim, "emd-$(emdid)_slices.gif", fps = 30) +``` + +![EMD-5778 slices](https://github.com/sethaxen/MRCFile.jl/blob/main/docs/src/assets/emd-5778_slices.gif) \ No newline at end of file diff --git a/docs/src/index.md b/docs/src/index.md deleted file mode 100644 index 9778506..0000000 --- a/docs/src/index.md +++ /dev/null @@ -1,7 +0,0 @@ -# MRCFile - -MRCFile.jl implements the [MRC2014 format](https://www.ccpem.ac.uk/mrc_format/mrc2014.php) for storing image and volume data such as those produced by electron microscopy. -It offers the ability to read, edit, and write MRC files, as well as utility functions for extracting useful information from the headers. - -The key type is `MRCData`, which contains the contents of the MRC file, accessible with `header` and `extendedheader`. -`MRCData` is an `AbstractArray` whose elements are those of the data portion of the file and can be accessed or modified accordingly. diff --git a/docs/src/index.md b/docs/src/index.md new file mode 120000 index 0000000..fe84005 --- /dev/null +++ b/docs/src/index.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/src/data.jl b/src/data.jl index 6270b97..45fd4d5 100644 --- a/src/data.jl +++ b/src/data.jl @@ -67,7 +67,7 @@ extendedheader(d::MRCData) = d.extendedheader """ data(d::MRCData) -> AbstractArray -Get header. +Get data. """ data(d::MRCData) = parent(d) @@ -143,21 +143,21 @@ end Return an iterator over map rows. """ -eachmaprow(d::MRCData) = eachslice(d; dims=ndims(d)-header(d).mapr+1) +eachmaprow(d::MRCData) = eachslice(d; dims=ndims(d) - header(d).mapr + 1) """ eachmapcol(d::MRCData) Return an iterator over columns. """ -eachmapcol(d::MRCData) = eachslice(d; dims=ndims(d)-header(d).mapc+1) +eachmapcol(d::MRCData) = eachslice(d; dims=ndims(d) - header(d).mapc + 1) """ eachmapsection(d::MRCData) Return an iterator over sections. """ -eachmapsection(d::MRCData) = eachslice(d; dims=ndims(d)-header(d).maps+1) +eachmapsection(d::MRCData) = eachslice(d; dims=ndims(d) - header(d).maps + 1) """ eachstackunit(d::MRCData) From 5005c3843a417f0c2968a71e6dfb799667b82e95 Mon Sep 17 00:00:00 2001 From: Shu Li Date: Mon, 17 Jul 2023 22:58:59 -0400 Subject: [PATCH 17/18] Format. --- src/data.jl | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/data.jl b/src/data.jl index 45fd4d5..202546d 100644 --- a/src/data.jl +++ b/src/data.jl @@ -113,7 +113,9 @@ function updateheader!(d::MRCData; statistics=true) end # Array overloads -@inline Base.parent(d::MRCData) = PermutedDimsArray(d.original_data, reverse(1:ndims(d.original_data))) +@inline function Base.parent(d::MRCData) + return PermutedDimsArray(d.original_data, reverse(1:ndims(d.original_data))) +end @inline Base.getindex(d::MRCData, idx::Int...) = getindex(parent(d), idx...) From c4a46c2e010a1755d1c23418615d3ea0ab953f53 Mon Sep 17 00:00:00 2001 From: Shu Li Date: Mon, 17 Jul 2023 23:08:57 -0400 Subject: [PATCH 18/18] Format again. --- src/io.jl | 1 - 1 file changed, 1 deletion(-) diff --git a/src/io.jl b/src/io.jl index bbe4109..bbb4b54 100644 --- a/src/io.jl +++ b/src/io.jl @@ -66,7 +66,6 @@ function read_mmap(io::IO, ::Type{MRCData}) arraytype = Array{datatype(head),ndims(head)} data = Mmap.mmap(io, arraytype, reverse(size(head))) return ReadMRCData(head, exthead, data) - end function read_mmap(path::AbstractString, T::Type{MRCData}) return open(path, "r") do io