Exclude test data files from crates.io package

Cargo.toml

@@ -5,6 +5,7 @@ version = "0.8.35"
 edition = '2018'
 authors = ["Henri Sivonen <hsivonen@hsivonen.fi>"]
 license = "(Apache-2.0 OR MIT) AND BSD-3-Clause"
+include = ["src/**/*.rs", "/data", "Cargo.toml", "COPYRIGHT", "LICENSE*", "README.md"]
 readme = "README.md"
 documentation = "https://docs.rs/encoding_rs/"
 homepage = "https://docs.rs/encoding_rs/"

generate-encoding-data.py

@@ -1620,7 +1620,7 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
 
 index = indexes["jis0208"]
 
-jis0208_in_file = open("src/test_data/jis0208_in.txt", "w")
+jis0208_in_file = open("tests/test_data/jis0208_in.txt", "w")
 jis0208_in_file.write(TEST_HEADER)
 for pointer in range(0, 94 * 94):
   (lead, trail) = divmod(pointer, 94)
@@ -1629,7 +1629,7 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
   jis0208_in_file.write("%s%s\n" % (chr(lead), chr(trail)))
 jis0208_in_file.close()
 
-jis0208_in_ref_file = open("src/test_data/jis0208_in_ref.txt", "w")
+jis0208_in_ref_file = open("tests/test_data/jis0208_in_ref.txt", "w")
 jis0208_in_ref_file.write(TEST_HEADER)
 for pointer in range(0, 94 * 94):
   code_point = index[pointer]
@@ -1639,8 +1639,8 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
     jis0208_in_ref_file.write(u"\uFFFD\n".encode("utf-8"))
 jis0208_in_ref_file.close()
 
-jis0208_out_file = open("src/test_data/jis0208_out.txt", "w")
-jis0208_out_ref_file = open("src/test_data/jis0208_out_ref.txt", "w")
+jis0208_out_file = open("tests/test_data/jis0208_out.txt", "w")
+jis0208_out_ref_file = open("tests/test_data/jis0208_out_ref.txt", "w")
 jis0208_out_file.write(TEST_HEADER)
 jis0208_out_ref_file.write(TEST_HEADER)
 for pointer in range(0, 94 * 94):
@@ -1657,7 +1657,7 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
 jis0208_out_file.close()
 jis0208_out_ref_file.close()
 
-shift_jis_in_file = open("src/test_data/shift_jis_in.txt", "w")
+shift_jis_in_file = open("tests/test_data/shift_jis_in.txt", "w")
 shift_jis_in_file.write(TEST_HEADER)
 for pointer in range(0, len(index)):
   (lead, trail) = divmod(pointer, 188)
@@ -1666,7 +1666,7 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
   shift_jis_in_file.write("%s%s\n" % (chr(lead), chr(trail)))
 shift_jis_in_file.close()
 
-shift_jis_in_ref_file = open("src/test_data/shift_jis_in_ref.txt", "w")
+shift_jis_in_ref_file = open("tests/test_data/shift_jis_in_ref.txt", "w")
 shift_jis_in_ref_file.write(TEST_HEADER)
 for pointer in range(0, len(index)):
   code_point = 0xE000 - 8836 + pointer if pointer >= 8836 and pointer <= 10715 else index[pointer]
@@ -1681,8 +1681,8 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
       shift_jis_in_ref_file.write(u"\uFFFD\n".encode("utf-8"))
 shift_jis_in_ref_file.close()
 
-shift_jis_out_file = open("src/test_data/shift_jis_out.txt", "w")
-shift_jis_out_ref_file = open("src/test_data/shift_jis_out_ref.txt", "w")
+shift_jis_out_file = open("tests/test_data/shift_jis_out.txt", "w")
+shift_jis_out_ref_file = open("tests/test_data/shift_jis_out_ref.txt", "w")
 shift_jis_out_file.write(TEST_HEADER)
 shift_jis_out_ref_file.write(TEST_HEADER)
 for pointer in range(0, 8272):
@@ -1710,7 +1710,7 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
 shift_jis_out_file.close()
 shift_jis_out_ref_file.close()
 
-iso_2022_jp_in_file = open("src/test_data/iso_2022_jp_in.txt", "w")
+iso_2022_jp_in_file = open("tests/test_data/iso_2022_jp_in.txt", "w")
 iso_2022_jp_in_file.write(TEST_HEADER)
 for pointer in range(0, 94 * 94):
   (lead, trail) = divmod(pointer, 94)
@@ -1719,7 +1719,7 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
   iso_2022_jp_in_file.write("\x1B$B%s%s\x1B(B\n" % (chr(lead), chr(trail)))
 iso_2022_jp_in_file.close()
 
-iso_2022_jp_in_ref_file = open("src/test_data/iso_2022_jp_in_ref.txt", "w")
+iso_2022_jp_in_ref_file = open("tests/test_data/iso_2022_jp_in_ref.txt", "w")
 iso_2022_jp_in_ref_file.write(TEST_HEADER)
 for pointer in range(0, 94 * 94):
   code_point = index[pointer]
@@ -1729,8 +1729,8 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
     iso_2022_jp_in_ref_file.write(u"\uFFFD\n".encode("utf-8"))
 iso_2022_jp_in_ref_file.close()
 
-iso_2022_jp_out_file = open("src/test_data/iso_2022_jp_out.txt", "w")
-iso_2022_jp_out_ref_file = open("src/test_data/iso_2022_jp_out_ref.txt", "w")
+iso_2022_jp_out_file = open("tests/test_data/iso_2022_jp_out.txt", "w")
+iso_2022_jp_out_ref_file = open("tests/test_data/iso_2022_jp_out_ref.txt", "w")
 iso_2022_jp_out_file.write(TEST_HEADER)
 iso_2022_jp_out_ref_file.write(TEST_HEADER)
 for pointer in range(0, 94 * 94):
@@ -1758,7 +1758,7 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
 
 index = indexes["euc-kr"]
 
-euc_kr_in_file = open("src/test_data/euc_kr_in.txt", "w")
+euc_kr_in_file = open("tests/test_data/euc_kr_in.txt", "w")
 euc_kr_in_file.write(TEST_HEADER)
 for pointer in range(0, len(index)):
   (lead, trail) = divmod(pointer, 190)
@@ -1767,7 +1767,7 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
   euc_kr_in_file.write("%s%s\n" % (chr(lead), chr(trail)))
 euc_kr_in_file.close()
 
-euc_kr_in_ref_file = open("src/test_data/euc_kr_in_ref.txt", "w")
+euc_kr_in_ref_file = open("tests/test_data/euc_kr_in_ref.txt", "w")
 euc_kr_in_ref_file.write(TEST_HEADER)
 for pointer in range(0, len(index)):
   code_point = index[pointer]
@@ -1782,8 +1782,8 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
       euc_kr_in_ref_file.write(u"\uFFFD\n".encode("utf-8"))
 euc_kr_in_ref_file.close()
 
-euc_kr_out_file = open("src/test_data/euc_kr_out.txt", "w")
-euc_kr_out_ref_file = open("src/test_data/euc_kr_out_ref.txt", "w")
+euc_kr_out_file = open("tests/test_data/euc_kr_out.txt", "w")
+euc_kr_out_ref_file = open("tests/test_data/euc_kr_out_ref.txt", "w")
 euc_kr_out_file.write(TEST_HEADER)
 euc_kr_out_ref_file.write(TEST_HEADER)
 for pointer in range(0, len(index)):
@@ -1799,7 +1799,7 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
 
 index = indexes["gb18030"]
 
-gb18030_in_file = open("src/test_data/gb18030_in.txt", "w")
+gb18030_in_file = open("tests/test_data/gb18030_in.txt", "w")
 gb18030_in_file.write(TEST_HEADER)
 for pointer in range(0, len(index)):
   (lead, trail) = divmod(pointer, 190)
@@ -1808,7 +1808,7 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
   gb18030_in_file.write("%s%s\n" % (chr(lead), chr(trail)))
 gb18030_in_file.close()
 
-gb18030_in_ref_file = open("src/test_data/gb18030_in_ref.txt", "w")
+gb18030_in_ref_file = open("tests/test_data/gb18030_in_ref.txt", "w")
 gb18030_in_ref_file.write(TEST_HEADER)
 for pointer in range(0, len(index)):
   code_point = index[pointer]
@@ -1823,8 +1823,8 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
       gb18030_in_ref_file.write(u"\uFFFD\n".encode("utf-8"))
 gb18030_in_ref_file.close()
 
-gb18030_out_file = open("src/test_data/gb18030_out.txt", "w")
-gb18030_out_ref_file = open("src/test_data/gb18030_out_ref.txt", "w")
+gb18030_out_file = open("tests/test_data/gb18030_out.txt", "w")
+gb18030_out_ref_file = open("tests/test_data/gb18030_out_ref.txt", "w")
 gb18030_out_file.write(TEST_HEADER)
 gb18030_out_ref_file.write(TEST_HEADER)
 for pointer in range(0, len(index)):
@@ -1842,7 +1842,7 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
 
 index = indexes["big5"]
 
-big5_in_file = open("src/test_data/big5_in.txt", "w")
+big5_in_file = open("tests/test_data/big5_in.txt", "w")
 big5_in_file.write(TEST_HEADER)
 for pointer in range(0, len(index)):
   (lead, trail) = divmod(pointer, 157)
@@ -1858,7 +1858,7 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
   1166: u"\u00EA\u030C",
 }
 
-big5_in_ref_file = open("src/test_data/big5_in_ref.txt", "w")
+big5_in_ref_file = open("tests/test_data/big5_in_ref.txt", "w")
 big5_in_ref_file.write(TEST_HEADER)
 for pointer in range(0, len(index)):
   if pointer in big5_two_characters.keys():
@@ -1895,8 +1895,8 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
        pointer_for_prefer_last.append(i)
        break
 
-big5_out_file = open("src/test_data/big5_out.txt", "w")
-big5_out_ref_file = open("src/test_data/big5_out_ref.txt", "w")
+big5_out_file = open("tests/test_data/big5_out.txt", "w")
+big5_out_ref_file = open("tests/test_data/big5_out_ref.txt", "w")
 big5_out_file.write(TEST_HEADER)
 big5_out_ref_file.write(TEST_HEADER)
 for pointer in range(((0xA1 - 0x81) * 157), len(index)):
@@ -1918,7 +1918,7 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
 
 index = indexes["jis0212"]
 
-jis0212_in_file = open("src/test_data/jis0212_in.txt", "w")
+jis0212_in_file = open("tests/test_data/jis0212_in.txt", "w")
 jis0212_in_file.write(TEST_HEADER)
 for pointer in range(0, len(index)):
   (lead, trail) = divmod(pointer, 94)
@@ -1927,7 +1927,7 @@ def write_variant_method(name, mut, arg_list, ret, variants, excludes, kind):
   jis0212_in_file.write("\x8F%s%s\n" % (chr(lead), chr(trail)))
 jis0212_in_file.close()
 
-jis0212_in_ref_file = open("src/test_data/jis0212_in_ref.txt", "w")
+jis0212_in_ref_file = open("tests/test_data/jis0212_in_ref.txt", "w")
 jis0212_in_ref_file.write(TEST_HEADER)
 for pointer in range(0, len(index)):
   code_point = index[pointer]

src/big5.rs

@@ -389,27 +389,6 @@ mod tests {
         encode_big5("\u{2550}", b"\xF9\xF9");
     }
 
-    #[test]
-    #[cfg_attr(miri, ignore)] // Miri is too slow
-    fn test_big5_decode_all() {
-        let input = include_bytes!("test_data/big5_in.txt");
-        let expectation = include_str!("test_data/big5_in_ref.txt");
-        let (cow, had_errors) = BIG5.decode_without_bom_handling(input);
-        assert!(had_errors, "Should have had errors.");
-        assert_eq!(&cow[..], expectation);
-    }
-
-    #[test]
-    #[cfg_attr(miri, ignore)] // Miri is too slow
-    fn test_big5_encode_all() {
-        let input = include_str!("test_data/big5_out.txt");
-        let expectation = include_bytes!("test_data/big5_out_ref.txt");
-        let (cow, encoding, had_errors) = BIG5.encode(input);
-        assert!(!had_errors, "Should not have had errors.");
-        assert_eq!(encoding, BIG5);
-        assert_eq!(&cow[..], &expectation[..]);
-    }
-
     #[test]
     #[cfg_attr(miri, ignore)] // Miri is too slow
     fn test_big5_encode_from_two_low_surrogates() {

src/euc_jp.rs

@@ -435,35 +435,4 @@ mod tests {
         encode_euc_jp("\u{3000}", b"\xA1\xA1");
         encode_euc_jp("\u{FF02}", b"\xFC\xFE");
     }
-
-    #[test]
-    #[cfg_attr(miri, ignore)] // Miri is too slow
-    fn test_jis0208_decode_all() {
-        let input = include_bytes!("test_data/jis0208_in.txt");
-        let expectation = include_str!("test_data/jis0208_in_ref.txt");
-        let (cow, had_errors) = EUC_JP.decode_without_bom_handling(input);
-        assert!(had_errors, "Should have had errors.");
-        assert_eq!(&cow[..], expectation);
-    }
-
-    #[test]
-    #[cfg_attr(miri, ignore)] // Miri is too slow
-    fn test_jis0208_encode_all() {
-        let input = include_str!("test_data/jis0208_out.txt");
-        let expectation = include_bytes!("test_data/jis0208_out_ref.txt");
-        let (cow, encoding, had_errors) = EUC_JP.encode(input);
-        assert!(!had_errors, "Should not have had errors.");
-        assert_eq!(encoding, EUC_JP);
-        assert_eq!(&cow[..], &expectation[..]);
-    }
-
-    #[test]
-    #[cfg_attr(miri, ignore)] // Miri is too slow
-    fn test_jis0212_decode_all() {
-        let input = include_bytes!("test_data/jis0212_in.txt");
-        let expectation = include_str!("test_data/jis0212_in_ref.txt");
-        let (cow, had_errors) = EUC_JP.decode_without_bom_handling(input);
-        assert!(had_errors, "Should have had errors.");
-        assert_eq!(&cow[..], expectation);
-    }
 }

src/euc_kr.rs

@@ -405,27 +405,6 @@ mod tests {
         encode_euc_kr("\u{8A70}", b"\xFD\xFE");
     }
 
-    #[test]
-    #[cfg_attr(miri, ignore)] // Miri is too slow
-    fn test_euc_kr_decode_all() {
-        let input = include_bytes!("test_data/euc_kr_in.txt");
-        let expectation = include_str!("test_data/euc_kr_in_ref.txt");
-        let (cow, had_errors) = EUC_KR.decode_without_bom_handling(input);
-        assert!(had_errors, "Should have had errors.");
-        assert_eq!(&cow[..], expectation);
-    }
-
-    #[test]
-    #[cfg_attr(miri, ignore)] // Miri is too slow
-    fn test_euc_kr_encode_all() {
-        let input = include_str!("test_data/euc_kr_out.txt");
-        let expectation = include_bytes!("test_data/euc_kr_out_ref.txt");
-        let (cow, encoding, had_errors) = EUC_KR.encode(input);
-        assert!(!had_errors, "Should not have had errors.");
-        assert_eq!(encoding, EUC_KR);
-        assert_eq!(&cow[..], &expectation[..]);
-    }
-
     #[test]
     fn test_euc_kr_encode_from_two_low_surrogates() {
         let expectation = b"��";

src/gb18030.rs

@@ -759,27 +759,6 @@ mod tests {
         encode_gb18030("\u{E817}", b"\xFE\x52");
     }
 
-    #[test]
-    #[cfg_attr(miri, ignore)] // Miri is too slow
-    fn test_gb18030_decode_all() {
-        let input = include_bytes!("test_data/gb18030_in.txt");
-        let expectation = include_str!("test_data/gb18030_in_ref.txt");
-        let (cow, had_errors) = GB18030.decode_without_bom_handling(input);
-        assert!(!had_errors, "Should not have had errors.");
-        assert_eq!(&cow[..], expectation);
-    }
-
-    #[test]
-    #[cfg_attr(miri, ignore)] // Miri is too slow
-    fn test_gb18030_encode_all() {
-        let input = include_str!("test_data/gb18030_out.txt");
-        let expectation = include_bytes!("test_data/gb18030_out_ref.txt");
-        let (cow, encoding, had_errors) = GB18030.encode(input);
-        assert!(!had_errors, "Should not have had errors.");
-        assert_eq!(encoding, GB18030);
-        assert_eq!(&cow[..], &expectation[..]);
-    }
-
     #[test]
     fn test_gb18030_encode_from_utf16_max_length() {
         let mut output = [0u8; 20];

src/iso_2022_jp.rs

@@ -974,27 +974,6 @@ mod tests {
         }
     }
 
-    #[test]
-    #[cfg_attr(miri, ignore)] // Miri is too slow
-    fn test_iso_2022_jp_decode_all() {
-        let input = include_bytes!("test_data/iso_2022_jp_in.txt");
-        let expectation = include_str!("test_data/iso_2022_jp_in_ref.txt");
-        let (cow, had_errors) = ISO_2022_JP.decode_without_bom_handling(input);
-        assert!(had_errors, "Should have had errors.");
-        assert_eq!(&cow[..], expectation);
-    }
-
-    #[test]
-    #[cfg_attr(miri, ignore)] // Miri is too slow
-    fn test_iso_2022_jp_encode_all() {
-        let input = include_str!("test_data/iso_2022_jp_out.txt");
-        let expectation = include_bytes!("test_data/iso_2022_jp_out_ref.txt");
-        let (cow, encoding, had_errors) = ISO_2022_JP.encode(input);
-        assert!(!had_errors, "Should not have had errors.");
-        assert_eq!(encoding, ISO_2022_JP);
-        assert_eq!(&cow[..], &expectation[..]);
-    }
-
     #[test]
     fn test_iso_2022_jp_half_width_katakana_length() {
         let mut output = [0u8; 20];

src/shift_jis.rs

@@ -384,27 +384,6 @@ mod tests {
         encode_shift_jis("\u{9ED1}", b"\xFC\x4B");
     }
 
-    #[test]
-    #[cfg_attr(miri, ignore)] // Miri is too slow
-    fn test_shift_jis_decode_all() {
-        let input = include_bytes!("test_data/shift_jis_in.txt");
-        let expectation = include_str!("test_data/shift_jis_in_ref.txt");
-        let (cow, had_errors) = SHIFT_JIS.decode_without_bom_handling(input);
-        assert!(had_errors, "Should have had errors.");
-        assert_eq!(&cow[..], expectation);
-    }
-
-    #[test]
-    #[cfg_attr(miri, ignore)] // Miri is too slow
-    fn test_shift_jis_encode_all() {
-        let input = include_str!("test_data/shift_jis_out.txt");
-        let expectation = include_bytes!("test_data/shift_jis_out_ref.txt");
-        let (cow, encoding, had_errors) = SHIFT_JIS.encode(input);
-        assert!(!had_errors, "Should not have had errors.");
-        assert_eq!(encoding, SHIFT_JIS);
-        assert_eq!(&cow[..], &expectation[..]);
-    }
-
     #[test]
     fn test_shift_jis_half_width_katakana_length() {
         let mut output = [0u8; 20];

tests/big5.rs

@@ -0,0 +1,22 @@
+use encoding_rs::*;
+
+#[test]
+#[cfg_attr(miri, ignore)] // Miri is too slow
+fn test_big5_decode_all() {
+    let input = include_bytes!("test_data/big5_in.txt");
+    let expectation = include_str!("test_data/big5_in_ref.txt");
+    let (cow, had_errors) = BIG5.decode_without_bom_handling(input);
+    assert!(had_errors, "Should have had errors.");
+    assert_eq!(&cow[..], expectation);
+}
+
+#[test]
+#[cfg_attr(miri, ignore)] // Miri is too slow
+fn test_big5_encode_all() {
+    let input = include_str!("test_data/big5_out.txt");
+    let expectation = include_bytes!("test_data/big5_out_ref.txt");
+    let (cow, encoding, had_errors) = BIG5.encode(input);
+    assert!(!had_errors, "Should not have had errors.");
+    assert_eq!(encoding, BIG5);
+    assert_eq!(&cow[..], &expectation[..]);
+}