From 9730117cead00bb3d1139ec35dbb09fd79e0f65b Mon Sep 17 00:00:00 2001 From: "rukayaj@gmail.com" Date: Mon, 1 Sep 2025 18:20:05 +0200 Subject: [PATCH] fix: handle decimal-formatted integers in date parsing --- .../org/filteredpush/qc/date/DwCEventDQ.java | 53 +++---- .../filteredpush/qc/date/DwCOtherDateDQ.java | 8 +- .../qc/date/LocalDateInterval.java | 4 +- .../qc/date/LocalDateTimeInterval.java | 4 +- .../java/org/filteredpush/qc/date/Runner.java | 3 +- .../filteredpush/qc/date/util/DateUtils.java | 43 +++--- .../qc/date/util/NumberUtils.java | 105 +++++++++++++ .../filteredpush/qc/date/DateUtilsTest.java | 25 ++- .../filteredpush/qc/date/DwcEventDQTest.java | 87 +++++++++++ .../qc/date/util/NumberUtilsTest.java | 144 ++++++++++++++++++ 10 files changed, 419 insertions(+), 57 deletions(-) create mode 100644 src/main/java/org/filteredpush/qc/date/util/NumberUtils.java create mode 100644 src/test/java/org/filteredpush/qc/date/util/NumberUtilsTest.java diff --git a/src/main/java/org/filteredpush/qc/date/DwCEventDQ.java b/src/main/java/org/filteredpush/qc/date/DwCEventDQ.java index 946ccfe..15ebf43 100644 --- a/src/main/java/org/filteredpush/qc/date/DwCEventDQ.java +++ b/src/main/java/org/filteredpush/qc/date/DwCEventDQ.java @@ -27,6 +27,7 @@ import org.datakurator.ffdq.api.result.NumericalValue; import org.datakurator.ffdq.model.ResultState; import org.filteredpush.qc.date.util.DateUtils; +import org.filteredpush.qc.date.util.NumberUtils; import java.time.LocalDate; import java.time.LocalDateTime; @@ -654,7 +655,7 @@ public static DQResponse validationDayStandard(@ActedUpon("dwc: result.setResultState(ResultState.INTERNAL_PREREQUISITES_NOT_MET); } else { try { - int numericDay = Integer.parseInt(day.trim()); + int numericDay = NumberUtils.parseInt(day); if (DateUtils.isDayInRange(numericDay)) { result.setValue(ComplianceValue.COMPLIANT); result.addComment("Provided value for day '" + day + "' is an integer in the range 1 to 31."); @@ -705,7 +706,7 @@ public static DQResponse validationMonthStandard(@ActedUpon("dw result.setResultState(ResultState.INTERNAL_PREREQUISITES_NOT_MET); } else { try { - int numericMonth = Integer.parseInt(month.trim()); + int numericMonth = NumberUtils.parseInt(month); if (DateUtils.isMonthInRange(numericMonth)) { result.setValue(ComplianceValue.COMPLIANT); result.addComment("Provided value for month '" + month + "' is an integer in the range 1 to 12."); @@ -863,7 +864,7 @@ public static DQResponse validationDayInrange( result.addComment("Provided value for dwc:day [" + day + "] is EMPTY.");; } else { try { - Integer dayInteger = Integer.parseInt(day.trim()); + Integer dayInteger = NumberUtils.parseInt(day); if (dayInteger>=1 && dayInteger<=28) { // COMP (a) the value of dwc:day is interpretable as an integer between 1 and 28 inclusive, result.setResultState(ResultState.RUN_HAS_RESULT); @@ -884,7 +885,7 @@ public static DQResponse validationDayInrange( if (DateUtils.isEmpty(month)) { throw new NumberFormatException(); } - Integer monthInteger = Integer.parseInt(month.trim()); + Integer monthInteger = NumberUtils.parseInt(month); if (dayInteger>=29 && dayInteger<=30 && (monthInteger==4 || monthInteger==6 || monthInteger==9 || monthInteger==11)) { @@ -908,7 +909,7 @@ public static DQResponse validationDayInrange( if (DateUtils.isEmpty(year)) { throw new NumberFormatException(); } - Integer yearInteger = Integer.parseInt(year.trim()); + Integer yearInteger = NumberUtils.parseInt(year); if ( ((yearInteger%400)==0) || ((yearInteger % 4)==0 && (yearInteger % 100)!=0)) { // COMP (d) dwc:day is interpretable as the // integer 29 and dwc:month is interpretable as the integer @@ -1034,8 +1035,8 @@ public static final DQResponse dayMonthTransposition(@ActedUpon( // month is integer, but out of range if (dayResult.getResultState().equals(ResultState.RUN_HAS_RESULT)) { // day is also integer - int dayNumeric = Integer.parseInt(day); - int monthNumeric = Integer.parseInt(month); + int dayNumeric = NumberUtils.parseInt(day); + int monthNumeric = NumberUtils.parseInt(month); if (DateUtils.isDayInRange(monthNumeric) && DateUtils.isMonthInRange(dayNumeric)) { // day is in range for months, and month is in range for days, so transpose. Map transposedValues = new HashMap<>(); @@ -1126,7 +1127,7 @@ public static final DQResponse validationStartdayofyearInrange( result.addComment("startDayOfYear was not provided."); } else { try { - Integer numericStartDay = Integer.parseInt(startDayOfYear); + Integer numericStartDay = NumberUtils.parseInt(startDayOfYear); if (numericStartDay>0 && numericStartDay<366) { result.setValue(ComplianceValue.COMPLIANT); result.setResultState(ResultState.RUN_HAS_RESULT); @@ -1228,7 +1229,7 @@ public static final DQResponse validationEnddayofyearInrange( result.addComment("endDayOfYear was not provided."); } else { try { - Integer numericEndDay = Integer.parseInt(endDay); + Integer numericEndDay = NumberUtils.parseInt(endDay); if (numericEndDay>0 && numericEndDay<366) { result.setValue(ComplianceValue.COMPLIANT); result.setResultState(ResultState.RUN_HAS_RESULT); @@ -1323,9 +1324,9 @@ public static final DQResponse amendmentEventdateFromYearstartda result.addComment("A value exists in dwc:eventDate, ammendment not attempted."); } else { try { - Integer numericYear = Integer.parseInt(year); - Integer numericStartDay = Integer.parseInt(startDay); - Integer numericEndDay = Integer.parseInt(endDay); + Integer numericYear = NumberUtils.parseInt(year); + Integer numericStartDay = NumberUtils.parseInt(startDay); + Integer numericEndDay = NumberUtils.parseInt(endDay); logger.debug(numericStartDay); if (numericStartDay < 1 || numericStartDay > 366 || numericEndDay < 1 || numericEndDay > 366) { // out of range for possible days of year, report and fail. @@ -1419,7 +1420,7 @@ public static final DQResponse amendmentEventDateFromYearMonthDa result.addComment("A value exists in dwc:eventDate, ammendment not attempted."); } else { try { - Integer.parseInt(year.trim()); + NumberUtils.parseInt(year); try { boolean hasMonth = false; boolean hasDay = false; @@ -1429,7 +1430,7 @@ public static final DQResponse amendmentEventDateFromYearMonthDa // Roman numeral month values are interpretable as numbers. logger.debug(month); if (DateUtils.romanMonthToInteger(month)==null) { - Integer monthInt = Integer.parseInt(month); + Integer monthInt = NumberUtils.parseInt(month); if (DateUtils.isMonthInRange(monthInt)) { hasMonth=true; } @@ -1447,7 +1448,7 @@ public static final DQResponse amendmentEventDateFromYearMonthDa logger.debug(month); } else { try { - Integer monthInt = Integer.parseInt(month); + Integer monthInt = NumberUtils.parseInt(month); if (DateUtils.isMonthInRange(monthInt)) { hasMonth=true; } @@ -1458,7 +1459,7 @@ public static final DQResponse amendmentEventDateFromYearMonthDa } if (!DateUtils.isEmpty(day)) { try { - Integer numericDay = Integer.parseInt(day.trim()); + Integer numericDay = NumberUtils.parseInt(day); if (!DateUtils.isDayInRange(numericDay)) { throw new NumberFormatException("The provided value for Day is out of range for a day"); } else { @@ -1552,7 +1553,7 @@ public static final DQResponse amendmentMonthStandardized(@Acted } else { try { - Integer monthNumeric = Integer.parseInt(month.trim()); + Integer monthNumeric = NumberUtils.parseInt(month); result.addComment("A value for dwc:month parsable as an integer was provided."); if (monthNumeric >= 1 && monthNumeric <=12) { result.addComment("Provided value for dwc:month was in the range 1-12."); @@ -1582,7 +1583,7 @@ public static final DQResponse amendmentMonthStandardized(@Acted } else { logger.debug(monthTrim); try { - Integer monthInteger = Integer.parseInt(monthTrim); + Integer monthInteger = NumberUtils.parseInt(monthTrim); result.setResultState(ResultState.AMENDED); Map values = new HashMap<>(); values.put("dwc:month", monthInteger.toString()); @@ -1648,7 +1649,7 @@ public static final DQResponse amendmentDayStandardized(@ActedUp } else { try { - Integer dayNumeric = Integer.parseInt(day); + Integer dayNumeric = NumberUtils.parseInt(day); result.addComment("A value for dwc:day parsable as an integer was provided."); String dayTrimmed = day.replaceAll("[^0-9]", ""); String dayCleaned = dayNumeric.toString(); @@ -1698,7 +1699,7 @@ public static final DQResponse amendmentDayStandardized(@ActedUp if (!failed) { // Try again try { - Integer dayNumeric = Integer.parseInt(dayTrimmed.trim()); + Integer dayNumeric = NumberUtils.parseInt(dayTrimmed); logger.debug(dayNumeric); if (dayNumeric>0 && dayNumeric<32) { result.setResultState(ResultState.AMENDED); @@ -1858,7 +1859,7 @@ public static DQResponse validationEventConsistent( result.addComment("The provided dwc:eventDate ["+eventDate+"] represents an interval of more than one day, and dwc:day contains a value ["+day+"] when it should not. "); inconsistencyFound = true; } - if (DateUtils.extractDate(eventDate).getDayOfMonth()!=Integer.parseInt(day)) { + if (DateUtils.extractDate(eventDate).getDayOfMonth()!=NumberUtils.parseInt(day)) { result.addComment("Provided value for dwc:eventDate ["+eventDate+"] is not consistent with the provided value of dwc:day ["+day+"]."); inconsistencyFound = true; } @@ -1883,7 +1884,7 @@ public static DQResponse validationEventConsistent( result.addComment("Unable to extract startDayOfYear from dwc:eventDate ["+eventDate+"] for comparision with provided dwc:startDayOfYear ["+startDayOfYear+"]."); interpretationProblem = true; } else { - if (extractedDate.getDayOfYear()!=Integer.parseInt(startDayOfYear)) { + if (extractedDate.getDayOfYear()!=NumberUtils.parseInt(startDayOfYear)) { result.addComment("Provided value for dwc:eventDate ["+eventDate+"] is not consistent with the provided value of dwc:startDayOfYear["+startDayOfYear+"]."); inconsistencyFound = true; } @@ -1900,7 +1901,7 @@ public static DQResponse validationEventConsistent( // range represented by dwc:eventDate; if (!DateUtils.isEmpty(endDayOfYear)) { if (DateUtils.hasResolutionDayOrFiner(eventDate)) { - if (DateUtils.extractDateInterval(eventDate).getEnd().getDayOfYear()!=Integer.parseInt(endDayOfYear)) { + if (DateUtils.extractDateInterval(eventDate).getEnd().getDayOfYear()!=NumberUtils.parseInt(endDayOfYear)) { result.addComment("Provided value for dwc:eventDate ["+eventDate+"] is not consistent with the provided value of dwc:endDayIfYear ["+endDayOfYear+"]."); inconsistencyFound = true; } @@ -2316,9 +2317,9 @@ public static DQResponse validationYearInrange( result.setResultState(ResultState.INTERNAL_PREREQUISITES_NOT_MET); } else { try { - int numericYear = Integer.parseInt(year.trim()); - int numericLowerBound = Integer.parseInt(lowerBound); - int numericUpperBound = Integer.parseInt(upperBound); + int numericYear = NumberUtils.parseInt(year); + int numericLowerBound = NumberUtils.parseInt(lowerBound); + int numericUpperBound = NumberUtils.parseInt(upperBound); if (numericYearnumericUpperBound) { result.setValue(ComplianceValue.NOT_COMPLIANT); result.addComment("Provided value for dwc:year '" + year + "' is not an integer in the range " + lowerBound + " to " + upperBound + " (current year)."); diff --git a/src/main/java/org/filteredpush/qc/date/DwCOtherDateDQ.java b/src/main/java/org/filteredpush/qc/date/DwCOtherDateDQ.java index 06efc9f..b087166 100644 --- a/src/main/java/org/filteredpush/qc/date/DwCOtherDateDQ.java +++ b/src/main/java/org/filteredpush/qc/date/DwCOtherDateDQ.java @@ -360,8 +360,8 @@ public static DQResponse amendmentDateidentifiedStandardized( if (dateIdentified.matches("^[0-9]{4}.[0-9]{2}.[0-9]{2}$")) { try { // try to see if this is yyyy-dd-mm error for yyyy-mm-dd - Integer secondBit = Integer.parseInt(dateIdentified.substring(5,7)); - Integer thirdBit = Integer.parseInt(dateIdentified.substring(8)); + Integer secondBit = org.filteredpush.qc.date.util.NumberUtils.parseInt(dateIdentified.substring(5,7)); + Integer thirdBit = org.filteredpush.qc.date.util.NumberUtils.parseInt(dateIdentified.substring(8)); if (secondBit>12 && thirdBit<12) { // try switching second and third parts of date. String toTest = dateIdentified.substring(0, 4).concat("-").concat(dateIdentified.substring(8)).concat("-").concat(dateIdentified.substring(5, 7)); @@ -729,8 +729,8 @@ public static DQResponse amendmentModifiedStandardized( if (modified.matches("^[0-9]{4}.[0-9]{2}.[0-9]{2}$")) { try { // try to see if this is yyyy-dd-mm error for yyyy-mm-dd - Integer secondBit = Integer.parseInt(modified.substring(5,7)); - Integer thirdBit = Integer.parseInt(modified.substring(8)); + Integer secondBit = org.filteredpush.qc.date.util.NumberUtils.parseInt(modified.substring(5,7)); + Integer thirdBit = org.filteredpush.qc.date.util.NumberUtils.parseInt(modified.substring(8)); if (secondBit>12 && thirdBit<12) { // try switching second and third parts of date. String toTest = modified.substring(0, 4).concat("-").concat(modified.substring(8)).concat("-").concat(modified.substring(5, 7)); diff --git a/src/main/java/org/filteredpush/qc/date/LocalDateInterval.java b/src/main/java/org/filteredpush/qc/date/LocalDateInterval.java index c4a136c..078559c 100644 --- a/src/main/java/org/filteredpush/qc/date/LocalDateInterval.java +++ b/src/main/java/org/filteredpush/qc/date/LocalDateInterval.java @@ -269,7 +269,7 @@ protected DatePair parseDateBit(String dateBit) throws EmptyDateException, Date .append(DateTimeFormatter.ISO_LOCAL_DATE) .toFormatter().withResolverStyle(ResolverStyle.STRICT); if (dateBit.matches("^[0-9]{1,3}$")) { - dateBit = String.format("%04d", Integer.parseInt(dateBit)); + dateBit = String.format("%04d", org.filteredpush.qc.date.util.NumberUtils.parseInt(dateBit)); } LocalDate startDateBit = LocalDate.parse(dateBit+"-01-01", formatter); result = new DatePair(startDateBit,startDateBit.with(TemporalAdjusters.lastDayOfYear())); @@ -278,7 +278,7 @@ protected DatePair parseDateBit(String dateBit) throws EmptyDateException, Date DateTimeFormatter formatter = new DateTimeFormatterBuilder() .append(DateTimeFormatter.ofPattern("yyyy-MM-ddGGGGG")) .toFormatter().withResolverStyle(ResolverStyle.STRICT); - dateBit = String.format("%04d", Integer.parseInt(dateBit.substring(0,4))); + dateBit = String.format("%04d", org.filteredpush.qc.date.util.NumberUtils.parseInt(dateBit.substring(0,4))); LocalDate startDateBit = LocalDate.parse(dateBit+"-01-01B", formatter); result = new DatePair(startDateBit.minusYears(1),startDateBit.minusYears(1).with(TemporalAdjusters.lastDayOfYear())); } else { diff --git a/src/main/java/org/filteredpush/qc/date/LocalDateTimeInterval.java b/src/main/java/org/filteredpush/qc/date/LocalDateTimeInterval.java index c7d6458..961e14e 100644 --- a/src/main/java/org/filteredpush/qc/date/LocalDateTimeInterval.java +++ b/src/main/java/org/filteredpush/qc/date/LocalDateTimeInterval.java @@ -258,7 +258,7 @@ protected DatePair parseDateBit(String dateBit) throws EmptyDateException, Date .append(DateTimeFormatter.ISO_LOCAL_DATE) .toFormatter().withResolverStyle(ResolverStyle.STRICT); if (dateBit.matches("^[0-9]{1,3}$")) { - dateBit = String.format("%04d", Integer.parseInt(dateBit)); + dateBit = String.format("%04d", org.filteredpush.qc.date.util.NumberUtils.parseInt(dateBit)); } LocalDateTime startDateBit = LocalDateTime.parse(dateBit+"-01-01", formatter); result = new DatePair(startDateBit,startDateBit.with(TemporalAdjusters.lastDayOfYear())); @@ -267,7 +267,7 @@ protected DatePair parseDateBit(String dateBit) throws EmptyDateException, Date DateTimeFormatter formatter = new DateTimeFormatterBuilder() .append(DateTimeFormatter.ofPattern("yyyy-MM-ddGGGGG")) .toFormatter().withResolverStyle(ResolverStyle.STRICT); - dateBit = String.format("%04d", Integer.parseInt(dateBit.substring(0,4))); + dateBit = String.format("%04d", org.filteredpush.qc.date.util.NumberUtils.parseInt(dateBit.substring(0,4))); LocalDateTime startDateBit = LocalDateTime.parse(dateBit+"-01-01B", formatter); result = new DatePair(startDateBit.minusYears(1),startDateBit.minusYears(1).with(TemporalAdjusters.lastDayOfYear())); } else { diff --git a/src/main/java/org/filteredpush/qc/date/Runner.java b/src/main/java/org/filteredpush/qc/date/Runner.java index 9ba5be2..2ab1b78 100644 --- a/src/main/java/org/filteredpush/qc/date/Runner.java +++ b/src/main/java/org/filteredpush/qc/date/Runner.java @@ -52,6 +52,7 @@ import org.datakurator.ffdq.api.result.NumericalValue; import org.datakurator.ffdq.model.ResultState; import org.filteredpush.qc.date.util.DateUtils; +import org.filteredpush.qc.date.util.NumberUtils; /** * Selfstanding execution of event_date_qc functionality. Can run TG2 Date related tests on flat DarwinCore @@ -106,7 +107,7 @@ public static void main(String[] args) { int limit = 0; if (limitValue!=null) { try { - limit = Integer.parseInt(limitValue); + limit = org.filteredpush.qc.date.util.NumberUtils.parseInt(limitValue); } catch (NumberFormatException nfe) { logger.error(nfe.getMessage()); } diff --git a/src/main/java/org/filteredpush/qc/date/util/DateUtils.java b/src/main/java/org/filteredpush/qc/date/util/DateUtils.java index d7aafc6..b2489b7 100644 --- a/src/main/java/org/filteredpush/qc/date/util/DateUtils.java +++ b/src/main/java/org/filteredpush/qc/date/util/DateUtils.java @@ -43,6 +43,7 @@ import org.filteredpush.qc.date.EventResult.EventQCResultState; import org.filteredpush.qc.date.LocalDateTimeInterval; import org.filteredpush.qc.date.TimeExtractionException; +import org.filteredpush.qc.date.util.NumberUtils; import java.time.Instant; import java.time.Duration; @@ -181,10 +182,10 @@ public static String createEventDateFromParts(String verbatimEventDate, String s try { StringBuffer assembly = new StringBuffer(); if (!isEmpty(endDayOfYear) && !startDayOfYear.trim().equals(endDayOfYear.trim())) { - assembly.append(year).append("-").append(String.format("%03d",Integer.parseInt(startDayOfYear))).append("/"); - assembly.append(year).append("-").append(String.format("%03d",Integer.parseInt(endDayOfYear))); + assembly.append(year).append("-").append(String.format("%03d",NumberUtils.parseInt(startDayOfYear))).append("/"); + assembly.append(year).append("-").append(String.format("%03d",NumberUtils.parseInt(endDayOfYear))); } else { - assembly.append(year).append("-").append(String.format("%03d",Integer.parseInt(startDayOfYear))); + assembly.append(year).append("-").append(String.format("%03d",NumberUtils.parseInt(startDayOfYear))); } EventResult verbatim = extractDateToDayFromVerbatimER(assembly.toString(), DateUtils.YEAR_BEFORE_SUSPECT) ; logger.debug(verbatim.getResultState().toString()); @@ -206,10 +207,10 @@ public static String createEventDateFromParts(String verbatimEventDate, String s try { StringBuffer assembly = new StringBuffer(); if (endDayOfYear !=null && endDayOfYear.trim().length() > 0 && !startDayOfYear.trim().equals(endDayOfYear.trim())) { - assembly.append(verbatimEventDate).append("-").append(String.format("%03d",Integer.parseInt(startDayOfYear))).append("/"); - assembly.append(verbatimEventDate).append("-").append(String.format("%03d",Integer.parseInt(endDayOfYear))); + assembly.append(verbatimEventDate).append("-").append(String.format("%03d",NumberUtils.parseInt(startDayOfYear))).append("/"); + assembly.append(verbatimEventDate).append("-").append(String.format("%03d",NumberUtils.parseInt(endDayOfYear))); } else { - assembly.append(verbatimEventDate).append("-").append(String.format("%03d",Integer.parseInt(startDayOfYear))); + assembly.append(verbatimEventDate).append("-").append(String.format("%03d",NumberUtils.parseInt(startDayOfYear))); } EventResult verbatim = extractDateToDayFromVerbatimER(assembly.toString(), DateUtils.YEAR_BEFORE_SUSPECT) ; logger.debug(verbatim.getResultState().toString()); @@ -234,9 +235,9 @@ public static String createEventDateFromParts(String verbatimEventDate, String s ) { try { - Integer yearInt = Integer.parseInt(year.trim()); - Integer startInt = Integer.parseInt(startDayOfYear.trim()); - Integer endInt = Integer.parseInt(endDayOfYear.trim()); + Integer yearInt = NumberUtils.parseInt(year); + Integer startInt = NumberUtils.parseInt(startDayOfYear); + Integer endInt = NumberUtils.parseInt(endDayOfYear); String toTest = String.format("%04d", yearInt).concat("-"); toTest = toTest.concat(String.format("%03d", startInt)); toTest = toTest.concat("/"); @@ -251,13 +252,13 @@ public static String createEventDateFromParts(String verbatimEventDate, String s logger.debug(result); } if (year!=null && year.matches("[0-9]{4}") && month!=null && month.matches("[0-9]{1,2}") &&( day==null || day.trim().length()==0 )) { - result = String.format("%04d",Integer.parseInt(year)) + "-" + String.format("%02d",Integer.parseInt(month)); + result = String.format("%04d",NumberUtils.parseInt(year)) + "-" + String.format("%02d",NumberUtils.parseInt(month)); logger.debug(result); } if (year!=null && year.matches("[0-9]{4}") && month!=null && month.matches("[0-9]{1,2}") && day!=null && day.matches("[0-9]{1,2}")) { - result = String.format("%04d",Integer.parseInt(year)) + "-" + - String.format("%02d",Integer.parseInt(month)) + "-" + - String.format("%02d",Integer.parseInt(day)); + result = String.format("%04d",NumberUtils.parseInt(year)) + "-" + + String.format("%02d",NumberUtils.parseInt(month)) + "-" + + String.format("%02d",NumberUtils.parseInt(day)); logger.debug(result); } if (!DateUtils.isEmpty(result)) { @@ -1366,7 +1367,7 @@ public static EventResult extractDateFromVerbatimER(String verbatimEventDate, in // 1815-16 won't parse in thls block as 16 is too large to be a month, passes to next block // 1805-06 could be month or abbreviated year. Suspect. // 1805-03 should to be month (3<=5, thus not likely to be year range). - if (Integer.parseInt(startBit)>=Integer.parseInt(century+endBit)) { + if (NumberUtils.parseInt(startBit)>=NumberUtils.parseInt(century+endBit)) { result.setResultState(EventResult.EventQCResultState.RANGE); result.setResult(resultDate); } else { @@ -2568,7 +2569,7 @@ public static boolean isConsistent(String eventDate, String startDayOfYear, Stri if (endDayOfYear==null || endDayOfYear.trim().length()==0 || startDayOfYear.trim().equals(endDayOfYear.trim())) { int startDayInt = -1; try { - startDayInt = Integer.parseInt(startDayOfYear); + startDayInt = NumberUtils.parseInt(startDayOfYear); } catch (NumberFormatException e) { logger.debug(e.getMessage()); logger.debug(startDayOfYear + " is not an integer."); @@ -2584,9 +2585,9 @@ public static boolean isConsistent(String eventDate, String startDayOfYear, Stri } else { int startDayInt = -1; int endDayInt = -1; - try { - startDayInt = Integer.parseInt(startDayOfYear); - endDayInt = Integer.parseInt(endDayOfYear); + try { + startDayInt = NumberUtils.parseInt(startDayOfYear); + endDayInt = NumberUtils.parseInt(endDayOfYear); } catch (NumberFormatException e) { logger.debug(e.getMessage()); result = false; @@ -2645,7 +2646,7 @@ public static boolean isConsistent(String eventDate, String year, String month, Integer dayBit = null; if (!isEmpty(day)) { try { - dayBit = Integer.parseInt(day); + dayBit = NumberUtils.parseInt(day); } catch (NumberFormatException e) { anyFails = true; } @@ -2653,7 +2654,7 @@ public static boolean isConsistent(String eventDate, String year, String month, Integer monthBit = null; if (!isEmpty(month)) { try { - monthBit = Integer.parseInt(month); + monthBit = NumberUtils.parseInt(month); } catch (NumberFormatException e) { anyFails = true; } @@ -2662,7 +2663,7 @@ public static boolean isConsistent(String eventDate, String year, String month, logger.debug(anyFails); if (!isEmpty(year)) { try { - yearBit = Integer.parseInt(year); + yearBit = NumberUtils.parseInt(year); } catch (NumberFormatException e) { anyFails = true; } diff --git a/src/main/java/org/filteredpush/qc/date/util/NumberUtils.java b/src/main/java/org/filteredpush/qc/date/util/NumberUtils.java new file mode 100644 index 0000000..2c2c02e --- /dev/null +++ b/src/main/java/org/filteredpush/qc/date/util/NumberUtils.java @@ -0,0 +1,105 @@ +/** NumberUtils.java + * + * Copyright 2015-2017 President and Fellows of Harvard College + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.filteredpush.qc.date.util; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +/** + * Utility class for parsing numeric values that may contain decimal places. + * This handles cases where integer values are stored as decimals (e.g., "29.0" -> 29). + */ +public class NumberUtils { + + private static final Log logger = LogFactory.getLog(NumberUtils.class); + + /** + * Parse a string to an integer, handling decimal values gracefully. + * If the string represents a decimal number that equals an integer (e.g., "29.0"), + * it will be parsed as that integer. Otherwise, throws NumberFormatException. + * + * @param value the string to parse + * @return the parsed integer value + * @throws NumberFormatException if the value cannot be parsed as an integer + */ + public static int parseInt(String value) throws NumberFormatException { + if (value == null) { + throw new NumberFormatException("Cannot parse null value"); + } + + String trimmed = value.trim(); + if (trimmed.isEmpty()) { + throw new NumberFormatException("Cannot parse empty string"); + } + + // Check if it's a decimal or scientific notation that equals an integer + if (trimmed.contains(".") || trimmed.toUpperCase().contains("E")) { + try { + double doubleValue = Double.parseDouble(trimmed); + int intValue = (int) doubleValue; + + // Verify that the double value equals the integer value + if (doubleValue == intValue) { + if (logger.isDebugEnabled()) { + logger.debug("Successfully parsed decimal/scientific '" + trimmed + "' as integer " + intValue); + } + return intValue; + } else { + throw new NumberFormatException("Decimal/scientific value '" + trimmed + "' does not represent an integer"); + } + } catch (NumberFormatException e) { + throw new NumberFormatException("Cannot parse '" + trimmed + "' as a number"); + } + } + + // Standard integer parsing + return Integer.parseInt(trimmed); + } + + /** + * Parse a string to an Integer, handling decimal values gracefully. + * Returns null if the value is null or empty, otherwise delegates to parseInt(). + * + * @param value the string to parse + * @return the parsed Integer value, or null if input is null/empty + * @throws NumberFormatException if the value cannot be parsed as an integer + */ + public static Integer parseIntOrNull(String value) throws NumberFormatException { + if (value == null || value.trim().isEmpty()) { + return null; + } + return parseInt(value); + } + + /** + * Safely parse a string to an integer, returning a default value on failure. + * + * @param value the string to parse + * @param defaultValue the value to return if parsing fails + * @return the parsed integer or the default value + */ + public static int parseIntOrDefault(String value, int defaultValue) { + try { + return parseInt(value); + } catch (NumberFormatException e) { + if (logger.isDebugEnabled()) { + logger.debug("Failed to parse '" + value + "' as integer, using default value " + defaultValue); + } + return defaultValue; + } + } +} diff --git a/src/test/java/org/filteredpush/qc/date/DateUtilsTest.java b/src/test/java/org/filteredpush/qc/date/DateUtilsTest.java index dc39195..1d20e2e 100644 --- a/src/test/java/org/filteredpush/qc/date/DateUtilsTest.java +++ b/src/test/java/org/filteredpush/qc/date/DateUtilsTest.java @@ -88,7 +88,7 @@ public void teststringIsISOFormattedDate() { } /** - * Test method for {@link org.filteredpush.qc.date.util.DateUtils#createEventDateFromParts(java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String)}. + * Test method for {@link org.filteredpush.qc.date.util.DateUtils#createEventDateFromParts(java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String, java.lang.String)}. */ @Test public void testCreateEventDateFromParts() { @@ -2233,6 +2233,29 @@ public void testromanMonthToInteger() { assertEquals(Integer.valueOf(11),DateUtils.romanMonthToInteger("Xi")); assertEquals(Integer.valueOf(11),DateUtils.romanMonthToInteger("xI")); } + + @Test + public void testAssembleEventDateFromDecimalYearMonthDay() { + String y = "2020.0"; + String m = "5.0"; + String d = "1.0"; + String assembled = DateUtils.createEventDateFromParts("", "", "", + Integer.toString(org.filteredpush.qc.date.util.NumberUtils.parseInt(y)), + Integer.toString(org.filteredpush.qc.date.util.NumberUtils.parseInt(m)), + Integer.toString(org.filteredpush.qc.date.util.NumberUtils.parseInt(d))); + assertEquals("2020-05-01", assembled); + } + + @Test + public void testAssembleEventDateFromDecimalYearAndMonth() { + String y = "2020.0"; + String m = "5.0"; + String assembled = DateUtils.createEventDateFromParts("", "", "", + Integer.toString(org.filteredpush.qc.date.util.NumberUtils.parseInt(y)), + Integer.toString(org.filteredpush.qc.date.util.NumberUtils.parseInt(m)), + ""); + assertEquals("2020-05", assembled); + } @Test public void hasResolutionDayOrFinerTest() { diff --git a/src/test/java/org/filteredpush/qc/date/DwcEventDQTest.java b/src/test/java/org/filteredpush/qc/date/DwcEventDQTest.java index 61984e0..362f39d 100644 --- a/src/test/java/org/filteredpush/qc/date/DwcEventDQTest.java +++ b/src/test/java/org/filteredpush/qc/date/DwcEventDQTest.java @@ -463,6 +463,32 @@ public void testIsMonthInRange() { assertEquals(ComplianceValue.NOT_COMPLIANT.getLabel(), result.getValue().getLabel()); } + @Test + public void testDecimalInputsAreAcceptedInValidations() { + DQResponse respDay = DwCEventDQ.validationDayStandard("29.0"); + assertEquals(ResultState.RUN_HAS_RESULT, respDay.getResultState()); + assertEquals(ComplianceValue.COMPLIANT, respDay.getValue()); + + DQResponse respMonth = DwCEventDQ.validationMonthStandard("12.0"); + assertEquals(ResultState.RUN_HAS_RESULT, respMonth.getResultState()); + assertEquals(ComplianceValue.COMPLIANT, respMonth.getValue()); + + DQResponse respDayRange = DwCEventDQ.validationDayInrange("2020.0", "2.0", "29.0"); + assertEquals(ResultState.RUN_HAS_RESULT, respDayRange.getResultState()); + assertEquals(ComplianceValue.COMPLIANT, respDayRange.getValue()); + } + + @Test + public void testDayOfYearValidationsAcceptDecimals() { + DQResponse start = DwCEventDQ.validationStartdayofyearInrange("60.0", "2020-03-01"); + assertEquals(ResultState.RUN_HAS_RESULT, start.getResultState()); + assertEquals(ComplianceValue.COMPLIANT, start.getValue()); + + DQResponse end = DwCEventDQ.validationEnddayofyearInrange("365.0", "2020-12-31"); + assertEquals(ResultState.RUN_HAS_RESULT, end.getResultState()); + assertEquals(ComplianceValue.COMPLIANT, end.getValue()); + } + @Test public void testDurationInYear() { DQResponse result = null; @@ -3264,4 +3290,65 @@ public void testInternalConsistency() { assertNotNull(vresult.getComment()); } + + @Test + public void testDecimalInputsAreHandledAcrossValidations() { + // Day Standard with decimal + DQResponse r1 = DwCEventDQ.validationDayStandard("29.0"); + assertEquals(ResultState.RUN_HAS_RESULT, r1.getResultState()); + assertEquals(ComplianceValue.COMPLIANT, r1.getValue()); + + // Month Standard with decimal + DQResponse r2 = DwCEventDQ.validationMonthStandard("12.0"); + assertEquals(ResultState.RUN_HAS_RESULT, r2.getResultState()); + assertEquals(ComplianceValue.COMPLIANT, r2.getValue()); + + // Day in range with decimals for month/day and integral year + DQResponse r3 = DwCEventDQ.validationDayInrange("2020", "2.0", "29.0"); + assertEquals(ResultState.RUN_HAS_RESULT, r3.getResultState()); + assertEquals(ComplianceValue.COMPLIANT, r3.getValue()); + + // Day in range with decimal year + DQResponse r4 = DwCEventDQ.validationDayInrange("2020.0", "2", "29"); + assertEquals(ResultState.RUN_HAS_RESULT, r4.getResultState()); + assertEquals(ComplianceValue.COMPLIANT, r4.getValue()); + + // startDayOfYear in range with decimal start day + DQResponse r5 = DwCEventDQ.validationStartdayofyearInrange("60.0", "2020-03-01"); + assertEquals(ResultState.RUN_HAS_RESULT, r5.getResultState()); + assertEquals(ComplianceValue.COMPLIANT, r5.getValue()); + + // endDayOfYear in range with decimal end day + DQResponse r6 = DwCEventDQ.validationEnddayofyearInrange("365.0", "2020-12-31"); + assertEquals(ResultState.RUN_HAS_RESULT, r6.getResultState()); + assertEquals(ComplianceValue.COMPLIANT, r6.getValue()); + + // Event date standard with parts that include decimals + DQResponse r7 = DwCEventDQ.validationEventdateStandard("2020-05-01"); + assertEquals(ResultState.RUN_HAS_RESULT, r7.getResultState()); + assertEquals(ComplianceValue.COMPLIANT, r7.getValue()); + } + + @Test + public void testDecimalInputsThatShouldFail() { + // Non-integer decimal day should be NOT_COMPLIANT under day standard + DQResponse r1 = DwCEventDQ.validationDayStandard("29.5"); + assertEquals(ResultState.RUN_HAS_RESULT, r1.getResultState()); + assertEquals(ComplianceValue.NOT_COMPLIANT, r1.getValue()); + + // Non-integer decimal month should be NOT_COMPLIANT under month standard + DQResponse r2 = DwCEventDQ.validationMonthStandard("6.2"); + assertEquals(ResultState.RUN_HAS_RESULT, r2.getResultState()); + assertEquals(ComplianceValue.NOT_COMPLIANT, r2.getValue()); + + // startDayOfYear with non-integer decimal should be NOT_COMPLIANT + DQResponse r3 = DwCEventDQ.validationStartdayofyearInrange("60.4", "2020-03-01"); + assertEquals(ResultState.RUN_HAS_RESULT, r3.getResultState()); + assertEquals(ComplianceValue.NOT_COMPLIANT, r3.getValue()); + + // endDayOfYear with non-integer decimal should be NOT_COMPLIANT + DQResponse r4 = DwCEventDQ.validationEnddayofyearInrange("200.9", "2020-12-31"); + assertEquals(ResultState.RUN_HAS_RESULT, r4.getResultState()); + assertEquals(ComplianceValue.NOT_COMPLIANT, r4.getValue()); + } } diff --git a/src/test/java/org/filteredpush/qc/date/util/NumberUtilsTest.java b/src/test/java/org/filteredpush/qc/date/util/NumberUtilsTest.java new file mode 100644 index 0000000..d00f0de --- /dev/null +++ b/src/test/java/org/filteredpush/qc/date/util/NumberUtilsTest.java @@ -0,0 +1,144 @@ +package org.filteredpush.qc.date.util; + +import org.junit.Test; +import static org.junit.Assert.*; + +/** + * Tests for NumberUtils class to ensure proper handling of decimal values + * that represent integers (e.g., "29.0" -> 29). + */ +public class NumberUtilsTest { + + @Test + public void testParseIntWithDecimalValues() { + // Test decimal values that equal integers + assertEquals(29, NumberUtils.parseInt("29.0")); + assertEquals(12, NumberUtils.parseInt("12.0")); + assertEquals(10, NumberUtils.parseInt("10.0")); + assertEquals(20, NumberUtils.parseInt("20.0")); + assertEquals(5, NumberUtils.parseInt("5.0")); + assertEquals(9, NumberUtils.parseInt("9.0")); + assertEquals(15, NumberUtils.parseInt("15.0")); + assertEquals(30, NumberUtils.parseInt("30.0")); + assertEquals(13, NumberUtils.parseInt("13.0")); + assertEquals(7, NumberUtils.parseInt("7.0")); + assertEquals(31, NumberUtils.parseInt("31.0")); + assertEquals(1, NumberUtils.parseInt("1.0")); + assertEquals(19, NumberUtils.parseInt("19.0")); + assertEquals(2, NumberUtils.parseInt("2.0")); + assertEquals(27, NumberUtils.parseInt("27.0")); + assertEquals(24, NumberUtils.parseInt("24.0")); + assertEquals(4, NumberUtils.parseInt("4.0")); + assertEquals(3, NumberUtils.parseInt("3.0")); + assertEquals(8, NumberUtils.parseInt("8.0")); + assertEquals(21, NumberUtils.parseInt("21.0")); + assertEquals(25, NumberUtils.parseInt("25.0")); + } + + @Test + public void testParseIntWithStandardIntegers() { + // Test standard integer values + assertEquals(29, NumberUtils.parseInt("29")); + assertEquals(12, NumberUtils.parseInt("12")); + assertEquals(10, NumberUtils.parseInt("10")); + assertEquals(0, NumberUtils.parseInt("0")); + assertEquals(-5, NumberUtils.parseInt("-5")); + assertEquals(1000, NumberUtils.parseInt("1000")); + } + + @Test + public void testParseIntWithTrimmedValues() { + // Test values with whitespace + assertEquals(29, NumberUtils.parseInt(" 29.0 ")); + assertEquals(12, NumberUtils.parseInt(" 12 ")); + assertEquals(10, NumberUtils.parseInt("\t10.0\n")); + } + + @Test(expected = NumberFormatException.class) + public void testParseIntWithNonIntegerDecimals() { + // Test decimal values that don't equal integers + NumberUtils.parseInt("29.5"); + } + + @Test(expected = NumberFormatException.class) + public void testParseIntWithNonIntegerDecimals2() { + NumberUtils.parseInt("12.7"); + } + + @Test(expected = NumberFormatException.class) + public void testParseIntWithNonIntegerDecimals3() { + NumberUtils.parseInt("10.1"); + } + + @Test(expected = NumberFormatException.class) + public void testParseIntWithNonIntegerDecimals4() { + NumberUtils.parseInt("5.99"); + } + + @Test(expected = NumberFormatException.class) + public void testParseIntWithNull() { + NumberUtils.parseInt(null); + } + + @Test(expected = NumberFormatException.class) + public void testParseIntWithEmptyString() { + NumberUtils.parseInt(""); + } + + @Test(expected = NumberFormatException.class) + public void testParseIntWithWhitespaceOnly() { + NumberUtils.parseInt(" "); + } + + @Test(expected = NumberFormatException.class) + public void testParseIntWithInvalidInput() { + NumberUtils.parseInt("abc"); + } + + @Test(expected = NumberFormatException.class) + public void testParseIntWithMixedInput() { + NumberUtils.parseInt("29abc"); + } + + @Test + public void testParseIntOrNull() { + // Test null/empty handling + assertNull(NumberUtils.parseIntOrNull(null)); + assertNull(NumberUtils.parseIntOrNull("")); + assertNull(NumberUtils.parseIntOrNull(" ")); + + // Test valid values + assertEquals((Integer) 29, NumberUtils.parseIntOrNull("29.0")); + assertEquals((Integer) 12, NumberUtils.parseIntOrNull("12")); + } + + @Test + public void testParseIntOrDefault() { + // Test default value handling + assertEquals(42, NumberUtils.parseIntOrDefault("invalid", 42)); + assertEquals(42, NumberUtils.parseIntOrDefault(null, 42)); + assertEquals(42, NumberUtils.parseIntOrDefault("", 42)); + + // Test valid values + assertEquals(29, NumberUtils.parseIntOrDefault("29.0", 42)); + assertEquals(12, NumberUtils.parseIntOrDefault("12", 42)); + } + + @Test + public void testEdgeCases() { + // Test edge cases + assertEquals(0, NumberUtils.parseInt("0.0")); + assertEquals(1, NumberUtils.parseInt("1.0")); + assertEquals(999, NumberUtils.parseInt("999.0")); + assertEquals(-1, NumberUtils.parseInt("-1.0")); + assertEquals(-999, NumberUtils.parseInt("-999.0")); + } + + @Test + public void testScientificNotation() { + // Test scientific notation that equals integers + assertEquals(100, NumberUtils.parseInt("1.0E2")); + assertEquals(1000, NumberUtils.parseInt("1.0E3")); + assertEquals(100, NumberUtils.parseInt("1E2")); + } +}