Skip to content

fix: handle decimal-formatted integers in date parsing #18

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
rukayaj wants to merge 1 commit into
base: master
Choose a base branch
from
Open

fix: handle decimal-formatted integers in date parsing #18

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 27 additions & 26 deletions src/main/java/org/filteredpush/qc/date/DwCEventDQ.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import org.datakurator.ffdq.api.result.NumericalValue;
import org.datakurator.ffdq.model.ResultState;
import org.filteredpush.qc.date.util.DateUtils;
import org.filteredpush.qc.date.util.NumberUtils;

import java.time.LocalDate;
import java.time.LocalDateTime;
Expand Down Expand Up @@ -654,7 +655,7 @@ public static DQResponse<ComplianceValue> validationDayStandard(@ActedUpon("dwc:
result.setResultState(ResultState.INTERNAL_PREREQUISITES_NOT_MET);
} else {
try {
int numericDay = Integer.parseInt(day.trim());
int numericDay = NumberUtils.parseInt(day);
if (DateUtils.isDayInRange(numericDay)) {
result.setValue(ComplianceValue.COMPLIANT);
result.addComment("Provided value for day '" + day + "' is an integer in the range 1 to 31.");
Expand Down Expand Up @@ -705,7 +706,7 @@ public static DQResponse<ComplianceValue> validationMonthStandard(@ActedUpon("dw
result.setResultState(ResultState.INTERNAL_PREREQUISITES_NOT_MET);
} else {
try {
int numericMonth = Integer.parseInt(month.trim());
int numericMonth = NumberUtils.parseInt(month);
if (DateUtils.isMonthInRange(numericMonth)) {
result.setValue(ComplianceValue.COMPLIANT);
result.addComment("Provided value for month '" + month + "' is an integer in the range 1 to 12.");
Expand Down Expand Up @@ -863,7 +864,7 @@ public static DQResponse<ComplianceValue> validationDayInrange(
result.addComment("Provided value for dwc:day [" + day + "] is EMPTY.");;
} else {
try {
Integer dayInteger = Integer.parseInt(day.trim());
Integer dayInteger = NumberUtils.parseInt(day);
if (dayInteger>=1 && dayInteger<=28) {
// COMP (a) the value of dwc:day is interpretable as an integer between 1 and 28 inclusive,
result.setResultState(ResultState.RUN_HAS_RESULT);
Expand All @@ -884,7 +885,7 @@ public static DQResponse<ComplianceValue> validationDayInrange(
if (DateUtils.isEmpty(month)) {
throw new NumberFormatException();
}
Integer monthInteger = Integer.parseInt(month.trim());
Integer monthInteger = NumberUtils.parseInt(month);
if (dayInteger>=29 && dayInteger<=30 &&
(monthInteger==4 || monthInteger==6 || monthInteger==9 || monthInteger==11))
{
Expand All @@ -908,7 +909,7 @@ public static DQResponse<ComplianceValue> validationDayInrange(
if (DateUtils.isEmpty(year)) {
throw new NumberFormatException();
}
Integer yearInteger = Integer.parseInt(year.trim());
Integer yearInteger = NumberUtils.parseInt(year);
if ( ((yearInteger%400)==0) || ((yearInteger % 4)==0 && (yearInteger % 100)!=0)) {
// COMP (d) dwc:day is interpretable as the
// integer 29 and dwc:month is interpretable as the integer
Expand Down Expand Up @@ -1034,8 +1035,8 @@ public static final DQResponse<AmendmentValue> dayMonthTransposition(@ActedUpon(
// month is integer, but out of range
if (dayResult.getResultState().equals(ResultState.RUN_HAS_RESULT)) {
// day is also integer
int dayNumeric = Integer.parseInt(day);
int monthNumeric = Integer.parseInt(month);
int dayNumeric = NumberUtils.parseInt(day);
int monthNumeric = NumberUtils.parseInt(month);
if (DateUtils.isDayInRange(monthNumeric) && DateUtils.isMonthInRange(dayNumeric)) {
// day is in range for months, and month is in range for days, so transpose.
Map<String, String> transposedValues = new HashMap<>();
Expand Down Expand Up @@ -1126,7 +1127,7 @@ public static final DQResponse<ComplianceValue> validationStartdayofyearInrange(
result.addComment("startDayOfYear was not provided.");
} else {
try {
Integer numericStartDay = Integer.parseInt(startDayOfYear);
Integer numericStartDay = NumberUtils.parseInt(startDayOfYear);
if (numericStartDay>0 && numericStartDay<366) {
result.setValue(ComplianceValue.COMPLIANT);
result.setResultState(ResultState.RUN_HAS_RESULT);
Expand Down Expand Up @@ -1228,7 +1229,7 @@ public static final DQResponse<ComplianceValue> validationEnddayofyearInrange(
result.addComment("endDayOfYear was not provided.");
} else {
try {
Integer numericEndDay = Integer.parseInt(endDay);
Integer numericEndDay = NumberUtils.parseInt(endDay);
if (numericEndDay>0 && numericEndDay<366) {
result.setValue(ComplianceValue.COMPLIANT);
result.setResultState(ResultState.RUN_HAS_RESULT);
Expand Down Expand Up @@ -1323,9 +1324,9 @@ public static final DQResponse<AmendmentValue> amendmentEventdateFromYearstartda
result.addComment("A value exists in dwc:eventDate, ammendment not attempted.");
} else {
try {
Integer numericYear = Integer.parseInt(year);
Integer numericStartDay = Integer.parseInt(startDay);
Integer numericEndDay = Integer.parseInt(endDay);
Integer numericYear = NumberUtils.parseInt(year);
Integer numericStartDay = NumberUtils.parseInt(startDay);
Integer numericEndDay = NumberUtils.parseInt(endDay);
logger.debug(numericStartDay);
if (numericStartDay < 1 || numericStartDay > 366 || numericEndDay < 1 || numericEndDay > 366) {
// out of range for possible days of year, report and fail.
Expand Down Expand Up @@ -1419,7 +1420,7 @@ public static final DQResponse<AmendmentValue> amendmentEventDateFromYearMonthDa
result.addComment("A value exists in dwc:eventDate, ammendment not attempted.");
} else {
try {
Integer.parseInt(year.trim());
NumberUtils.parseInt(year);
try {
boolean hasMonth = false;
boolean hasDay = false;
Expand All @@ -1429,7 +1430,7 @@ public static final DQResponse<AmendmentValue> amendmentEventDateFromYearMonthDa
// Roman numeral month values are interpretable as numbers.
logger.debug(month);
if (DateUtils.romanMonthToInteger(month)==null) {
Integer monthInt = Integer.parseInt(month);
Integer monthInt = NumberUtils.parseInt(month);
if (DateUtils.isMonthInRange(monthInt)) {
hasMonth=true;
}
Expand All @@ -1447,7 +1448,7 @@ public static final DQResponse<AmendmentValue> amendmentEventDateFromYearMonthDa
logger.debug(month);
} else {
try {
Integer monthInt = Integer.parseInt(month);
Integer monthInt = NumberUtils.parseInt(month);
if (DateUtils.isMonthInRange(monthInt)) {
hasMonth=true;
}
Expand All @@ -1458,7 +1459,7 @@ public static final DQResponse<AmendmentValue> amendmentEventDateFromYearMonthDa
}
if (!DateUtils.isEmpty(day)) {
try {
Integer numericDay = Integer.parseInt(day.trim());
Integer numericDay = NumberUtils.parseInt(day);
if (!DateUtils.isDayInRange(numericDay)) {
throw new NumberFormatException("The provided value for Day is out of range for a day");
} else {
Expand Down Expand Up @@ -1552,7 +1553,7 @@ public static final DQResponse<AmendmentValue> amendmentMonthStandardized(@Acted
} else {

try {
Integer monthNumeric = Integer.parseInt(month.trim());
Integer monthNumeric = NumberUtils.parseInt(month);
result.addComment("A value for dwc:month parsable as an integer was provided.");
if (monthNumeric >= 1 && monthNumeric <=12) {
result.addComment("Provided value for dwc:month was in the range 1-12.");
Expand Down Expand Up @@ -1582,7 +1583,7 @@ public static final DQResponse<AmendmentValue> amendmentMonthStandardized(@Acted
} else {
logger.debug(monthTrim);
try {
Integer monthInteger = Integer.parseInt(monthTrim);
Integer monthInteger = NumberUtils.parseInt(monthTrim);
result.setResultState(ResultState.AMENDED);
Map<String, String> values = new HashMap<>();
values.put("dwc:month", monthInteger.toString());
Expand Down Expand Up @@ -1648,7 +1649,7 @@ public static final DQResponse<AmendmentValue> amendmentDayStandardized(@ActedUp
} else {

try {
Integer dayNumeric = Integer.parseInt(day);
Integer dayNumeric = NumberUtils.parseInt(day);
result.addComment("A value for dwc:day parsable as an integer was provided.");
String dayTrimmed = day.replaceAll("[^0-9]", "");
String dayCleaned = dayNumeric.toString();
Expand Down Expand Up @@ -1698,7 +1699,7 @@ public static final DQResponse<AmendmentValue> amendmentDayStandardized(@ActedUp
if (!failed) {
// Try again
try {
Integer dayNumeric = Integer.parseInt(dayTrimmed.trim());
Integer dayNumeric = NumberUtils.parseInt(dayTrimmed);
logger.debug(dayNumeric);
if (dayNumeric>0 && dayNumeric<32) {
result.setResultState(ResultState.AMENDED);
Expand Down Expand Up @@ -1858,7 +1859,7 @@ public static DQResponse<ComplianceValue> validationEventConsistent(
result.addComment("The provided dwc:eventDate ["+eventDate+"] represents an interval of more than one day, and dwc:day contains a value ["+day+"] when it should not. ");
inconsistencyFound = true;
}
if (DateUtils.extractDate(eventDate).getDayOfMonth()!=Integer.parseInt(day)) {
if (DateUtils.extractDate(eventDate).getDayOfMonth()!=NumberUtils.parseInt(day)) {
result.addComment("Provided value for dwc:eventDate ["+eventDate+"] is not consistent with the provided value of dwc:day ["+day+"].");
inconsistencyFound = true;
}
Expand All @@ -1883,7 +1884,7 @@ public static DQResponse<ComplianceValue> validationEventConsistent(
result.addComment("Unable to extract startDayOfYear from dwc:eventDate ["+eventDate+"] for comparision with provided dwc:startDayOfYear ["+startDayOfYear+"].");
interpretationProblem = true;
} else {
if (extractedDate.getDayOfYear()!=Integer.parseInt(startDayOfYear)) {
if (extractedDate.getDayOfYear()!=NumberUtils.parseInt(startDayOfYear)) {
result.addComment("Provided value for dwc:eventDate ["+eventDate+"] is not consistent with the provided value of dwc:startDayOfYear["+startDayOfYear+"].");
inconsistencyFound = true;
}
Expand All @@ -1900,7 +1901,7 @@ public static DQResponse<ComplianceValue> validationEventConsistent(
// range represented by dwc:eventDate;
if (!DateUtils.isEmpty(endDayOfYear)) {
if (DateUtils.hasResolutionDayOrFiner(eventDate)) {
if (DateUtils.extractDateInterval(eventDate).getEnd().getDayOfYear()!=Integer.parseInt(endDayOfYear)) {
if (DateUtils.extractDateInterval(eventDate).getEnd().getDayOfYear()!=NumberUtils.parseInt(endDayOfYear)) {
result.addComment("Provided value for dwc:eventDate ["+eventDate+"] is not consistent with the provided value of dwc:endDayIfYear ["+endDayOfYear+"].");
inconsistencyFound = true;
}
Expand Down Expand Up @@ -2316,9 +2317,9 @@ public static DQResponse<ComplianceValue> validationYearInrange(
result.setResultState(ResultState.INTERNAL_PREREQUISITES_NOT_MET);
} else {
try {
int numericYear = Integer.parseInt(year.trim());
int numericLowerBound = Integer.parseInt(lowerBound);
int numericUpperBound = Integer.parseInt(upperBound);
int numericYear = NumberUtils.parseInt(year);
int numericLowerBound = NumberUtils.parseInt(lowerBound);
int numericUpperBound = NumberUtils.parseInt(upperBound);
if (numericYear<numericLowerBound || numericYear>numericUpperBound) {
result.setValue(ComplianceValue.NOT_COMPLIANT);
result.addComment("Provided value for dwc:year '" + year + "' is not an integer in the range " + lowerBound + " to " + upperBound + " (current year).");
Expand Down
8 changes: 4 additions & 4 deletions src/main/java/org/filteredpush/qc/date/DwCOtherDateDQ.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -360,8 +360,8 @@ public static DQResponse<AmendmentValue> amendmentDateidentifiedStandardized(
if (dateIdentified.matches("^[0-9]{4}.[0-9]{2}.[0-9]{2}$")) {
try {
// try to see if this is yyyy-dd-mm error for yyyy-mm-dd
Integer secondBit = Integer.parseInt(dateIdentified.substring(5,7));
Integer thirdBit = Integer.parseInt(dateIdentified.substring(8));
Integer secondBit = org.filteredpush.qc.date.util.NumberUtils.parseInt(dateIdentified.substring(5,7));
Integer thirdBit = org.filteredpush.qc.date.util.NumberUtils.parseInt(dateIdentified.substring(8));
if (secondBit>12 && thirdBit<12) {
// try switching second and third parts of date.
String toTest = dateIdentified.substring(0, 4).concat("-").concat(dateIdentified.substring(8)).concat("-").concat(dateIdentified.substring(5, 7));
Expand Down Expand Up @@ -729,8 +729,8 @@ public static DQResponse<AmendmentValue> amendmentModifiedStandardized(
if (modified.matches("^[0-9]{4}.[0-9]{2}.[0-9]{2}$")) {
try {
// try to see if this is yyyy-dd-mm error for yyyy-mm-dd
Integer secondBit = Integer.parseInt(modified.substring(5,7));
Integer thirdBit = Integer.parseInt(modified.substring(8));
Integer secondBit = org.filteredpush.qc.date.util.NumberUtils.parseInt(modified.substring(5,7));
Integer thirdBit = org.filteredpush.qc.date.util.NumberUtils.parseInt(modified.substring(8));
if (secondBit>12 && thirdBit<12) {
// try switching second and third parts of date.
String toTest = modified.substring(0, 4).concat("-").concat(modified.substring(8)).concat("-").concat(modified.substring(5, 7));
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/org/filteredpush/qc/date/LocalDateInterval.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,7 @@ protected DatePair parseDateBit(String dateBit) throws EmptyDateException, Date
.append(DateTimeFormatter.ISO_LOCAL_DATE)
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
if (dateBit.matches("^[0-9]{1,3}$")) {
dateBit = String.format("%04d", Integer.parseInt(dateBit));
dateBit = String.format("%04d", org.filteredpush.qc.date.util.NumberUtils.parseInt(dateBit));
}
LocalDate startDateBit = LocalDate.parse(dateBit+"-01-01", formatter);
result = new DatePair(startDateBit,startDateBit.with(TemporalAdjusters.lastDayOfYear()));
Expand All @@ -278,7 +278,7 @@ protected DatePair parseDateBit(String dateBit) throws EmptyDateException, Date
DateTimeFormatter formatter = new DateTimeFormatterBuilder()
.append(DateTimeFormatter.ofPattern("yyyy-MM-ddGGGGG"))
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
dateBit = String.format("%04d", Integer.parseInt(dateBit.substring(0,4)));
dateBit = String.format("%04d", org.filteredpush.qc.date.util.NumberUtils.parseInt(dateBit.substring(0,4)));
LocalDate startDateBit = LocalDate.parse(dateBit+"-01-01B", formatter);
result = new DatePair(startDateBit.minusYears(1),startDateBit.minusYears(1).with(TemporalAdjusters.lastDayOfYear()));
} else {
Expand Down
4 changes: 2 additions & 2 deletions src/main/java/org/filteredpush/qc/date/LocalDateTimeInterval.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -258,7 +258,7 @@ protected DatePair parseDateBit(String dateBit) throws EmptyDateException, Date
.append(DateTimeFormatter.ISO_LOCAL_DATE)
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
if (dateBit.matches("^[0-9]{1,3}$")) {
dateBit = String.format("%04d", Integer.parseInt(dateBit));
dateBit = String.format("%04d", org.filteredpush.qc.date.util.NumberUtils.parseInt(dateBit));
}
LocalDateTime startDateBit = LocalDateTime.parse(dateBit+"-01-01", formatter);
result = new DatePair(startDateBit,startDateBit.with(TemporalAdjusters.lastDayOfYear()));
Expand All @@ -267,7 +267,7 @@ protected DatePair parseDateBit(String dateBit) throws EmptyDateException, Date
DateTimeFormatter formatter = new DateTimeFormatterBuilder()
.append(DateTimeFormatter.ofPattern("yyyy-MM-ddGGGGG"))
.toFormatter().withResolverStyle(ResolverStyle.STRICT);
dateBit = String.format("%04d", Integer.parseInt(dateBit.substring(0,4)));
dateBit = String.format("%04d", org.filteredpush.qc.date.util.NumberUtils.parseInt(dateBit.substring(0,4)));
LocalDateTime startDateBit = LocalDateTime.parse(dateBit+"-01-01B", formatter);
result = new DatePair(startDateBit.minusYears(1),startDateBit.minusYears(1).with(TemporalAdjusters.lastDayOfYear()));
} else {
Expand Down
3 changes: 2 additions & 1 deletion src/main/java/org/filteredpush/qc/date/Runner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
import org.datakurator.ffdq.api.result.NumericalValue;
import org.datakurator.ffdq.model.ResultState;
import org.filteredpush.qc.date.util.DateUtils;
import org.filteredpush.qc.date.util.NumberUtils;

/**
* Selfstanding execution of event_date_qc functionality. Can run TG2 Date related tests on flat DarwinCore
Expand Down Expand Up @@ -106,7 +107,7 @@ public static void main(String[] args) {
int limit = 0;
if (limitValue!=null) {
try {
limit = Integer.parseInt(limitValue);
limit = org.filteredpush.qc.date.util.NumberUtils.parseInt(limitValue);
} catch (NumberFormatException nfe) {
logger.error(nfe.getMessage());
}
Expand Down
Loading