From 65386429d6fbd3eb34ad6ee31bc88d67f713c62f Mon Sep 17 00:00:00 2001 From: Tom Wright Date: Sun, 31 Jan 2016 20:57:41 +0000 Subject: [PATCH 1/2] number-as-id -- use qif number as id My statements seem to have perfectly good id's don't impose your made up ids upon me :P --- fixofx.py | 74 ++++++++++++++++---------------- lib/ofxtools/qif_converter.py | 29 +++++++++---- test/ofxtools_qif_converter.py | 78 ++++++++++++++++++++-------------- 3 files changed, 103 insertions(+), 78 deletions(-) diff --git a/fixofx.py b/fixofx.py index 0ebbb2a..626b41f 100755 --- a/fixofx.py +++ b/fixofx.py @@ -1,13 +1,13 @@ #!/usr/bin/env python # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -57,67 +57,66 @@ def fixpath(filename): pass -def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN", +def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN", bankid="UNKNOWN", accttype="UNKNOWN", acctid="UNKNOWN", - balance="UNKNOWN", curdef=None, lang="ENG", dayfirst=False, - debug=False): - + balance="UNKNOWN", curdef=None, lang="ENG", dayfirst=False, + number_as_id=False, debug=False): + # This finishes a verbosity message started by the caller, where the # caller explains the source command-line option and this explains the # source format. - if verbose: + if verbose: sys.stderr.write("Converting from %s format.\n" % filetype) if options.debug and (filetype in ["OFC", "QIF"] or filetype.startswith("OFX")): sys.stderr.write("Starting work on raw text:\n") sys.stderr.write(rawtext + "\n\n") - + if filetype.startswith("OFX/2"): if verbose: sys.stderr.write("No conversion needed; returning unmodified.\n") - + # The file is already OFX 2 -- return it unaltered, ignoring # any of the parameters passed to this method. return text - + elif filetype.startswith("OFX"): if verbose: sys.stderr.write("Converting to OFX/2.0...\n") - + # This will throw a ParseException if it is unable to recognize # the source format. - response = ofx.Response(text, debug=debug) + response = ofx.Response(text, debug=debug) return response.as_xml(original_format=filetype) - + elif filetype == "OFC": if verbose: sys.stderr.write("Beginning OFC conversion...\n") converter = ofxtools.OfcConverter(text, fid=fid, org=org, curdef=curdef, lang=lang, debug=debug) - + # This will throw a ParseException if it is unable to recognize # the source format. - if verbose: + if verbose: sys.stderr.write("Converting to OFX/1.02...\n\n%s\n\n" % converter.to_ofx102()) sys.stderr.write("Converting to OFX/2.0...\n") - + return converter.to_xml() - + elif filetype == "QIF": if verbose: sys.stderr.write("Beginning QIF conversion...\n") converter = ofxtools.QifConverter(text, fid=fid, org=org, - bankid=bankid, accttype=accttype, - acctid=acctid, balance=balance, + bankid=bankid, accttype=accttype, + acctid=acctid, balance=balance, curdef=curdef, lang=lang, dayfirst=dayfirst, - debug=debug) - + debug=debug, number_as_id=number_as_id) # This will throw a ParseException if it is unable to recognize # the source format. - if verbose: + if verbose: sys.stderr.write("Converting to OFX/1.02...\n\n%s\n\n" % converter.to_ofx102()) sys.stderr.write("Converting to OFX/2.0...\n") - + return converter.to_xml() - + else: raise TypeError("Unable to convert source format '%s'." % filetype) @@ -148,6 +147,8 @@ def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN", help="(QIF only) Account balance to use in output") parser.add_option("--dayfirst", action="store_true", dest="dayfirst", default=False, help="(QIF only) Parse dates day first (UK format)") +parser.add_option("--number-as-id", action="store_true", dest="number_as_id", default=False, + help="(QIF only) Use the number as id") (options, args) = parser.parse_args() # @@ -168,9 +169,9 @@ def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN", if options.filename: if os.path.isfile(options.filename): - if options.verbose: + if options.verbose: sys.stderr.write("Reading from '%s'\n." % options.filename) - + try: srcfile = open(options.filename, 'rU') rawtext = srcfile.read() @@ -180,19 +181,19 @@ def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN", print "Exiting." sys.stderr.write("fixofx failed with error code 1\n") sys.exit(1) - + else: print "'%s' does not appear to be a file. Try --help." % options.filename sys.stderr.write("fixofx failed with error code 2\n") sys.exit(2) else: - if options.verbose: + if options.verbose: sys.stderr.write("Reading from standard input.\n") - + stdin_universal = os.fdopen(os.dup(sys.stdin.fileno()), "rU") rawtext = stdin_universal.read() - + if rawtext == "" or rawtext is None: print "No input. Pipe a file to convert to the script,\n" + \ "or call with -f. Call with --help for more info." @@ -208,18 +209,19 @@ def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN", # rather than parsing the file to make sure. (Parsing will fail # below if the guess is wrong on OFX/1 and QIF.) filetype = ofx.FileTyper(rawtext).trust() - + if options.type: print "Input file type is %s." % filetype sys.exit(0) elif options.debug: sys.stderr.write("Input file type is %s.\n" % filetype) - - converted = convert(rawtext, filetype, verbose=options.verbose, - fid=options.fid, org=options.org, bankid=options.bankid, - accttype=options.accttype, acctid=options.acctid, + + converted = convert(rawtext, filetype, verbose=options.verbose, + fid=options.fid, org=options.org, bankid=options.bankid, + accttype=options.accttype, acctid=options.acctid, balance=options.balance, curdef=options.curdef, lang=options.lang, dayfirst=options.dayfirst, + number_as_id=options.number_as_id, debug=options.debug) print converted sys.exit(0) diff --git a/lib/ofxtools/qif_converter.py b/lib/ofxtools/qif_converter.py index 25251e2..b061e77 100644 --- a/lib/ofxtools/qif_converter.py +++ b/lib/ofxtools/qif_converter.py @@ -30,7 +30,8 @@ class QifConverter: def __init__(self, qif, fid="UNKNOWN", org="UNKNOWN", bankid="UNKNOWN", accttype="UNKNOWN", acctid="UNKNOWN", balance="UNKNOWN", - curdef=None, lang="ENG", dayfirst=False, debug=False): + curdef=None, lang="ENG", dayfirst=False, debug=False, + number_as_id=False): self.qif = qif self.fid = fid self.org = org @@ -42,6 +43,7 @@ def __init__(self, qif, fid="UNKNOWN", org="UNKNOWN", bankid="UNKNOWN", self.lang = lang self.debug = debug self.dayfirst = dayfirst + self.number_as_id = number_as_id self.parsed_qif = None @@ -227,7 +229,15 @@ def _check_date_format(self, parsed_date): def _clean_txn_list(self, txn_list): for txn_obj in txn_list: try: - txn = self._clean_txn(txn_obj) + txn = txn_obj.asDict() + + if self.number_as_id: + number = txn.get("Number", None) + if number is not None: + txn["ID"] = number + del txn["Number"] + + txn = self._clean_txn(txn) txn_date = txn["Date"] txn_date_list = self.txns_by_date.get(txn_date, []) txn_date_list.append(txn) @@ -259,7 +269,7 @@ def _clean_txn_list(self, txn_list): self.start_date = strftime("%Y%m%d", localtime()) self.end_date = self.start_date - def _clean_txn(self, txn_obj): + def _clean_txn(self, txn): # This is sort of the brute-force method of the converter. It # looks at the data we get from the bank and tries as hard as # possible to make best-effort guesses about what the OFX 2.0 @@ -269,7 +279,6 @@ def _clean_txn(self, txn_obj): # the txn_obj shouldn't be in the data, it will throw a ValueError. # Otherwise, it will return a transaction cleaned to the best # of our abilities. - txn = txn_obj.asDict() self._clean_txn_date(txn) self._clean_txn_amount(txn) self._clean_txn_number(txn) @@ -589,11 +598,13 @@ def _ofx_txns(self): txn_date = txn.get("Date", "UNKNOWN") txn_amt = txn.get("Amount", "00.00") - # Make a synthetic transaction ID using as many - # uniqueness guarantors as possible. - txn["ID"] = "%s-%s-%s-%s-%s" % (self.org, self.accttype, - txn_date, txn_index, - txn_amt) + if not self.number_as_id: + # Make a synthetic transaction ID using as many + # uniqueness guarantors as possible. + txn["ID"] = "%s-%s-%s-%s-%s" % (self.org, self.accttype, + txn_date, txn_index, + txn_amt) + txns += self._ofx_txn(txn) txn_index -= 1 diff --git a/test/ofxtools_qif_converter.py b/test/ofxtools_qif_converter.py index ac00261..9ecd02b 100644 --- a/test/ofxtools_qif_converter.py +++ b/test/ofxtools_qif_converter.py @@ -1,11 +1,11 @@ # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,7 +25,7 @@ class QifConverterTests(unittest.TestCase): def setUp(self): pass - + def test_bank_stmttype(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -34,7 +34,7 @@ def test_bank_stmttype(self): ''') converter = ofxtools.QifConverter(qiftext) self.assertEqual(converter.accttype, "CHECKING") - + def test_ccard_stmttype(self): qiftext = textwrap.dedent('''\ !Type:CCard @@ -43,7 +43,7 @@ def test_ccard_stmttype(self): ''') converter = ofxtools.QifConverter(qiftext) self.assertEqual(converter.accttype, "CREDITCARD") - + def test_no_stmttype(self): qiftext = textwrap.dedent('''\ D01/13/2005 @@ -51,7 +51,7 @@ def test_no_stmttype(self): ''') converter = ofxtools.QifConverter(qiftext) self.assertEqual(converter.accttype, "CHECKING") - + def test_no_txns(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -60,7 +60,7 @@ def test_no_txns(self): converter = ofxtools.QifConverter(qiftext) self.assertEqual(converter.start_date, today) self.assertEqual(converter.end_date, today) - + def test_us_date(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -69,7 +69,7 @@ def test_us_date(self): ''') converter = ofxtools.QifConverter(qiftext) self.assertTrue(converter.txns_by_date.has_key("20050113")) - + def test_uk_date(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -78,7 +78,7 @@ def test_uk_date(self): ''') converter = ofxtools.QifConverter(qiftext) self.assertTrue(converter.txns_by_date.has_key("20050113")) - + def test_ambiguous_date(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -87,7 +87,7 @@ def test_ambiguous_date(self): ''') converter = ofxtools.QifConverter(qiftext) self.assertTrue(converter.txns_by_date.has_key("20051201")) - + def test_mixed_us_dates(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -99,7 +99,7 @@ def test_mixed_us_dates(self): converter = ofxtools.QifConverter(qiftext) self.assertTrue(converter.txns_by_date.has_key("20050112")) self.assertTrue(converter.txns_by_date.has_key("20050113")) - + def test_mixed_uk_dates(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -111,7 +111,7 @@ def test_mixed_uk_dates(self): converter = ofxtools.QifConverter(qiftext) self.assertTrue(converter.txns_by_date.has_key("20050112")) self.assertTrue(converter.txns_by_date.has_key("20050113")) - + def test_slashfree_date(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -120,7 +120,7 @@ def test_slashfree_date(self): ''') converter = ofxtools.QifConverter(qiftext) self.assertTrue(converter.txns_by_date.has_key("20051201")) - + def test_unparseable_date(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -128,7 +128,7 @@ def test_unparseable_date(self): ^ ''') self.assertRaises(ValueError, ofxtools.QifConverter, qiftext) - + def test_len_eight_no_int_date(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -136,7 +136,7 @@ def test_len_eight_no_int_date(self): ^ ''') self.assertRaises(ValueError, ofxtools.QifConverter, qiftext) - + def test_asc_dates(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -147,7 +147,7 @@ def test_asc_dates(self): D02/01/2005 ^ D02/01/2005 - ^ + ^ D02/13/2005 ^ ''') @@ -155,7 +155,7 @@ def test_asc_dates(self): self.assertEqual(converter.start_date, "20050113") self.assertEqual(converter.end_date, "20050213") self.assertEqual(len(converter.txns_by_date.keys()), 4) - + def test_desc_dates(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -164,7 +164,7 @@ def test_desc_dates(self): D02/01/2005 ^ D02/01/2005 - ^ + ^ D01/27/2005 ^ D01/13/2005 @@ -174,7 +174,7 @@ def test_desc_dates(self): self.assertEqual(converter.start_date, "20050113") self.assertEqual(converter.end_date, "20050213") self.assertEqual(len(converter.txns_by_date.keys()), 4) - + def test_mixed_dates(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -185,7 +185,7 @@ def test_mixed_dates(self): D01/13/2005 ^ D02/01/2005 - ^ + ^ D01/27/2005 ^ ''') @@ -193,7 +193,7 @@ def test_mixed_dates(self): self.assertEqual(converter.start_date, "20050113") self.assertEqual(converter.end_date, "20050213") self.assertEqual(len(converter.txns_by_date.keys()), 4) - + def test_default_currency(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -204,7 +204,7 @@ def test_default_currency(self): converter = ofxtools.QifConverter(qiftext) ofx102 = converter.to_ofx102() self.assertTrue(ofx102.find('USD') != -1) - + def test_found_currency(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -215,7 +215,7 @@ def test_found_currency(self): converter = ofxtools.QifConverter(qiftext) ofx102 = converter.to_ofx102() self.assertTrue(ofx102.find('EUR') != -1) - + def test_explicit_currency(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -226,7 +226,7 @@ def test_explicit_currency(self): converter = ofxtools.QifConverter(qiftext, curdef='GBP') ofx102 = converter.to_ofx102() self.assertTrue(ofx102.find('GBP') != -1) - + def test_amount2(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -237,7 +237,7 @@ def test_amount2(self): converter = ofxtools.QifConverter(qiftext) txn = converter.txns_by_date["20050201"][0] self.assertEqual(txn["Amount"], "25.42") - + def test_bad_amount_precision(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -248,7 +248,19 @@ def test_bad_amount_precision(self): converter = ofxtools.QifConverter(qiftext) txn = converter.txns_by_date["20070125"][0] self.assertEqual(txn["Amount"], "417.93") - + + def test_id(self): + qiftext = textwrap.dedent('''\ + !Type:Bank + D01/25/2007 + T417.930 + NI_AM_ID + ^ + ''') + converter = ofxtools.QifConverter(qiftext, number_as_id=True) + txn = converter.txns_by_date["20070125"][0] + self.assertEqual(txn["ID"], "I_AM_ID") + def test_dash_amount(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -264,7 +276,7 @@ def test_dash_amount(self): self.assertEqual(len(txn_list), 1) txn = txn_list[0] self.assertEqual(txn["Amount"], "25.42") - + def test_trailing_minus(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -275,7 +287,7 @@ def test_trailing_minus(self): converter = ofxtools.QifConverter(qiftext) txn = converter.txns_by_date["20080806"][0] self.assertEqual(txn["Amount"], "-26.24") - + def test_n_a_number(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -287,7 +299,7 @@ def test_n_a_number(self): converter = ofxtools.QifConverter(qiftext) txn = converter.txns_by_date["20070125"][0] self.assertEqual(txn.has_key("Number"), False) - + def test_creditcard_number(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -299,7 +311,7 @@ def test_creditcard_number(self): converter = ofxtools.QifConverter(qiftext) txn = converter.txns_by_date["20070125"][0] self.assertEqual(txn.has_key("Number"), False) - + def test_creditcard_stmt_number(self): qiftext = textwrap.dedent('''\ !Type:CCard @@ -311,7 +323,7 @@ def test_creditcard_stmt_number(self): converter = ofxtools.QifConverter(qiftext) txn = converter.txns_by_date["20070125"][0] self.assertEqual(txn.has_key("Number"), False) - + def test_check_stmt_number(self): qiftext = textwrap.dedent('''\ !Type:Bank @@ -323,7 +335,7 @@ def test_check_stmt_number(self): converter = ofxtools.QifConverter(qiftext) txn = converter.txns_by_date["20070125"][0] self.assertEqual(txn.get("Type"), "CHECK") - + def test_usaa_check(self): qiftext = textwrap.dedent('''\ !Type:Bank From 007f9a5b348285544f075b7fb4f0fa7585ab2568 Mon Sep 17 00:00:00 2001 From: Tom Wright Date: Sun, 31 Jan 2016 20:52:31 +0000 Subject: [PATCH 2/2] my editor still doesn't like your whitespace --- lib/ofxtools/qif_parser.py | 59 +++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 30 deletions(-) diff --git a/lib/ofxtools/qif_parser.py b/lib/ofxtools/qif_parser.py index 8ab4398..3c99353 100644 --- a/lib/ofxtools/qif_parser.py +++ b/lib/ofxtools/qif_parser.py @@ -1,11 +1,11 @@ # Copyright 2005-2010 Wesabe, Inc. -# +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at -# +# # http://www.apache.org/licenses/LICENSE-2.0 -# +# # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -31,7 +31,7 @@ def __init__(self, debug=False): 'B' : "Balance", '/' : "BalanceDate", '$' : "Balance" } - + noninvestment_items = { 'D' : "Date", 'T' : "Amount", 'U' : "Amount2", @@ -45,7 +45,7 @@ def __init__(self, debug=False): 'E' : "SplitMemo", '$' : "SplitAmount", '-' : "NegativeSplitAmount" } - + investment_items = { 'D' : "Date", 'N' : "Action", 'Y' : "Security", @@ -58,7 +58,7 @@ def __init__(self, debug=False): 'O' : "Commission", 'L' : "TransferAccount", '$' : "TransferAmount" } - + category_items = { 'N' : "Name", 'D' : "Description", 'T' : "TaxRelated", @@ -66,61 +66,61 @@ def __init__(self, debug=False): 'E' : "ExpenseCategory", 'B' : "BudgetAmount", 'R' : "TaxSchedule" } - + class_items = { 'N' : "Name", 'D' : "Description" } - + options = Group(CaselessLiteral('!Option:') + restOfLine).suppress() - - banktxns = Group(CaselessLiteral('!Type:Bank').suppress() + + + banktxns = Group(CaselessLiteral('!Type:Bank').suppress() + ZeroOrMore(Or([self._items(noninvestment_items), options])) ).setResultsName("BankTransactions") - - cashtxns = Group(CaselessLiteral('!Type:Cash').suppress() + + + cashtxns = Group(CaselessLiteral('!Type:Cash').suppress() + ZeroOrMore(Or([self._items(noninvestment_items), options])) ).setResultsName("CashTransactions") - + ccardtxns = Group(Or([CaselessLiteral('!Type:CCard').suppress(), - CaselessLiteral('!Type!CCard').suppress()]) + + CaselessLiteral('!Type!CCard').suppress()]) + ZeroOrMore(Or([self._items(noninvestment_items), options])) ).setResultsName("CreditCardTransactions") - - liabilitytxns = Group(CaselessLiteral('!Type:Oth L').suppress() + + + liabilitytxns = Group(CaselessLiteral('!Type:Oth L').suppress() + ZeroOrMore(Or([self._items(noninvestment_items), options])) ).setResultsName("CreditCardTransactions") - - invsttxns = Group(CaselessLiteral('!Type:Invst').suppress() + + + invsttxns = Group(CaselessLiteral('!Type:Invst').suppress() + ZeroOrMore(self._items(investment_items)) ).setResultsName("InvestmentTransactions") - + acctlist = Group(CaselessLiteral('!Account').suppress() + ZeroOrMore(Or([self._items(account_items, name="AccountInfo")])) ).setResultsName("AccountList") - + category = Group(CaselessLiteral('!Type:Cat').suppress() + ZeroOrMore(self._items(category_items)) ).setResultsName("CategoryList") - + classlist = Group(CaselessLiteral('!Type:Class').suppress() + ZeroOrMore(self._items(category_items)) ).setResultsName("ClassList") - + self.parser = Group(ZeroOrMore(White()).suppress() + ZeroOrMore(acctlist).suppress() + OneOrMore(ccardtxns | cashtxns | banktxns | liabilitytxns | invsttxns) + ZeroOrMore(White()).suppress() ).setResultsName("QifStatement") - + if (debug): - self.parser.setDebugActions(ofxtools._ofxtoolsStartDebugAction, - ofxtools._ofxtoolsSuccessDebugAction, + self.parser.setDebugActions(ofxtools._ofxtoolsStartDebugAction, + ofxtools._ofxtoolsSuccessDebugAction, ofxtools._ofxtoolsExceptionDebugAction) - - + + def _items(self, items, name="Transaction"): item_list = [] for (code, name) in items.iteritems(): @@ -130,12 +130,11 @@ def _items(self, items, name="Transaction"): oneOf('^EUR ^').setResultsName('Currency') + LineEnd().suppress() ).setResultsName(name) - + def _item(self, code, name): return CaselessLiteral(code).suppress() + \ restOfLine.setResultsName(name) + \ LineEnd().suppress() - + def parse(self, qif): return self.parser.parseString(qif) -