Skip to content
This repository was archived by the owner on May 8, 2018. It is now read-only.

Support using qif numbers as fitid's #9

Open
talwrii wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Support using qif numbers as fitid's #9

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6538642
number-as-id -- use qif number as id
Jan 31, 2016
007f9a5
my editor still doesn't like your whitespace
Jan 31, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 38 additions & 36 deletions fixofx.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,13 +1,13 @@
#!/usr/bin/env python

# Copyright 2005-2010 Wesabe, Inc.
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#
# http://www.apache.org/licenses/LICENSE-2.0
#
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Expand Down Expand Up @@ -57,67 +57,66 @@ def fixpath(filename):
pass


def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN",
def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN",
bankid="UNKNOWN", accttype="UNKNOWN", acctid="UNKNOWN",
balance="UNKNOWN", curdef=None, lang="ENG", dayfirst=False,
debug=False):
balance="UNKNOWN", curdef=None, lang="ENG", dayfirst=False,
number_as_id=False, debug=False):

# This finishes a verbosity message started by the caller, where the
# caller explains the source command-line option and this explains the
# source format.
if verbose:
if verbose:
sys.stderr.write("Converting from %s format.\n" % filetype)

if options.debug and (filetype in ["OFC", "QIF"] or filetype.startswith("OFX")):
sys.stderr.write("Starting work on raw text:\n")
sys.stderr.write(rawtext + "\n\n")

if filetype.startswith("OFX/2"):
if verbose: sys.stderr.write("No conversion needed; returning unmodified.\n")

# The file is already OFX 2 -- return it unaltered, ignoring
# any of the parameters passed to this method.
return text

elif filetype.startswith("OFX"):
if verbose: sys.stderr.write("Converting to OFX/2.0...\n")

# This will throw a ParseException if it is unable to recognize
# the source format.
response = ofx.Response(text, debug=debug)
response = ofx.Response(text, debug=debug)
return response.as_xml(original_format=filetype)

elif filetype == "OFC":
if verbose: sys.stderr.write("Beginning OFC conversion...\n")
converter = ofxtools.OfcConverter(text, fid=fid, org=org, curdef=curdef,
lang=lang, debug=debug)

# This will throw a ParseException if it is unable to recognize
# the source format.
if verbose:
if verbose:
sys.stderr.write("Converting to OFX/1.02...\n\n%s\n\n" %
converter.to_ofx102())
sys.stderr.write("Converting to OFX/2.0...\n")

return converter.to_xml()

elif filetype == "QIF":
if verbose: sys.stderr.write("Beginning QIF conversion...\n")
converter = ofxtools.QifConverter(text, fid=fid, org=org,
bankid=bankid, accttype=accttype,
acctid=acctid, balance=balance,
bankid=bankid, accttype=accttype,
acctid=acctid, balance=balance,
curdef=curdef, lang=lang, dayfirst=dayfirst,
debug=debug)

debug=debug, number_as_id=number_as_id)
# This will throw a ParseException if it is unable to recognize
# the source format.
if verbose:
if verbose:
sys.stderr.write("Converting to OFX/1.02...\n\n%s\n\n" %
converter.to_ofx102())
sys.stderr.write("Converting to OFX/2.0...\n")

return converter.to_xml()

else:
raise TypeError("Unable to convert source format '%s'." % filetype)

Expand Down Expand Up @@ -148,6 +147,8 @@ def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN",
help="(QIF only) Account balance to use in output")
parser.add_option("--dayfirst", action="store_true", dest="dayfirst", default=False,
help="(QIF only) Parse dates day first (UK format)")
parser.add_option("--number-as-id", action="store_true", dest="number_as_id", default=False,
help="(QIF only) Use the number as id")
(options, args) = parser.parse_args()

#
Expand All @@ -168,9 +169,9 @@ def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN",

if options.filename:
if os.path.isfile(options.filename):
if options.verbose:
if options.verbose:
sys.stderr.write("Reading from '%s'\n." % options.filename)

try:
srcfile = open(options.filename, 'rU')
rawtext = srcfile.read()
Expand All @@ -180,19 +181,19 @@ def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN",
print "Exiting."
sys.stderr.write("fixofx failed with error code 1\n")
sys.exit(1)

else:
print "'%s' does not appear to be a file. Try --help." % options.filename
sys.stderr.write("fixofx failed with error code 2\n")
sys.exit(2)

else:
if options.verbose:
if options.verbose:
sys.stderr.write("Reading from standard input.\n")

stdin_universal = os.fdopen(os.dup(sys.stdin.fileno()), "rU")
rawtext = stdin_universal.read()

if rawtext == "" or rawtext is None:
print "No input. Pipe a file to convert to the script,\n" + \
"or call with -f. Call with --help for more info."
Expand All @@ -208,18 +209,19 @@ def convert(text, filetype, verbose=False, fid="UNKNOWN", org="UNKNOWN",
# rather than parsing the file to make sure. (Parsing will fail
# below if the guess is wrong on OFX/1 and QIF.)
filetype = ofx.FileTyper(rawtext).trust()

if options.type:
print "Input file type is %s." % filetype
sys.exit(0)
elif options.debug:
sys.stderr.write("Input file type is %s.\n" % filetype)
converted = convert(rawtext, filetype, verbose=options.verbose,
fid=options.fid, org=options.org, bankid=options.bankid,
accttype=options.accttype, acctid=options.acctid,

converted = convert(rawtext, filetype, verbose=options.verbose,
fid=options.fid, org=options.org, bankid=options.bankid,
accttype=options.accttype, acctid=options.acctid,
balance=options.balance, curdef=options.curdef,
lang=options.lang, dayfirst=options.dayfirst,
number_as_id=options.number_as_id,
debug=options.debug)
print converted
sys.exit(0)
Expand Down
29 changes: 20 additions & 9 deletions lib/ofxtools/qif_converter.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@
class QifConverter:
def __init__(self, qif, fid="UNKNOWN", org="UNKNOWN", bankid="UNKNOWN",
accttype="UNKNOWN", acctid="UNKNOWN", balance="UNKNOWN",
curdef=None, lang="ENG", dayfirst=False, debug=False):
curdef=None, lang="ENG", dayfirst=False, debug=False,
number_as_id=False):
self.qif = qif
self.fid = fid
self.org = org
Expand All @@ -42,6 +43,7 @@ def __init__(self, qif, fid="UNKNOWN", org="UNKNOWN", bankid="UNKNOWN",
self.lang = lang
self.debug = debug
self.dayfirst = dayfirst
self.number_as_id = number_as_id

self.parsed_qif = None

Expand Down Expand Up @@ -227,7 +229,15 @@ def _check_date_format(self, parsed_date):
def _clean_txn_list(self, txn_list):
for txn_obj in txn_list:
try:
txn = self._clean_txn(txn_obj)
txn = txn_obj.asDict()

if self.number_as_id:
number = txn.get("Number", None)
if number is not None:
txn["ID"] = number
del txn["Number"]

txn = self._clean_txn(txn)
txn_date = txn["Date"]
txn_date_list = self.txns_by_date.get(txn_date, [])
txn_date_list.append(txn)
Expand Down Expand Up @@ -259,7 +269,7 @@ def _clean_txn_list(self, txn_list):
self.start_date = strftime("%Y%m%d", localtime())
self.end_date = self.start_date

def _clean_txn(self, txn_obj):
def _clean_txn(self, txn):
# This is sort of the brute-force method of the converter. It
# looks at the data we get from the bank and tries as hard as
# possible to make best-effort guesses about what the OFX 2.0
Expand All @@ -269,7 +279,6 @@ def _clean_txn(self, txn_obj):
# the txn_obj shouldn't be in the data, it will throw a ValueError.
# Otherwise, it will return a transaction cleaned to the best
# of our abilities.
txn = txn_obj.asDict()
self._clean_txn_date(txn)
self._clean_txn_amount(txn)
self._clean_txn_number(txn)
Expand Down Expand Up @@ -589,11 +598,13 @@ def _ofx_txns(self):
txn_date = txn.get("Date", "UNKNOWN")
txn_amt = txn.get("Amount", "00.00")

# Make a synthetic transaction ID using as many
# uniqueness guarantors as possible.
txn["ID"] = "%s-%s-%s-%s-%s" % (self.org, self.accttype,
txn_date, txn_index,
txn_amt)
if not self.number_as_id:
# Make a synthetic transaction ID using as many
# uniqueness guarantors as possible.
txn["ID"] = "%s-%s-%s-%s-%s" % (self.org, self.accttype,
txn_date, txn_index,
txn_amt)

txns += self._ofx_txn(txn)
txn_index -= 1

Expand Down
59 changes: 29 additions & 30 deletions lib/ofxtools/qif_parser.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
# Copyright 2005-2010 Wesabe, Inc.
#
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#
# http://www.apache.org/licenses/LICENSE-2.0
#
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
Expand All @@ -31,7 +31,7 @@ def __init__(self, debug=False):
'B' : "Balance",
'/' : "BalanceDate",
'$' : "Balance" }

noninvestment_items = { 'D' : "Date",
'T' : "Amount",
'U' : "Amount2",
Expand All @@ -45,7 +45,7 @@ def __init__(self, debug=False):
'E' : "SplitMemo",
'$' : "SplitAmount",
'-' : "NegativeSplitAmount" }

investment_items = { 'D' : "Date",
'N' : "Action",
'Y' : "Security",
Expand All @@ -58,69 +58,69 @@ def __init__(self, debug=False):
'O' : "Commission",
'L' : "TransferAccount",
'$' : "TransferAmount" }

category_items = { 'N' : "Name",
'D' : "Description",
'T' : "TaxRelated",
'I' : "IncomeCategory",
'E' : "ExpenseCategory",
'B' : "BudgetAmount",
'R' : "TaxSchedule" }

class_items = { 'N' : "Name",
'D' : "Description" }

options = Group(CaselessLiteral('!Option:') + restOfLine).suppress()
banktxns = Group(CaselessLiteral('!Type:Bank').suppress() +

banktxns = Group(CaselessLiteral('!Type:Bank').suppress() +
ZeroOrMore(Or([self._items(noninvestment_items),
options]))
).setResultsName("BankTransactions")
cashtxns = Group(CaselessLiteral('!Type:Cash').suppress() +

cashtxns = Group(CaselessLiteral('!Type:Cash').suppress() +
ZeroOrMore(Or([self._items(noninvestment_items),
options]))
).setResultsName("CashTransactions")

ccardtxns = Group(Or([CaselessLiteral('!Type:CCard').suppress(),
CaselessLiteral('!Type!CCard').suppress()]) +
CaselessLiteral('!Type!CCard').suppress()]) +
ZeroOrMore(Or([self._items(noninvestment_items),
options]))
).setResultsName("CreditCardTransactions")
liabilitytxns = Group(CaselessLiteral('!Type:Oth L').suppress() +

liabilitytxns = Group(CaselessLiteral('!Type:Oth L').suppress() +
ZeroOrMore(Or([self._items(noninvestment_items),
options]))
).setResultsName("CreditCardTransactions")
invsttxns = Group(CaselessLiteral('!Type:Invst').suppress() +

invsttxns = Group(CaselessLiteral('!Type:Invst').suppress() +
ZeroOrMore(self._items(investment_items))
).setResultsName("InvestmentTransactions")

acctlist = Group(CaselessLiteral('!Account').suppress() +
ZeroOrMore(Or([self._items(account_items, name="AccountInfo")]))
).setResultsName("AccountList")

category = Group(CaselessLiteral('!Type:Cat').suppress() +
ZeroOrMore(self._items(category_items))
).setResultsName("CategoryList")

classlist = Group(CaselessLiteral('!Type:Class').suppress() +
ZeroOrMore(self._items(category_items))
).setResultsName("ClassList")

self.parser = Group(ZeroOrMore(White()).suppress() +
ZeroOrMore(acctlist).suppress() +
OneOrMore(ccardtxns | cashtxns | banktxns | liabilitytxns | invsttxns) +
ZeroOrMore(White()).suppress()
).setResultsName("QifStatement")

if (debug):
self.parser.setDebugActions(ofxtools._ofxtoolsStartDebugAction,
ofxtools._ofxtoolsSuccessDebugAction,
self.parser.setDebugActions(ofxtools._ofxtoolsStartDebugAction,
ofxtools._ofxtoolsSuccessDebugAction,
ofxtools._ofxtoolsExceptionDebugAction)


def _items(self, items, name="Transaction"):
item_list = []
for (code, name) in items.iteritems():
Expand All @@ -130,12 +130,11 @@ def _items(self, items, name="Transaction"):
oneOf('^EUR ^').setResultsName('Currency') +
LineEnd().suppress()
).setResultsName(name)

def _item(self, code, name):
return CaselessLiteral(code).suppress() + \
restOfLine.setResultsName(name) + \
LineEnd().suppress()

def parse(self, qif):
return self.parser.parseString(qif)

Loading