diff --git a/firefly/_version.py b/firefly/_version.py old mode 100644 new mode 100755 index 5d75144..5ee46cc --- a/firefly/_version.py +++ b/firefly/_version.py @@ -1,9 +1,9 @@ -#coding:utf8 -''' -Created on 2013-10-21 - -@author: lan (www.9miao.com) -''' - -from twisted.python import versions -version = versions.Version('firefly', 1, 3, 3) \ No newline at end of file +#coding:utf8 +''' +Created on 2013-10-21 + +@author: lan (www.9miao.com) +''' + +from twisted.python import versions +version = versions.Version('firefly', 1, 3, 4) \ No newline at end of file diff --git a/firefly/dbentrust/__init__.py b/firefly/dbentrust/__init__.py old mode 100644 new mode 100755 diff --git a/firefly/dbentrust/dbpool.py b/firefly/dbentrust/dbpool.py old mode 100644 new mode 100755 index 97307d3..518359b --- a/firefly/dbentrust/dbpool.py +++ b/firefly/dbentrust/dbpool.py @@ -1,26 +1,26 @@ -#coding:utf8 -''' -Created on 2013-5-8 - -@author: lan (www.9miao.com) -''' -from DBUtils.PooledDB import PooledDB -import MySQLdb - -DBCS = {'mysql':MySQLdb,} - -class DBPool(object): - ''' - ''' - def initPool(self,**kw): - ''' - ''' - self.config = kw - creator = DBCS.get(kw.get('engine','mysql'),MySQLdb) - self.pool = PooledDB(creator,5,**kw) - - def connection(self): - return self.pool.connection() - -dbpool = DBPool() - +#coding:utf8 +''' +Created on 2013-5-8 + +@author: lan (www.9miao.com) +''' +from DBUtils.PooledDB import PooledDB +import MySQLdb + +DBCS = {'mysql':MySQLdb,} + +class DBPool(object): + ''' + ''' + def initPool(self,**kw): + ''' + ''' + self.config = kw + creator = DBCS.get(kw.get('engine','mysql'),MySQLdb) + self.pool = PooledDB(creator,5,**kw) + + def connection(self): + return self.pool.connection() + +dbpool = DBPool() + diff --git a/firefly/dbentrust/dbutils.py b/firefly/dbentrust/dbutils.py old mode 100644 new mode 100755 index 470158f..160de6b --- a/firefly/dbentrust/dbutils.py +++ b/firefly/dbentrust/dbutils.py @@ -1,700 +1,699 @@ -#coding:utf8 -''' -Created on 2013-8-21 - -@author: lan (www.9miao.com) -''' -import itertools -import datetime - - -def safeunicode(obj, encoding='utf-8'): - r""" - Converts any given object to unicode string. - - >>> safeunicode('hello') - u'hello' - >>> safeunicode(2) - u'2' - >>> safeunicode('\xe1\x88\xb4') - u'\u1234' - """ - t = type(obj) - if t is unicode: - return obj - elif t is str: - return obj.decode(encoding) - elif t in [int, float, bool]: - return unicode(obj) - elif hasattr(obj, '__unicode__') or isinstance(obj, unicode): - return unicode(obj) - else: - return str(obj).decode(encoding) - -def safestr(obj, encoding='utf-8'): - r""" - Converts any given object to utf-8 encoded string. - - >>> safestr('hello') - 'hello' - >>> safestr(u'\u1234') - '\xe1\x88\xb4' - >>> safestr(2) - '2' - """ - if isinstance(obj, unicode): - return obj.encode(encoding) - elif isinstance(obj, str): - return obj - elif hasattr(obj, 'next'): # iterator - return itertools.imap(safestr, obj) - else: - return str(obj) - -def sqlify(obj): - """ - converts `obj` to its proper SQL version - - >>> sqlify(None) - 'NULL' - >>> sqlify(True) - "'t'" - >>> sqlify(3) - '3' - """ - # because `1 == True and hash(1) == hash(True)` - # we have to do this the hard way... - - if obj is None: - return 'NULL' - elif obj is True: - return "'t'" - elif obj is False: - return "'f'" - elif datetime and isinstance(obj, datetime.datetime): - return repr(obj.isoformat()) - else: - if isinstance(obj, unicode): obj = obj.encode('utf8') - return repr(obj) - -def sqllist(lst): - """ - Converts the arguments for use in something like a WHERE clause. - - >>> sqllist(['a', 'b']) - 'a, b' - >>> sqllist('a') - 'a' - >>> sqllist(u'abc') - u'abc' - """ - if isinstance(lst, basestring): - return lst - else: - return ', '.join(lst) - -def _sqllist(values): - """ - >>> _sqllist([1, 2, 3]) - - """ - items = [] - items.append('(') - for i, v in enumerate(values): - if i != 0: - items.append(', ') - items.append(sqlparam(v)) - items.append(')') - return SQLQuery(items) - -def sqlquote(a): - """ - Ensures `a` is quoted properly for use in a SQL query. - - >>> 'WHERE x = ' + sqlquote(True) + ' AND y = ' + sqlquote(3) - - >>> 'WHERE x = ' + sqlquote(True) + ' AND y IN ' + sqlquote([2, 3]) - - """ - if isinstance(a, list): - return _sqllist(a) - else: - return sqlparam(a).sqlquery() - -def _interpolate(sformat): - """ - Takes a format string and returns a list of 2-tuples of the form - (boolean, string) where boolean says whether string should be evaled - or not. - - from (public domain, Ka-Ping Yee) - """ - from tokenize import tokenprog - - tokenprog = tokenprog - - def matchorfail(text, pos): - match = tokenprog.match(text, pos) - if match is None: - raise _ItplError(text, pos) - return match, match.end() - - namechars = "abcdefghijklmnopqrstuvwxyz" \ - "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"; - chunks = [] - pos = 0 - - while 1: - dollar = sformat.find("$", pos) - if dollar < 0: - break - nextchar = sformat[dollar + 1] - - if nextchar == "{": - chunks.append((0, sformat[pos:dollar])) - pos, level = dollar + 2, 1 - while level: - match, pos = matchorfail(sformat, pos) - tstart, tend = match.regs[3] - token = sformat[tstart:tend] - if token == "{": - level = level + 1 - elif token == "}": - level = level - 1 - chunks.append((1, sformat[dollar + 2:pos - 1])) - - elif nextchar in namechars: - chunks.append((0, sformat[pos:dollar])) - match, pos = matchorfail(sformat, dollar + 1) - while pos < len(sformat): - if sformat[pos] == "." and \ - pos + 1 < len(sformat) and sformat[pos + 1] in namechars: - match, pos = matchorfail(sformat, pos + 1) - elif sformat[pos] in "([": - pos, level = pos + 1, 1 - while level: - match, pos = matchorfail(sformat, pos) - tstart, tend = match.regs[3] - token = sformat[tstart:tend] - if token[0] in "([": - level = level + 1 - elif token[0] in ")]": - level = level - 1 - else: - break - chunks.append((1, sformat[dollar + 1:pos])) - else: - chunks.append((0, sformat[pos:dollar + 1])) - pos = dollar + 1 + (nextchar == "$") - - if pos < len(sformat): - chunks.append((0, sformat[pos:])) - return chunks - -def sqlwhere(dictionary, grouping=' AND '): - """ - Converts a `dictionary` to an SQL WHERE clause `SQLQuery`. - - >>> sqlwhere({'cust_id': 2, 'order_id':3}) - - >>> sqlwhere({'cust_id': 2, 'order_id':3}, grouping=', ') - - >>> sqlwhere({'a': 'a', 'b': 'b'}).query() - 'a = %s AND b = %s' - """ - return SQLQuery.join([k + ' = ' + sqlparam(v) for k, v in dictionary.items()], grouping) - -def reparam(string_, dictionary): - """ - Takes a string and a dictionary and interpolates the string - using values from the dictionary. Returns an `SQLQuery` for the result. - - >>> reparam("s = $s", dict(s=True)) - - >>> reparam("s IN $s", dict(s=[1, 2])) - - """ - dictionary = dictionary.copy() # eval mucks with it - result = [] - for live, chunk in _interpolate(string_): - if live: - v = eval(chunk, dictionary) - result.append(sqlquote(v)) - else: - result.append(chunk) - return SQLQuery.join(result, '') - -class UnknownParamstyle(Exception): - """ - raised for unsupported db paramstyles - - (currently supported: qmark, numeric, format, pyformat) - """ - pass - -class _ItplError(ValueError): - def __init__(self, text, pos): - ValueError.__init__(self) - self.text = text - self.pos = pos - def __str__(self): - return "unfinished expression in %s at char %d" % ( - repr(self.text), self.pos) - -class SQLParam(object): - """ - Parameter in SQLQuery. - - >>> q = SQLQuery(["SELECT * FROM test WHERE name=", SQLParam("joe")]) - >>> q - - >>> q.query() - 'SELECT * FROM test WHERE name=%s' - >>> q.values() - ['joe'] - """ - __slots__ = ["value"] - - def __init__(self, value): - self.value = value - - def get_marker(self, paramstyle='pyformat'): - if paramstyle == 'qmark': - return '?' - elif paramstyle == 'numeric': - return ':1' - elif paramstyle is None or paramstyle in ['format', 'pyformat']: - return '%s' - raise UnknownParamstyle, paramstyle - - def sqlquery(self): - return SQLQuery([self]) - - def __add__(self, other): - return self.sqlquery() + other - - def __radd__(self, other): - return other + self.sqlquery() - - def __str__(self): - return str(self.value) - - def __repr__(self): - return '' % repr(self.value) - -sqlparam = SQLParam - -class SQLQuery(object): - """ - You can pass this sort of thing as a clause in any db function. - Otherwise, you can pass a dictionary to the keyword argument `vars` - and the function will call reparam for you. - - Internally, consists of `items`, which is a list of strings and - SQLParams, which get concatenated to produce the actual query. - """ - __slots__ = ["items"] - - # tested in sqlquote's docstring - def __init__(self, items=None): - r"""Creates a new SQLQuery. - - >>> SQLQuery("x") - - >>> q = SQLQuery(['SELECT * FROM ', 'test', ' WHERE x=', SQLParam(1)]) - >>> q - - >>> q.query(), q.values() - ('SELECT * FROM test WHERE x=%s', [1]) - >>> SQLQuery(SQLParam(1)) - - """ - if items is None: - self.items = [] - elif isinstance(items, list): - self.items = items - elif isinstance(items, SQLParam): - self.items = [items] - elif isinstance(items, SQLQuery): - self.items = list(items.items) - else: - self.items = [items] - - # Take care of SQLLiterals - for i, item in enumerate(self.items): - if isinstance(item, SQLParam) and isinstance(item.value, SQLLiteral): - self.items[i] = item.value.v - - def append(self, value): - self.items.append(value) - - def __add__(self, other): - if isinstance(other, basestring): - items = [other] - elif isinstance(other, SQLQuery): - items = other.items - else: - return NotImplemented - return SQLQuery(self.items + items) - - def __radd__(self, other): - if isinstance(other, basestring): - items = [other] - else: - return NotImplemented - - return SQLQuery(items + self.items) - - def __iadd__(self, other): - if isinstance(other, (basestring, SQLParam)): - self.items.append(other) - elif isinstance(other, SQLQuery): - self.items.extend(other.items) - else: - return NotImplemented - return self - - def __len__(self): - return len(self.query()) - - def query(self, paramstyle=None): - """ - Returns the query part of the sql query. - >>> q = SQLQuery(["SELECT * FROM test WHERE name=", SQLParam('joe')]) - >>> q.query() - 'SELECT * FROM test WHERE name=%s' - >>> q.query(paramstyle='qmark') - 'SELECT * FROM test WHERE name=?' - """ - s = [] - for x in self.items: - if isinstance(x, SQLParam): - x = x.get_marker(paramstyle) - s.append(safestr(x)) - else: - x = safestr(x) - # automatically escape % characters in the query - # For backward compatability, ignore escaping when the query looks already escaped - if paramstyle in ['format', 'pyformat']: - if '%' in x and '%%' not in x: - x = x.replace('%', '%%') - s.append(x) - return "".join(s) - - def values(self): - """ - Returns the values of the parameters used in the sql query. - >>> q = SQLQuery(["SELECT * FROM test WHERE name=", SQLParam('joe')]) - >>> q.values() - ['joe'] - """ - return [i.value for i in self.items if isinstance(i, SQLParam)] - - def join(items, sep=' ', prefix=None, suffix=None, target=None): - """ - Joins multiple queries. - - >>> SQLQuery.join(['a', 'b'], ', ') - - - Optinally, prefix and suffix arguments can be provided. - - >>> SQLQuery.join(['a', 'b'], ', ', prefix='(', suffix=')') - - - If target argument is provided, the items are appended to target instead of creating a new SQLQuery. - """ - if target is None: - target = SQLQuery() - - target_items = target.items - - if prefix: - target_items.append(prefix) - - for i, item in enumerate(items): - if i != 0: - target_items.append(sep) - if isinstance(item, SQLQuery): - target_items.extend(item.items) - else: - target_items.append(item) - - if suffix: - target_items.append(suffix) - return target - - join = staticmethod(join) - - def _str(self): - try: - return self.query() % tuple([sqlify(x) for x in self.values()]) - except (ValueError, TypeError): - return self.query() - - def __str__(self): - return safestr(self._str()) - - def __unicode__(self): - return safeunicode(self._str()) - - def __repr__(self): - return '' % repr(str(self)) - -class SQLLiteral: - """ - Protects a string from `sqlquote`. - - >>> sqlquote('NOW()') - - >>> sqlquote(SQLLiteral('NOW()')) - - """ - def __init__(self, v): - self.v = v - - def __repr__(self): - return self.v - -class SQLProducer: - """Database""" - def __init__(self): - """Creates a database. - """ - pass - - def query(self, sql_query,processed=False, svars=None): - """ - Execute SQL query `sql_query` using dictionary `vars` to interpolate it. - If `processed=True`, `vars` is a `reparam`-style list to use - instead of interpolating. - - >>> db = DB(None, {}) - >>> db.query("SELECT * FROM foo", _test=True) - - >>> db.query("SELECT * FROM foo WHERE x = $x", vars=dict(x='f'), _test=True) - - >>> db.query("SELECT * FROM foo WHERE x = " + sqlquote('f'), _test=True) - - """ - if svars is None: - svars = {} - - if not processed and not isinstance(sql_query, SQLQuery): - sql_query = reparam(sql_query, svars) - - return sql_query - - def sql_clauses(self, what, tables, where, group, order, limit, offset): - return ( - ('SELECT', what), - ('FROM', sqllist(tables)), - ('WHERE', where), - ('GROUP BY', group), - ('ORDER BY', order), - ('LIMIT', limit), - ('OFFSET', offset)) - - def gen_clause(self, sql, val, svars): - if isinstance(val, (int, long)): - if sql == 'WHERE': - nout = 'id = ' + sqlquote(val) - else: - nout = SQLQuery(val) - - elif isinstance(val, (list, tuple)) and len(val) == 2: - nout = SQLQuery(val[0], val[1]) # backwards-compatibility - elif isinstance(val, SQLQuery): - nout = val - else: - nout = reparam(val, svars) - - def xjoin(a, b): - if a and b: return a + ' ' + b - else: return a or b - - return xjoin(sql, nout) - - def _where(self, where, svars): - if isinstance(where, (int, long)): - where = "id = " + sqlparam(where) - elif isinstance(where, (list, tuple)) and len(where) == 2: - where = SQLQuery(where[0], where[1]) - elif isinstance(where, SQLQuery): - pass - else: - where = reparam(where, svars) - return where - - def select(self, tables, svars=None, what='*', where=None, order=None, group=None, - limit=None, offset=None, _test=False): - """ - Selects `what` from `tables` with clauses `where`, `order`, - `group`, `limit`, and `offset`. Uses vars to interpolate. - Otherwise, each clause can be a SQLQuery. - - >>> db = DB(None, {}) - >>> db.select('foo', _test=True) - - >>> db.select(['foo', 'bar'], where="foo.bar_id = bar.id", limit=5, _test=True) - - """ - if svars is None: svars = {} - sql_clauses = self.sql_clauses(what, tables, where, group, order, limit, offset) - clauses = [self.gen_clause(sql, val, svars) for sql, val in sql_clauses if val is not None] - qout = SQLQuery.join(clauses) - if _test: return qout - return self.query(qout, processed=True) - - def insert(self, tablename, seqname=None, _test=False, **values): - """ - Inserts `values` into `tablename`. Returns current sequence ID. - Set `seqname` to the ID if it's not the default, or to `False` - if there isn't one. - - >>> db = DB(None, {}) - >>> q = db.insert('foo', name='bob', age=2, created=SQLLiteral('NOW()'), _test=True) - >>> q - - >>> q.query() - 'INSERT INTO foo (age, name, created) VALUES (%s, %s, NOW())' - >>> q.values() - [2, 'bob'] - """ - def q(x): return "(" + x + ")" - - if values: - _keys = SQLQuery.join(values.keys(), ', ') - _values = SQLQuery.join([sqlparam(v) for v in values.values()], ', ') - sql_query = "INSERT INTO %s " % tablename + q(_keys) + ' VALUES ' + q(_values) - else: - sql_query = SQLQuery(self._get_insert_default_values_query(tablename)) - - return sql_query - - def _get_insert_default_values_query(self, table): - return "INSERT INTO %s DEFAULT VALUES" % table - - def multiple_insert(self, tablename, values, seqname=None, _test=False): - """ - Inserts multiple rows into `tablename`. The `values` must be a list of dictioanries, - one for each row to be inserted, each with the same set of keys. - Returns the list of ids of the inserted rows. - Set `seqname` to the ID if it's not the default, or to `False` - if there isn't one. - - >>> db = DB(None, {}) - >>> db.supports_multiple_insert = True - >>> values = [{"name": "foo", "email": "foo@example.com"}, {"name": "bar", "email": "bar@example.com"}] - >>> db.multiple_insert('person', values=values, _test=True) - - """ - if not values: - return [] - - if not self.supports_multiple_insert: - out = [self.insert(tablename, seqname=seqname, _test=_test, **v) for v in values] - if seqname is False: - return None - else: - return out - - keys = values[0].keys() - #@@ make sure all keys are valid - - # make sure all rows have same keys. - for v in values: - if v.keys() != keys: - raise ValueError, 'Bad data' - - sql_query = SQLQuery('INSERT INTO %s (%s) VALUES ' % (tablename, ', '.join(keys))) - - for i, row in enumerate(values): - if i != 0: - sql_query.append(", ") - SQLQuery.join([SQLParam(row[k]) for k in keys], sep=", ", target=sql_query, prefix="(", suffix=")") - - if _test: return sql_query - - db_cursor = self._db_cursor() - if seqname is not False: - sql_query = self._process_insert_query(sql_query, tablename, seqname) - - if isinstance(sql_query, tuple): - # for some databases, a separate query has to be made to find - # the id of the inserted row. - q1, q2 = sql_query - self._db_execute(db_cursor, q1) - self._db_execute(db_cursor, q2) - else: - self._db_execute(db_cursor, sql_query) - - try: - out = db_cursor.fetchone()[0] - out = range(out-len(values)+1, out+1) - except Exception: - out = None - - if not self.ctx.transactions: - self.ctx.commit() - return out - - - def update(self, tables, where, svars=None, _test=False, **values): - """ - Update `tables` with clause `where` (interpolated using `vars`) - and setting `values`. - - >>> db = DB(None, {}) - >>> name = 'Joseph' - >>> q = db.update('foo', where='name = $name', name='bob', age=2, - ... created=SQLLiteral('NOW()'), vars=locals(), _test=True) - >>> q - - >>> q.query() - 'UPDATE foo SET age = %s, name = %s, created = NOW() WHERE name = %s' - >>> q.values() - [2, 'bob', 'Joseph'] - """ - if svars is None: svars = {} - where = self._where(where, svars) - - query = ( - "UPDATE " + sqllist(tables) + - " SET " + sqlwhere(values, ', ') + - " WHERE " + where) - - if _test: return query - - db_cursor = self._db_cursor() - self._db_execute(db_cursor, query) - if not self.ctx.transactions: - self.ctx.commit() - return db_cursor.rowcount - - def delete(self, table, where, using=None, svars=None, _test=False): - """ - Deletes from `table` with clauses `where` and `using`. - - >>> db = DB(None, {}) - >>> name = 'Joe' - >>> db.delete('foo', where='name = $name', vars=locals(), _test=True) - - """ - if svars is None: - svars = {} - where = self._where(where, svars) - - q = 'DELETE FROM ' + table - if using: - q += ' USING ' + sqllist(using) - if where: - q += ' WHERE ' + where - - return q - -sqlproducer = SQLProducer() - - -print sqlproducer.delete("tb_item", where="id=123") - +#coding:utf8 +''' +Created on 2013-8-21 + +@author: lan (www.9miao.com) +''' +import itertools +import datetime + + +def safeunicode(obj, encoding='utf-8'): + r""" + Converts any given object to unicode string. + + >>> safeunicode('hello') + u'hello' + >>> safeunicode(2) + u'2' + >>> safeunicode('\xe1\x88\xb4') + u'\u1234' + """ + t = type(obj) + if t is unicode: + return obj + elif t is str: + return obj.decode(encoding) + elif t in [int, float, bool]: + return unicode(obj) + elif hasattr(obj, '__unicode__') or isinstance(obj, unicode): + return unicode(obj) + else: + return str(obj).decode(encoding) + +def safestr(obj, encoding='utf-8'): + r""" + Converts any given object to utf-8 encoded string. + + >>> safestr('hello') + 'hello' + >>> safestr(u'\u1234') + '\xe1\x88\xb4' + >>> safestr(2) + '2' + """ + if isinstance(obj, unicode): + return obj.encode(encoding) + elif isinstance(obj, str): + return obj + elif hasattr(obj, 'next'): # iterator + return itertools.imap(safestr, obj) + else: + return str(obj) + +def sqlify(obj): + """ + converts `obj` to its proper SQL version + + >>> sqlify(None) + 'NULL' + >>> sqlify(True) + "'t'" + >>> sqlify(3) + '3' + """ + # because `1 == True and hash(1) == hash(True)` + # we have to do this the hard way... + + if obj is None: + return 'NULL' + elif obj is True: + return "'t'" + elif obj is False: + return "'f'" + elif datetime and isinstance(obj, datetime.datetime): + return repr(obj.isoformat()) + else: + if isinstance(obj, unicode): obj = obj.encode('utf8') + return repr(obj) + +def sqllist(lst): + """ + Converts the arguments for use in something like a WHERE clause. + + >>> sqllist(['a', 'b']) + 'a, b' + >>> sqllist('a') + 'a' + >>> sqllist(u'abc') + u'abc' + """ + if isinstance(lst, basestring): + return lst + else: + return ', '.join(lst) + +def _sqllist(values): + """ + >>> _sqllist([1, 2, 3]) + + """ + items = [] + items.append('(') + for i, v in enumerate(values): + if i != 0: + items.append(', ') + items.append(sqlparam(v)) + items.append(')') + return SQLQuery(items) + +def sqlquote(a): + """ + Ensures `a` is quoted properly for use in a SQL query. + + >>> 'WHERE x = ' + sqlquote(True) + ' AND y = ' + sqlquote(3) + + >>> 'WHERE x = ' + sqlquote(True) + ' AND y IN ' + sqlquote([2, 3]) + + """ + if isinstance(a, list): + return _sqllist(a) + else: + return sqlparam(a).sqlquery() + +def _interpolate(sformat): + """ + Takes a format string and returns a list of 2-tuples of the form + (boolean, string) where boolean says whether string should be evaled + or not. + + from (public domain, Ka-Ping Yee) + """ + from tokenize import tokenprog + + tokenprog = tokenprog + + def matchorfail(text, pos): + match = tokenprog.match(text, pos) + if match is None: + raise _ItplError(text, pos) + return match, match.end() + + namechars = "abcdefghijklmnopqrstuvwxyz" \ + "ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789_"; + chunks = [] + pos = 0 + + while 1: + dollar = sformat.find("$", pos) + if dollar < 0: + break + nextchar = sformat[dollar + 1] + + if nextchar == "{": + chunks.append((0, sformat[pos:dollar])) + pos, level = dollar + 2, 1 + while level: + match, pos = matchorfail(sformat, pos) + tstart, tend = match.regs[3] + token = sformat[tstart:tend] + if token == "{": + level = level + 1 + elif token == "}": + level = level - 1 + chunks.append((1, sformat[dollar + 2:pos - 1])) + + elif nextchar in namechars: + chunks.append((0, sformat[pos:dollar])) + match, pos = matchorfail(sformat, dollar + 1) + while pos < len(sformat): + if sformat[pos] == "." and \ + pos + 1 < len(sformat) and sformat[pos + 1] in namechars: + match, pos = matchorfail(sformat, pos + 1) + elif sformat[pos] in "([": + pos, level = pos + 1, 1 + while level: + match, pos = matchorfail(sformat, pos) + tstart, tend = match.regs[3] + token = sformat[tstart:tend] + if token[0] in "([": + level = level + 1 + elif token[0] in ")]": + level = level - 1 + else: + break + chunks.append((1, sformat[dollar + 1:pos])) + else: + chunks.append((0, sformat[pos:dollar + 1])) + pos = dollar + 1 + (nextchar == "$") + + if pos < len(sformat): + chunks.append((0, sformat[pos:])) + return chunks + +def sqlwhere(dictionary, grouping=' AND '): + """ + Converts a `dictionary` to an SQL WHERE clause `SQLQuery`. + + >>> sqlwhere({'cust_id': 2, 'order_id':3}) + + >>> sqlwhere({'cust_id': 2, 'order_id':3}, grouping=', ') + + >>> sqlwhere({'a': 'a', 'b': 'b'}).query() + 'a = %s AND b = %s' + """ + return SQLQuery.join([k + ' = ' + sqlparam(v) for k, v in dictionary.items()], grouping) + +def reparam(string_, dictionary): + """ + Takes a string and a dictionary and interpolates the string + using values from the dictionary. Returns an `SQLQuery` for the result. + + >>> reparam("s = $s", dict(s=True)) + + >>> reparam("s IN $s", dict(s=[1, 2])) + + """ + dictionary = dictionary.copy() # eval mucks with it + result = [] + for live, chunk in _interpolate(string_): + if live: + v = eval(chunk, dictionary) + result.append(sqlquote(v)) + else: + result.append(chunk) + return SQLQuery.join(result, '') + +class UnknownParamstyle(Exception): + """ + raised for unsupported db paramstyles + + (currently supported: qmark, numeric, format, pyformat) + """ + pass + +class _ItplError(ValueError): + def __init__(self, text, pos): + ValueError.__init__(self) + self.text = text + self.pos = pos + def __str__(self): + return "unfinished expression in %s at char %d" % ( + repr(self.text), self.pos) + +class SQLParam(object): + """ + Parameter in SQLQuery. + + >>> q = SQLQuery(["SELECT * FROM test WHERE name=", SQLParam("joe")]) + >>> q + + >>> q.query() + 'SELECT * FROM test WHERE name=%s' + >>> q.values() + ['joe'] + """ + __slots__ = ["value"] + + def __init__(self, value): + self.value = value + + def get_marker(self, paramstyle='pyformat'): + if paramstyle == 'qmark': + return '?' + elif paramstyle == 'numeric': + return ':1' + elif paramstyle is None or paramstyle in ['format', 'pyformat']: + return '%s' + raise UnknownParamstyle, paramstyle + + def sqlquery(self): + return SQLQuery([self]) + + def __add__(self, other): + return self.sqlquery() + other + + def __radd__(self, other): + return other + self.sqlquery() + + def __str__(self): + return str(self.value) + + def __repr__(self): + return '' % repr(self.value) + +sqlparam = SQLParam + +class SQLQuery(object): + """ + You can pass this sort of thing as a clause in any db function. + Otherwise, you can pass a dictionary to the keyword argument `vars` + and the function will call reparam for you. + + Internally, consists of `items`, which is a list of strings and + SQLParams, which get concatenated to produce the actual query. + """ + __slots__ = ["items"] + + # tested in sqlquote's docstring + def __init__(self, items=None): + r"""Creates a new SQLQuery. + + >>> SQLQuery("x") + + >>> q = SQLQuery(['SELECT * FROM ', 'test', ' WHERE x=', SQLParam(1)]) + >>> q + + >>> q.query(), q.values() + ('SELECT * FROM test WHERE x=%s', [1]) + >>> SQLQuery(SQLParam(1)) + + """ + if items is None: + self.items = [] + elif isinstance(items, list): + self.items = items + elif isinstance(items, SQLParam): + self.items = [items] + elif isinstance(items, SQLQuery): + self.items = list(items.items) + else: + self.items = [items] + + # Take care of SQLLiterals + for i, item in enumerate(self.items): + if isinstance(item, SQLParam) and isinstance(item.value, SQLLiteral): + self.items[i] = item.value.v + + def append(self, value): + self.items.append(value) + + def __add__(self, other): + if isinstance(other, basestring): + items = [other] + elif isinstance(other, SQLQuery): + items = other.items + else: + return NotImplemented + return SQLQuery(self.items + items) + + def __radd__(self, other): + if isinstance(other, basestring): + items = [other] + else: + return NotImplemented + + return SQLQuery(items + self.items) + + def __iadd__(self, other): + if isinstance(other, (basestring, SQLParam)): + self.items.append(other) + elif isinstance(other, SQLQuery): + self.items.extend(other.items) + else: + return NotImplemented + return self + + def __len__(self): + return len(self.query()) + + def query(self, paramstyle=None): + """ + Returns the query part of the sql query. + >>> q = SQLQuery(["SELECT * FROM test WHERE name=", SQLParam('joe')]) + >>> q.query() + 'SELECT * FROM test WHERE name=%s' + >>> q.query(paramstyle='qmark') + 'SELECT * FROM test WHERE name=?' + """ + s = [] + for x in self.items: + if isinstance(x, SQLParam): + x = x.get_marker(paramstyle) + s.append(safestr(x)) + else: + x = safestr(x) + # automatically escape % characters in the query + # For backward compatability, ignore escaping when the query looks already escaped + if paramstyle in ['format', 'pyformat']: + if '%' in x and '%%' not in x: + x = x.replace('%', '%%') + s.append(x) + return "".join(s) + + def values(self): + """ + Returns the values of the parameters used in the sql query. + >>> q = SQLQuery(["SELECT * FROM test WHERE name=", SQLParam('joe')]) + >>> q.values() + ['joe'] + """ + return [i.value for i in self.items if isinstance(i, SQLParam)] + + def join(items, sep=' ', prefix=None, suffix=None, target=None): + """ + Joins multiple queries. + + >>> SQLQuery.join(['a', 'b'], ', ') + + + Optinally, prefix and suffix arguments can be provided. + + >>> SQLQuery.join(['a', 'b'], ', ', prefix='(', suffix=')') + + + If target argument is provided, the items are appended to target instead of creating a new SQLQuery. + """ + if target is None: + target = SQLQuery() + + target_items = target.items + + if prefix: + target_items.append(prefix) + + for i, item in enumerate(items): + if i != 0: + target_items.append(sep) + if isinstance(item, SQLQuery): + target_items.extend(item.items) + else: + target_items.append(item) + + if suffix: + target_items.append(suffix) + return target + + join = staticmethod(join) + + def _str(self): + try: + return self.query() % tuple([sqlify(x) for x in self.values()]) + except (ValueError, TypeError): + return self.query() + + def __str__(self): + return safestr(self._str()) + + def __unicode__(self): + return safeunicode(self._str()) + + def __repr__(self): + return '' % repr(str(self)) + +class SQLLiteral: + """ + Protects a string from `sqlquote`. + + >>> sqlquote('NOW()') + + >>> sqlquote(SQLLiteral('NOW()')) + + """ + def __init__(self, v): + self.v = v + + def __repr__(self): + return self.v + +class SQLProducer: + """Database""" + def __init__(self): + """Creates a database. + """ + pass + + def query(self, sql_query,processed=False, svars=None): + """ + Execute SQL query `sql_query` using dictionary `vars` to interpolate it. + If `processed=True`, `vars` is a `reparam`-style list to use + instead of interpolating. + + >>> db = DB(None, {}) + >>> db.query("SELECT * FROM foo", _test=True) + + >>> db.query("SELECT * FROM foo WHERE x = $x", vars=dict(x='f'), _test=True) + + >>> db.query("SELECT * FROM foo WHERE x = " + sqlquote('f'), _test=True) + + """ + if svars is None: + svars = {} + + if not processed and not isinstance(sql_query, SQLQuery): + sql_query = reparam(sql_query, svars) + + return sql_query + + def sql_clauses(self, what, tables, where, group, order, limit, offset): + return ( + ('SELECT', what), + ('FROM', sqllist(tables)), + ('WHERE', where), + ('GROUP BY', group), + ('ORDER BY', order), + ('LIMIT', limit), + ('OFFSET', offset)) + + def gen_clause(self, sql, val, svars): + if isinstance(val, (int, long)): + if sql == 'WHERE': + nout = 'id = ' + sqlquote(val) + else: + nout = SQLQuery(val) + + elif isinstance(val, (list, tuple)) and len(val) == 2: + nout = SQLQuery(val[0], val[1]) # backwards-compatibility + elif isinstance(val, SQLQuery): + nout = val + else: + nout = reparam(val, svars) + + def xjoin(a, b): + if a and b: return a + ' ' + b + else: return a or b + + return xjoin(sql, nout) + + def _where(self, where, svars): + if isinstance(where, (int, long)): + where = "id = " + sqlparam(where) + elif isinstance(where, (list, tuple)) and len(where) == 2: + where = SQLQuery(where[0], where[1]) + elif isinstance(where, SQLQuery): + pass + else: + where = reparam(where, svars) + return where + + def select(self, tables, svars=None, what='*', where=None, order=None, group=None, + limit=None, offset=None, _test=False): + """ + Selects `what` from `tables` with clauses `where`, `order`, + `group`, `limit`, and `offset`. Uses vars to interpolate. + Otherwise, each clause can be a SQLQuery. + + >>> db = DB(None, {}) + >>> db.select('foo', _test=True) + + >>> db.select(['foo', 'bar'], where="foo.bar_id = bar.id", limit=5, _test=True) + + """ + if svars is None: svars = {} + sql_clauses = self.sql_clauses(what, tables, where, group, order, limit, offset) + clauses = [self.gen_clause(sql, val, svars) for sql, val in sql_clauses if val is not None] + qout = SQLQuery.join(clauses) + if _test: return qout + return self.query(qout, processed=True) + + def insert(self, tablename, seqname=None, _test=False, **values): + """ + Inserts `values` into `tablename`. Returns current sequence ID. + Set `seqname` to the ID if it's not the default, or to `False` + if there isn't one. + + >>> db = DB(None, {}) + >>> q = db.insert('foo', name='bob', age=2, created=SQLLiteral('NOW()'), _test=True) + >>> q + + >>> q.query() + 'INSERT INTO foo (age, name, created) VALUES (%s, %s, NOW())' + >>> q.values() + [2, 'bob'] + """ + def q(x): return "(" + x + ")" + + if values: + _keys = SQLQuery.join(values.keys(), ', ') + _values = SQLQuery.join([sqlparam(v) for v in values.values()], ', ') + sql_query = "INSERT INTO %s " % tablename + q(_keys) + ' VALUES ' + q(_values) + else: + sql_query = SQLQuery(self._get_insert_default_values_query(tablename)) + + return sql_query + + def _get_insert_default_values_query(self, table): + return "INSERT INTO %s DEFAULT VALUES" % table + + def multiple_insert(self, tablename, values, seqname=None, _test=False): + """ + Inserts multiple rows into `tablename`. The `values` must be a list of dictioanries, + one for each row to be inserted, each with the same set of keys. + Returns the list of ids of the inserted rows. + Set `seqname` to the ID if it's not the default, or to `False` + if there isn't one. + + >>> db = DB(None, {}) + >>> db.supports_multiple_insert = True + >>> values = [{"name": "foo", "email": "foo@example.com"}, {"name": "bar", "email": "bar@example.com"}] + >>> db.multiple_insert('person', values=values, _test=True) + + """ + if not values: + return [] + + if not self.supports_multiple_insert: + out = [self.insert(tablename, seqname=seqname, _test=_test, **v) for v in values] + if seqname is False: + return None + else: + return out + + keys = values[0].keys() + #@@ make sure all keys are valid + + # make sure all rows have same keys. + for v in values: + if v.keys() != keys: + raise ValueError, 'Bad data' + + sql_query = SQLQuery('INSERT INTO %s (%s) VALUES ' % (tablename, ', '.join(keys))) + + for i, row in enumerate(values): + if i != 0: + sql_query.append(", ") + SQLQuery.join([SQLParam(row[k]) for k in keys], sep=", ", target=sql_query, prefix="(", suffix=")") + + if _test: return sql_query + + db_cursor = self._db_cursor() + if seqname is not False: + sql_query = self._process_insert_query(sql_query, tablename, seqname) + + if isinstance(sql_query, tuple): + # for some databases, a separate query has to be made to find + # the id of the inserted row. + q1, q2 = sql_query + self._db_execute(db_cursor, q1) + self._db_execute(db_cursor, q2) + else: + self._db_execute(db_cursor, sql_query) + + try: + out = db_cursor.fetchone()[0] + out = range(out-len(values)+1, out+1) + except Exception: + out = None + + if not self.ctx.transactions: + self.ctx.commit() + return out + + + def update(self, tables, where, svars=None, _test=False, **values): + """ + Update `tables` with clause `where` (interpolated using `vars`) + and setting `values`. + + >>> db = DB(None, {}) + >>> name = 'Joseph' + >>> q = db.update('foo', where='name = $name', name='bob', age=2, + ... created=SQLLiteral('NOW()'), vars=locals(), _test=True) + >>> q + + >>> q.query() + 'UPDATE foo SET age = %s, name = %s, created = NOW() WHERE name = %s' + >>> q.values() + [2, 'bob', 'Joseph'] + """ + if svars is None: svars = {} + where = self._where(where, svars) + + query = ( + "UPDATE " + sqllist(tables) + + " SET " + sqlwhere(values, ', ') + + " WHERE " + where) + + if _test: return query + + db_cursor = self._db_cursor() + self._db_execute(db_cursor, query) + if not self.ctx.transactions: + self.ctx.commit() + return db_cursor.rowcount + + def delete(self, table, where, using=None, svars=None, _test=False): + """ + Deletes from `table` with clauses `where` and `using`. + + >>> db = DB(None, {}) + >>> name = 'Joe' + >>> db.delete('foo', where='name = $name', vars=locals(), _test=True) + + """ + if svars is None: + svars = {} + where = self._where(where, svars) + + q = 'DELETE FROM ' + table + if using: + q += ' USING ' + sqllist(using) + if where: + q += ' WHERE ' + where + + return q + +sqlproducer = SQLProducer() + + + diff --git a/firefly/dbentrust/madminanager.py b/firefly/dbentrust/madminanager.py old mode 100644 new mode 100755 index 014a517..342bfb9 --- a/firefly/dbentrust/madminanager.py +++ b/firefly/dbentrust/madminanager.py @@ -1,41 +1,41 @@ -#coding:utf8 -''' -Created on 2013-5-22 - -@author: lan (www.9miao.com) -''' -from firefly.utils.singleton import Singleton - -class MAdminManager: - __metaclass__ = Singleton - - def __init__(self): - """ - """ - self.admins = {} - - def registe(self,admin): - """ - """ - self.admins[admin._name] = admin - - def dropAdmin(self,adminname): - """ - """ - if self.admins.has_key(adminname): - del self.admins[adminname] - - def getAdmin(self,adminname): - """ - """ - return self.admins.get(adminname) - - def checkAdmins(self): - """ - """ - for admin in self.admins.values(): - admin.checkAll() - - - +#coding:utf8 +''' +Created on 2013-5-22 + +@author: lan (www.9miao.com) +''' +from firefly.utils.singleton import Singleton + +class MAdminManager: + __metaclass__ = Singleton + + def __init__(self): + """ + """ + self.admins = {} + + def registe(self,admin): + """ + """ + self.admins[admin._name] = admin + + def dropAdmin(self,adminname): + """ + """ + if self.admins.has_key(adminname): + del self.admins[adminname] + + def getAdmin(self,adminname): + """ + """ + return self.admins.get(adminname) + + def checkAdmins(self): + """ + """ + for admin in self.admins.values(): + admin.checkAll() + + + \ No newline at end of file diff --git a/firefly/dbentrust/memclient.py b/firefly/dbentrust/memclient.py old mode 100644 new mode 100755 index 4289939..fea64b7 --- a/firefly/dbentrust/memclient.py +++ b/firefly/dbentrust/memclient.py @@ -1,104 +1,104 @@ -#coding:utf8 -''' -Created on 2013-7-10 -memcached client -@author: lan (www.9miao.com) -''' -import memcache - -class MemConnError(Exception): - """ - """ - def __str__(self): - return "memcache connect error" - -class MemClient: - '''memcached - ''' - - def __init__(self,timeout = 0): - ''' - ''' - self._hostname = "" - self._urls = [] - self.connection = None - - def connect(self,urls,hostname): - '''memcached connect - ''' - self._hostname = hostname - self._urls = urls - self.connection = memcache.Client(self._urls,debug=0) - if not self.connection.set("__testkey__",1): - raise MemConnError() - - def produceKey(self,keyname): - ''' - ''' - if isinstance(keyname, basestring): - return ''.join([self._hostname,':',keyname]) - else: - raise "type error" - - def get(self,key): - ''' - ''' - key = self.produceKey(key) - return self.connection.get(key) - - def get_multi(self,keys): - ''' - ''' - keynamelist = [self.produceKey(keyname) for keyname in keys] - olddict = self.connection.get_multi(keynamelist) - newdict = dict(zip([keyname.split(':')[-1] for keyname in olddict.keys()], - olddict.values())) - return newdict - - def set(self,keyname,value): - ''' - ''' - key = self.produceKey(keyname) - result = self.connection.set(key,value) - if not result:#如果写入失败 - self.connect(self._urls,self._hostname)#重新连接 - return self.connection.set(key,value) - return result - - def set_multi(self,mapping): - ''' - ''' - newmapping = dict(zip([self.produceKey(keyname) for keyname in mapping.keys()], - mapping.values())) - result = self.connection.set_multi(newmapping) - if result:#如果写入失败 - self.connect(self._urls,self._hostname)#重新连接 - return self.connection.set_multi(newmapping) - return result - - def incr(self,key,delta): - ''' - ''' - key = self.produceKey(key) - return self.connection.incr(key, delta) - - def delete(self,key): - ''' - ''' - key = self.produceKey(key) - return self.connection.delete(key) - - def delete_multi(self,keys): - """ - """ - keys = [self.produceKey(key) for key in keys] - return self.connection.delete_multi(keys) - - def flush_all(self): - ''' - ''' - self.connection.flush_all() - -mclient = MemClient() - - +#coding:utf8 +''' +Created on 2013-7-10 +memcached client +@author: lan (www.9miao.com) +''' +import memcache + +class MemConnError(Exception): + """ + """ + def __str__(self): + return "memcache connect error" + +class MemClient: + '''memcached + ''' + + def __init__(self,timeout = 0): + ''' + ''' + self._hostname = "" + self._urls = [] + self.connection = None + + def connect(self,urls,hostname): + '''memcached connect + ''' + self._hostname = hostname + self._urls = urls + self.connection = memcache.Client(self._urls,debug=0) + if not self.connection.set("__testkey__",1): + raise MemConnError() + + def produceKey(self,keyname): + ''' + ''' + if isinstance(keyname, basestring): + return ''.join([self._hostname,':',keyname]) + else: + raise "type error" + + def get(self,key): + ''' + ''' + key = self.produceKey(key) + return self.connection.get(key) + + def get_multi(self,keys): + ''' + ''' + keynamelist = [self.produceKey(keyname) for keyname in keys] + olddict = self.connection.get_multi(keynamelist) + newdict = dict(zip([keyname.split(':')[-1] for keyname in olddict.keys()], + olddict.values())) + return newdict + + def set(self,keyname,value): + ''' + ''' + key = self.produceKey(keyname) + result = self.connection.set(key,value) + if not result:#如果写入失败 + self.connect(self._urls,self._hostname)#重新连接 + return self.connection.set(key,value) + return result + + def set_multi(self,mapping): + ''' + ''' + newmapping = dict(zip([self.produceKey(keyname) for keyname in mapping.keys()], + mapping.values())) + result = self.connection.set_multi(newmapping) + if result:#如果写入失败 + self.connect(self._urls,self._hostname)#重新连接 + return self.connection.set_multi(newmapping) + return result + + def incr(self,key,delta): + ''' + ''' + key = self.produceKey(key) + return self.connection.incr(key, delta) + + def delete(self,key): + ''' + ''' + key = self.produceKey(key) + return self.connection.delete(key) + + def delete_multi(self,keys): + """ + """ + keys = [self.produceKey(key) for key in keys] + return self.connection.delete_multi(keys) + + def flush_all(self): + ''' + ''' + self.connection.flush_all() + +mclient = MemClient() + + diff --git a/firefly/dbentrust/memobject.py b/firefly/dbentrust/memobject.py old mode 100644 new mode 100755 index bc8505e..50f9720 --- a/firefly/dbentrust/memobject.py +++ b/firefly/dbentrust/memobject.py @@ -1,108 +1,108 @@ -#coding:utf8 -''' -Created on 2012-7-10 -memcached 关系对象 -通过key键的名称前缀来建立 -各个key-value 直接的关系 -@author: lan (www.9miao.com) -''' - -class MemObject: - '''memcached 关系对象 - ''' - - def __init__(self,name,mc): - ''' - @param name: str 对象的名称 - @param _lock: int 对象锁 为1时表示对象被锁定无法进行修改 - ''' - self._client = mc - self._name = name - self._lock = 0 - - def produceKey(self,keyname): - '''重新生成key - ''' - if isinstance(keyname, basestring): - return ''.join([self._name,':',keyname]) - else: - raise "type error" - - def locked(self): - '''检测对象是否被锁定 - ''' - key = self.produceKey('_lock') - return self._client.get(key) - - def lock(self): - '''锁定对象 - ''' - key = self.produceKey('_lock') - self._client.set(key, 1) - - def release(self): - '''释放锁 - ''' - key = self.produceKey('_lock') - self._client.set(key, 0) - - def get(self,key): - '''获取对象值 - ''' - key = self.produceKey(key) - return self._client.get(key) - - def get_multi(self,keys): - '''一次获取多个key的值 - @param keys: list(str) key的列表 - ''' - keynamelist = [self.produceKey(keyname) for keyname in keys] - olddict = self._client.get_multi(keynamelist) - newdict = dict(zip([keyname.split(':')[-1] for keyname in olddict.keys()], - olddict.values())) - return newdict - - def update(self,key,values): - '''修改对象的值 - ''' - if self.locked(): - return False - key = self.produceKey(key) - return self._client.set(key,values) - - def update_multi(self,mapping): - '''同时修改多个key值 - ''' - if self.locked(): - return False - newmapping = dict(zip([self.produceKey(keyname) for keyname in mapping.keys()], - mapping.values())) - return self._client.set_multi(newmapping) - - def mdelete(self): - '''删除memcache中的数据 - ''' - nowdict = dict(self.__dict__) - del nowdict['_client'] - keys = nowdict.keys() - keys = [self.produceKey(key) for key in keys] - self._client.delete_multi(keys) - - def incr(self, key, delta): - '''自增 - ''' - key = self.produceKey(key) - return self._client.incr( key, delta) - - def insert(self): - '''插入对象记录 - ''' - nowdict = dict(self.__dict__) - del nowdict['_client'] - newmapping = dict(zip([self.produceKey(keyname) for keyname in nowdict.keys()], - nowdict.values())) - self._client.set_multi(newmapping) - - - - +#coding:utf8 +''' +Created on 2012-7-10 +memcached 关系对象 +通过key键的名称前缀来建立 +各个key-value 直接的关系 +@author: lan (www.9miao.com) +''' + +class MemObject: + '''memcached 关系对象 + ''' + + def __init__(self,name,mc): + ''' + @param name: str 对象的名称 + @param _lock: int 对象锁 为1时表示对象被锁定无法进行修改 + ''' + self._client = mc + self._name = name + self._lock = 0 + + def produceKey(self,keyname): + '''重新生成key + ''' + if isinstance(keyname, basestring): + return ''.join([self._name,':',keyname]) + else: + raise "type error" + + def locked(self): + '''检测对象是否被锁定 + ''' + key = self.produceKey('_lock') + return self._client.get(key) + + def lock(self): + '''锁定对象 + ''' + key = self.produceKey('_lock') + self._client.set(key, 1) + + def release(self): + '''释放锁 + ''' + key = self.produceKey('_lock') + self._client.set(key, 0) + + def get(self,key): + '''获取对象值 + ''' + key = self.produceKey(key) + return self._client.get(key) + + def get_multi(self,keys): + '''一次获取多个key的值 + @param keys: list(str) key的列表 + ''' + keynamelist = [self.produceKey(keyname) for keyname in keys] + olddict = self._client.get_multi(keynamelist) + newdict = dict(zip([keyname.split(':')[-1] for keyname in olddict.keys()], + olddict.values())) + return newdict + + def update(self,key,values): + '''修改对象的值 + ''' + if self.locked(): + return False + key = self.produceKey(key) + return self._client.set(key,values) + + def update_multi(self,mapping): + '''同时修改多个key值 + ''' + if self.locked(): + return False + newmapping = dict(zip([self.produceKey(keyname) for keyname in mapping.keys()], + mapping.values())) + return self._client.set_multi(newmapping) + + def mdelete(self): + '''删除memcache中的数据 + ''' + nowdict = dict(self.__dict__) + del nowdict['_client'] + keys = nowdict.keys() + keys = [self.produceKey(key) for key in keys] + self._client.delete_multi(keys) + + def incr(self, key, delta): + '''自增 + ''' + key = self.produceKey(key) + return self._client.incr( key, delta) + + def insert(self): + '''插入对象记录 + ''' + nowdict = dict(self.__dict__) + del nowdict['_client'] + newmapping = dict(zip([self.produceKey(keyname) for keyname in nowdict.keys()], + nowdict.values())) + self._client.set_multi(newmapping) + + + + diff --git a/firefly/dbentrust/mmode.py b/firefly/dbentrust/mmode.py old mode 100644 new mode 100755 index f1e6ed6..8e039be --- a/firefly/dbentrust/mmode.py +++ b/firefly/dbentrust/mmode.py @@ -1,310 +1,328 @@ -#coding:utf8 -''' -Created on 2013-5-8 - -@author: lan (www.9miao.com) -''' -from memclient import mclient -from memobject import MemObject -import util -import time - -MMODE_STATE_ORI = 0 #未变更 -MMODE_STATE_NEW = 1 #创建 -MMODE_STATE_UPDATE = 2 #更新 -MMODE_STATE_DEL = 3 #删除 - - - -TIMEOUT = 1800 - -def _insert(args): - record,pkname,mmname,cls = args - pk = record[pkname] - mm = cls(mmname+':%s'%pk,pkname,data=record) - mm.insert() - return pk - -class PKValueError(ValueError): - """ - """ - def __init__(self, data): - ValueError.__init__(self) - self.data = data - def __str__(self): - return "new record has no 'PK': %s" % (self.data) - -class MMode(MemObject): - """内存数据模型 - """ - def __init__(self, name,pk,data={}): - """ - """ - MemObject.__init__(self, name, mclient) - self._state = MMODE_STATE_ORI#对象的状态 0未变更 1新建 2更新 3删除 - self._pk = pk - self.data = data - self._time = time.time() - - def update(self, key, values): - data = self.get_multi(['data','_state']) - ntime = time.time() - data['data'].update({key:values}) - if data.get('_state')==MMODE_STATE_NEW: - props = {'data':data.get('data'),'_time':ntime} - else: - props = {'_state':MMODE_STATE_UPDATE,'data':data.get('data'),'_time':ntime} - return MemObject.update_multi(self, props) - - def update_multi(self, mapping): - ntime = time.time() - data = self.get_multi(['data','_state']) - data['data'].update(mapping) - if data.get('_state')==MMODE_STATE_NEW: - props = {'data':data.get('data'),'_time':ntime} - else: - props = {'_state':MMODE_STATE_UPDATE,'data':data.get('data'),'_time':ntime} - return MemObject.update_multi(self, props) - - def get(self, key): - ntime = time.time() - MemObject.update(self, "_time", ntime) - return MemObject.get(self, key) - - def get_multi(self, keys): - ntime = time.time() - MemObject.update(self, "_time", ntime) - return MemObject.get_multi(self, keys) - - def delete(self): - '''删除对象 - ''' - return MemObject.update(self,'_state',MMODE_STATE_DEL) - - def mdelete(self): - """清理对象 - """ - self.syncDB() - MemObject.mdelete(self) - - def IsEffective(self): - '''检测对象是否有效 - ''' - if self.get('_state')==MMODE_STATE_DEL: - return False - return True - - def syncDB(self): - """同步到数据库 - """ - state = self.get('_state') - tablename = self._name.split(':')[0] - if state==MMODE_STATE_ORI: - return - elif state==MMODE_STATE_NEW: - props = self.get('data') - pk = self.get('_pk') - result = util.InsertIntoDB(tablename, props) - elif state==MMODE_STATE_UPDATE: - props = self.get('data') - pk = self.get('_pk') - prere = {pk:props.get(pk)} - util.UpdateWithDict(tablename, props, prere) - result = True - else: - pk = self.get('_pk') - props = self.get('data') - prere = {pk:props.get(pk)} - result = util.DeleteFromDB(tablename,prere) - if result: - MemObject.update(self,'_state', MMODE_STATE_ORI) - - def checkSync(self,timeout=TIMEOUT): - """检测同步 - """ - ntime = time.time() - objtime = MemObject.get(self, '_time') - if ntime -objtime>=timeout and timeout: - self.mdelete() - else: - self.syncDB() - - -class MFKMode(MemObject): - """内存数据模型 - """ - def __init__(self, name,pklist = []): - MemObject.__init__(self, name, mclient) - self.pklist = pklist - -class MAdmin(MemObject): - - def __init__(self, name,pk,timeout=TIMEOUT,**kw): - MemObject.__init__(self, name, mclient) - self._pk = pk - self._fk = kw.get('fk','') - self._incrkey = kw.get('incrkey','') - self._incrvalue = kw.get('incrvalue',0) - self._timeout = timeout - - def insert(self): - if self._incrkey and not self.get("_incrvalue"): - self._incrvalue = util.GetTableIncrValue(self._name) - MemObject.insert(self) - - def load(self): - '''读取数据到数据库中 - ''' - mmname = self._name - recordlist = util.ReadDataFromDB(mmname) - for record in recordlist: - pk = record[self._pk] - mm = MMode(self._name+':%s'%pk,self._pk,data=record) - mm.insert() - - @property - def madmininfo(self): - keys = self.__dict__.keys() - info = self.get_multi(keys) - return info - - def getAllPkByFk(self,fk): - '''根据外键获取主键列表 - ''' - name = '%s_fk:%s'%(self._name,fk) - fkmm = MFKMode(name) - pklist = fkmm.get('pklist') - if pklist is not None: - return pklist - props = {self._fk:fk} - dbkeylist = util.getAllPkByFkInDB(self._name, self._pk, props) - name = '%s_fk:%s'%(self._name,fk) - fkmm = MFKMode(name, pklist = dbkeylist) - fkmm.insert() - return dbkeylist - - def getObj(self,pk): - ''' - ''' - mm = MMode(self._name+':%s'%pk,self._pk) - if not mm.IsEffective(): - return None - if mm.get('data'): - return mm - props = {self._pk:pk} - record = util.GetOneRecordInfo(self._name,props) - if not record: - return None - mm = MMode(self._name+':%s'%pk,self._pk,data = record) - mm.insert() - return mm - - def getObjData(self,pk): - ''' - ''' - mm = MMode(self._name+':%s'%pk,self._pk) - if not mm.IsEffective(): - return None - data = mm.get('data') - if mm.get('data'): - return data - props = {self._pk:pk} - record = util.GetOneRecordInfo(self._name,props) - if not record: - return None - mm = MMode(self._name+':%s'%pk,self._pk,data = record) - mm.insert() - return record - - - def getObjList(self,pklist): - ''' - ''' - _pklist = [] - objlist = [] - for pk in pklist: - mm = MMode(self._name+':%s'%pk,self._pk) - if not mm.IsEffective(): - continue - if mm.get('data'): - objlist.append(mm) - else: - _pklist.append(pk) - if _pklist: - recordlist = util.GetRecordList(self._name, self._pk,_pklist) - for record in recordlist: - pk = record[self._pk] - mm = MMode(self._name+':%s'%pk,self._pk,data = record) - mm.insert() - objlist.append(mm) - return objlist - - def deleteMode(self,pk): - ''' - ''' - mm = self.getObj(pk) - if mm: - if self._fk: - data = mm.get('data') - if data: - fk = data.get(self._fk,0) - name = '%s_fk:%s'%(self._name,fk) - fkmm = MFKMode(name) - pklist = fkmm.get('pklist') - if pklist and pk in pklist: - pklist.remove(pk) - fkmm.update('pklist', pklist) - mm.delete() - return True - - def checkAll(self): - key = '%s:%s:'%(mclient._hostname,self._name) - _pklist = util.getallkeys(key, mclient.connection) - for pk in _pklist: - mm = MMode(self._name+':%s'%pk,self._pk) - if not mm.IsEffective(): - mm.mdelete() - continue - if not mm.get('data'): - continue - mm.checkSync(timeout=self._timeout) - self.deleteAllFk() - - def deleteAllFk(self): - """删除所有的外键 - """ - key = '%s:%s_fk:'%(mclient._hostname,self._name) - _fklist = util.getallkeys(key, mclient.connection) - for fk in _fklist: - name = '%s_fk:%s'%(self._name,fk) - fkmm = MFKMode(name) - fkmm.mdelete() - - def new(self,data): - """创建一个新的对象 - """ - incrkey = self._incrkey - if incrkey: - incrvalue = self.incr('_incrvalue', 1) - data[incrkey] = incrvalue - 1 - pk = data.get(self._pk) - if pk is None: - raise PKValueError(data) - mm = MMode(self._name+':%s'%pk,self._pk,data=data) - setattr(mm,incrkey,pk) - else: - pk = data.get(self._pk) - mm = MMode(self._name+':%s'%pk,self._pk,data=data) - if self._fk: - fk = data.get(self._fk,0) - name = '%s_fk:%s'%(self._name,fk) - fkmm = MFKMode(name) - pklist = fkmm.get('pklist') - if pklist is None: - pklist = self.getAllPkByFk(fk) - pklist.append(pk) - fkmm.update('pklist', pklist) - setattr(mm,'_state',MMODE_STATE_NEW) - mm.insert() - return mm - +#coding:utf8 +''' +Created on 2013-5-8 + +@author: lan (www.9miao.com) +''' +from memclient import mclient +from memobject import MemObject +import util +import time + +MMODE_STATE_ORI = 0 #未变更 +MMODE_STATE_NEW = 1 #创建 +MMODE_STATE_UPDATE = 2 #更新 +MMODE_STATE_DEL = 3 #删除 + + + +TIMEOUT = 1800 + +def _insert(args): + record,pkname,mmname,cls = args + pk = record[pkname] + mm = cls(mmname+':%s'%pk,pkname,data=record) + mm.insert() + return pk + +class PKValueError(ValueError): + """ + """ + def __init__(self, data): + ValueError.__init__(self) + self.data = data + def __str__(self): + return "new record has no 'PK': %s" % (self.data) + +class MMode(MemObject): + """内存数据模型,最终对应到的是表中的一条记录 + """ + def __init__(self, name,pk,data={}): + """ + """ + MemObject.__init__(self, name, mclient) + self._state = MMODE_STATE_ORI#对象的状态 0未变更 1新建 2更新 3删除 + self._pk = pk + self.data = data + self._time = time.time() + + def update(self, key, values): + data = self.get_multi(['data','_state']) + ntime = time.time() + data['data'].update({key:values}) + if data.get('_state')==MMODE_STATE_NEW: + props = {'data':data.get('data'),'_time':ntime} + else: + props = {'_state':MMODE_STATE_UPDATE,'data':data.get('data'),'_time':ntime} + return MemObject.update_multi(self, props) + + def update_multi(self, mapping): + ntime = time.time() + data = self.get_multi(['data','_state']) + data['data'].update(mapping) + if data.get('_state')==MMODE_STATE_NEW: + props = {'data':data.get('data'),'_time':ntime} + else: + props = {'_state':MMODE_STATE_UPDATE,'data':data.get('data'),'_time':ntime} + return MemObject.update_multi(self, props) + + def get(self, key): + ntime = time.time() + MemObject.update(self, "_time", ntime) + return MemObject.get(self, key) + + def get_multi(self, keys): + ntime = time.time() + MemObject.update(self, "_time", ntime) + return MemObject.get_multi(self, keys) + + def delete(self): + '''删除对象 + ''' + return MemObject.update(self,'_state',MMODE_STATE_DEL) + + def mdelete(self): + """清理对象 + """ + self.syncDB() + MemObject.mdelete(self) + + def IsEffective(self): + '''检测对象是否有效 + ''' + if self.get('_state')==MMODE_STATE_DEL: + return False + return True + + def syncDB(self): + """同步到数据库 + """ + state = self.get('_state') + tablename = self._name.split(':')[0] + if state==MMODE_STATE_ORI: + return + elif state==MMODE_STATE_NEW: + props = self.get('data') + pk = self.get('_pk') + result = util.InsertIntoDB(tablename, props) + elif state==MMODE_STATE_UPDATE: + props = self.get('data') + pk = self.get('_pk') + prere = {pk:props.get(pk)} + util.UpdateWithDict(tablename, props, prere) + result = True + else: + pk = self.get('_pk') + props = self.get('data') + prere = {pk:props.get(pk)} + result = util.DeleteFromDB(tablename,prere) + #根据9秒@qindavid 同学的建议修改了这里可能会出现db与memcached数据不统一的bug。 + if result and state<>MMODE_STATE_DEL: + MemObject.update(self,'_state', MMODE_STATE_ORI) + ###################################################################### + + def checkSync(self,timeout=TIMEOUT): + """检测同步 + """ + ntime = time.time() + objtime = MemObject.get(self, '_time') + if ntime -objtime>=timeout and timeout: + self.mdelete() + else: + self.syncDB() + + +class MFKMode(MemObject): + """外键内存数据模型 + """ + def __init__(self, name,pklist = []): + MemObject.__init__(self, name, mclient) + self.pklist = pklist + +class MAdmin(MemObject): + """MMode对象管理,同一个MAdmin管理同一类的MMode,对应的是数据库中的某一种表 + """ + + def __init__(self, name,pk,timeout=TIMEOUT,**kw): + MemObject.__init__(self, name, mclient) + self._pk = pk + self._fk = kw.get('fk','') + self._incrkey = kw.get('incrkey','') + self._incrvalue = kw.get('incrvalue',0) + self._timeout = timeout + + def insert(self): + """将MAdmin配置的信息写入memcached中保存。\n当在其他的进程中实例化相同的配置的MAdmin,可以使得数据同步。 + """ + if self._incrkey and not self.get("_incrvalue"): + self._incrvalue = util.GetTableIncrValue(self._name) + MemObject.insert(self) + + def load(self): + '''读取数据到数据库中 + ''' + mmname = self._name + recordlist = util.ReadDataFromDB(mmname) + for record in recordlist: + pk = record[self._pk] + mm = MMode(self._name+':%s'%pk,self._pk,data=record) + mm.insert() + + @property + def madmininfo(self): + """作为一个特性属性。可以获取这个madmin的相关信息 + """ + keys = self.__dict__.keys() + info = self.get_multi(keys) + return info + + def getAllPkByFk(self,fk): + '''根据外键获取主键列表 + ''' + name = '%s_fk:%s'%(self._name,fk) + fkmm = MFKMode(name) + pklist = fkmm.get('pklist') + if pklist is not None: + return pklist + props = {self._fk:fk} + dbkeylist = util.getAllPkByFkInDB(self._name, self._pk, props) + name = '%s_fk:%s'%(self._name,fk) + fkmm = MFKMode(name, pklist = dbkeylist) + fkmm.insert() + return dbkeylist + + def getObj(self,pk): + '''根据主键,可以获得mmode对象的实例.\n + >>> m = madmin.getObj(1) + ''' + mm = MMode(self._name+':%s'%pk,self._pk) + if not mm.IsEffective(): + return None + if mm.get('data'): + return mm + props = {self._pk:pk} + record = util.GetOneRecordInfo(self._name,props) + if not record: + return None + mm = MMode(self._name+':%s'%pk,self._pk,data = record) + mm.insert() + return mm + + def getObjData(self,pk): + '''根据主键,可以获得mmode对象的实例的数据.\n + >>> m = madmin.getObjData(1) + ''' + mm = MMode(self._name+':%s'%pk,self._pk) + if not mm.IsEffective(): + return None + data = mm.get('data') + if mm.get('data'): + return data + props = {self._pk:pk} + record = util.GetOneRecordInfo(self._name,props) + if not record: + return None + mm = MMode(self._name+':%s'%pk,self._pk,data = record) + mm.insert() + return record + + + def getObjList(self,pklist): + '''根据主键列表获取mmode对象的列表.\n + >>> m = madmin.getObjList([1,2,3,4,5]) + ''' + _pklist = [] + objlist = [] + for pk in pklist: + mm = MMode(self._name+':%s'%pk,self._pk) + if not mm.IsEffective(): + continue + if mm.get('data'): + objlist.append(mm) + else: + _pklist.append(pk) + if _pklist: + recordlist = util.GetRecordList(self._name, self._pk,_pklist) + for record in recordlist: + pk = record[self._pk] + mm = MMode(self._name+':%s'%pk,self._pk,data = record) + mm.insert() + objlist.append(mm) + return objlist + + def deleteMode(self,pk): + '''根据主键删除内存中的某条记录信息,\n这里只是修改内存中的记录状态_state为删除状态.\n + >>> m = madmin.deleteMode(1) + ''' + mm = self.getObj(pk) + if mm: + if self._fk: + data = mm.get('data') + if data: + fk = data.get(self._fk,0) + name = '%s_fk:%s'%(self._name,fk) + fkmm = MFKMode(name) + pklist = fkmm.get('pklist') + if pklist and pk in pklist: + pklist.remove(pk) + fkmm.update('pklist', pklist) + mm.delete() + return True + + def checkAll(self): + """同步内存中的数据到对应的数据表中。\n + >>> m = madmin.checkAll() + """ + key = '%s:%s:'%(mclient._hostname,self._name) + _pklist = util.getallkeys(key, mclient.connection) + for pk in _pklist: + mm = MMode(self._name+':%s'%pk,self._pk) + if not mm.IsEffective(): + mm.mdelete() + continue + if not mm.get('data'): + continue + mm.checkSync(timeout=self._timeout) + self.deleteAllFk() + + def deleteAllFk(self): + """删除所有的外键 + """ + key = '%s:%s_fk:'%(mclient._hostname,self._name) + _fklist = util.getallkeys(key, mclient.connection) + for fk in _fklist: + name = '%s_fk:%s'%(self._name,fk) + fkmm = MFKMode(name) + fkmm.mdelete() + + def new(self,data): + """创建一个新的对象 + """ + incrkey = self._incrkey + if incrkey: + incrvalue = self.incr('_incrvalue', 1) + if not incrvalue: + _incrvalue = util.GetTableIncrValue(self._name)+1 + self.update("_incrvalue", incrvalue) + data[incrkey] = incrvalue - 1 + pk = data.get(self._pk) + if pk is None: + raise PKValueError(data) + mm = MMode(self._name+':%s'%pk,self._pk,data=data) + setattr(mm,incrkey,pk) + else: + pk = data.get(self._pk) + mm = MMode(self._name+':%s'%pk,self._pk,data=data) + if self._fk: + fk = data.get(self._fk,0) + name = '%s_fk:%s'%(self._name,fk) + fkmm = MFKMode(name) + pklist = fkmm.get('pklist') + if pklist is None: + pklist = self.getAllPkByFk(fk) + pklist.append(pk) + fkmm.update('pklist', pklist) + setattr(mm,'_state',MMODE_STATE_NEW) + mm.insert() + return mm + diff --git a/firefly/dbentrust/util.py b/firefly/dbentrust/util.py old mode 100644 new mode 100755 index 570e862..00eabbf --- a/firefly/dbentrust/util.py +++ b/firefly/dbentrust/util.py @@ -1,249 +1,249 @@ -#coding:utf8 -''' -Created on 2013-5-8 - -@author: lan (www.9miao.com) -''' - -from dbpool import dbpool -from MySQLdb.cursors import DictCursor -from numbers import Number -from twisted.python import log - - -def forEachPlusInsertProps(tablename,props): - assert type(props) == dict - pkeysstr = str(tuple(props.keys())).replace('\'','`') - pvaluesstr = ["%s,"%val if isinstance(val,Number) else - "'%s',"%str(val).replace("'", "\\'") for val in props.values()] - pvaluesstr = ''.join(pvaluesstr)[:-1] - sqlstr = """INSERT INTO `%s` %s values (%s);"""%(tablename,pkeysstr,pvaluesstr) - return sqlstr - -def FormatCondition(props): - """生成查询条件字符串 - """ - items = props.items() - itemstrlist = [] - for _item in items: - if isinstance(_item[1],Number): - sqlstr = " `%s`=%s AND"%_item - else: - sqlstr = " `%s`='%s' AND "%(_item[0],str(_item[1]).replace("'", "\\'")) - itemstrlist.append(sqlstr) - sqlstr = ''.join(itemstrlist) - return sqlstr[:-4] - -def FormatUpdateStr(props): - """生成更新语句 - """ - items = props.items() - itemstrlist = [] - for _item in items: - if isinstance(_item[1],Number): - sqlstr = " `%s`=%s,"%_item - else: - sqlstr = " `%s`='%s',"%(_item[0],str(_item[1]).replace("'", "\\'")) - itemstrlist.append(sqlstr) - sqlstr = ''.join(itemstrlist) - return sqlstr[:-1] - -def forEachUpdateProps(tablename,props,prere): - '''遍历所要修改的属性,以生成sql语句''' - assert type(props) == dict - pro = FormatUpdateStr(props) - pre = FormatCondition(prere) - sqlstr = """UPDATE `%s` SET %s WHERE %s;"""%(tablename,pro,pre) - return sqlstr - -def EachQueryProps(props): - '''遍历字段列表生成sql语句 - ''' - sqlstr = "" - if props == '*': - return '*' - elif type(props) == type([0]): - for prop in props: - sqlstr = sqlstr + prop +',' - sqlstr = sqlstr[:-1] - return sqlstr - else: - raise Exception('props to query must be dict') - return - -def forEachQueryProps(sqlstr, props): - '''遍历所要查询属性,以生成sql语句''' - if props == '*': - sqlstr += ' *' - elif type(props) == type([0]): - i = 0 - for prop in props: - if(i == 0): - sqlstr += ' ' + prop - else: - sqlstr += ', ' + prop - i += 1 - else: - raise Exception('props to query must be list') - return - return sqlstr - -def GetTableIncrValue(tablename): - """ - """ - database = dbpool.config.get('db') - sql = """SELECT AUTO_INCREMENT FROM information_schema.`TABLES` \ - WHERE TABLE_SCHEMA='%s' AND TABLE_NAME='%s';"""%(database,tablename) - conn = dbpool.connection() - cursor = conn.cursor() - cursor.execute(sql) - result = cursor.fetchone() - cursor.close() - conn.close() - if result: - return result[0] - return result - -def ReadDataFromDB(tablename): - """ - """ - sql = """select * from %s"""%tablename - conn = dbpool.connection() - cursor = conn.cursor(cursorclass = DictCursor) - cursor.execute(sql) - result=cursor.fetchall() - cursor.close() - conn.close() - return result - -def DeleteFromDB(tablename,props): - '''从数据库中删除 - ''' - prers = FormatCondition(props) - sql = """DELETE FROM %s WHERE %s ;"""%(tablename,prers) - conn = dbpool.connection() - cursor = conn.cursor() - count = 0 - try: - count = cursor.execute(sql) - conn.commit() - except Exception,e: - log.err(e) - log.err(sql) - cursor.close() - conn.close() - return bool(count) - -def InsertIntoDB(tablename,data): - """写入数据库 - """ - sql = forEachPlusInsertProps(tablename,data) - conn = dbpool.connection() - cursor = conn.cursor() - count = 0 - try: - count = cursor.execute(sql) - conn.commit() - except Exception,e: - log.err(e) - log.err(sql) - cursor.close() - conn.close() - return bool(count) - -def UpdateWithDict(tablename,props,prere): - """更新记录 - """ - sql = forEachUpdateProps(tablename, props, prere) - conn = dbpool.connection() - cursor = conn.cursor() - count = 0 - try: - count = cursor.execute(sql) - conn.commit() - except Exception,e: - log.err(e) - log.err(sql) - cursor.close() - conn.close() - if(count >= 1): - return True - return False - -def getAllPkByFkInDB(tablename,pkname,props): - """根据所有的外键获取主键ID - """ - props = FormatCondition(props) - sql = """Select `%s` from `%s` where %s"""%(pkname,tablename,props) - conn = dbpool.connection() - cursor = conn.cursor() - cursor.execute(sql) - result = cursor.fetchall() - cursor.close() - conn.close() - return [key[0] for key in result] - -def GetOneRecordInfo(tablename,props): - '''获取单条数据的信息 - ''' - props = FormatCondition(props) - sql = """Select * from `%s` where %s"""%(tablename,props) - conn = dbpool.connection() - cursor = conn.cursor(cursorclass = DictCursor) - cursor.execute(sql) - result = cursor.fetchone() - cursor.close() - conn.close() - return result - -def GetRecordList(tablename,pkname,pklist): - """ - """ - pkliststr = "" - for pkid in pklist: - pkliststr+="%s,"%pkid - pkliststr = "(%s)"%pkliststr[:-1] - sql = """SELECT * FROM `%s` WHERE `%s` IN %s;"""%(tablename,pkname,pkliststr) - conn = dbpool.connection() - cursor = conn.cursor(cursorclass = DictCursor) - cursor.execute(sql) - result = cursor.fetchall() - cursor.close() - conn.close() - return result - -def DBTest(): - sql = """SELECT * FROM tb_item WHERE characterId=1000001;""" - conn = dbpool.connection() - cursor = conn.cursor(cursorclass = DictCursor) - cursor.execute(sql) - result=cursor.fetchall() - cursor.close() - conn.close() - return result - -def getallkeys(key,mem): - itemsinfo = mem.get_stats('items') - itemindex = [] - for items in itemsinfo: - itemindex += [ _key.split(':')[1] for _key in items[1].keys()] - s = set(itemindex) - itemss = [mem.get_stats('cachedump %s 0'%i) for i in s] - allkeys = set([]) - for item in itemss: - for _item in item: - nowlist = set([]) - for _key in _item[1].keys(): - try: - keysplit = _key.split(':') - pk = keysplit[2] - except: - continue - if _key.startswith(key) and not pk.startswith('_'): - nowlist.add(pk) - allkeys = allkeys.union(nowlist) - return allkeys - -def getAllPkByFkInMEM(key,fk,mem): - - pass +#coding:utf8 +''' +Created on 2013-5-8 + +@author: lan (www.9miao.com) +''' + +from dbpool import dbpool +from MySQLdb.cursors import DictCursor +from numbers import Number +from twisted.python import log + + +def forEachPlusInsertProps(tablename,props): + assert type(props) == dict + pkeysstr = str(tuple(props.keys())).replace('\'','`') + pvaluesstr = ["%s,"%val if isinstance(val,Number) else + "'%s',"%str(val).replace("'", "\\'") for val in props.values()] + pvaluesstr = ''.join(pvaluesstr)[:-1] + sqlstr = """INSERT INTO `%s` %s values (%s);"""%(tablename,pkeysstr,pvaluesstr) + return sqlstr + +def FormatCondition(props): + """生成查询条件字符串 + """ + items = props.items() + itemstrlist = [] + for _item in items: + if isinstance(_item[1],Number): + sqlstr = " `%s`=%s AND"%_item + else: + sqlstr = " `%s`='%s' AND "%(_item[0],str(_item[1]).replace("'", "\\'")) + itemstrlist.append(sqlstr) + sqlstr = ''.join(itemstrlist) + return sqlstr[:-4] + +def FormatUpdateStr(props): + """生成更新语句 + """ + items = props.items() + itemstrlist = [] + for _item in items: + if isinstance(_item[1],Number): + sqlstr = " `%s`=%s,"%_item + else: + sqlstr = " `%s`='%s',"%(_item[0],str(_item[1]).replace("'", "\\'")) + itemstrlist.append(sqlstr) + sqlstr = ''.join(itemstrlist) + return sqlstr[:-1] + +def forEachUpdateProps(tablename,props,prere): + '''遍历所要修改的属性,以生成sql语句''' + assert type(props) == dict + pro = FormatUpdateStr(props) + pre = FormatCondition(prere) + sqlstr = """UPDATE `%s` SET %s WHERE %s;"""%(tablename,pro,pre) + return sqlstr + +def EachQueryProps(props): + '''遍历字段列表生成sql语句 + ''' + sqlstr = "" + if props == '*': + return '*' + elif type(props) == type([0]): + for prop in props: + sqlstr = sqlstr + prop +',' + sqlstr = sqlstr[:-1] + return sqlstr + else: + raise Exception('props to query must be dict') + return + +def forEachQueryProps(sqlstr, props): + '''遍历所要查询属性,以生成sql语句''' + if props == '*': + sqlstr += ' *' + elif type(props) == type([0]): + i = 0 + for prop in props: + if(i == 0): + sqlstr += ' ' + prop + else: + sqlstr += ', ' + prop + i += 1 + else: + raise Exception('props to query must be list') + return + return sqlstr + +def GetTableIncrValue(tablename): + """ + """ + database = dbpool.config.get('db') + sql = """SELECT AUTO_INCREMENT FROM information_schema.`TABLES` \ + WHERE TABLE_SCHEMA='%s' AND TABLE_NAME='%s';"""%(database,tablename) + conn = dbpool.connection() + cursor = conn.cursor() + cursor.execute(sql) + result = cursor.fetchone() + cursor.close() + conn.close() + if result: + return result[0] + return result + +def ReadDataFromDB(tablename): + """ + """ + sql = """select * from %s"""%tablename + conn = dbpool.connection() + cursor = conn.cursor(cursorclass = DictCursor) + cursor.execute(sql) + result=cursor.fetchall() + cursor.close() + conn.close() + return result + +def DeleteFromDB(tablename,props): + '''从数据库中删除 + ''' + prers = FormatCondition(props) + sql = """DELETE FROM %s WHERE %s ;"""%(tablename,prers) + conn = dbpool.connection() + cursor = conn.cursor() + count = 0 + try: + count = cursor.execute(sql) + conn.commit() + except Exception,e: + log.err(e) + log.err(sql) + cursor.close() + conn.close() + return bool(count) + +def InsertIntoDB(tablename,data): + """写入数据库 + """ + sql = forEachPlusInsertProps(tablename,data) + conn = dbpool.connection() + cursor = conn.cursor() + count = 0 + try: + count = cursor.execute(sql) + conn.commit() + except Exception,e: + log.err(e) + log.err(sql) + cursor.close() + conn.close() + return bool(count) + +def UpdateWithDict(tablename,props,prere): + """更新记录 + """ + sql = forEachUpdateProps(tablename, props, prere) + conn = dbpool.connection() + cursor = conn.cursor() + count = 0 + try: + count = cursor.execute(sql) + conn.commit() + except Exception,e: + log.err(e) + log.err(sql) + cursor.close() + conn.close() + if(count >= 1): + return True + return False + +def getAllPkByFkInDB(tablename,pkname,props): + """根据所有的外键获取主键ID + """ + props = FormatCondition(props) + sql = """Select `%s` from `%s` where %s"""%(pkname,tablename,props) + conn = dbpool.connection() + cursor = conn.cursor() + cursor.execute(sql) + result = cursor.fetchall() + cursor.close() + conn.close() + return [key[0] for key in result] + +def GetOneRecordInfo(tablename,props): + '''获取单条数据的信息 + ''' + props = FormatCondition(props) + sql = """Select * from `%s` where %s"""%(tablename,props) + conn = dbpool.connection() + cursor = conn.cursor(cursorclass = DictCursor) + cursor.execute(sql) + result = cursor.fetchone() + cursor.close() + conn.close() + return result + +def GetRecordList(tablename,pkname,pklist): + """ + """ + pkliststr = "" + for pkid in pklist: + pkliststr+="%s,"%pkid + pkliststr = "(%s)"%pkliststr[:-1] + sql = """SELECT * FROM `%s` WHERE `%s` IN %s;"""%(tablename,pkname,pkliststr) + conn = dbpool.connection() + cursor = conn.cursor(cursorclass = DictCursor) + cursor.execute(sql) + result = cursor.fetchall() + cursor.close() + conn.close() + return result + +def DBTest(): + sql = """SELECT * FROM tb_item WHERE characterId=1000001;""" + conn = dbpool.connection() + cursor = conn.cursor(cursorclass = DictCursor) + cursor.execute(sql) + result=cursor.fetchall() + cursor.close() + conn.close() + return result + +def getallkeys(key,mem): + itemsinfo = mem.get_stats('items') + itemindex = [] + for items in itemsinfo: + itemindex += [ _key.split(':')[1] for _key in items[1].keys()] + s = set(itemindex) + itemss = [mem.get_stats('cachedump %s 0'%i) for i in s] + allkeys = set([]) + for item in itemss: + for _item in item: + nowlist = set([]) + for _key in _item[1].keys(): + try: + keysplit = _key.split(':') + pk = keysplit[2] + except: + continue + if _key.startswith(key) and not pk.startswith('_'): + nowlist.add(pk) + allkeys = allkeys.union(nowlist) + return allkeys + +def getAllPkByFkInMEM(key,fk,mem): + + pass diff --git a/setup.py b/setup.py index 388106b..2d2a621 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages import sys, os -version = '1.3.3' +version = '1.3.4' setup(name='firefly', version=version,