Add Error Reporting for Transaction Posts & Bug Fixes/Optimizations

lib/services/address/index.js

@@ -13,6 +13,17 @@ var Encoding = require('./encoding');
 var Transform = require('stream').Transform;
 var assert = require('assert');
 var utils = require('../../utils');
+var LRU = require('lru-cache');
+var XXHash = require('xxhash');
+
+
+
+// See rationale about this cache at function getTxList(next)
+const TXID_LIST_CACHE_ITEMS = 250;              // nr of items (this translates to: consecutive 
+                                                // clients downloading their tx history)
+const TXID_LIST_CACHE_EXPIRATION = 1000 * 30;   // ms
+const TXID_LIST_CACHE_MIN = 100;                // Min items to cache
+const TXID_LIST_CACHE_SEED = 0x3233DE;                // Min items to cache
 
 var AddressService = function(options) {
 
@@ -24,6 +35,11 @@ var AddressService = function(options) {
   this._network = this.node.network;
   this._db = this.node.services.db;
   this._mempool = this.node.services.mempool;
+  this._txIdListCache = new LRU({
+    max: TXID_LIST_CACHE_ITEMS,        
+    maxAge: TXID_LIST_CACHE_EXPIRATION 
+  });
+
 
   if (this._network === 'livenet') {
     this._network = 'main';
@@ -49,8 +65,12 @@ AddressService.dependencies = [
 // for example if the query /api/addrs/txs?from=0&to=5&noAsm=1&noScriptSig=1&noSpent=1, and the addresses passed
 // in are [addr1, addr2, addr3], then if addr3 has tx1 at height 10, addr2 has tx2 at height 9 and tx1 has no txs,
 // then I would pass back [tx1, tx2] in that order
+//
+// Instead of passing addresses, with from>0, options.cacheKey can be used to define the address set.
+//
 AddressService.prototype.getAddressHistory = function(addresses, options, callback) {
   var self = this;
+  var cacheUsed = false;
 
   addresses = utils.convertAddressArrayToMostCommonFormat(addresses); // For Litecoin
 
@@ -67,31 +87,75 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
     addresses = [addresses];
   }
 
-  async.eachLimit(addresses, 4, function(address, next) {
 
-    self._getAddressTxidHistory(address, options, next);
+  function getTxList(next) {
 
-  }, function(err) {
 
-    if(err) {
-      return callback(err);
-    }
+    function hashAddresses(addresses) {
 
-    var unique = {};
-    var list = [];
+      // Given there are only TXID_LIST_CACHE_ITEMS ~ 250 items cached at the sametime
+      // a 32 bits hash is secure enough
 
-    for (let i = 0; i < options.txIdList.length; i++) {
-      unique[options.txIdList[i].txid + options.txIdList[i].height] = options.txIdList[i];
-    }
+      return XXHash.hash(Buffer.from(addresses.join('')), TXID_LIST_CACHE_SEED);
+    };
 
-    for (var prop in unique) {
-      list.push(unique[prop]);
+    var calculatedCacheKey;
+
+    // We use the cache ONLY on from > 0 queries.
+    //
+    // Rationale: The a full history is downloaded, the client do
+    // from =0, to=x
+    // then from =x+1 to=y
+    // then [...]
+    // The objective of this cache is to speed up the from>0 queries, and also
+    // "freeze" the txid list during download.
+    //
+    if (options.from >0 ) {
+
+      let cacheKey  = options.cacheKey;
+      if (!cacheKey) {
+         calculatedCacheKey = hashAddresses(addresses);
+         cacheKey =  calculatedCacheKey;
+      }
+
+      var txIdList  = self._txIdListCache.get(cacheKey);
+      if (txIdList) {
+        options.txIdList = txIdList;
+        cacheUsed = true;
+        return next();
+      }
     }
 
-    options.txIdList = list.sort(function(a, b) {
-     return b.height - a.height;
+    // Get the list from the db
+    async.eachLimit(addresses, 4, function(address, next) {
+      self._getAddressTxidHistory(address, options, next);
+    }, function(err) {
+      if (err) return next(err);
+
+      var list = lodash.uniqBy(options.txIdList, function(x) { 
+        return x.txid + x.height;
+      });
+
+
+      options.txIdList = lodash.orderBy(list,['height','txid'], ['desc','asc']);
+
+      if (list.length > TXID_LIST_CACHE_MIN) {
+        calculatedCacheKey  = calculatedCacheKey || hashAddresses(addresses);
+
+        self._txIdListCache.set(calculatedCacheKey, options.txIdList);
+      }
+
+      return next();
     });
 
+  };
+
+
+  getTxList(function(err) {
+    if(err) {
+      return callback(err);
+    }
+
     self._getAddressTxHistory(options, function(err, txList) {
 
       if (err) {
@@ -100,10 +164,11 @@ AddressService.prototype.getAddressHistory = function(addresses, options, callba
 
       var results = {
         totalCount: options.txIdList.length || 0,
-        items: txList
+        items: txList,
       };
 
-      callback(null, results);
+      // cacheUsed is returned for testing
+      callback(null, results, cacheUsed);
 
     });
   });
@@ -426,14 +491,13 @@ AddressService.prototype._getAddressTxHistory = function(options, callback) {
       });
     }
 
-    self._transaction.getTransaction(id.txid, next);
+    self._transaction.getDetailedTransaction(id.txid, options, next);
 
   }, callback);
 
 };
 
 AddressService.prototype._getAddressTxidHistory = function(address, options, callback) {
-
   var self = this;
 
   options = options || {};

lib/services/header/index.js

@@ -573,11 +573,15 @@ HeaderService.prototype._startBlockSubscription = function() {
 };
 
 HeaderService.prototype._syncComplete = function() {
+  // Check if we have reached the last block that we can download.
+  var bestHeight = Math.max(this._bestHeight, this._lastHeader.height);
+
+  var syncComplete = bestHeight === this._tip.height
 
-  // we always ask for the max number of headers, which is 2000.
-  // so any response with < 2000 means we have reached the end of the headers list.
-  // we could make an extra call if the number of total headers is multiple of 2000.
-  return this._lastHeaderCount < 2000;
+  if (syncComplete && this._initialSync)
+    log.info("Header Service: Sync Complete!")
+
+  return syncComplete
 
 };
 

lib/services/p2p/index.js

@@ -10,6 +10,7 @@ var assert = require('assert');
 var Bcoin = require('./bcoin'); // !!!: DO not change this to lcoin
 var BcoinTx = require('lcoin').tx;
 var Networks = require('litecore-lib').Networks;
+var BitcoreRPC = require('litecoind-rpc');
 var LRU = require('lru-cache');
 var constants = require('../../constants');
 
@@ -22,6 +23,7 @@ var P2P = function(options) {
   BaseService.call(this, options);
   this._options = options;
 
+  this._initRPC(options);
   this._initP2P();
   this._initPubSub();
   this._bcoin = null;
@@ -129,26 +131,7 @@ P2P.prototype.getPublishEvents = function() {
 
 
 P2P.prototype.sendTransaction = function(tx, callback) {
-  var peer = this._getPeer();
-
-  var bcoinTx;
-  try {
-    bcoinTx = BcoinTx.fromRaw(tx, 'hex');
-  } catch(e) {
-    return callback(e);
-  }
-
-  log.info('P2P Service: sending transaction: ' + bcoinTx.txid());
-
-  this._outgoingTxs.set(bcoinTx.txid(), bcoinTx);
-  var inv = p2p.Inventory.forTransaction(bcoinTx.txid(), tx.hasWitness());
-  var txMessage = this.messages.Inventory([inv]);
-
-  peer.sendMessage(txMessage);
-
-  this._onPeerTx(peer, { transaction: bcoinTx });
-
-  return callback(null, bcoinTx.txid());
+  return this._client.sendRawTransaction(tx, callback);
 };
 
 
@@ -270,6 +253,17 @@ P2P.prototype._initCache = function() {
   this._inv = LRU(1000);
 };
 
+P2P.prototype._initRPC = function (options) {
+this._config = options.rpc || {
+    user: 'litecoin',
+    pass: 'local321',
+    host: 'localhost',
+    protocol: 'http',
+    port: 9332
+  };
+  this._client = new BitcoreRPC(this._config);
+}
+
 P2P.prototype._initP2P = function() {
   this._maxPeers = this._options.maxPeers || 60;
   this._minPeers = this._options.minPeers || 0;

lib/services/transaction/index.js

@@ -16,7 +16,6 @@ function TransactionService(options) {
   this._header = this.node.services.header;
   this._p2p = this.node.services.p2p;
   this._timestamp = this.node.services.timestamp;
-  this._address = this.node.services.address;
   this._network = this.node.network;
 
   if (this._network === 'livenet') {
@@ -131,7 +130,8 @@ TransactionService.prototype.getTransaction = function(txid, options, callback)
     if (err) {
       return callback(err);
     }
-    if (tx) {
+    // Only add to cache if there are at least 3 confirmations, this prevents it getting stuck showing 0 confirmations
+    if (tx && tx.confirmations >= 3) {
       self._cacheTx.set(txid, tx);
     }
     callback(err, tx);